@rent-scraper/scrape-listings 1.0.30 → 1.0.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md
CHANGED
|
@@ -1,3 +1,50 @@
|
|
|
1
1
|
# @rent-scraper/scrape-listings
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Core scraping orchestrator. Fetches listing results by zip code, scrapes listing detail HTML, and exports everything to CSV.
|
|
4
|
+
|
|
5
|
+
## Usage
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
# Start (production build)
|
|
9
|
+
pnpm run scrape-listings
|
|
10
|
+
|
|
11
|
+
# Start (dev, no build)
|
|
12
|
+
pnpm run scrape-listings:dev
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
The browser server must already be running before calling this.
|
|
16
|
+
|
|
17
|
+
## Scraping Phases (Zillow)
|
|
18
|
+
|
|
19
|
+
1. **Cookie refresh** — opens Zillow in the browser to get a fresh session cookie before any fetches
|
|
20
|
+
2. **Results** — fetches listing search results JSON for each zip code using the captured cookie
|
|
21
|
+
3. **HTML** — fetches the detail page HTML for each valid listing
|
|
22
|
+
4. **Parse** — extracts structured listing data from the HTML
|
|
23
|
+
5. **CSV export** — writes all listings to a single timestamped CSV file
|
|
24
|
+
|
|
25
|
+
## CLI Flags
|
|
26
|
+
|
|
27
|
+
| Flag | Default | Description |
|
|
28
|
+
| --------------- | -------- | --------------------------------------------------------------------- |
|
|
29
|
+
| `--source` | `zillow` | `zillow` or `redfin` |
|
|
30
|
+
| `--days-listed` | `1` | Listings posted within N days |
|
|
31
|
+
| `--runs` | `1` | Number of full scrape passes |
|
|
32
|
+
| `--reruns` | `0` | Retries per failed zip code fetch |
|
|
33
|
+
| `--timeout-ms` | `60000` | Per-request timeout in milliseconds |
|
|
34
|
+
| `--limit` | _(none)_ | Only scrape first N zip codes |
|
|
35
|
+
| `--offset` | `0` | Skip first N zip codes |
|
|
36
|
+
| `--retry` | `false` | Retry hard bot-filtered (403) zip codes inline |
|
|
37
|
+
| `--rerun` | _(none)_ | Resume from a previous run's timestamp, or `true` for the most recent |
|
|
38
|
+
|
|
39
|
+
## Output
|
|
40
|
+
|
|
41
|
+
Written under `outputPath` from config:
|
|
42
|
+
|
|
43
|
+
```
|
|
44
|
+
outputPath/
|
|
45
|
+
zillow/
|
|
46
|
+
results/YYYY-MM-DD-HHmm/ # raw JSON per zip code
|
|
47
|
+
listings/YYYY-MM-DD-HHmm/ # parsed HTML per listing
|
|
48
|
+
csv/YYYY-MM-DD-HHmm.csv # final export
|
|
49
|
+
logs/ # scraping summaries and error zip codes
|
|
50
|
+
```
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{parseError as t}from"@rent-scraper/utils";import{r as i}from"../shared/scrape-listings.
|
|
1
|
+
import{parseError as t}from"@rent-scraper/utils";import{r as i}from"../shared/scrape-listings.DgphD-7P.mjs";import{log as m}from"@clack/prompts";import"minimist";import"dayjs";import"path";import"axios";import"fs/promises";import"@rent-scraper/api";import"@rent-scraper/api/config";import"node:timers/promises";import"crypto";import"@rent-scraper/utils/config";import"picocolors";i().then(()=>{process.exit(0)}).catch(r=>{const{message:o}=t(r);m.error(o),process.exit(1)});
|
package/dist/index.d.mts
CHANGED
|
@@ -17,7 +17,10 @@ declare const scrapeListingHtmlByZipCodesAndListingDetails: (source: ListingsSou
|
|
|
17
17
|
declare const scrapeListingHtmlByInputDirectory: (source: ListingsSource, inputDirectory: string, outputDirectory?: string, options?: Pick<ScrapeListingHtmlOptions, "skipBotCheck">) => Promise<void>;
|
|
18
18
|
declare const scrapeListingHtmlByIds: (source: ListingsSource, ids: string[], outputDirectory: string, options?: Pick<ScrapeListingHtmlOptions, "skipBotCheck">) => Promise<void>;
|
|
19
19
|
|
|
20
|
-
declare function runScrapeListings(): Promise<
|
|
20
|
+
declare function runScrapeListings(): Promise<{
|
|
21
|
+
success: boolean;
|
|
22
|
+
totalCsvListings: number;
|
|
23
|
+
}>;
|
|
21
24
|
|
|
22
25
|
declare const scrapeZillowListingsToCsv: (listingsDirectory: string, outputFilePath: string) => Promise<number>;
|
|
23
26
|
|
package/dist/index.mjs
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export{f as fetchListingHtmlByUrlAndExport,r as runScrapeListings,s as scrapeListingDetailsFromHtmlByFilePaths,a as scrapeListingDetailsFromHtmlByZipCodes,e as scrapeListingHtmlByIds,d as scrapeListingHtmlByInputDirectory,b as scrapeListingHtmlByZipCodes,c as scrapeListingHtmlByZipCodesAndListingDetails,h as scrapeRedfinListingResultsByZipCodes,i as scrapeZillowListingResultsByZipCodes,g as scrapeZillowListingsToCsv}from"./shared/scrape-listings.
|
|
1
|
+
export{f as fetchListingHtmlByUrlAndExport,r as runScrapeListings,s as scrapeListingDetailsFromHtmlByFilePaths,a as scrapeListingDetailsFromHtmlByZipCodes,e as scrapeListingHtmlByIds,d as scrapeListingHtmlByInputDirectory,b as scrapeListingHtmlByZipCodes,c as scrapeListingHtmlByZipCodesAndListingDetails,h as scrapeRedfinListingResultsByZipCodes,i as scrapeZillowListingResultsByZipCodes,g as scrapeZillowListingsToCsv}from"./shared/scrape-listings.DgphD-7P.mjs";import"minimist";import"dayjs";import"path";import"axios";import"fs/promises";import"@rent-scraper/api";import"@rent-scraper/utils";import"@rent-scraper/api/config";import"@clack/prompts";import"node:timers/promises";import"crypto";import"@rent-scraper/utils/config";import"picocolors";
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import kt from"minimist";import lt from"dayjs";import o from"path";import ct from"axios";import{mkdir as S,writeFile as M,readFile as W,unlink as X,readdir as et}from"fs/promises";import{checkForZillowBotFiltering as q,isZillowBotFiltering as dt,fetchHtmlFromRedfinListingUrl as Bt,fetchHtmlFromZillowListingUrl as Mt,scrapeDataFromRedfinListingHtml as Dt,scrapeDataFromZillowListingHtml as Ft,getZillowListingResults as Rt,getRedfinListingResults as Pt,waitForZillowCaptchaSolve as Ut,waitForSolvedZillowCaptcha as ut,getZillowForceCaptcha as Ht,isBrowserShowingCaptcha as Nt}from"@rent-scraper/api";import{ErrorLog as O,throwError as Z,checkForFile as L,parseJsonFile as Ot,parseError as R,readFilesInDirectory as K,roundValue as st,chunkArray as Tt,parsePercentage as It,compareArrays as wt}from"@rent-scraper/utils";import{getZillowOutputPath as T,getRedfinOutputPath as I,getZillowDaysListed as Et,getRedfinDaysListed as At,getZillowLimit as Yt,getZillowOffset as qt,getRedfinZipCodes as Jt,getZillowZipCodes as Vt}from"@rent-scraper/api/config";import{spinner as pt,log as u,progress as Gt,intro as Wt}from"@clack/prompts";import{setTimeout as gt}from"node:timers/promises";import{createHash as Xt}from"crypto";import{checkBrowserServer as mt}from"@rent-scraper/utils/config";import Kt from"picocolors";const P=process.env.DEBUG,E=async(t,e,s,i)=>{const{timeoutMs:r}=i??{};if(await L(s))P&&u.warn(`${s} exists, skipping`);else try{P&&u.info(`writing ${s}`);const n=t==="redfin"?await Bt(e):await Mt(e,{timeoutMs:r});await M(`${s}`,n)}catch(n){Z(`error fetching html for ${e}`,n)}},Qt=async(t,{timeoutMs:e})=>{if(await L(t)){const s=JSON.parse(await W(t,"utf8"))||{},{hdpUrl:i}=s||{};if(i){const r=`https://www.zillow.com${i}`,n=t.replace(".json",".html");await L(n)?P&&u.warn(`file already exists, ${n}`):await E("zillow",r,n,{timeoutMs:e})}else Z(`file is empty, ${t}`)}else P&&u.warn(`file does not exist at this path, ${t}, skipping`)},ht=async(t,e)=>{const{timeoutMs:s}=e??{};await Promise.all(t.map(async i=>await Qt(i,{timeoutMs:s})))},te=(t,e="zillow")=>{if(e==="zillow"){const{zpid:s,detailUrl:i}=t||{},r=i&&!i.startsWith("http")?`https://www.zillow.com${i}`:i;return{id:s,url:r}}else if(e==="redfin"){const{propertyId:s,url:i}=t?.homeData||{};return{id:s,url:`https://www.redfin.com${i}`}}},it=async(t,e,s,i=s,r)=>{const{timeoutMs:n,run:a=1,reruns:l=0,skipBotCheck:f=!1}=r??{},c=new O;s||Z("inputDirectory is required"),t==="zillow"&&!f&&await q();const w=[],y=[],d=pt();d.start("Downloading listings html files");for(let g=1;g<=l+1;g++){l>0&&g>1&&c.add(`rerun ${g-1} of ${l}`);const m=w.length;(g===1||m)&&await Promise.all((m?w:e).map(async v=>{const b=`${s}/${v}.json`;if(await L(b))try{const{results:x}=await Ot(b)||{};if(!x)c.add(`empty file, ${b}`);else{const j=i?`${i}/${v}`:`${s}/${v}`;await S(j,{recursive:!0}),x?.length?await Promise.all(x.map(async k=>{const{id:B,url:z}=te(k,t)??{};if(!z)return c.add(`url missing for ${B}`);const F=`${B}.html`,C=`${j}/${F}`;try{await E(t,z,C,{timeoutMs:n})}catch(N){const{status:A,message:U}=R(N);dt(A,U)||y.push({url:z,filePath:C}),c.add("scrape listing html error: "+(U??`error fetching listing for id, ${N}`))}})):c.add(`no results for file, ${b}`)}}catch(x){w.push(v);const{message:j}=R(x);c.add("scrape listing html error: "+(j??`error reading json data, ${b}, ${x}`))}else P&&u.warn(`file does not exist, ${b}, skipping`)}))}y.length>0&&(await gt(2e3),await Promise.all(y.map(async({url:g,filePath:m})=>{try{await E(t,g,m,{timeoutMs:n})}catch(v){const{message:b}=R(v);c.add("scrape listing html retry error: "+(b??`error fetching ${g}`))}})));const $=await T(),h=await I(),p=t==="zillow"?$:h;e.length>0?(d.stop("Listings HTML files have been saved to:"),u.message(o.join(p,t,"listings",o.basename(s)))):d.stop("No listings HTML files saved.");const _=o.join(p,t,"logs");if(await S(_,{recursive:!0}),c.get().filter(g=>!g.includes("rerun ")).length>l){const g=`${o.basename(s)}-html-errors-${a}.txt`,m=o.join(_,g);await c.write(m,[...new Set(c.get())].join(`
|
|
2
|
+
`)),P&&u.error(`There were errors during processing, see ${o.resolve(m)}`)}},ee=async(t,e,s,i)=>{const{timeoutMs:r,run:n=1,reruns:a=0,skipBotCheck:l=!1}=i??{},f=new O;s||Z("inputDirectory is required"),t==="zillow"&&!l&&await q();const c=[];for(let h=1;h<=a+1;h++){a>0&&h>1&&f.add(`rerun ${h-1} of ${a}`);const p=c.length;(h===1||p)&&await Promise.all((p?c:e).map(async _=>{const g=`${s}/${_}`;if(await L(g))try{const m=await K(g,{extension:".json",prependDirectory:!0});await ht(m,{timeoutMs:r})}catch(m){c.push(_);const{message:v}=R(m);f.add("scrape listing html error: "+(v??`Error during fetch for ${_}, ${m}`))}else f.add(`listing directory does not exist, ${g}`)}))}const w=await T(),y=await I(),d=t==="zillow"?w:y,$=o.join(d,t,"logs");if(await S($,{recursive:!0}),f.get().filter(h=>!h.includes("rerun ")).length>a){const h=`${o.basename(s)}-html-errors-${n}.txt`,p=o.join($,h);await f.write(p,[...new Set(f.get())].join(`
|
|
3
|
+
`)),P&&u.error(`There were errors during processing, see ${o.resolve(p)}`)}},se=async(t,e,s=e,i)=>{const{skipBotCheck:r=!1}=i??{},n=new O;if(t==="zillow"&&!r&&await q(),e||Z("inputDirectory is required"),await S(s,{recursive:!0}),await L(e))try{const d=await K(e,{extension:".json",prependDirectory:!0});await ht(d)}catch(d){const{message:$}=R(d);n.add("scrape listing html error: "+($??`Error during fetch for ${e}, ${d}`))}else n.add(`inputDirectory does not exist, ${e}`);const a=await T(),l=await I(),f=t==="zillow"?a:l,c=o.join(f,t,"logs");await S(c,{recursive:!0});const w=`${o.basename(e)}-html-errors.txt`,y=o.join(c,w);n.get().length>0&&(await n.write(y,[...new Set(n.get())].join(`
|
|
4
|
+
`)),P&&u.error(`There were errors during processing, see ${o.resolve(y)}`))},ie=(t,e)=>t==="redfin"?`https://www.redfin.com/home/${e}`:t==="zillow"?`https://www.zillow.com/homedetails/${e}_zpid`:null,re=async(t,e,s,i)=>{const{skipBotCheck:r=!1}=i??{};t==="zillow"&&!r&&await q(),s||Z("outputDirectory is required"),await S(s,{recursive:!0}),await Promise.all(e.map(async n=>{const a=ie(t,n),l=`${s}/${n}.html`;a&&await E(t,a,l)}))},D=process.env.DEBUG,ft=async t=>{if(await L(t)){const e=t.replace(".html",".json");if(await L(e))D&&u.warning(`file already exists, ${e}`);else{D&&u.message(`scraping data for ${t}`);const s=(await W(t)).toString();(!s||s.trim()==="")&&(await X(t),Z(`empty file found at ${t}, deleted for retry`)),s.includes("px-captcha")&&(await X(t),Z(`captcha page found in ${t}, deleted for retry`));const i=await Ft(s);if(!i?.priceHistory&&i?.bestMatchedUnit?.hdpUrl){D&&u.warning(`rescraping ${t} - https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`),await L(t)&&(D&&u.warning(`deleting ${t}`),await X(t)),await L(e)&&(D&&u.warning(`deleting ${e}`),await X(e));const r=`https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`;await E("zillow",r,t),await ft(t)}else i?(D&&u.info(`writing ${e}`),await M(`${e}`,JSON.stringify(i))):Z(`problem scraping data for ${t}`)}}},ae=async t=>{if(await L(t)){const e=t.replace(".html",".json");if(await L(e))D&&u.warning(`file already exists, ${e}`);else{D&&u.message(`scraping data for ${t}`);const s=(await W(t)).toString(),i=Dt(s);i?(D&&u.warning(`writing ${e}`),await M(`${e}`,JSON.stringify(i))):Z(`problem scraping data for ${t}`)}}},yt=async(t,e,s)=>(await Promise.all(e.map(async i=>{try{t==="redfin"?await ae(i):await ft(i);const r=i.replace(".html",".json");return await L(r)?1:0}catch(r){const{message:n}=R(r);return s?.add(n),0}}))).reduce((i,r)=>i+r,0),rt=async(t,e,s)=>{const i=new O;let r=0;s||Z("inputDirectory is required");const n=Gt({style:"heavy",max:100,size:50});n.start("Scraping listings data");const a=e.length,l=a<20?a+1:20,f=a>l?st(a/l,1):a,c=Tt(e,f);for(const[h,p]of c.entries()){const _=st((Number(h)+1)/l*100,1);n.advance(st(1/l*100,1),`Scraping listings (${It(_)})`),await Promise.all(p.map(async g=>{D&&u.message(`processing files for ${g}`);const m=`${s}/${g}`;if(await L(m)){const v=await K(m,{extension:".html",prependDirectory:!0}),b=await yt(t,v,i);r=r+b}else i.add(`listing directory does not exist, ${m}`)}))}const w=await T(),y=await I(),d=t==="zillow"?w:y;r>0?(n.stop("Listings data has been saved to:"),u.message(o.join(d,t,"listings",o.basename(s)))):n.stop("No listings data saved.");const $=o.join(d,t,"logs");if(await S($,{recursive:!0}),i.get().length>0){const h=`${o.basename(s)}-listing-errors.txt`,p=o.join($,h);await i.write(p,[...new Set(i.get())].join(`
|
|
5
|
+
`)),u.error(`There were errors during processing, see ${o.resolve(p)}`)}return{numListings:r}},at=process.env.DEBUG,ne=async(t,e,s,i,r)=>{const{daysOnZillow:n,timeoutMs:a}=r??{};if(await L(e))at&&u.warning(`${t} exists, skipping`),s.push(t);else{const l=await Rt({zipCode:t,daysOnZillow:n,mergePageResults:!0,timeoutMs:a});l?(at&&u.info(`writing ${e}`),await M(e,JSON.stringify(l)),s.push(t)):(at&&u.warning(`no results for ${t}`),i.push(t))}},nt=async(t,e,s)=>{const{daysListed:i,timeoutMs:r,run:n=1,reruns:a=0,fetchListings:l=!1,skipBotCheck:f=!1,silent:c=!1}=s??{},w=new O;f||await q({fetchListings:l});const y=[],d=[],$=[],h=[];await S(e,{recursive:!0});const p=pt();c||p.start("Scraping Zillow search results");for(let m=1;m<=a+1;m++){a>0&&m>1&&w.add(`rerun ${m-1} of ${a}`);const v=d.length;(m===1||v)&&await Promise.all((v?d:t).map(async b=>{const x=`${b}.json`,j=`${e}/${x}`;try{await ne(b,j,y,h,{daysOnZillow:i,timeoutMs:r})}catch(k){d.push(b);const{status:B,message:z}=R(k);dt(B,z)&&$.push(b),w.add("scrape listing results error: "+(z??`Error during fetch for ${b}, ${k}`))}}))}const _=await T();c||(y.length>0?(p.stop("Zillow search results have been saved to:"),u.message(o.join(_,"zillow","results",o.basename(e)))):p.stop("No results saved."));const g=o.join(_,"zillow","logs");if(await S(g,{recursive:!0}),w.get().filter(m=>!m.includes("rerun ")).length>a){const m=`${o.basename(e)}-results-errors-${n}.txt`,v=o.join(g,m);await w.write(v,[...new Set(w.get())].join(`
|
|
6
|
+
`)),c||u.error(`There were errors during processing, see ${o.resolve(v)}`)}return{validZipCodes:y,botFilteredZipCodes:$,noResultsZipCodes:h}},oe=async(t,e,s,i)=>{const{daysListed:r,timeoutMs:n}=i??{};if(await L(e))console.log(`${t} exists, skipping`),s.push(t);else{const a=await Pt({zipCode:t,daysListed:r,timeoutMs:n});a?(console.log(`writing ${e}`),await M(e,JSON.stringify(a)),s.push(t)):console.log(`no results for ${t}`)}},$t=async(t,e,s)=>{const{daysListed:i,timeoutMs:r,run:n=1,reruns:a=0}=s??{},l=new O,f=[],c=[];await S(e,{recursive:!0});for(let d=1;d<=a+1;d++){a>0&&d>1&&l.add(`rerun ${d-1} of ${a}`);const $=c.length;(d===1||$)&&await Promise.all(($?c:t).map(async h=>{const p=`${h}.json`,_=`${e}/${p}`;try{await oe(h,_,f,{daysListed:i,timeoutMs:r})}catch(g){c.push(h);const{message:m}=R(g);l.add("scrape listing results error: "+(m??`Error during fetch for ${h}, ${g}`))}}))}const w=await I(),y=o.join(w,"redfin","logs");if(await S(y,{recursive:!0}),l.get().filter(d=>!d.includes("rerun ")).length>a){const d=`${o.basename(e)}-results-errors-${n}.txt`,$=o.join(y,d);await l.write($,[...new Set(l.get())].join(`
|
|
7
|
+
`)),console.log(`\x1B[41m
|
|
8
|
+
%s\x1B[0m`,`There were errors during processing, see ${o.resolve($)}`)}return{validZipCodes:f}},_t=["listing_id","listing_source","source_listing_id","parcel_number","listing_url","street_address","cleaned_address","cleaned_unit_number","city","state","zipcode","county","neighborhood_region","is_undisclosed_address","home_status","home_type","bedrooms","year_built","living_area","living_area_units","is_income_restricted","price_at_source","platform_rent_estimate","agent_name","agent_phone_number","broker_name","broker_phone_number","is_owned_by_listing_platform","date_last_updated_at_source","date_scraped","scrape_job_name"],le=t=>{if(t==null)return"";const e=typeof t=="object"?JSON.stringify(t):String(t);return e.includes(",")||e.includes('"')||e.includes(`
|
|
9
|
+
`)?`"${e.replace(/"/g,'""')}"`:e},ce=t=>{const e=/^(.*?)\s+((?:#|APT|UNIT|STE|Suite|Apt|Unit)\s*.+)$/i.exec(t);if(e){const s=e[1].trim().toUpperCase();let i=e[2].trim();return i.startsWith("#")&&(i=`# ${i.replace(/^#\s*/,"").trim()}`),{cleaned:s,unit:i}}return{cleaned:t.toUpperCase(),unit:""}},de=(t,e)=>{const s=String(t.zpid),i=Xt("md5").update(s).digest("hex"),r=t.streetAddress||t.address?.streetAddress||"",{cleaned:n,unit:a}=ce(r),l=t.hdpUrl?`https://www.zillow.com${t.hdpUrl}`:`https://www.zillow.com/homedetails/${s}_zpid/`;return{listing_id:i,listing_source:"zillow",source_listing_id:s,parcel_number:t.resoFacts?.parcelNumber??"",listing_url:l,street_address:r,cleaned_address:n,cleaned_unit_number:a,city:t.city||t.address?.city||"",state:t.state||t.address?.state||"",zipcode:t.zipcode||t.address?.zipcode||"",county:t.county||"",neighborhood_region:t.neighborhoodRegion?.name||t.parentRegion?.name||"",is_undisclosed_address:t.isUndisclosedAddress??"",home_status:t.homeStatus||"",home_type:t.homeType||"",bedrooms:t.bedrooms??"",year_built:t.yearBuilt||t.resoFacts?.yearBuilt||"",living_area:t.livingAreaValue??"",living_area_units:t.livingAreaUnits||"Square Feet",is_income_restricted:t.isIncomeRestricted??"",price_at_source:t.price??"",platform_rent_estimate:t.rentZestimate??"",agent_name:t.attributionInfo?.agentName||"",agent_phone_number:t.attributionInfo?.agentPhoneNumber||"",broker_name:t.attributionInfo?.brokerName||"",broker_phone_number:t.attributionInfo?.brokerPhoneNumber||"",is_owned_by_listing_platform:t.attributionInfo?.mlsName==="Zillow Rentals",date_last_updated_at_source:t.attributionInfo?.lastUpdated||"",date_scraped:t.timestamp?lt(t.timestamp).format("YYYY-MM-DD HH:mm:ss"):"",scrape_job_name:e}},bt=async(t,e)=>{const s=o.basename(t),i=[_t.join(",")];let r;try{r=await et(t,{withFileTypes:!0})}catch{return 0}const n=r.filter(a=>a.isDirectory()).map(a=>a.name);for(const a of n){const l=o.join(t,a);if(!await L(l))continue;const f=await K(l,{extension:".json",prependDirectory:!0});for(const c of f)try{const w=JSON.parse(await W(c,"utf8"));if(!w.zpid)continue;const y=de(w,s);i.push(_t.map(d=>le(y[d])).join(","))}catch{}}return await S(o.dirname(e),{recursive:!0}),await M(e,i.join(`
|
|
10
|
+
`)),i.length-1},ue=async t=>{try{const e=(await et(t,{withFileTypes:!0})).filter(s=>s.isDirectory()).map(s=>s.name).sort();return e.length>0?e[e.length-1]:null}catch{return null}},vt=async()=>{await ct.post("http://localhost:8082/browser/close")},we=async()=>{await ct.post("http://localhost:8082/server/shutdown")},pe=async(t,e,s,{daysListed:i,timeoutMs:r,run:n,reruns:a,preValidatedZipCodes:l=[],retry:f=!1})=>{let c=[],w=[],y=[];if(t.length>0){if(await(await Ht()?Ut():ut()),await vt(),{validZipCodes:c,botFilteredZipCodes:w,noResultsZipCodes:y}=await nt(t,e,{daysListed:i,timeoutMs:r,run:n,reruns:a,skipBotCheck:!0}),f&&w.length>0){u.warn(`Bot filtering hit ${w.length} of ${t.length} zip code(s) \u2014 will retry after listings are fetched`),await Nt()?(await ut(),await vt()):await gt(3e3);const h=w.length,p=await nt(w,e,{daysListed:i,timeoutMs:r,run:n,reruns:a,skipBotCheck:!0,silent:!0});c=[...c,...p.validZipCodes],w=p.botFilteredZipCodes,y=[...y,...p.noResultsZipCodes];const _=p.validZipCodes.length;u.info(`Inline retry: recovered ${_} of ${h} \xB7 ${w.length} still bot-filtered`)}c.length===0&&t.length>0&&u.warn("No results returned for any zip codes \u2014 possible soft bot filtering. Please try again shortly."),w.length>0&&u.warn(`${w.length} zip code(s) still bot-filtered \u2014 use --rerun to retry`)}const d=[...l,...c];await it("zillow",d,e,s,{timeoutMs:r,run:n,reruns:a,skipBotCheck:!0});const{numListings:$}=await rt("zillow",d,s);return{numListings:$,validZipCodes:d,noResultsZipCodes:y}},ge=async(t,e,s,{daysListed:i,timeoutMs:r,run:n,reruns:a})=>{const{validZipCodes:l}=await $t(t,e,{daysListed:i,timeoutMs:r,run:n,reruns:a});await it("redfin",l,e,s,{timeoutMs:r,run:n,reruns:a});const{numListings:f}=await rt("redfin",l,s);return{numListings:f,validZipCodes:l}};async function me(){Wt(Kt.inverse(" scrape listings "));const t=kt(process.argv.slice(2)),e=t.source??"zillow";e==="zillow"&&(await mt()||Z("Please launch the browser server before scraping."));const s=await T(),i=await I(),r=e==="zillow"?s:i,n=t["days-listed"]??(e==="zillow"?await Et():await At())??1,a=t.runs??1,l=t.reruns??0,f=t["timeout-ms"]??6e4,c=t.limit??await Yt()??void 0,w=t.offset??await qt()??0,y=t.retry??!1,d=t.rerun,$=typeof d=="string"?d:d?await ue(o.join(r,e,"results")):null;d&&!$&&Z("No previous run found to rerun."),$&&u.info(`Rerunning from: ${$}`);const h=$??lt().format("YYYY-MM-DD-HHmm"),p=t["results-directory"]??o.join(r,e,"results",h),_=t["listings-directory"]??o.join(r,e,"listings",h),g=t["logs-directory"]??o.join(r,e,"logs");let m=0;const v=async()=>{(e==="zillow"||e==="redfin")&&await mt()&&await we()};try{if(e==="redfin"){const x=await Jt();for(let j=1;j<=a;j++){x||Z("zip codes required, please run the createConfig script");const k=x,{numListings:B,validZipCodes:z}=await ge(k,p,_,{daysListed:n,timeoutMs:f,run:j,reruns:l});if(j===a){const F=Object.entries({numListings:B}).map(([Q,J])=>`${Q}: ${J}`).join(`
|
|
11
|
+
`),C=wt(k,z).join(`
|
|
12
|
+
`);await S(g,{recursive:!0});const N=`${o.basename(p)}-scraping-results.txt`,A=o.join(g,N),U=`${o.basename(p)}-invalid-zipcodes.txt`,H=o.join(g,U);await M(A,F),await M(H,C)}}}else if(e==="zillow"){const x=await Vt();x||Z("zip codes required, please run the createConfig script");const j=x.slice(w,c?w+c:void 0);let k=j,B=[];if($){const z=await et(p).catch(()=>[]),F=new Set(z.filter(C=>C.endsWith(".json")).map(C=>C.replace(".json","")));B=j.filter(C=>F.has(String(C))),k=j.filter(C=>!F.has(String(C))),k.length>0?u.info(`${B.length} already fetched \xB7 retrying ${k.length} missing`):u.info(`All ${B.length} zip codes already fetched \xB7 proceeding to HTML scraping`)}for(let z=1;z<=a;z++){const{numListings:F,validZipCodes:C,noResultsZipCodes:N}=await pe(k,p,_,{daysListed:n,timeoutMs:f,run:z,reruns:l,preValidatedZipCodes:B,retry:y});if(z===a){const A=o.basename(_),U=o.join(s,"zillow","csv",`${A}.csv`),H=await bt(_,U);m=H,H>0&&u.info(`CSV exported: ${H} listings \u2192 ${U}`);const Q=Object.entries({numListings:F,numCsvListings:H}).map(([Y,St])=>`${Y}: ${St}`).join(`
|
|
13
|
+
`),J=wt(j,C),ot=new Set(N),V=J.filter(Y=>!ot.has(Y)),G=J.filter(Y=>ot.has(Y));await S(g,{recursive:!0});const jt=`${o.basename(p)}-scraping-results.txt`,Lt=o.join(g,jt),zt=`${o.basename(p)}-zipcodes-no-results.txt`,Ct=o.join(g,zt),Zt=`${o.basename(p)}-zipcodes-errored.txt`,xt=o.join(g,Zt);await M(Lt,Q),G.length>0&&await M(Ct,G.join(`
|
|
14
|
+
`)),V.length>0&&await M(xt,V.join(`
|
|
15
|
+
`));const tt=[`${C.length}/${j.length} with results`];G.length>0&&tt.push(`${G.length} no results`),V.length>0&&tt.push(`${V.length} errored`),u.info(`Zip codes: ${tt.join(" \xB7 ")}`),u.info(`Listings: ${F} parsed \xB7 ${H} exported to CSV`)}}}const b=e==="zillow"?m>0:!0;return b?u.success("Scraping complete!"):u.warn("Scraping finished but no listings were exported \u2014 please try again shortly."),{success:b,totalCsvListings:m}}finally{await v()}}export{rt as a,it as b,ee as c,se as d,re as e,E as f,bt as g,$t as h,nt as i,me as r,yt as s};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rent-scraper/scrape-listings",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.32",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": {
|
|
@@ -30,8 +30,8 @@
|
|
|
30
30
|
"dayjs": "^1.11.13",
|
|
31
31
|
"minimist": "^1.2.8",
|
|
32
32
|
"picocolors": "^1.1.1",
|
|
33
|
-
"@rent-scraper/
|
|
34
|
-
"@rent-scraper/
|
|
33
|
+
"@rent-scraper/api": "1.0.32",
|
|
34
|
+
"@rent-scraper/utils": "1.0.32"
|
|
35
35
|
},
|
|
36
36
|
"devDependencies": {
|
|
37
37
|
"@types/minimist": "^1.2.5",
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
import kt from"minimist";import ot from"dayjs";import o from"path";import lt from"axios";import{mkdir as k,writeFile as M,readFile as G,unlink as W,readdir as tt}from"fs/promises";import{checkForZillowBotFiltering as Y,isZillowBotFiltering as ct,fetchHtmlFromRedfinListingUrl as St,fetchHtmlFromZillowListingUrl as Bt,scrapeDataFromRedfinListingHtml as Mt,scrapeDataFromZillowListingHtml as Dt,getZillowListingResults as Ft,getRedfinListingResults as Pt,waitForSolvedZillowCaptcha as dt,isBrowserShowingCaptcha as Rt}from"@rent-scraper/api";import{ErrorLog as T,throwError as z,checkForFile as L,parseJsonFile as Ut,parseError as P,readFilesInDirectory as X,roundValue as et,chunkArray as Ht,parsePercentage as Nt,compareArrays as ut}from"@rent-scraper/utils";import{getZillowOutputPath as O,getRedfinOutputPath as I,getZillowDaysListed as Tt,getRedfinDaysListed as Ot,getZillowLimit as It,getZillowOffset as Et,getRedfinZipCodes as At,getZillowZipCodes as Yt}from"@rent-scraper/api/config";import{spinner as pt,log as u,progress as qt,intro as Jt}from"@clack/prompts";import{setTimeout as wt}from"node:timers/promises";import{createHash as Vt}from"crypto";import{checkBrowserServer as gt}from"@rent-scraper/utils/config";import Gt from"picocolors";const R=process.env.DEBUG,E=async(t,e,s,i)=>{const{timeoutMs:r}=i??{};if(await L(s))R&&u.warn(`${s} exists, skipping`);else try{R&&u.info(`writing ${s}`);const a=t==="redfin"?await St(e):await Bt(e,{timeoutMs:r});await M(`${s}`,a)}catch(a){z(`error fetching html for ${e}`,a)}},Wt=async(t,{timeoutMs:e})=>{if(await L(t)){const s=JSON.parse(await G(t,"utf8"))||{},{hdpUrl:i}=s||{};if(i){const r=`https://www.zillow.com${i}`,a=t.replace(".json",".html");await L(a)?R&&u.warn(`file already exists, ${a}`):await E("zillow",r,a,{timeoutMs:e})}else z(`file is empty, ${t}`)}else R&&u.warn(`file does not exist at this path, ${t}, skipping`)},mt=async(t,e)=>{const{timeoutMs:s}=e??{};await Promise.all(t.map(async i=>await Wt(i,{timeoutMs:s})))},Xt=(t,e="zillow")=>{if(e==="zillow"){const{zpid:s,detailUrl:i}=t||{},r=i&&!i.startsWith("http")?`https://www.zillow.com${i}`:i;return{id:s,url:r}}else if(e==="redfin"){const{propertyId:s,url:i}=t?.homeData||{};return{id:s,url:`https://www.redfin.com${i}`}}},st=async(t,e,s,i=s,r)=>{const{timeoutMs:a,run:n=1,reruns:l=0,skipBotCheck:h=!1}=r??{},c=new T;s||z("inputDirectory is required"),t==="zillow"&&!h&&await Y();const p=[],y=[],d=pt();d.start("Downloading listings html files");for(let g=1;g<=l+1;g++){l>0&&g>1&&c.add(`rerun ${g-1} of ${l}`);const m=p.length;(g===1||m)&&await Promise.all((m?p:e).map(async j=>{const _=`${s}/${j}.json`;if(await L(_))try{const{results:v}=await Ut(_)||{};if(!v)c.add(`empty file, ${_}`);else{const Z=i?`${i}/${j}`:`${s}/${j}`;await k(Z,{recursive:!0}),v?.length?await Promise.all(v.map(async B=>{const{id:S,url:C}=Xt(B,t)??{};if(!C)return c.add(`url missing for ${S}`);const x=`${S}.html`,U=`${Z}/${x}`;try{await E(t,C,U,{timeoutMs:a})}catch(H){const{status:N,message:D}=P(H);ct(N,D)||y.push({url:C,filePath:U}),c.add("scrape listing html error: "+(D??`error fetching listing for id, ${H}`))}})):c.add(`no results for file, ${_}`)}}catch(v){p.push(j);const{message:Z}=P(v);c.add("scrape listing html error: "+(Z??`error reading json data, ${_}, ${v}`))}else R&&u.warn(`file does not exist, ${_}, skipping`)}))}y.length>0&&(await wt(2e3),await Promise.all(y.map(async({url:g,filePath:m})=>{try{await E(t,g,m,{timeoutMs:a})}catch(j){const{message:_}=P(j);c.add("scrape listing html retry error: "+(_??`error fetching ${g}`))}})));const $=await O(),f=await I(),w=t==="zillow"?$:f;e.length>0?(d.stop("Listings HTML files have been saved to:"),u.message(o.join(w,t,"listings",o.basename(s)))):d.stop("No listings HTML files saved.");const b=o.join(w,t,"logs");if(await k(b,{recursive:!0}),c.get().filter(g=>!g.includes("rerun ")).length>l){const g=`${o.basename(s)}-html-errors-${n}.txt`,m=o.join(b,g);await c.write(m,[...new Set(c.get())].join(`
|
|
2
|
-
`)),R&&u.error(`There were errors during processing, see ${o.resolve(m)}`)}},Qt=async(t,e,s,i)=>{const{timeoutMs:r,run:a=1,reruns:n=0,skipBotCheck:l=!1}=i??{},h=new T;s||z("inputDirectory is required"),t==="zillow"&&!l&&await Y();const c=[];for(let f=1;f<=n+1;f++){n>0&&f>1&&h.add(`rerun ${f-1} of ${n}`);const w=c.length;(f===1||w)&&await Promise.all((w?c:e).map(async b=>{const g=`${s}/${b}`;if(await L(g))try{const m=await X(g,{extension:".json",prependDirectory:!0});await mt(m,{timeoutMs:r})}catch(m){c.push(b);const{message:j}=P(m);h.add("scrape listing html error: "+(j??`Error during fetch for ${b}, ${m}`))}else h.add(`listing directory does not exist, ${g}`)}))}const p=await O(),y=await I(),d=t==="zillow"?p:y,$=o.join(d,t,"logs");if(await k($,{recursive:!0}),h.get().filter(f=>!f.includes("rerun ")).length>n){const f=`${o.basename(s)}-html-errors-${a}.txt`,w=o.join($,f);await h.write(w,[...new Set(h.get())].join(`
|
|
3
|
-
`)),R&&u.error(`There were errors during processing, see ${o.resolve(w)}`)}},Kt=async(t,e,s=e,i)=>{const{skipBotCheck:r=!1}=i??{},a=new T;if(t==="zillow"&&!r&&await Y(),e||z("inputDirectory is required"),await k(s,{recursive:!0}),await L(e))try{const d=await X(e,{extension:".json",prependDirectory:!0});await mt(d)}catch(d){const{message:$}=P(d);a.add("scrape listing html error: "+($??`Error during fetch for ${e}, ${d}`))}else a.add(`inputDirectory does not exist, ${e}`);const n=await O(),l=await I(),h=t==="zillow"?n:l,c=o.join(h,t,"logs");await k(c,{recursive:!0});const p=`${o.basename(e)}-html-errors.txt`,y=o.join(c,p);a.get().length>0&&(await a.write(y,[...new Set(a.get())].join(`
|
|
4
|
-
`)),R&&u.error(`There were errors during processing, see ${o.resolve(y)}`))},te=(t,e)=>t==="redfin"?`https://www.redfin.com/home/${e}`:t==="zillow"?`https://www.zillow.com/homedetails/${e}_zpid`:null,ee=async(t,e,s,i)=>{const{skipBotCheck:r=!1}=i??{};t==="zillow"&&!r&&await Y(),s||z("outputDirectory is required"),await k(s,{recursive:!0}),await Promise.all(e.map(async a=>{const n=te(t,a),l=`${s}/${a}.html`;n&&await E(t,n,l)}))},F=process.env.DEBUG,ft=async t=>{if(await L(t)){const e=t.replace(".html",".json");if(await L(e))F&&u.warning(`file already exists, ${e}`);else{F&&u.message(`scraping data for ${t}`);const s=(await G(t)).toString();(!s||s.trim()==="")&&(await W(t),z(`empty file found at ${t}, deleted for retry`)),s.includes("px-captcha")&&(await W(t),z(`captcha page found in ${t}, deleted for retry`));const i=await Dt(s);if(!i?.priceHistory&&i?.bestMatchedUnit?.hdpUrl){F&&u.warning(`rescraping ${t} - https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`),await L(t)&&(F&&u.warning(`deleting ${t}`),await W(t)),await L(e)&&(F&&u.warning(`deleting ${e}`),await W(e));const r=`https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`;await E("zillow",r,t),await ft(t)}else i?(F&&u.info(`writing ${e}`),await M(`${e}`,JSON.stringify(i))):z(`problem scraping data for ${t}`)}}},se=async t=>{if(await L(t)){const e=t.replace(".html",".json");if(await L(e))F&&u.warning(`file already exists, ${e}`);else{F&&u.message(`scraping data for ${t}`);const s=(await G(t)).toString(),i=Mt(s);i?(F&&u.warning(`writing ${e}`),await M(`${e}`,JSON.stringify(i))):z(`problem scraping data for ${t}`)}}},ht=async(t,e,s)=>(await Promise.all(e.map(async i=>{try{t==="redfin"?await se(i):await ft(i);const r=i.replace(".html",".json");return await L(r)?1:0}catch(r){const{message:a}=P(r);return s?.add(a),0}}))).reduce((i,r)=>i+r,0),it=async(t,e,s)=>{const i=new T;let r=0;s||z("inputDirectory is required");const a=qt({style:"heavy",max:100,size:50});a.start("Scraping listings data");const n=e.length,l=n<20?n+1:20,h=n>l?et(n/l,1):n,c=Ht(e,h);for(const[f,w]of c.entries()){const b=et((Number(f)+1)/l*100,1);a.advance(et(1/l*100,1),`Scraping listings (${Nt(b)})`),await Promise.all(w.map(async g=>{F&&u.message(`processing files for ${g}`);const m=`${s}/${g}`;if(await L(m)){const j=await X(m,{extension:".html",prependDirectory:!0}),_=await ht(t,j,i);r=r+_}else i.add(`listing directory does not exist, ${m}`)}))}const p=await O(),y=await I(),d=t==="zillow"?p:y;r>0?(a.stop("Listings data has been saved to:"),u.message(o.join(d,t,"listings",o.basename(s)))):a.stop("No listings data saved.");const $=o.join(d,t,"logs");if(await k($,{recursive:!0}),i.get().length>0){const f=`${o.basename(s)}-listing-errors.txt`,w=o.join($,f);await i.write(w,[...new Set(i.get())].join(`
|
|
5
|
-
`)),u.error(`There were errors during processing, see ${o.resolve(w)}`)}return{numListings:r}},rt=process.env.DEBUG,ie=async(t,e,s,i,r)=>{const{daysOnZillow:a,timeoutMs:n}=r??{};if(await L(e))rt&&u.warning(`${t} exists, skipping`),s.push(t);else{const l=await Ft({zipCode:t,daysOnZillow:a,mergePageResults:!0,timeoutMs:n});l?(rt&&u.info(`writing ${e}`),await M(e,JSON.stringify(l)),s.push(t)):(rt&&u.warning(`no results for ${t}`),i.push(t))}},nt=async(t,e,s)=>{const{daysListed:i,timeoutMs:r,run:a=1,reruns:n=0,fetchListings:l=!1,skipBotCheck:h=!1,silent:c=!1}=s??{},p=new T;h||await Y({fetchListings:l});const y=[],d=[],$=[],f=[];await k(e,{recursive:!0});const w=pt();c||w.start("Scraping Zillow search results");for(let m=1;m<=n+1;m++){n>0&&m>1&&p.add(`rerun ${m-1} of ${n}`);const j=d.length;(m===1||j)&&await Promise.all((j?d:t).map(async _=>{const v=`${_}.json`,Z=`${e}/${v}`;try{await ie(_,Z,y,f,{daysOnZillow:i,timeoutMs:r})}catch(B){d.push(_);const{status:S,message:C}=P(B);ct(S,C)&&$.push(_),p.add("scrape listing results error: "+(C??`Error during fetch for ${_}, ${B}`))}}))}const b=await O();c||(y.length>0?(w.stop("Zillow search results have been saved to:"),u.message(o.join(b,"zillow","results",o.basename(e)))):w.stop("No results saved."));const g=o.join(b,"zillow","logs");if(await k(g,{recursive:!0}),p.get().filter(m=>!m.includes("rerun ")).length>n){const m=`${o.basename(e)}-results-errors-${a}.txt`,j=o.join(g,m);await p.write(j,[...new Set(p.get())].join(`
|
|
6
|
-
`)),c||u.error(`There were errors during processing, see ${o.resolve(j)}`)}return{validZipCodes:y,botFilteredZipCodes:$,noResultsZipCodes:f}},re=async(t,e,s,i)=>{const{daysListed:r,timeoutMs:a}=i??{};if(await L(e))console.log(`${t} exists, skipping`),s.push(t);else{const n=await Pt({zipCode:t,daysListed:r,timeoutMs:a});n?(console.log(`writing ${e}`),await M(e,JSON.stringify(n)),s.push(t)):console.log(`no results for ${t}`)}},yt=async(t,e,s)=>{const{daysListed:i,timeoutMs:r,run:a=1,reruns:n=0}=s??{},l=new T,h=[],c=[];await k(e,{recursive:!0});for(let d=1;d<=n+1;d++){n>0&&d>1&&l.add(`rerun ${d-1} of ${n}`);const $=c.length;(d===1||$)&&await Promise.all(($?c:t).map(async f=>{const w=`${f}.json`,b=`${e}/${w}`;try{await re(f,b,h,{daysListed:i,timeoutMs:r})}catch(g){c.push(f);const{message:m}=P(g);l.add("scrape listing results error: "+(m??`Error during fetch for ${f}, ${g}`))}}))}const p=await I(),y=o.join(p,"redfin","logs");if(await k(y,{recursive:!0}),l.get().filter(d=>!d.includes("rerun ")).length>n){const d=`${o.basename(e)}-results-errors-${a}.txt`,$=o.join(y,d);await l.write($,[...new Set(l.get())].join(`
|
|
7
|
-
`)),console.log(`\x1B[41m
|
|
8
|
-
%s\x1B[0m`,`There were errors during processing, see ${o.resolve($)}`)}return{validZipCodes:h}},$t=["listing_id","listing_source","source_listing_id","parcel_number","listing_url","street_address","cleaned_address","cleaned_unit_number","city","state","zipcode","county","neighborhood_region","is_undisclosed_address","home_status","home_type","bedrooms","year_built","living_area","living_area_units","is_income_restricted","price_at_source","platform_rent_estimate","agent_name","agent_phone_number","broker_name","broker_phone_number","is_owned_by_listing_platform","date_last_updated_at_source","date_scraped","scrape_job_name"],ne=t=>{if(t==null)return"";const e=typeof t=="object"?JSON.stringify(t):String(t);return e.includes(",")||e.includes('"')||e.includes(`
|
|
9
|
-
`)?`"${e.replace(/"/g,'""')}"`:e},ae=t=>{const e=/^(.*?)\s+((?:#|APT|UNIT|STE|Suite|Apt|Unit)\s*.+)$/i.exec(t);if(e){const s=e[1].trim().toUpperCase();let i=e[2].trim();return i.startsWith("#")&&(i=`# ${i.replace(/^#\s*/,"").trim()}`),{cleaned:s,unit:i}}return{cleaned:t.toUpperCase(),unit:""}},oe=(t,e)=>{const s=String(t.zpid),i=Vt("md5").update(s).digest("hex"),r=t.streetAddress||t.address?.streetAddress||"",{cleaned:a,unit:n}=ae(r),l=t.hdpUrl?`https://www.zillow.com${t.hdpUrl}`:`https://www.zillow.com/homedetails/${s}_zpid/`;return{listing_id:i,listing_source:"zillow",source_listing_id:s,parcel_number:t.resoFacts?.parcelNumber??"",listing_url:l,street_address:r,cleaned_address:a,cleaned_unit_number:n,city:t.city||t.address?.city||"",state:t.state||t.address?.state||"",zipcode:t.zipcode||t.address?.zipcode||"",county:t.county||"",neighborhood_region:t.neighborhoodRegion?.name||t.parentRegion?.name||"",is_undisclosed_address:t.isUndisclosedAddress??"",home_status:t.homeStatus||"",home_type:t.homeType||"",bedrooms:t.bedrooms??"",year_built:t.yearBuilt||t.resoFacts?.yearBuilt||"",living_area:t.livingAreaValue??"",living_area_units:t.livingAreaUnits||"Square Feet",is_income_restricted:t.isIncomeRestricted??"",price_at_source:t.price??"",platform_rent_estimate:t.rentZestimate??"",agent_name:t.attributionInfo?.agentName||"",agent_phone_number:t.attributionInfo?.agentPhoneNumber||"",broker_name:t.attributionInfo?.brokerName||"",broker_phone_number:t.attributionInfo?.brokerPhoneNumber||"",is_owned_by_listing_platform:t.attributionInfo?.mlsName==="Zillow Rentals",date_last_updated_at_source:t.attributionInfo?.lastUpdated||"",date_scraped:t.timestamp?ot(t.timestamp).format("YYYY-MM-DD HH:mm:ss"):"",scrape_job_name:e}},_t=async(t,e)=>{const s=o.basename(t),i=[$t.join(",")];let r;try{r=await tt(t,{withFileTypes:!0})}catch{return 0}const a=r.filter(n=>n.isDirectory()).map(n=>n.name);for(const n of a){const l=o.join(t,n);if(!await L(l))continue;const h=await X(l,{extension:".json",prependDirectory:!0});for(const c of h)try{const p=JSON.parse(await G(c,"utf8"));if(!p.zpid)continue;const y=oe(p,s);i.push($t.map(d=>ne(y[d])).join(","))}catch{}}return await k(o.dirname(e),{recursive:!0}),await M(e,i.join(`
|
|
10
|
-
`)),i.length-1},le=async t=>{try{const e=(await tt(t,{withFileTypes:!0})).filter(s=>s.isDirectory()).map(s=>s.name).sort();return e.length>0?e[e.length-1]:null}catch{return null}},bt=async()=>{await lt.post("http://localhost:8082/browser/close")},ce=async()=>{await lt.post("http://localhost:8082/server/shutdown")},de=async(t,e,s,{daysListed:i,timeoutMs:r,run:a,reruns:n,preValidatedZipCodes:l=[],retry:h=!1})=>{let c=[],p=[],y=[];if(t.length>0){if(await dt(),await bt(),{validZipCodes:c,botFilteredZipCodes:p,noResultsZipCodes:y}=await nt(t,e,{daysListed:i,timeoutMs:r,run:a,reruns:n,skipBotCheck:!0}),h&&p.length>0){u.warn(`Bot filtering hit ${p.length} of ${t.length} zip code(s) \u2014 will retry after listings are fetched`),await Rt()?(await dt(),await bt()):await wt(3e3);const f=p.length,w=await nt(p,e,{daysListed:i,timeoutMs:r,run:a,reruns:n,skipBotCheck:!0,silent:!0});c=[...c,...w.validZipCodes],p=w.botFilteredZipCodes,y=[...y,...w.noResultsZipCodes];const b=w.validZipCodes.length;u.info(`Inline retry: recovered ${b} of ${f} \xB7 ${p.length} still bot-filtered`)}c.length===0&&t.length>0&&u.warn("No results returned for any zip codes \u2014 possible soft bot filtering. Please try again shortly."),p.length>0&&u.warn(`${p.length} zip code(s) still bot-filtered \u2014 use --rerun to retry`)}const d=[...l,...c];await st("zillow",d,e,s,{timeoutMs:r,run:a,reruns:n,skipBotCheck:!0});const{numListings:$}=await it("zillow",d,s);return{numListings:$,validZipCodes:d,noResultsZipCodes:y}},ue=async(t,e,s,{daysListed:i,timeoutMs:r,run:a,reruns:n})=>{const{validZipCodes:l}=await yt(t,e,{daysListed:i,timeoutMs:r,run:a,reruns:n});await st("redfin",l,e,s,{timeoutMs:r,run:a,reruns:n});const{numListings:h}=await it("redfin",l,s);return{numListings:h,validZipCodes:l}};async function pe(){Jt(Gt.inverse(" scrape listings "));const t=kt(process.argv.slice(2)),e=t.source??"zillow";e==="zillow"&&(await gt()||z("Please launch the browser server before scraping."));const s=await O(),i=await I(),r=e==="zillow"?s:i,a=t["days-listed"]??(e==="zillow"?await Tt():await Ot())??1,n=t.runs??1,l=t.reruns??0,h=t["timeout-ms"]??6e4,c=t.limit??await It()??void 0,p=t.offset??await Et()??0,y=t.retry??!1,d=t.rerun,$=typeof d=="string"?d:d?await le(o.join(r,e,"results")):null;d&&!$&&z("No previous run found to rerun."),$&&u.info(`Rerunning from: ${$}`);const f=$??ot().format("YYYY-MM-DD-HHmm"),w=t["results-directory"]??o.join(r,e,"results",f),b=t["listings-directory"]??o.join(r,e,"listings",f),g=t["logs-directory"]??o.join(r,e,"logs");let m=0;const j=async()=>{(e==="zillow"||e==="redfin")&&await gt()&&await ce()};try{if(e==="redfin"){const _=await At();for(let v=1;v<=n;v++){_||z("zip codes required, please run the createConfig script");const Z=_,{numListings:B,validZipCodes:S}=await ue(Z,w,b,{daysListed:a,timeoutMs:h,run:v,reruns:l});if(v===n){const C=Object.entries({numListings:B}).map(([Q,q])=>`${Q}: ${q}`).join(`
|
|
11
|
-
`),x=ut(Z,S).join(`
|
|
12
|
-
`);await k(g,{recursive:!0});const U=`${o.basename(w)}-scraping-results.txt`,H=o.join(g,U),N=`${o.basename(w)}-invalid-zipcodes.txt`,D=o.join(g,N);await M(H,C),await M(D,x)}}}else if(e==="zillow"){const _=await Yt();_||z("zip codes required, please run the createConfig script");const v=_.slice(p,c?p+c:void 0);let Z=v,B=[];if($){const S=await tt(w).catch(()=>[]),C=new Set(S.filter(x=>x.endsWith(".json")).map(x=>x.replace(".json","")));B=v.filter(x=>C.has(String(x))),Z=v.filter(x=>!C.has(String(x))),Z.length>0?u.info(`${B.length} already fetched \xB7 retrying ${Z.length} missing`):u.info(`All ${B.length} zip codes already fetched \xB7 proceeding to HTML scraping`)}for(let S=1;S<=n;S++){const{numListings:C,validZipCodes:x,noResultsZipCodes:U}=await de(Z,w,b,{daysListed:a,timeoutMs:h,run:S,reruns:l,preValidatedZipCodes:B,retry:y});if(S===n){const H=o.basename(b),N=o.join(s,"zillow","csv",`${H}.csv`),D=await _t(b,N);m=D,D>0&&u.info(`CSV exported: ${D} listings \u2192 ${N}`);const Q=Object.entries({numListings:C,numCsvListings:D}).map(([A,xt])=>`${A}: ${xt}`).join(`
|
|
13
|
-
`),q=ut(v,x),at=new Set(U),J=q.filter(A=>!at.has(A)),V=q.filter(A=>at.has(A));await k(g,{recursive:!0});const vt=`${o.basename(w)}-scraping-results.txt`,jt=o.join(g,vt),Lt=`${o.basename(w)}-zipcodes-no-results.txt`,zt=o.join(g,Lt),Zt=`${o.basename(w)}-zipcodes-errored.txt`,Ct=o.join(g,Zt);await M(jt,Q),V.length>0&&await M(zt,V.join(`
|
|
14
|
-
`)),J.length>0&&await M(Ct,J.join(`
|
|
15
|
-
`));const K=[`${x.length}/${v.length} with results`];V.length>0&&K.push(`${V.length} no results`),J.length>0&&K.push(`${J.length} errored`),u.info(`Zip codes: ${K.join(" \xB7 ")}`),u.info(`Listings: ${C} parsed \xB7 ${D} exported to CSV`)}}}e!=="zillow"||m>0?u.success("Scraping complete!"):u.warn("Scraping finished but no listings were exported \u2014 please try again shortly.")}finally{await j()}}export{it as a,st as b,Qt as c,Kt as d,ee as e,E as f,_t as g,yt as h,nt as i,pe as r,ht as s};
|