@rent-scraper/scrape-listings 1.0.31 → 1.0.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md
CHANGED
|
@@ -1,3 +1,50 @@
|
|
|
1
1
|
# @rent-scraper/scrape-listings
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Core scraping orchestrator. Fetches listing results by zip code, scrapes listing detail HTML, and exports everything to CSV.
|
|
4
|
+
|
|
5
|
+
## Usage
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
# Start (production build)
|
|
9
|
+
pnpm run scrape-listings
|
|
10
|
+
|
|
11
|
+
# Start (dev, no build)
|
|
12
|
+
pnpm run scrape-listings:dev
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
The browser server must already be running before calling this.
|
|
16
|
+
|
|
17
|
+
## Scraping Phases (Zillow)
|
|
18
|
+
|
|
19
|
+
1. **Cookie refresh** — opens Zillow in the browser to get a fresh session cookie before any fetches
|
|
20
|
+
2. **Results** — fetches listing search results JSON for each zip code using the captured cookie
|
|
21
|
+
3. **HTML** — fetches the detail page HTML for each valid listing
|
|
22
|
+
4. **Parse** — extracts structured listing data from the HTML
|
|
23
|
+
5. **CSV export** — writes all listings to a single timestamped CSV file
|
|
24
|
+
|
|
25
|
+
## CLI Flags
|
|
26
|
+
|
|
27
|
+
| Flag | Default | Description |
|
|
28
|
+
| --------------- | -------- | --------------------------------------------------------------------- |
|
|
29
|
+
| `--source` | `zillow` | `zillow` or `redfin` |
|
|
30
|
+
| `--days-listed` | `1` | Listings posted within N days |
|
|
31
|
+
| `--runs` | `1` | Number of full scrape passes |
|
|
32
|
+
| `--reruns` | `0` | Retries per failed zip code fetch |
|
|
33
|
+
| `--timeout-ms` | `60000` | Per-request timeout in milliseconds |
|
|
34
|
+
| `--limit` | _(none)_ | Only scrape first N zip codes |
|
|
35
|
+
| `--offset` | `0` | Skip first N zip codes |
|
|
36
|
+
| `--retry` | `false` | Retry hard bot-filtered (403) zip codes inline |
|
|
37
|
+
| `--rerun` | _(none)_ | Resume from a previous run's timestamp, or `true` for the most recent |
|
|
38
|
+
|
|
39
|
+
## Output
|
|
40
|
+
|
|
41
|
+
Written under `outputPath` from config:
|
|
42
|
+
|
|
43
|
+
```
|
|
44
|
+
outputPath/
|
|
45
|
+
zillow/
|
|
46
|
+
results/YYYY-MM-DD-HHmm/ # raw JSON per zip code
|
|
47
|
+
listings/YYYY-MM-DD-HHmm/ # parsed HTML per listing
|
|
48
|
+
csv/YYYY-MM-DD-HHmm.csv # final export
|
|
49
|
+
logs/ # scraping summaries and error zip codes
|
|
50
|
+
```
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{parseError as t}from"@rent-scraper/utils";import{r as i}from"../shared/scrape-listings.
|
|
1
|
+
import{parseError as t}from"@rent-scraper/utils";import{r as i}from"../shared/scrape-listings.DgphD-7P.mjs";import{log as m}from"@clack/prompts";import"minimist";import"dayjs";import"path";import"axios";import"fs/promises";import"@rent-scraper/api";import"@rent-scraper/api/config";import"node:timers/promises";import"crypto";import"@rent-scraper/utils/config";import"picocolors";i().then(()=>{process.exit(0)}).catch(r=>{const{message:o}=t(r);m.error(o),process.exit(1)});
|
package/dist/index.mjs
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export{f as fetchListingHtmlByUrlAndExport,r as runScrapeListings,s as scrapeListingDetailsFromHtmlByFilePaths,a as scrapeListingDetailsFromHtmlByZipCodes,e as scrapeListingHtmlByIds,d as scrapeListingHtmlByInputDirectory,b as scrapeListingHtmlByZipCodes,c as scrapeListingHtmlByZipCodesAndListingDetails,h as scrapeRedfinListingResultsByZipCodes,i as scrapeZillowListingResultsByZipCodes,g as scrapeZillowListingsToCsv}from"./shared/scrape-listings.
|
|
1
|
+
export{f as fetchListingHtmlByUrlAndExport,r as runScrapeListings,s as scrapeListingDetailsFromHtmlByFilePaths,a as scrapeListingDetailsFromHtmlByZipCodes,e as scrapeListingHtmlByIds,d as scrapeListingHtmlByInputDirectory,b as scrapeListingHtmlByZipCodes,c as scrapeListingHtmlByZipCodesAndListingDetails,h as scrapeRedfinListingResultsByZipCodes,i as scrapeZillowListingResultsByZipCodes,g as scrapeZillowListingsToCsv}from"./shared/scrape-listings.DgphD-7P.mjs";import"minimist";import"dayjs";import"path";import"axios";import"fs/promises";import"@rent-scraper/api";import"@rent-scraper/utils";import"@rent-scraper/api/config";import"@clack/prompts";import"node:timers/promises";import"crypto";import"@rent-scraper/utils/config";import"picocolors";
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import kt from"minimist";import lt from"dayjs";import o from"path";import ct from"axios";import{mkdir as S,writeFile as M,readFile as W,unlink as X,readdir as et}from"fs/promises";import{checkForZillowBotFiltering as q,isZillowBotFiltering as dt,fetchHtmlFromRedfinListingUrl as Bt,fetchHtmlFromZillowListingUrl as Mt,scrapeDataFromRedfinListingHtml as Dt,scrapeDataFromZillowListingHtml as Ft,getZillowListingResults as Rt,getRedfinListingResults as Pt,waitForZillowCaptchaSolve as Ut,waitForSolvedZillowCaptcha as ut,getZillowForceCaptcha as Ht,isBrowserShowingCaptcha as Nt}from"@rent-scraper/api";import{ErrorLog as O,throwError as Z,checkForFile as L,parseJsonFile as Ot,parseError as R,readFilesInDirectory as K,roundValue as st,chunkArray as Tt,parsePercentage as It,compareArrays as wt}from"@rent-scraper/utils";import{getZillowOutputPath as T,getRedfinOutputPath as I,getZillowDaysListed as Et,getRedfinDaysListed as At,getZillowLimit as Yt,getZillowOffset as qt,getRedfinZipCodes as Jt,getZillowZipCodes as Vt}from"@rent-scraper/api/config";import{spinner as pt,log as u,progress as Gt,intro as Wt}from"@clack/prompts";import{setTimeout as gt}from"node:timers/promises";import{createHash as Xt}from"crypto";import{checkBrowserServer as mt}from"@rent-scraper/utils/config";import Kt from"picocolors";const P=process.env.DEBUG,E=async(t,e,s,i)=>{const{timeoutMs:r}=i??{};if(await L(s))P&&u.warn(`${s} exists, skipping`);else try{P&&u.info(`writing ${s}`);const n=t==="redfin"?await Bt(e):await Mt(e,{timeoutMs:r});await M(`${s}`,n)}catch(n){Z(`error fetching html for ${e}`,n)}},Qt=async(t,{timeoutMs:e})=>{if(await L(t)){const s=JSON.parse(await W(t,"utf8"))||{},{hdpUrl:i}=s||{};if(i){const r=`https://www.zillow.com${i}`,n=t.replace(".json",".html");await L(n)?P&&u.warn(`file already exists, ${n}`):await E("zillow",r,n,{timeoutMs:e})}else Z(`file is empty, ${t}`)}else P&&u.warn(`file does not exist at this path, ${t}, skipping`)},ht=async(t,e)=>{const{timeoutMs:s}=e??{};await Promise.all(t.map(async i=>await Qt(i,{timeoutMs:s})))},te=(t,e="zillow")=>{if(e==="zillow"){const{zpid:s,detailUrl:i}=t||{},r=i&&!i.startsWith("http")?`https://www.zillow.com${i}`:i;return{id:s,url:r}}else if(e==="redfin"){const{propertyId:s,url:i}=t?.homeData||{};return{id:s,url:`https://www.redfin.com${i}`}}},it=async(t,e,s,i=s,r)=>{const{timeoutMs:n,run:a=1,reruns:l=0,skipBotCheck:f=!1}=r??{},c=new O;s||Z("inputDirectory is required"),t==="zillow"&&!f&&await q();const w=[],y=[],d=pt();d.start("Downloading listings html files");for(let g=1;g<=l+1;g++){l>0&&g>1&&c.add(`rerun ${g-1} of ${l}`);const m=w.length;(g===1||m)&&await Promise.all((m?w:e).map(async v=>{const b=`${s}/${v}.json`;if(await L(b))try{const{results:x}=await Ot(b)||{};if(!x)c.add(`empty file, ${b}`);else{const j=i?`${i}/${v}`:`${s}/${v}`;await S(j,{recursive:!0}),x?.length?await Promise.all(x.map(async k=>{const{id:B,url:z}=te(k,t)??{};if(!z)return c.add(`url missing for ${B}`);const F=`${B}.html`,C=`${j}/${F}`;try{await E(t,z,C,{timeoutMs:n})}catch(N){const{status:A,message:U}=R(N);dt(A,U)||y.push({url:z,filePath:C}),c.add("scrape listing html error: "+(U??`error fetching listing for id, ${N}`))}})):c.add(`no results for file, ${b}`)}}catch(x){w.push(v);const{message:j}=R(x);c.add("scrape listing html error: "+(j??`error reading json data, ${b}, ${x}`))}else P&&u.warn(`file does not exist, ${b}, skipping`)}))}y.length>0&&(await gt(2e3),await Promise.all(y.map(async({url:g,filePath:m})=>{try{await E(t,g,m,{timeoutMs:n})}catch(v){const{message:b}=R(v);c.add("scrape listing html retry error: "+(b??`error fetching ${g}`))}})));const $=await T(),h=await I(),p=t==="zillow"?$:h;e.length>0?(d.stop("Listings HTML files have been saved to:"),u.message(o.join(p,t,"listings",o.basename(s)))):d.stop("No listings HTML files saved.");const _=o.join(p,t,"logs");if(await S(_,{recursive:!0}),c.get().filter(g=>!g.includes("rerun ")).length>l){const g=`${o.basename(s)}-html-errors-${a}.txt`,m=o.join(_,g);await c.write(m,[...new Set(c.get())].join(`
|
|
2
|
+
`)),P&&u.error(`There were errors during processing, see ${o.resolve(m)}`)}},ee=async(t,e,s,i)=>{const{timeoutMs:r,run:n=1,reruns:a=0,skipBotCheck:l=!1}=i??{},f=new O;s||Z("inputDirectory is required"),t==="zillow"&&!l&&await q();const c=[];for(let h=1;h<=a+1;h++){a>0&&h>1&&f.add(`rerun ${h-1} of ${a}`);const p=c.length;(h===1||p)&&await Promise.all((p?c:e).map(async _=>{const g=`${s}/${_}`;if(await L(g))try{const m=await K(g,{extension:".json",prependDirectory:!0});await ht(m,{timeoutMs:r})}catch(m){c.push(_);const{message:v}=R(m);f.add("scrape listing html error: "+(v??`Error during fetch for ${_}, ${m}`))}else f.add(`listing directory does not exist, ${g}`)}))}const w=await T(),y=await I(),d=t==="zillow"?w:y,$=o.join(d,t,"logs");if(await S($,{recursive:!0}),f.get().filter(h=>!h.includes("rerun ")).length>a){const h=`${o.basename(s)}-html-errors-${n}.txt`,p=o.join($,h);await f.write(p,[...new Set(f.get())].join(`
|
|
3
|
+
`)),P&&u.error(`There were errors during processing, see ${o.resolve(p)}`)}},se=async(t,e,s=e,i)=>{const{skipBotCheck:r=!1}=i??{},n=new O;if(t==="zillow"&&!r&&await q(),e||Z("inputDirectory is required"),await S(s,{recursive:!0}),await L(e))try{const d=await K(e,{extension:".json",prependDirectory:!0});await ht(d)}catch(d){const{message:$}=R(d);n.add("scrape listing html error: "+($??`Error during fetch for ${e}, ${d}`))}else n.add(`inputDirectory does not exist, ${e}`);const a=await T(),l=await I(),f=t==="zillow"?a:l,c=o.join(f,t,"logs");await S(c,{recursive:!0});const w=`${o.basename(e)}-html-errors.txt`,y=o.join(c,w);n.get().length>0&&(await n.write(y,[...new Set(n.get())].join(`
|
|
4
|
+
`)),P&&u.error(`There were errors during processing, see ${o.resolve(y)}`))},ie=(t,e)=>t==="redfin"?`https://www.redfin.com/home/${e}`:t==="zillow"?`https://www.zillow.com/homedetails/${e}_zpid`:null,re=async(t,e,s,i)=>{const{skipBotCheck:r=!1}=i??{};t==="zillow"&&!r&&await q(),s||Z("outputDirectory is required"),await S(s,{recursive:!0}),await Promise.all(e.map(async n=>{const a=ie(t,n),l=`${s}/${n}.html`;a&&await E(t,a,l)}))},D=process.env.DEBUG,ft=async t=>{if(await L(t)){const e=t.replace(".html",".json");if(await L(e))D&&u.warning(`file already exists, ${e}`);else{D&&u.message(`scraping data for ${t}`);const s=(await W(t)).toString();(!s||s.trim()==="")&&(await X(t),Z(`empty file found at ${t}, deleted for retry`)),s.includes("px-captcha")&&(await X(t),Z(`captcha page found in ${t}, deleted for retry`));const i=await Ft(s);if(!i?.priceHistory&&i?.bestMatchedUnit?.hdpUrl){D&&u.warning(`rescraping ${t} - https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`),await L(t)&&(D&&u.warning(`deleting ${t}`),await X(t)),await L(e)&&(D&&u.warning(`deleting ${e}`),await X(e));const r=`https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`;await E("zillow",r,t),await ft(t)}else i?(D&&u.info(`writing ${e}`),await M(`${e}`,JSON.stringify(i))):Z(`problem scraping data for ${t}`)}}},ae=async t=>{if(await L(t)){const e=t.replace(".html",".json");if(await L(e))D&&u.warning(`file already exists, ${e}`);else{D&&u.message(`scraping data for ${t}`);const s=(await W(t)).toString(),i=Dt(s);i?(D&&u.warning(`writing ${e}`),await M(`${e}`,JSON.stringify(i))):Z(`problem scraping data for ${t}`)}}},yt=async(t,e,s)=>(await Promise.all(e.map(async i=>{try{t==="redfin"?await ae(i):await ft(i);const r=i.replace(".html",".json");return await L(r)?1:0}catch(r){const{message:n}=R(r);return s?.add(n),0}}))).reduce((i,r)=>i+r,0),rt=async(t,e,s)=>{const i=new O;let r=0;s||Z("inputDirectory is required");const n=Gt({style:"heavy",max:100,size:50});n.start("Scraping listings data");const a=e.length,l=a<20?a+1:20,f=a>l?st(a/l,1):a,c=Tt(e,f);for(const[h,p]of c.entries()){const _=st((Number(h)+1)/l*100,1);n.advance(st(1/l*100,1),`Scraping listings (${It(_)})`),await Promise.all(p.map(async g=>{D&&u.message(`processing files for ${g}`);const m=`${s}/${g}`;if(await L(m)){const v=await K(m,{extension:".html",prependDirectory:!0}),b=await yt(t,v,i);r=r+b}else i.add(`listing directory does not exist, ${m}`)}))}const w=await T(),y=await I(),d=t==="zillow"?w:y;r>0?(n.stop("Listings data has been saved to:"),u.message(o.join(d,t,"listings",o.basename(s)))):n.stop("No listings data saved.");const $=o.join(d,t,"logs");if(await S($,{recursive:!0}),i.get().length>0){const h=`${o.basename(s)}-listing-errors.txt`,p=o.join($,h);await i.write(p,[...new Set(i.get())].join(`
|
|
5
|
+
`)),u.error(`There were errors during processing, see ${o.resolve(p)}`)}return{numListings:r}},at=process.env.DEBUG,ne=async(t,e,s,i,r)=>{const{daysOnZillow:n,timeoutMs:a}=r??{};if(await L(e))at&&u.warning(`${t} exists, skipping`),s.push(t);else{const l=await Rt({zipCode:t,daysOnZillow:n,mergePageResults:!0,timeoutMs:a});l?(at&&u.info(`writing ${e}`),await M(e,JSON.stringify(l)),s.push(t)):(at&&u.warning(`no results for ${t}`),i.push(t))}},nt=async(t,e,s)=>{const{daysListed:i,timeoutMs:r,run:n=1,reruns:a=0,fetchListings:l=!1,skipBotCheck:f=!1,silent:c=!1}=s??{},w=new O;f||await q({fetchListings:l});const y=[],d=[],$=[],h=[];await S(e,{recursive:!0});const p=pt();c||p.start("Scraping Zillow search results");for(let m=1;m<=a+1;m++){a>0&&m>1&&w.add(`rerun ${m-1} of ${a}`);const v=d.length;(m===1||v)&&await Promise.all((v?d:t).map(async b=>{const x=`${b}.json`,j=`${e}/${x}`;try{await ne(b,j,y,h,{daysOnZillow:i,timeoutMs:r})}catch(k){d.push(b);const{status:B,message:z}=R(k);dt(B,z)&&$.push(b),w.add("scrape listing results error: "+(z??`Error during fetch for ${b}, ${k}`))}}))}const _=await T();c||(y.length>0?(p.stop("Zillow search results have been saved to:"),u.message(o.join(_,"zillow","results",o.basename(e)))):p.stop("No results saved."));const g=o.join(_,"zillow","logs");if(await S(g,{recursive:!0}),w.get().filter(m=>!m.includes("rerun ")).length>a){const m=`${o.basename(e)}-results-errors-${n}.txt`,v=o.join(g,m);await w.write(v,[...new Set(w.get())].join(`
|
|
6
|
+
`)),c||u.error(`There were errors during processing, see ${o.resolve(v)}`)}return{validZipCodes:y,botFilteredZipCodes:$,noResultsZipCodes:h}},oe=async(t,e,s,i)=>{const{daysListed:r,timeoutMs:n}=i??{};if(await L(e))console.log(`${t} exists, skipping`),s.push(t);else{const a=await Pt({zipCode:t,daysListed:r,timeoutMs:n});a?(console.log(`writing ${e}`),await M(e,JSON.stringify(a)),s.push(t)):console.log(`no results for ${t}`)}},$t=async(t,e,s)=>{const{daysListed:i,timeoutMs:r,run:n=1,reruns:a=0}=s??{},l=new O,f=[],c=[];await S(e,{recursive:!0});for(let d=1;d<=a+1;d++){a>0&&d>1&&l.add(`rerun ${d-1} of ${a}`);const $=c.length;(d===1||$)&&await Promise.all(($?c:t).map(async h=>{const p=`${h}.json`,_=`${e}/${p}`;try{await oe(h,_,f,{daysListed:i,timeoutMs:r})}catch(g){c.push(h);const{message:m}=R(g);l.add("scrape listing results error: "+(m??`Error during fetch for ${h}, ${g}`))}}))}const w=await I(),y=o.join(w,"redfin","logs");if(await S(y,{recursive:!0}),l.get().filter(d=>!d.includes("rerun ")).length>a){const d=`${o.basename(e)}-results-errors-${n}.txt`,$=o.join(y,d);await l.write($,[...new Set(l.get())].join(`
|
|
7
|
+
`)),console.log(`\x1B[41m
|
|
8
|
+
%s\x1B[0m`,`There were errors during processing, see ${o.resolve($)}`)}return{validZipCodes:f}},_t=["listing_id","listing_source","source_listing_id","parcel_number","listing_url","street_address","cleaned_address","cleaned_unit_number","city","state","zipcode","county","neighborhood_region","is_undisclosed_address","home_status","home_type","bedrooms","year_built","living_area","living_area_units","is_income_restricted","price_at_source","platform_rent_estimate","agent_name","agent_phone_number","broker_name","broker_phone_number","is_owned_by_listing_platform","date_last_updated_at_source","date_scraped","scrape_job_name"],le=t=>{if(t==null)return"";const e=typeof t=="object"?JSON.stringify(t):String(t);return e.includes(",")||e.includes('"')||e.includes(`
|
|
9
|
+
`)?`"${e.replace(/"/g,'""')}"`:e},ce=t=>{const e=/^(.*?)\s+((?:#|APT|UNIT|STE|Suite|Apt|Unit)\s*.+)$/i.exec(t);if(e){const s=e[1].trim().toUpperCase();let i=e[2].trim();return i.startsWith("#")&&(i=`# ${i.replace(/^#\s*/,"").trim()}`),{cleaned:s,unit:i}}return{cleaned:t.toUpperCase(),unit:""}},de=(t,e)=>{const s=String(t.zpid),i=Xt("md5").update(s).digest("hex"),r=t.streetAddress||t.address?.streetAddress||"",{cleaned:n,unit:a}=ce(r),l=t.hdpUrl?`https://www.zillow.com${t.hdpUrl}`:`https://www.zillow.com/homedetails/${s}_zpid/`;return{listing_id:i,listing_source:"zillow",source_listing_id:s,parcel_number:t.resoFacts?.parcelNumber??"",listing_url:l,street_address:r,cleaned_address:n,cleaned_unit_number:a,city:t.city||t.address?.city||"",state:t.state||t.address?.state||"",zipcode:t.zipcode||t.address?.zipcode||"",county:t.county||"",neighborhood_region:t.neighborhoodRegion?.name||t.parentRegion?.name||"",is_undisclosed_address:t.isUndisclosedAddress??"",home_status:t.homeStatus||"",home_type:t.homeType||"",bedrooms:t.bedrooms??"",year_built:t.yearBuilt||t.resoFacts?.yearBuilt||"",living_area:t.livingAreaValue??"",living_area_units:t.livingAreaUnits||"Square Feet",is_income_restricted:t.isIncomeRestricted??"",price_at_source:t.price??"",platform_rent_estimate:t.rentZestimate??"",agent_name:t.attributionInfo?.agentName||"",agent_phone_number:t.attributionInfo?.agentPhoneNumber||"",broker_name:t.attributionInfo?.brokerName||"",broker_phone_number:t.attributionInfo?.brokerPhoneNumber||"",is_owned_by_listing_platform:t.attributionInfo?.mlsName==="Zillow Rentals",date_last_updated_at_source:t.attributionInfo?.lastUpdated||"",date_scraped:t.timestamp?lt(t.timestamp).format("YYYY-MM-DD HH:mm:ss"):"",scrape_job_name:e}},bt=async(t,e)=>{const s=o.basename(t),i=[_t.join(",")];let r;try{r=await et(t,{withFileTypes:!0})}catch{return 0}const n=r.filter(a=>a.isDirectory()).map(a=>a.name);for(const a of n){const l=o.join(t,a);if(!await L(l))continue;const f=await K(l,{extension:".json",prependDirectory:!0});for(const c of f)try{const w=JSON.parse(await W(c,"utf8"));if(!w.zpid)continue;const y=de(w,s);i.push(_t.map(d=>le(y[d])).join(","))}catch{}}return await S(o.dirname(e),{recursive:!0}),await M(e,i.join(`
|
|
10
|
+
`)),i.length-1},ue=async t=>{try{const e=(await et(t,{withFileTypes:!0})).filter(s=>s.isDirectory()).map(s=>s.name).sort();return e.length>0?e[e.length-1]:null}catch{return null}},vt=async()=>{await ct.post("http://localhost:8082/browser/close")},we=async()=>{await ct.post("http://localhost:8082/server/shutdown")},pe=async(t,e,s,{daysListed:i,timeoutMs:r,run:n,reruns:a,preValidatedZipCodes:l=[],retry:f=!1})=>{let c=[],w=[],y=[];if(t.length>0){if(await(await Ht()?Ut():ut()),await vt(),{validZipCodes:c,botFilteredZipCodes:w,noResultsZipCodes:y}=await nt(t,e,{daysListed:i,timeoutMs:r,run:n,reruns:a,skipBotCheck:!0}),f&&w.length>0){u.warn(`Bot filtering hit ${w.length} of ${t.length} zip code(s) \u2014 will retry after listings are fetched`),await Nt()?(await ut(),await vt()):await gt(3e3);const h=w.length,p=await nt(w,e,{daysListed:i,timeoutMs:r,run:n,reruns:a,skipBotCheck:!0,silent:!0});c=[...c,...p.validZipCodes],w=p.botFilteredZipCodes,y=[...y,...p.noResultsZipCodes];const _=p.validZipCodes.length;u.info(`Inline retry: recovered ${_} of ${h} \xB7 ${w.length} still bot-filtered`)}c.length===0&&t.length>0&&u.warn("No results returned for any zip codes \u2014 possible soft bot filtering. Please try again shortly."),w.length>0&&u.warn(`${w.length} zip code(s) still bot-filtered \u2014 use --rerun to retry`)}const d=[...l,...c];await it("zillow",d,e,s,{timeoutMs:r,run:n,reruns:a,skipBotCheck:!0});const{numListings:$}=await rt("zillow",d,s);return{numListings:$,validZipCodes:d,noResultsZipCodes:y}},ge=async(t,e,s,{daysListed:i,timeoutMs:r,run:n,reruns:a})=>{const{validZipCodes:l}=await $t(t,e,{daysListed:i,timeoutMs:r,run:n,reruns:a});await it("redfin",l,e,s,{timeoutMs:r,run:n,reruns:a});const{numListings:f}=await rt("redfin",l,s);return{numListings:f,validZipCodes:l}};async function me(){Wt(Kt.inverse(" scrape listings "));const t=kt(process.argv.slice(2)),e=t.source??"zillow";e==="zillow"&&(await mt()||Z("Please launch the browser server before scraping."));const s=await T(),i=await I(),r=e==="zillow"?s:i,n=t["days-listed"]??(e==="zillow"?await Et():await At())??1,a=t.runs??1,l=t.reruns??0,f=t["timeout-ms"]??6e4,c=t.limit??await Yt()??void 0,w=t.offset??await qt()??0,y=t.retry??!1,d=t.rerun,$=typeof d=="string"?d:d?await ue(o.join(r,e,"results")):null;d&&!$&&Z("No previous run found to rerun."),$&&u.info(`Rerunning from: ${$}`);const h=$??lt().format("YYYY-MM-DD-HHmm"),p=t["results-directory"]??o.join(r,e,"results",h),_=t["listings-directory"]??o.join(r,e,"listings",h),g=t["logs-directory"]??o.join(r,e,"logs");let m=0;const v=async()=>{(e==="zillow"||e==="redfin")&&await mt()&&await we()};try{if(e==="redfin"){const x=await Jt();for(let j=1;j<=a;j++){x||Z("zip codes required, please run the createConfig script");const k=x,{numListings:B,validZipCodes:z}=await ge(k,p,_,{daysListed:n,timeoutMs:f,run:j,reruns:l});if(j===a){const F=Object.entries({numListings:B}).map(([Q,J])=>`${Q}: ${J}`).join(`
|
|
11
|
+
`),C=wt(k,z).join(`
|
|
12
|
+
`);await S(g,{recursive:!0});const N=`${o.basename(p)}-scraping-results.txt`,A=o.join(g,N),U=`${o.basename(p)}-invalid-zipcodes.txt`,H=o.join(g,U);await M(A,F),await M(H,C)}}}else if(e==="zillow"){const x=await Vt();x||Z("zip codes required, please run the createConfig script");const j=x.slice(w,c?w+c:void 0);let k=j,B=[];if($){const z=await et(p).catch(()=>[]),F=new Set(z.filter(C=>C.endsWith(".json")).map(C=>C.replace(".json","")));B=j.filter(C=>F.has(String(C))),k=j.filter(C=>!F.has(String(C))),k.length>0?u.info(`${B.length} already fetched \xB7 retrying ${k.length} missing`):u.info(`All ${B.length} zip codes already fetched \xB7 proceeding to HTML scraping`)}for(let z=1;z<=a;z++){const{numListings:F,validZipCodes:C,noResultsZipCodes:N}=await pe(k,p,_,{daysListed:n,timeoutMs:f,run:z,reruns:l,preValidatedZipCodes:B,retry:y});if(z===a){const A=o.basename(_),U=o.join(s,"zillow","csv",`${A}.csv`),H=await bt(_,U);m=H,H>0&&u.info(`CSV exported: ${H} listings \u2192 ${U}`);const Q=Object.entries({numListings:F,numCsvListings:H}).map(([Y,St])=>`${Y}: ${St}`).join(`
|
|
13
|
+
`),J=wt(j,C),ot=new Set(N),V=J.filter(Y=>!ot.has(Y)),G=J.filter(Y=>ot.has(Y));await S(g,{recursive:!0});const jt=`${o.basename(p)}-scraping-results.txt`,Lt=o.join(g,jt),zt=`${o.basename(p)}-zipcodes-no-results.txt`,Ct=o.join(g,zt),Zt=`${o.basename(p)}-zipcodes-errored.txt`,xt=o.join(g,Zt);await M(Lt,Q),G.length>0&&await M(Ct,G.join(`
|
|
14
|
+
`)),V.length>0&&await M(xt,V.join(`
|
|
15
|
+
`));const tt=[`${C.length}/${j.length} with results`];G.length>0&&tt.push(`${G.length} no results`),V.length>0&&tt.push(`${V.length} errored`),u.info(`Zip codes: ${tt.join(" \xB7 ")}`),u.info(`Listings: ${F} parsed \xB7 ${H} exported to CSV`)}}}const b=e==="zillow"?m>0:!0;return b?u.success("Scraping complete!"):u.warn("Scraping finished but no listings were exported \u2014 please try again shortly."),{success:b,totalCsvListings:m}}finally{await v()}}export{rt as a,it as b,ee as c,se as d,re as e,E as f,bt as g,$t as h,nt as i,me as r,yt as s};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rent-scraper/scrape-listings",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.32",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": {
|
|
@@ -30,8 +30,8 @@
|
|
|
30
30
|
"dayjs": "^1.11.13",
|
|
31
31
|
"minimist": "^1.2.8",
|
|
32
32
|
"picocolors": "^1.1.1",
|
|
33
|
-
"@rent-scraper/api": "1.0.
|
|
34
|
-
"@rent-scraper/utils": "1.0.
|
|
33
|
+
"@rent-scraper/api": "1.0.32",
|
|
34
|
+
"@rent-scraper/utils": "1.0.32"
|
|
35
35
|
},
|
|
36
36
|
"devDependencies": {
|
|
37
37
|
"@types/minimist": "^1.2.5",
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
import kt from"minimist";import lt from"dayjs";import o from"path";import ct from"axios";import{mkdir as S,writeFile as M,readFile as W,unlink as X,readdir as et}from"fs/promises";import{checkForZillowBotFiltering as q,isZillowBotFiltering as dt,fetchHtmlFromRedfinListingUrl as Bt,fetchHtmlFromZillowListingUrl as Mt,scrapeDataFromRedfinListingHtml as Dt,scrapeDataFromZillowListingHtml as Ft,getZillowListingResults as Pt,getRedfinListingResults as Rt,waitForSolvedZillowCaptcha as ut,isBrowserShowingCaptcha as Ut}from"@rent-scraper/api";import{ErrorLog as O,throwError as Z,checkForFile as L,parseJsonFile as Ht,parseError as P,readFilesInDirectory as K,roundValue as st,chunkArray as Nt,parsePercentage as Ot,compareArrays as pt}from"@rent-scraper/utils";import{getZillowOutputPath as T,getRedfinOutputPath as I,getZillowDaysListed as Tt,getRedfinDaysListed as It,getZillowLimit as Et,getZillowOffset as At,getRedfinZipCodes as Yt,getZillowZipCodes as qt}from"@rent-scraper/api/config";import{spinner as wt,log as u,progress as Jt,intro as Vt}from"@clack/prompts";import{setTimeout as gt}from"node:timers/promises";import{createHash as Gt}from"crypto";import{checkBrowserServer as mt}from"@rent-scraper/utils/config";import Wt from"picocolors";const R=process.env.DEBUG,E=async(t,e,s,i)=>{const{timeoutMs:r}=i??{};if(await L(s))R&&u.warn(`${s} exists, skipping`);else try{R&&u.info(`writing ${s}`);const a=t==="redfin"?await Bt(e):await Mt(e,{timeoutMs:r});await M(`${s}`,a)}catch(a){Z(`error fetching html for ${e}`,a)}},Xt=async(t,{timeoutMs:e})=>{if(await L(t)){const s=JSON.parse(await W(t,"utf8"))||{},{hdpUrl:i}=s||{};if(i){const r=`https://www.zillow.com${i}`,a=t.replace(".json",".html");await L(a)?R&&u.warn(`file already exists, ${a}`):await E("zillow",r,a,{timeoutMs:e})}else Z(`file is empty, ${t}`)}else R&&u.warn(`file does not exist at this path, ${t}, skipping`)},ft=async(t,e)=>{const{timeoutMs:s}=e??{};await Promise.all(t.map(async i=>await Xt(i,{timeoutMs:s})))},Kt=(t,e="zillow")=>{if(e==="zillow"){const{zpid:s,detailUrl:i}=t||{},r=i&&!i.startsWith("http")?`https://www.zillow.com${i}`:i;return{id:s,url:r}}else if(e==="redfin"){const{propertyId:s,url:i}=t?.homeData||{};return{id:s,url:`https://www.redfin.com${i}`}}},it=async(t,e,s,i=s,r)=>{const{timeoutMs:a,run:n=1,reruns:l=0,skipBotCheck:h=!1}=r??{},c=new O;s||Z("inputDirectory is required"),t==="zillow"&&!h&&await q();const p=[],y=[],d=wt();d.start("Downloading listings html files");for(let g=1;g<=l+1;g++){l>0&&g>1&&c.add(`rerun ${g-1} of ${l}`);const m=p.length;(g===1||m)&&await Promise.all((m?p:e).map(async v=>{const b=`${s}/${v}.json`;if(await L(b))try{const{results:x}=await Ht(b)||{};if(!x)c.add(`empty file, ${b}`);else{const j=i?`${i}/${v}`:`${s}/${v}`;await S(j,{recursive:!0}),x?.length?await Promise.all(x.map(async k=>{const{id:B,url:z}=Kt(k,t)??{};if(!z)return c.add(`url missing for ${B}`);const F=`${B}.html`,C=`${j}/${F}`;try{await E(t,z,C,{timeoutMs:a})}catch(N){const{status:A,message:U}=P(N);dt(A,U)||y.push({url:z,filePath:C}),c.add("scrape listing html error: "+(U??`error fetching listing for id, ${N}`))}})):c.add(`no results for file, ${b}`)}}catch(x){p.push(v);const{message:j}=P(x);c.add("scrape listing html error: "+(j??`error reading json data, ${b}, ${x}`))}else R&&u.warn(`file does not exist, ${b}, skipping`)}))}y.length>0&&(await gt(2e3),await Promise.all(y.map(async({url:g,filePath:m})=>{try{await E(t,g,m,{timeoutMs:a})}catch(v){const{message:b}=P(v);c.add("scrape listing html retry error: "+(b??`error fetching ${g}`))}})));const $=await T(),f=await I(),w=t==="zillow"?$:f;e.length>0?(d.stop("Listings HTML files have been saved to:"),u.message(o.join(w,t,"listings",o.basename(s)))):d.stop("No listings HTML files saved.");const _=o.join(w,t,"logs");if(await S(_,{recursive:!0}),c.get().filter(g=>!g.includes("rerun ")).length>l){const g=`${o.basename(s)}-html-errors-${n}.txt`,m=o.join(_,g);await c.write(m,[...new Set(c.get())].join(`
|
|
2
|
-
`)),R&&u.error(`There were errors during processing, see ${o.resolve(m)}`)}},Qt=async(t,e,s,i)=>{const{timeoutMs:r,run:a=1,reruns:n=0,skipBotCheck:l=!1}=i??{},h=new O;s||Z("inputDirectory is required"),t==="zillow"&&!l&&await q();const c=[];for(let f=1;f<=n+1;f++){n>0&&f>1&&h.add(`rerun ${f-1} of ${n}`);const w=c.length;(f===1||w)&&await Promise.all((w?c:e).map(async _=>{const g=`${s}/${_}`;if(await L(g))try{const m=await K(g,{extension:".json",prependDirectory:!0});await ft(m,{timeoutMs:r})}catch(m){c.push(_);const{message:v}=P(m);h.add("scrape listing html error: "+(v??`Error during fetch for ${_}, ${m}`))}else h.add(`listing directory does not exist, ${g}`)}))}const p=await T(),y=await I(),d=t==="zillow"?p:y,$=o.join(d,t,"logs");if(await S($,{recursive:!0}),h.get().filter(f=>!f.includes("rerun ")).length>n){const f=`${o.basename(s)}-html-errors-${a}.txt`,w=o.join($,f);await h.write(w,[...new Set(h.get())].join(`
|
|
3
|
-
`)),R&&u.error(`There were errors during processing, see ${o.resolve(w)}`)}},te=async(t,e,s=e,i)=>{const{skipBotCheck:r=!1}=i??{},a=new O;if(t==="zillow"&&!r&&await q(),e||Z("inputDirectory is required"),await S(s,{recursive:!0}),await L(e))try{const d=await K(e,{extension:".json",prependDirectory:!0});await ft(d)}catch(d){const{message:$}=P(d);a.add("scrape listing html error: "+($??`Error during fetch for ${e}, ${d}`))}else a.add(`inputDirectory does not exist, ${e}`);const n=await T(),l=await I(),h=t==="zillow"?n:l,c=o.join(h,t,"logs");await S(c,{recursive:!0});const p=`${o.basename(e)}-html-errors.txt`,y=o.join(c,p);a.get().length>0&&(await a.write(y,[...new Set(a.get())].join(`
|
|
4
|
-
`)),R&&u.error(`There were errors during processing, see ${o.resolve(y)}`))},ee=(t,e)=>t==="redfin"?`https://www.redfin.com/home/${e}`:t==="zillow"?`https://www.zillow.com/homedetails/${e}_zpid`:null,se=async(t,e,s,i)=>{const{skipBotCheck:r=!1}=i??{};t==="zillow"&&!r&&await q(),s||Z("outputDirectory is required"),await S(s,{recursive:!0}),await Promise.all(e.map(async a=>{const n=ee(t,a),l=`${s}/${a}.html`;n&&await E(t,n,l)}))},D=process.env.DEBUG,ht=async t=>{if(await L(t)){const e=t.replace(".html",".json");if(await L(e))D&&u.warning(`file already exists, ${e}`);else{D&&u.message(`scraping data for ${t}`);const s=(await W(t)).toString();(!s||s.trim()==="")&&(await X(t),Z(`empty file found at ${t}, deleted for retry`)),s.includes("px-captcha")&&(await X(t),Z(`captcha page found in ${t}, deleted for retry`));const i=await Ft(s);if(!i?.priceHistory&&i?.bestMatchedUnit?.hdpUrl){D&&u.warning(`rescraping ${t} - https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`),await L(t)&&(D&&u.warning(`deleting ${t}`),await X(t)),await L(e)&&(D&&u.warning(`deleting ${e}`),await X(e));const r=`https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`;await E("zillow",r,t),await ht(t)}else i?(D&&u.info(`writing ${e}`),await M(`${e}`,JSON.stringify(i))):Z(`problem scraping data for ${t}`)}}},ie=async t=>{if(await L(t)){const e=t.replace(".html",".json");if(await L(e))D&&u.warning(`file already exists, ${e}`);else{D&&u.message(`scraping data for ${t}`);const s=(await W(t)).toString(),i=Dt(s);i?(D&&u.warning(`writing ${e}`),await M(`${e}`,JSON.stringify(i))):Z(`problem scraping data for ${t}`)}}},yt=async(t,e,s)=>(await Promise.all(e.map(async i=>{try{t==="redfin"?await ie(i):await ht(i);const r=i.replace(".html",".json");return await L(r)?1:0}catch(r){const{message:a}=P(r);return s?.add(a),0}}))).reduce((i,r)=>i+r,0),rt=async(t,e,s)=>{const i=new O;let r=0;s||Z("inputDirectory is required");const a=Jt({style:"heavy",max:100,size:50});a.start("Scraping listings data");const n=e.length,l=n<20?n+1:20,h=n>l?st(n/l,1):n,c=Nt(e,h);for(const[f,w]of c.entries()){const _=st((Number(f)+1)/l*100,1);a.advance(st(1/l*100,1),`Scraping listings (${Ot(_)})`),await Promise.all(w.map(async g=>{D&&u.message(`processing files for ${g}`);const m=`${s}/${g}`;if(await L(m)){const v=await K(m,{extension:".html",prependDirectory:!0}),b=await yt(t,v,i);r=r+b}else i.add(`listing directory does not exist, ${m}`)}))}const p=await T(),y=await I(),d=t==="zillow"?p:y;r>0?(a.stop("Listings data has been saved to:"),u.message(o.join(d,t,"listings",o.basename(s)))):a.stop("No listings data saved.");const $=o.join(d,t,"logs");if(await S($,{recursive:!0}),i.get().length>0){const f=`${o.basename(s)}-listing-errors.txt`,w=o.join($,f);await i.write(w,[...new Set(i.get())].join(`
|
|
5
|
-
`)),u.error(`There were errors during processing, see ${o.resolve(w)}`)}return{numListings:r}},nt=process.env.DEBUG,re=async(t,e,s,i,r)=>{const{daysOnZillow:a,timeoutMs:n}=r??{};if(await L(e))nt&&u.warning(`${t} exists, skipping`),s.push(t);else{const l=await Pt({zipCode:t,daysOnZillow:a,mergePageResults:!0,timeoutMs:n});l?(nt&&u.info(`writing ${e}`),await M(e,JSON.stringify(l)),s.push(t)):(nt&&u.warning(`no results for ${t}`),i.push(t))}},at=async(t,e,s)=>{const{daysListed:i,timeoutMs:r,run:a=1,reruns:n=0,fetchListings:l=!1,skipBotCheck:h=!1,silent:c=!1}=s??{},p=new O;h||await q({fetchListings:l});const y=[],d=[],$=[],f=[];await S(e,{recursive:!0});const w=wt();c||w.start("Scraping Zillow search results");for(let m=1;m<=n+1;m++){n>0&&m>1&&p.add(`rerun ${m-1} of ${n}`);const v=d.length;(m===1||v)&&await Promise.all((v?d:t).map(async b=>{const x=`${b}.json`,j=`${e}/${x}`;try{await re(b,j,y,f,{daysOnZillow:i,timeoutMs:r})}catch(k){d.push(b);const{status:B,message:z}=P(k);dt(B,z)&&$.push(b),p.add("scrape listing results error: "+(z??`Error during fetch for ${b}, ${k}`))}}))}const _=await T();c||(y.length>0?(w.stop("Zillow search results have been saved to:"),u.message(o.join(_,"zillow","results",o.basename(e)))):w.stop("No results saved."));const g=o.join(_,"zillow","logs");if(await S(g,{recursive:!0}),p.get().filter(m=>!m.includes("rerun ")).length>n){const m=`${o.basename(e)}-results-errors-${a}.txt`,v=o.join(g,m);await p.write(v,[...new Set(p.get())].join(`
|
|
6
|
-
`)),c||u.error(`There were errors during processing, see ${o.resolve(v)}`)}return{validZipCodes:y,botFilteredZipCodes:$,noResultsZipCodes:f}},ne=async(t,e,s,i)=>{const{daysListed:r,timeoutMs:a}=i??{};if(await L(e))console.log(`${t} exists, skipping`),s.push(t);else{const n=await Rt({zipCode:t,daysListed:r,timeoutMs:a});n?(console.log(`writing ${e}`),await M(e,JSON.stringify(n)),s.push(t)):console.log(`no results for ${t}`)}},$t=async(t,e,s)=>{const{daysListed:i,timeoutMs:r,run:a=1,reruns:n=0}=s??{},l=new O,h=[],c=[];await S(e,{recursive:!0});for(let d=1;d<=n+1;d++){n>0&&d>1&&l.add(`rerun ${d-1} of ${n}`);const $=c.length;(d===1||$)&&await Promise.all(($?c:t).map(async f=>{const w=`${f}.json`,_=`${e}/${w}`;try{await ne(f,_,h,{daysListed:i,timeoutMs:r})}catch(g){c.push(f);const{message:m}=P(g);l.add("scrape listing results error: "+(m??`Error during fetch for ${f}, ${g}`))}}))}const p=await I(),y=o.join(p,"redfin","logs");if(await S(y,{recursive:!0}),l.get().filter(d=>!d.includes("rerun ")).length>n){const d=`${o.basename(e)}-results-errors-${a}.txt`,$=o.join(y,d);await l.write($,[...new Set(l.get())].join(`
|
|
7
|
-
`)),console.log(`\x1B[41m
|
|
8
|
-
%s\x1B[0m`,`There were errors during processing, see ${o.resolve($)}`)}return{validZipCodes:h}},_t=["listing_id","listing_source","source_listing_id","parcel_number","listing_url","street_address","cleaned_address","cleaned_unit_number","city","state","zipcode","county","neighborhood_region","is_undisclosed_address","home_status","home_type","bedrooms","year_built","living_area","living_area_units","is_income_restricted","price_at_source","platform_rent_estimate","agent_name","agent_phone_number","broker_name","broker_phone_number","is_owned_by_listing_platform","date_last_updated_at_source","date_scraped","scrape_job_name"],ae=t=>{if(t==null)return"";const e=typeof t=="object"?JSON.stringify(t):String(t);return e.includes(",")||e.includes('"')||e.includes(`
|
|
9
|
-
`)?`"${e.replace(/"/g,'""')}"`:e},oe=t=>{const e=/^(.*?)\s+((?:#|APT|UNIT|STE|Suite|Apt|Unit)\s*.+)$/i.exec(t);if(e){const s=e[1].trim().toUpperCase();let i=e[2].trim();return i.startsWith("#")&&(i=`# ${i.replace(/^#\s*/,"").trim()}`),{cleaned:s,unit:i}}return{cleaned:t.toUpperCase(),unit:""}},le=(t,e)=>{const s=String(t.zpid),i=Gt("md5").update(s).digest("hex"),r=t.streetAddress||t.address?.streetAddress||"",{cleaned:a,unit:n}=oe(r),l=t.hdpUrl?`https://www.zillow.com${t.hdpUrl}`:`https://www.zillow.com/homedetails/${s}_zpid/`;return{listing_id:i,listing_source:"zillow",source_listing_id:s,parcel_number:t.resoFacts?.parcelNumber??"",listing_url:l,street_address:r,cleaned_address:a,cleaned_unit_number:n,city:t.city||t.address?.city||"",state:t.state||t.address?.state||"",zipcode:t.zipcode||t.address?.zipcode||"",county:t.county||"",neighborhood_region:t.neighborhoodRegion?.name||t.parentRegion?.name||"",is_undisclosed_address:t.isUndisclosedAddress??"",home_status:t.homeStatus||"",home_type:t.homeType||"",bedrooms:t.bedrooms??"",year_built:t.yearBuilt||t.resoFacts?.yearBuilt||"",living_area:t.livingAreaValue??"",living_area_units:t.livingAreaUnits||"Square Feet",is_income_restricted:t.isIncomeRestricted??"",price_at_source:t.price??"",platform_rent_estimate:t.rentZestimate??"",agent_name:t.attributionInfo?.agentName||"",agent_phone_number:t.attributionInfo?.agentPhoneNumber||"",broker_name:t.attributionInfo?.brokerName||"",broker_phone_number:t.attributionInfo?.brokerPhoneNumber||"",is_owned_by_listing_platform:t.attributionInfo?.mlsName==="Zillow Rentals",date_last_updated_at_source:t.attributionInfo?.lastUpdated||"",date_scraped:t.timestamp?lt(t.timestamp).format("YYYY-MM-DD HH:mm:ss"):"",scrape_job_name:e}},bt=async(t,e)=>{const s=o.basename(t),i=[_t.join(",")];let r;try{r=await et(t,{withFileTypes:!0})}catch{return 0}const a=r.filter(n=>n.isDirectory()).map(n=>n.name);for(const n of a){const l=o.join(t,n);if(!await L(l))continue;const h=await K(l,{extension:".json",prependDirectory:!0});for(const c of h)try{const p=JSON.parse(await W(c,"utf8"));if(!p.zpid)continue;const y=le(p,s);i.push(_t.map(d=>ae(y[d])).join(","))}catch{}}return await S(o.dirname(e),{recursive:!0}),await M(e,i.join(`
|
|
10
|
-
`)),i.length-1},ce=async t=>{try{const e=(await et(t,{withFileTypes:!0})).filter(s=>s.isDirectory()).map(s=>s.name).sort();return e.length>0?e[e.length-1]:null}catch{return null}},vt=async()=>{await ct.post("http://localhost:8082/browser/close")},de=async()=>{await ct.post("http://localhost:8082/server/shutdown")},ue=async(t,e,s,{daysListed:i,timeoutMs:r,run:a,reruns:n,preValidatedZipCodes:l=[],retry:h=!1})=>{let c=[],p=[],y=[];if(t.length>0){if(await ut(),await vt(),{validZipCodes:c,botFilteredZipCodes:p,noResultsZipCodes:y}=await at(t,e,{daysListed:i,timeoutMs:r,run:a,reruns:n,skipBotCheck:!0}),h&&p.length>0){u.warn(`Bot filtering hit ${p.length} of ${t.length} zip code(s) \u2014 will retry after listings are fetched`),await Ut()?(await ut(),await vt()):await gt(3e3);const f=p.length,w=await at(p,e,{daysListed:i,timeoutMs:r,run:a,reruns:n,skipBotCheck:!0,silent:!0});c=[...c,...w.validZipCodes],p=w.botFilteredZipCodes,y=[...y,...w.noResultsZipCodes];const _=w.validZipCodes.length;u.info(`Inline retry: recovered ${_} of ${f} \xB7 ${p.length} still bot-filtered`)}c.length===0&&t.length>0&&u.warn("No results returned for any zip codes \u2014 possible soft bot filtering. Please try again shortly."),p.length>0&&u.warn(`${p.length} zip code(s) still bot-filtered \u2014 use --rerun to retry`)}const d=[...l,...c];await it("zillow",d,e,s,{timeoutMs:r,run:a,reruns:n,skipBotCheck:!0});const{numListings:$}=await rt("zillow",d,s);return{numListings:$,validZipCodes:d,noResultsZipCodes:y}},pe=async(t,e,s,{daysListed:i,timeoutMs:r,run:a,reruns:n})=>{const{validZipCodes:l}=await $t(t,e,{daysListed:i,timeoutMs:r,run:a,reruns:n});await it("redfin",l,e,s,{timeoutMs:r,run:a,reruns:n});const{numListings:h}=await rt("redfin",l,s);return{numListings:h,validZipCodes:l}};async function we(){Vt(Wt.inverse(" scrape listings "));const t=kt(process.argv.slice(2)),e=t.source??"zillow";e==="zillow"&&(await mt()||Z("Please launch the browser server before scraping."));const s=await T(),i=await I(),r=e==="zillow"?s:i,a=t["days-listed"]??(e==="zillow"?await Tt():await It())??1,n=t.runs??1,l=t.reruns??0,h=t["timeout-ms"]??6e4,c=t.limit??await Et()??void 0,p=t.offset??await At()??0,y=t.retry??!1,d=t.rerun,$=typeof d=="string"?d:d?await ce(o.join(r,e,"results")):null;d&&!$&&Z("No previous run found to rerun."),$&&u.info(`Rerunning from: ${$}`);const f=$??lt().format("YYYY-MM-DD-HHmm"),w=t["results-directory"]??o.join(r,e,"results",f),_=t["listings-directory"]??o.join(r,e,"listings",f),g=t["logs-directory"]??o.join(r,e,"logs");let m=0;const v=async()=>{(e==="zillow"||e==="redfin")&&await mt()&&await de()};try{if(e==="redfin"){const x=await Yt();for(let j=1;j<=n;j++){x||Z("zip codes required, please run the createConfig script");const k=x,{numListings:B,validZipCodes:z}=await pe(k,w,_,{daysListed:a,timeoutMs:h,run:j,reruns:l});if(j===n){const F=Object.entries({numListings:B}).map(([Q,J])=>`${Q}: ${J}`).join(`
|
|
11
|
-
`),C=pt(k,z).join(`
|
|
12
|
-
`);await S(g,{recursive:!0});const N=`${o.basename(w)}-scraping-results.txt`,A=o.join(g,N),U=`${o.basename(w)}-invalid-zipcodes.txt`,H=o.join(g,U);await M(A,F),await M(H,C)}}}else if(e==="zillow"){const x=await qt();x||Z("zip codes required, please run the createConfig script");const j=x.slice(p,c?p+c:void 0);let k=j,B=[];if($){const z=await et(w).catch(()=>[]),F=new Set(z.filter(C=>C.endsWith(".json")).map(C=>C.replace(".json","")));B=j.filter(C=>F.has(String(C))),k=j.filter(C=>!F.has(String(C))),k.length>0?u.info(`${B.length} already fetched \xB7 retrying ${k.length} missing`):u.info(`All ${B.length} zip codes already fetched \xB7 proceeding to HTML scraping`)}for(let z=1;z<=n;z++){const{numListings:F,validZipCodes:C,noResultsZipCodes:N}=await ue(k,w,_,{daysListed:a,timeoutMs:h,run:z,reruns:l,preValidatedZipCodes:B,retry:y});if(z===n){const A=o.basename(_),U=o.join(s,"zillow","csv",`${A}.csv`),H=await bt(_,U);m=H,H>0&&u.info(`CSV exported: ${H} listings \u2192 ${U}`);const Q=Object.entries({numListings:F,numCsvListings:H}).map(([Y,St])=>`${Y}: ${St}`).join(`
|
|
13
|
-
`),J=pt(j,C),ot=new Set(N),V=J.filter(Y=>!ot.has(Y)),G=J.filter(Y=>ot.has(Y));await S(g,{recursive:!0});const jt=`${o.basename(w)}-scraping-results.txt`,Lt=o.join(g,jt),zt=`${o.basename(w)}-zipcodes-no-results.txt`,Ct=o.join(g,zt),Zt=`${o.basename(w)}-zipcodes-errored.txt`,xt=o.join(g,Zt);await M(Lt,Q),G.length>0&&await M(Ct,G.join(`
|
|
14
|
-
`)),V.length>0&&await M(xt,V.join(`
|
|
15
|
-
`));const tt=[`${C.length}/${j.length} with results`];G.length>0&&tt.push(`${G.length} no results`),V.length>0&&tt.push(`${V.length} errored`),u.info(`Zip codes: ${tt.join(" \xB7 ")}`),u.info(`Listings: ${F} parsed \xB7 ${H} exported to CSV`)}}}const b=e==="zillow"?m>0:!0;return b?u.success("Scraping complete!"):u.warn("Scraping finished but no listings were exported \u2014 please try again shortly."),{success:b,totalCsvListings:m}}finally{await v()}}export{rt as a,it as b,Qt as c,te as d,se as e,E as f,bt as g,$t as h,at as i,we as r,yt as s};
|