@rent-scraper/scrape-listings 1.0.33 → 1.0.35

package/dist/bin/run-scrape-listings.mjs

@@ -1 +1 @@
-import{parseError as t}from"@rent-scraper/utils";import{r as i}from"../shared/scrape-listings.DgphD-7P.mjs";import{log as m}from"@clack/prompts";import"minimist";import"dayjs";import"path";import"axios";import"fs/promises";import"@rent-scraper/api";import"@rent-scraper/api/config";import"node:timers/promises";import"crypto";import"@rent-scraper/utils/config";import"picocolors";i().then(()=>{process.exit(0)}).catch(r=>{const{message:o}=t(r);m.error(o),process.exit(1)});
+import{parseError as t}from"@rent-scraper/utils";import{r as i}from"../shared/scrape-listings.D0kKRGJj.mjs";import{log as m}from"@clack/prompts";import"minimist";import"dayjs";import"path";import"axios";import"fs/promises";import"@rent-scraper/api";import"@rent-scraper/api/config";import"node:timers/promises";import"crypto";import"@rent-scraper/utils/config";import"picocolors";i().then(()=>{process.exit(0)}).catch(r=>{const{message:o}=t(r);m.error(o),process.exit(1)});

package/dist/index.d.mts

@@ -20,6 +20,9 @@ declare const scrapeListingHtmlByIds: (source: ListingsSource, ids: string[], ou
 declare function runScrapeListings(): Promise<{
     success: boolean;
     totalCsvListings: number;
+    missedZipCodes: number;
+    resultsDirectory: any;
+    listingsDirectory: any;
 }>;
 
 declare const scrapeZillowListingsToCsv: (listingsDirectory: string, outputFilePath: string) => Promise<number>;

package/dist/index.mjs

@@ -1 +1 @@
-export{f as fetchListingHtmlByUrlAndExport,r as runScrapeListings,s as scrapeListingDetailsFromHtmlByFilePaths,a as scrapeListingDetailsFromHtmlByZipCodes,e as scrapeListingHtmlByIds,d as scrapeListingHtmlByInputDirectory,b as scrapeListingHtmlByZipCodes,c as scrapeListingHtmlByZipCodesAndListingDetails,h as scrapeRedfinListingResultsByZipCodes,i as scrapeZillowListingResultsByZipCodes,g as scrapeZillowListingsToCsv}from"./shared/scrape-listings.DgphD-7P.mjs";import"minimist";import"dayjs";import"path";import"axios";import"fs/promises";import"@rent-scraper/api";import"@rent-scraper/utils";import"@rent-scraper/api/config";import"@clack/prompts";import"node:timers/promises";import"crypto";import"@rent-scraper/utils/config";import"picocolors";
+export{f as fetchListingHtmlByUrlAndExport,r as runScrapeListings,s as scrapeListingDetailsFromHtmlByFilePaths,a as scrapeListingDetailsFromHtmlByZipCodes,e as scrapeListingHtmlByIds,d as scrapeListingHtmlByInputDirectory,b as scrapeListingHtmlByZipCodes,c as scrapeListingHtmlByZipCodesAndListingDetails,h as scrapeRedfinListingResultsByZipCodes,i as scrapeZillowListingResultsByZipCodes,g as scrapeZillowListingsToCsv}from"./shared/scrape-listings.D0kKRGJj.mjs";import"minimist";import"dayjs";import"path";import"axios";import"fs/promises";import"@rent-scraper/api";import"@rent-scraper/utils";import"@rent-scraper/api/config";import"@clack/prompts";import"node:timers/promises";import"crypto";import"@rent-scraper/utils/config";import"picocolors";

package/dist/shared/scrape-listings.D0kKRGJj.mjs

@@ -0,0 +1,15 @@
+import Mt from"minimist";import ct from"dayjs";import o from"path";import dt from"axios";import{mkdir as Z,writeFile as B,readFile as X,unlink as K,readdir as st}from"fs/promises";import{checkForZillowBotFiltering as J,isZillowBotFiltering as ut,fetchHtmlFromRedfinListingUrl as Bt,fetchHtmlFromZillowListingUrl as Dt,scrapeDataFromRedfinListingHtml as Ft,scrapeDataFromZillowListingHtml as Pt,getZillowListingResults as Rt,getRedfinListingResults as Ut,waitForZillowCaptchaSolve as Nt,waitForSolvedZillowCaptcha as pt,getZillowForceCaptcha as Ht,isBrowserShowingCaptcha as Ot}from"@rent-scraper/api";import{ErrorLog as O,throwError as C,checkForFile as j,parseJsonFile as Tt,parseError as R,readFilesInDirectory as Q,roundValue as it,chunkArray as It,parsePercentage as Et,compareArrays as wt}from"@rent-scraper/utils";import{getZillowOutputPath as T,getRedfinOutputPath as I,getZillowDaysListed as At,getRedfinDaysListed as Yt,getZillowLimit as qt,getZillowOffset as Jt,getRedfinZipCodes as Vt,getZillowZipCodes as Gt}from"@rent-scraper/api/config";import{spinner as gt,log as d,progress as Wt,intro as Xt}from"@clack/prompts";import{setTimeout as mt}from"node:timers/promises";import{createHash as Kt}from"crypto";import{checkBrowserServer as ht}from"@rent-scraper/utils/config";import Qt from"picocolors";const U=process.env.DEBUG,E=async(t,e,s,i)=>{const{timeoutMs:r}=i??{};if(await j(s))U&&d.warn(`${s} exists, skipping`);else try{U&&d.info(`writing ${s}`);const a=t==="redfin"?await Bt(e):await Dt(e,{timeoutMs:r});await B(`${s}`,a)}catch(a){C(`error fetching html for ${e}`,a)}},te=async(t,{timeoutMs:e})=>{if(await j(t)){const s=JSON.parse(await X(t,"utf8"))||{},{hdpUrl:i}=s||{};if(i){const r=`https://www.zillow.com${i}`,a=t.replace(".json",".html");await j(a)?U&&d.warn(`file already exists, ${a}`):await E("zillow",r,a,{timeoutMs:e})}else C(`file is empty, ${t}`)}else U&&d.warn(`file does not exist at this path, ${t}, skipping`)},ft=async(t,e)=>{const{timeoutMs:s}=e??{};await Promise.all(t.map(async i=>await te(i,{timeoutMs:s})))},ee=(t,e="zillow")=>{if(e==="zillow"){const{zpid:s,detailUrl:i}=t||{},r=i&&!i.startsWith("http")?`https://www.zillow.com${i}`:i;return{id:s,url:r}}else if(e==="redfin"){const{propertyId:s,url:i}=t?.homeData||{};return{id:s,url:`https://www.redfin.com${i}`}}},rt=async(t,e,s,i=s,r)=>{const{timeoutMs:a,run:n=1,reruns:l=0,skipBotCheck:f=!1}=r??{},c=new O;s||C("inputDirectory is required"),t==="zillow"&&!f&&await J();const w=[],y=[],u=gt();u.start("Downloading listings html files");for(let g=1;g<=l+1;g++){l>0&&g>1&&c.add(`rerun ${g-1} of ${l}`);const m=w.length;(g===1||m)&&await Promise.all((m?w:e).map(async b=>{const v=`${s}/${b}.json`;if(await j(v))try{const{results:M}=await Tt(v)||{};if(!M)c.add(`empty file, ${v}`);else{const S=i?`${i}/${b}`:`${s}/${b}`;await Z(S,{recursive:!0}),M?.length?await Promise.all(M.map(async L=>{const{id:k,url:x}=ee(L,t)??{};if(!x)return c.add(`url missing for ${k}`);const D=`${k}.html`,F=`${S}/${D}`;try{await E(t,x,F,{timeoutMs:a})}catch(z){const{status:A,message:H}=R(z);ut(A,H)||y.push({url:x,filePath:F}),c.add("scrape listing html error: "+(H??`error fetching listing for id, ${z}`))}})):c.add(`no results for file, ${v}`)}}catch(M){w.push(b);const{message:S}=R(M);c.add("scrape listing html error: "+(S??`error reading json data, ${v}, ${M}`))}else U&&d.warn(`file does not exist, ${v}, skipping`)}))}y.length>0&&(await mt(2e3),await Promise.all(y.map(async({url:g,filePath:m})=>{try{await E(t,g,m,{timeoutMs:a})}catch(b){const{message:v}=R(b);c.add("scrape listing html retry error: "+(v??`error fetching ${g}`))}})));const $=await T(),h=await I(),p=t==="zillow"?$:h;e.length>0?(u.stop("Listings HTML files have been saved to:"),d.message(o.join(p,t,"listings",o.basename(s)))):u.stop("No listings HTML files saved.");const _=o.join(p,t,"logs");if(await Z(_,{recursive:!0}),c.get().filter(g=>!g.includes("rerun ")).length>l){const g=`${o.basename(s)}-html-errors-${n}.txt`,m=o.join(_,g);await c.write(m,[...new Set(c.get())].join(`
+`)),U&&d.error(`There were errors during processing, see ${o.resolve(m)}`)}},se=async(t,e,s,i)=>{const{timeoutMs:r,run:a=1,reruns:n=0,skipBotCheck:l=!1}=i??{},f=new O;s||C("inputDirectory is required"),t==="zillow"&&!l&&await J();const c=[];for(let h=1;h<=n+1;h++){n>0&&h>1&&f.add(`rerun ${h-1} of ${n}`);const p=c.length;(h===1||p)&&await Promise.all((p?c:e).map(async _=>{const g=`${s}/${_}`;if(await j(g))try{const m=await Q(g,{extension:".json",prependDirectory:!0});await ft(m,{timeoutMs:r})}catch(m){c.push(_);const{message:b}=R(m);f.add("scrape listing html error: "+(b??`Error during fetch for ${_}, ${m}`))}else f.add(`listing directory does not exist, ${g}`)}))}const w=await T(),y=await I(),u=t==="zillow"?w:y,$=o.join(u,t,"logs");if(await Z($,{recursive:!0}),f.get().filter(h=>!h.includes("rerun ")).length>n){const h=`${o.basename(s)}-html-errors-${a}.txt`,p=o.join($,h);await f.write(p,[...new Set(f.get())].join(`
+`)),U&&d.error(`There were errors during processing, see ${o.resolve(p)}`)}},ie=async(t,e,s=e,i)=>{const{skipBotCheck:r=!1}=i??{},a=new O;if(t==="zillow"&&!r&&await J(),e||C("inputDirectory is required"),await Z(s,{recursive:!0}),await j(e))try{const u=await Q(e,{extension:".json",prependDirectory:!0});await ft(u)}catch(u){const{message:$}=R(u);a.add("scrape listing html error: "+($??`Error during fetch for ${e}, ${u}`))}else a.add(`inputDirectory does not exist, ${e}`);const n=await T(),l=await I(),f=t==="zillow"?n:l,c=o.join(f,t,"logs");await Z(c,{recursive:!0});const w=`${o.basename(e)}-html-errors.txt`,y=o.join(c,w);a.get().length>0&&(await a.write(y,[...new Set(a.get())].join(`
+`)),U&&d.error(`There were errors during processing, see ${o.resolve(y)}`))},re=(t,e)=>t==="redfin"?`https://www.redfin.com/home/${e}`:t==="zillow"?`https://www.zillow.com/homedetails/${e}_zpid`:null,ne=async(t,e,s,i)=>{const{skipBotCheck:r=!1}=i??{};t==="zillow"&&!r&&await J(),s||C("outputDirectory is required"),await Z(s,{recursive:!0}),await Promise.all(e.map(async a=>{const n=re(t,a),l=`${s}/${a}.html`;n&&await E(t,n,l)}))},P=process.env.DEBUG,yt=async t=>{if(await j(t)){const e=t.replace(".html",".json");if(await j(e))P&&d.warning(`file already exists, ${e}`);else{P&&d.message(`scraping data for ${t}`);const s=(await X(t)).toString();(!s||s.trim()==="")&&(await K(t),C(`empty file found at ${t}, deleted for retry`)),s.includes("px-captcha")&&(await K(t),C(`captcha page found in ${t}, deleted for retry`));const i=await Pt(s);if(!i?.priceHistory&&i?.bestMatchedUnit?.hdpUrl){P&&d.warning(`rescraping ${t} - https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`),await j(t)&&(P&&d.warning(`deleting ${t}`),await K(t)),await j(e)&&(P&&d.warning(`deleting ${e}`),await K(e));const r=`https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`;await E("zillow",r,t),await yt(t)}else i?(P&&d.info(`writing ${e}`),await B(`${e}`,JSON.stringify(i))):C(`problem scraping data for ${t}`)}}},ae=async t=>{if(await j(t)){const e=t.replace(".html",".json");if(await j(e))P&&d.warning(`file already exists, ${e}`);else{P&&d.message(`scraping data for ${t}`);const s=(await X(t)).toString(),i=Ft(s);i?(P&&d.warning(`writing ${e}`),await B(`${e}`,JSON.stringify(i))):C(`problem scraping data for ${t}`)}}},$t=async(t,e,s)=>(await Promise.all(e.map(async i=>{try{t==="redfin"?await ae(i):await yt(i);const r=i.replace(".html",".json");return await j(r)?1:0}catch(r){const{message:a}=R(r);return s?.add(a),0}}))).reduce((i,r)=>i+r,0),nt=async(t,e,s)=>{const i=new O;let r=0;s||C("inputDirectory is required");const a=Wt({style:"heavy",max:100,size:50});a.start("Scraping listings data");const n=e.length,l=n<20?n+1:20,f=n>l?it(n/l,1):n,c=It(e,f);for(const[h,p]of c.entries()){const _=it((Number(h)+1)/c.length*100,1);a.advance(it(1/c.length*100,1),`Scraping listings (${Et(_)})`),await Promise.all(p.map(async g=>{P&&d.message(`processing files for ${g}`);const m=`${s}/${g}`;if(await j(m)){const b=await Q(m,{extension:".html",prependDirectory:!0}),v=await $t(t,b,i);r=r+v}else i.add(`listing directory does not exist, ${m}`)}))}const w=await T(),y=await I(),u=t==="zillow"?w:y;r>0?(a.stop("Listings data has been saved to:"),d.message(o.join(u,t,"listings",o.basename(s)))):a.stop("No listings data saved.");const $=o.join(u,t,"logs");if(await Z($,{recursive:!0}),i.get().length>0){const h=`${o.basename(s)}-listing-errors.txt`,p=o.join($,h);await i.write(p,[...new Set(i.get())].join(`
+`)),d.error(`There were errors during processing, see ${o.resolve(p)}`)}return{numListings:r}},at=process.env.DEBUG,oe=async(t,e,s,i,r)=>{const{daysOnZillow:a,timeoutMs:n}=r??{};if(await j(e))at&&d.warning(`${t} exists, skipping`),s.push(t);else{const l=await Rt({zipCode:t,daysOnZillow:a,mergePageResults:!0,timeoutMs:n});l?(at&&d.info(`writing ${e}`),await B(e,JSON.stringify(l)),s.push(t)):(at&&d.warning(`no results for ${t}`),i.push(t))}},ot=async(t,e,s)=>{const{daysListed:i,timeoutMs:r,run:a=1,reruns:n=0,fetchListings:l=!1,skipBotCheck:f=!1,silent:c=!1}=s??{},w=new O;f||await J({fetchListings:l});const y=[],u=[],$=[],h=[];await Z(e,{recursive:!0});const p=gt();c||p.start("Scraping Zillow search results");for(let m=1;m<=n+1;m++){n>0&&m>1&&w.add(`rerun ${m-1} of ${n}`);const b=u.length;(m===1||b)&&await Promise.all((b?u:t).map(async v=>{const M=`${v}.json`,S=`${e}/${M}`;try{await oe(v,S,y,h,{daysOnZillow:i,timeoutMs:r})}catch(L){u.push(v);const{status:k,message:x}=R(L);ut(k,x)&&$.push(v),w.add("scrape listing results error: "+(x??`Error during fetch for ${v}, ${L}`))}}))}const _=await T();c||(y.length>0?(p.stop("Zillow search results have been saved to:"),d.message(o.join(_,"zillow","results",o.basename(e)))):p.stop("No results saved."));const g=o.join(_,"zillow","logs");if(await Z(g,{recursive:!0}),w.get().filter(m=>!m.includes("rerun ")).length>n){const m=`${o.basename(e)}-results-errors-${a}.txt`,b=o.join(g,m);await w.write(b,[...new Set(w.get())].join(`
+`)),c||d.error(`There were errors during processing, see ${o.resolve(b)}`)}return{validZipCodes:y,botFilteredZipCodes:$,noResultsZipCodes:h}},le=async(t,e,s,i)=>{const{daysListed:r,timeoutMs:a}=i??{};if(await j(e))console.log(`${t} exists, skipping`),s.push(t);else{const n=await Ut({zipCode:t,daysListed:r,timeoutMs:a});n?(console.log(`writing ${e}`),await B(e,JSON.stringify(n)),s.push(t)):console.log(`no results for ${t}`)}},_t=async(t,e,s)=>{const{daysListed:i,timeoutMs:r,run:a=1,reruns:n=0}=s??{},l=new O,f=[],c=[];await Z(e,{recursive:!0});for(let u=1;u<=n+1;u++){n>0&&u>1&&l.add(`rerun ${u-1} of ${n}`);const $=c.length;(u===1||$)&&await Promise.all(($?c:t).map(async h=>{const p=`${h}.json`,_=`${e}/${p}`;try{await le(h,_,f,{daysListed:i,timeoutMs:r})}catch(g){c.push(h);const{message:m}=R(g);l.add("scrape listing results error: "+(m??`Error during fetch for ${h}, ${g}`))}}))}const w=await I(),y=o.join(w,"redfin","logs");if(await Z(y,{recursive:!0}),l.get().filter(u=>!u.includes("rerun ")).length>n){const u=`${o.basename(e)}-results-errors-${a}.txt`,$=o.join(y,u);await l.write($,[...new Set(l.get())].join(`
+`)),console.log(`\x1B[41m
+%s\x1B[0m`,`There were errors during processing, see ${o.resolve($)}`)}return{validZipCodes:f}},bt=["listing_id","listing_source","source_listing_id","parcel_number","listing_url","street_address","cleaned_address","cleaned_unit_number","city","state","zipcode","county","neighborhood_region","is_undisclosed_address","home_status","home_type","bedrooms","year_built","living_area","living_area_units","is_income_restricted","price_at_source","platform_rent_estimate","agent_name","agent_phone_number","broker_name","broker_phone_number","is_owned_by_listing_platform","date_last_updated_at_source","date_scraped","scrape_job_name"],ce=t=>{if(t==null)return"";const e=typeof t=="object"?JSON.stringify(t):String(t);return e.includes(",")||e.includes('"')||e.includes(`
+`)?`"${e.replace(/"/g,'""')}"`:e},de=t=>{const e=/^(.*?)\s+((?:#|APT|UNIT|STE|Suite|Apt|Unit)\s*.+)$/i.exec(t);if(e){const s=e[1].trim().toUpperCase();let i=e[2].trim();return i.startsWith("#")&&(i=`# ${i.replace(/^#\s*/,"").trim()}`),{cleaned:s,unit:i}}return{cleaned:t.toUpperCase(),unit:""}},ue=(t,e)=>{const s=String(t.zpid),i=Kt("md5").update(s).digest("hex"),r=t.streetAddress||t.address?.streetAddress||"",{cleaned:a,unit:n}=de(r),l=t.hdpUrl?`https://www.zillow.com${t.hdpUrl}`:`https://www.zillow.com/homedetails/${s}_zpid/`;return{listing_id:i,listing_source:"zillow",source_listing_id:s,parcel_number:t.resoFacts?.parcelNumber??"",listing_url:l,street_address:r,cleaned_address:a,cleaned_unit_number:n,city:t.city||t.address?.city||"",state:t.state||t.address?.state||"",zipcode:t.zipcode||t.address?.zipcode||"",county:t.county||"",neighborhood_region:t.neighborhoodRegion?.name||t.parentRegion?.name||"",is_undisclosed_address:t.isUndisclosedAddress??"",home_status:t.homeStatus||"",home_type:t.homeType||"",bedrooms:t.bedrooms??"",year_built:t.yearBuilt||t.resoFacts?.yearBuilt||"",living_area:t.livingAreaValue??"",living_area_units:t.livingAreaUnits||"Square Feet",is_income_restricted:t.isIncomeRestricted??"",price_at_source:t.price??"",platform_rent_estimate:t.rentZestimate??"",agent_name:t.attributionInfo?.agentName||"",agent_phone_number:t.attributionInfo?.agentPhoneNumber||"",broker_name:t.attributionInfo?.brokerName||"",broker_phone_number:t.attributionInfo?.brokerPhoneNumber||"",is_owned_by_listing_platform:t.attributionInfo?.mlsName==="Zillow Rentals",date_last_updated_at_source:t.attributionInfo?.lastUpdated||"",date_scraped:t.timestamp?ct(t.timestamp).format("YYYY-MM-DD HH:mm:ss"):"",scrape_job_name:e}},vt=async(t,e)=>{const s=o.basename(t),i=[bt.join(",")];let r;try{r=await st(t,{withFileTypes:!0})}catch{return 0}const a=r.filter(n=>n.isDirectory()).map(n=>n.name);for(const n of a){const l=o.join(t,n);if(!await j(l))continue;const f=await Q(l,{extension:".json",prependDirectory:!0});for(const c of f)try{const w=JSON.parse(await X(c,"utf8"));if(!w.zpid)continue;const y=ue(w,s);i.push(bt.map(u=>ce(y[u])).join(","))}catch{}}return await Z(o.dirname(e),{recursive:!0}),await B(e,i.join(`
+`)),i.length-1},pe=async t=>{try{const e=(await st(t,{withFileTypes:!0})).filter(s=>s.isDirectory()).map(s=>s.name).sort();return e.length>0?e[e.length-1]:null}catch{return null}},jt=async()=>{await dt.post("http://localhost:8082/browser/close")},we=async()=>{await dt.post("http://localhost:8082/server/shutdown")},ge=async(t,e,s,{daysListed:i,timeoutMs:r,run:a,reruns:n,preValidatedZipCodes:l=[],retry:f=!1})=>{let c=[],w=[],y=[];if(t.length>0){if(await(await Ht()?Nt():pt()),await jt(),{validZipCodes:c,botFilteredZipCodes:w,noResultsZipCodes:y}=await ot(t,e,{daysListed:i,timeoutMs:r,run:a,reruns:n,skipBotCheck:!0}),f&&w.length>0){d.warn(`Bot filtering hit ${w.length} of ${t.length} zip code(s) \u2014 will retry after listings are fetched`),await Ot()?(await pt(),await jt()):await mt(3e3);const h=w.length,p=await ot(w,e,{daysListed:i,timeoutMs:r,run:a,reruns:n,skipBotCheck:!0,silent:!0});c=[...c,...p.validZipCodes],w=p.botFilteredZipCodes,y=[...y,...p.noResultsZipCodes];const _=p.validZipCodes.length;d.info(`Inline retry: recovered ${_} of ${h} \xB7 ${w.length} still bot-filtered`)}c.length===0&&t.length>0&&(l.length>0?d.warn(`No new results for re-fetched zip codes \u2014 proceeding with ${l.length} previously fetched`):d.warn("No results returned for any zip codes \u2014 possible soft bot filtering. Please try again shortly.")),w.length>0&&d.warn(`${w.length} zip code(s) still bot-filtered \u2014 use --rerun to retry`)}const u=[...l,...c];await rt("zillow",u,e,s,{timeoutMs:r,run:a,reruns:n,skipBotCheck:!0});const{numListings:$}=await nt("zillow",u,s);return{numListings:$,validZipCodes:u,noResultsZipCodes:y}},me=async(t,e,s,{daysListed:i,timeoutMs:r,run:a,reruns:n})=>{const{validZipCodes:l}=await _t(t,e,{daysListed:i,timeoutMs:r,run:a,reruns:n});await rt("redfin",l,e,s,{timeoutMs:r,run:a,reruns:n});const{numListings:f}=await nt("redfin",l,s);return{numListings:f,validZipCodes:l}};async function he(){Xt(Qt.inverse(" scrape listings "));const t=Mt(process.argv.slice(2)),e=t.source??"zillow";e==="zillow"&&(await ht()||C("Please launch the browser server before scraping."));const s=await T(),i=await I(),r=e==="zillow"?s:i,a=t["days-listed"]??(e==="zillow"?await At():await Yt())??1,n=t.runs??1,l=t.reruns??0,f=t["timeout-ms"]??6e4,c=t.limit??await qt()??void 0,w=t.offset??await Jt()??0,y=t.retry??!1,u=t.rerun,$=typeof u=="string"?u:u?await pe(o.join(r,e,"results")):null;u&&!$&&C("No previous run found to rerun."),$&&d.info(`Rerunning from: ${$}`);const h=$??ct().format("YYYY-MM-DD-HHmm"),p=t["results-directory"]??o.join(r,e,"results",h),_=t["listings-directory"]??o.join(r,e,"listings",h),g=t["logs-directory"]??o.join(r,e,"logs");let m=0,b=0;const v=async()=>{(e==="zillow"||e==="redfin")&&await ht()&&await we()};try{if(e==="redfin"){const S=await Vt();for(let L=1;L<=n;L++){S||C("zip codes required, please run the createConfig script");const k=S,{numListings:x,validZipCodes:D}=await me(k,p,_,{daysListed:a,timeoutMs:f,run:L,reruns:l});if(L===n){const F=Object.entries({numListings:x}).map(([tt,G])=>`${tt}: ${G}`).join(`
+`),z=wt(k,D).join(`
+`);await Z(g,{recursive:!0});const A=`${o.basename(p)}-scraping-results.txt`,H=o.join(g,A),V=`${o.basename(p)}-invalid-zipcodes.txt`,N=o.join(g,V);await B(H,F),await B(N,z)}}}else if(e==="zillow"){const S=await Gt();S||C("zip codes required, please run the createConfig script");const L=S.slice(w,c?w+c:void 0);let k=L,x=[];if($){const D=await st(p).catch(()=>[]),F=new Set(D.filter(z=>z.endsWith(".json")).map(z=>z.replace(".json","")));x=L.filter(z=>F.has(String(z))),k=L.filter(z=>!F.has(String(z))),k.length>0?d.info(`${x.length} already fetched \xB7 retrying ${k.length} missing`):d.info(`All ${x.length} zip codes already fetched \xB7 proceeding to HTML scraping`)}for(let D=1;D<=n;D++){const{numListings:F,validZipCodes:z,noResultsZipCodes:A}=await ge(k,p,_,{daysListed:a,timeoutMs:f,run:D,reruns:l,preValidatedZipCodes:x,retry:y});if(D===n){const H=o.basename(_),V=o.join(s,"zillow","csv",`${H}.csv`),N=await vt(_,V);m=N,N>0&&d.info(`CSV exported: ${N} listings \u2192 ${V}`);const tt=Object.entries({numListings:F,numCsvListings:N}).map(([q,kt])=>`${q}: ${kt}`).join(`
+`),G=wt(L,z),lt=new Set(A),Y=G.filter(q=>!lt.has(q)),W=G.filter(q=>lt.has(q));await Z(g,{recursive:!0});const Lt=`${o.basename(p)}-scraping-results.txt`,zt=o.join(g,Lt),Ct=`${o.basename(p)}-zipcodes-no-results.txt`,Zt=o.join(g,Ct),xt=`${o.basename(p)}-zipcodes-errored.txt`,St=o.join(g,xt);await B(zt,tt),W.length>0&&await B(Zt,W.join(`
+`)),Y.length>0&&await B(St,Y.join(`
+`)),b=Y.length;const et=[`${z.length}/${L.length} with results`];W.length>0&&et.push(`${W.length} no results`),Y.length>0&&et.push(`${Y.length} errored`),d.info(`Zip codes: ${et.join(" \xB7 ")}`),d.info(`Listings:  ${F} parsed \xB7 ${N} exported to CSV`)}}}const M=e==="zillow"?m>0:!0;return M?d.success("Scraping complete!"):d.warn("Scraping finished but no listings were exported \u2014 please try again shortly."),{success:M,totalCsvListings:m,missedZipCodes:b,resultsDirectory:p,listingsDirectory:_}}finally{await v()}}export{nt as a,rt as b,se as c,ie as d,ne as e,E as f,vt as g,_t as h,ot as i,he as r,$t as s};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rent-scraper/scrape-listings",
-  "version": "1.0.33",
+  "version": "1.0.35",
   "type": "module",
   "exports": {
     ".": {
@@ -30,8 +30,8 @@
     "dayjs": "^1.11.13",
     "minimist": "^1.2.8",
     "picocolors": "^1.1.1",
-    "@rent-scraper/api": "1.0.33",
-    "@rent-scraper/utils": "1.0.33"
+    "@rent-scraper/utils": "1.0.35",
+    "@rent-scraper/api": "1.0.35"
   },
   "devDependencies": {
     "@types/minimist": "^1.2.5",

package/dist/shared/scrape-listings.DgphD-7P.mjs

@@ -1,15 +0,0 @@
-import kt from"minimist";import lt from"dayjs";import o from"path";import ct from"axios";import{mkdir as S,writeFile as M,readFile as W,unlink as X,readdir as et}from"fs/promises";import{checkForZillowBotFiltering as q,isZillowBotFiltering as dt,fetchHtmlFromRedfinListingUrl as Bt,fetchHtmlFromZillowListingUrl as Mt,scrapeDataFromRedfinListingHtml as Dt,scrapeDataFromZillowListingHtml as Ft,getZillowListingResults as Rt,getRedfinListingResults as Pt,waitForZillowCaptchaSolve as Ut,waitForSolvedZillowCaptcha as ut,getZillowForceCaptcha as Ht,isBrowserShowingCaptcha as Nt}from"@rent-scraper/api";import{ErrorLog as O,throwError as Z,checkForFile as L,parseJsonFile as Ot,parseError as R,readFilesInDirectory as K,roundValue as st,chunkArray as Tt,parsePercentage as It,compareArrays as wt}from"@rent-scraper/utils";import{getZillowOutputPath as T,getRedfinOutputPath as I,getZillowDaysListed as Et,getRedfinDaysListed as At,getZillowLimit as Yt,getZillowOffset as qt,getRedfinZipCodes as Jt,getZillowZipCodes as Vt}from"@rent-scraper/api/config";import{spinner as pt,log as u,progress as Gt,intro as Wt}from"@clack/prompts";import{setTimeout as gt}from"node:timers/promises";import{createHash as Xt}from"crypto";import{checkBrowserServer as mt}from"@rent-scraper/utils/config";import Kt from"picocolors";const P=process.env.DEBUG,E=async(t,e,s,i)=>{const{timeoutMs:r}=i??{};if(await L(s))P&&u.warn(`${s} exists, skipping`);else try{P&&u.info(`writing ${s}`);const n=t==="redfin"?await Bt(e):await Mt(e,{timeoutMs:r});await M(`${s}`,n)}catch(n){Z(`error fetching html for ${e}`,n)}},Qt=async(t,{timeoutMs:e})=>{if(await L(t)){const s=JSON.parse(await W(t,"utf8"))||{},{hdpUrl:i}=s||{};if(i){const r=`https://www.zillow.com${i}`,n=t.replace(".json",".html");await L(n)?P&&u.warn(`file already exists, ${n}`):await E("zillow",r,n,{timeoutMs:e})}else Z(`file is empty, ${t}`)}else P&&u.warn(`file does not exist at this path, ${t}, skipping`)},ht=async(t,e)=>{const{timeoutMs:s}=e??{};await Promise.all(t.map(async i=>await Qt(i,{timeoutMs:s})))},te=(t,e="zillow")=>{if(e==="zillow"){const{zpid:s,detailUrl:i}=t||{},r=i&&!i.startsWith("http")?`https://www.zillow.com${i}`:i;return{id:s,url:r}}else if(e==="redfin"){const{propertyId:s,url:i}=t?.homeData||{};return{id:s,url:`https://www.redfin.com${i}`}}},it=async(t,e,s,i=s,r)=>{const{timeoutMs:n,run:a=1,reruns:l=0,skipBotCheck:f=!1}=r??{},c=new O;s||Z("inputDirectory is required"),t==="zillow"&&!f&&await q();const w=[],y=[],d=pt();d.start("Downloading listings html files");for(let g=1;g<=l+1;g++){l>0&&g>1&&c.add(`rerun ${g-1} of ${l}`);const m=w.length;(g===1||m)&&await Promise.all((m?w:e).map(async v=>{const b=`${s}/${v}.json`;if(await L(b))try{const{results:x}=await Ot(b)||{};if(!x)c.add(`empty file, ${b}`);else{const j=i?`${i}/${v}`:`${s}/${v}`;await S(j,{recursive:!0}),x?.length?await Promise.all(x.map(async k=>{const{id:B,url:z}=te(k,t)??{};if(!z)return c.add(`url missing for ${B}`);const F=`${B}.html`,C=`${j}/${F}`;try{await E(t,z,C,{timeoutMs:n})}catch(N){const{status:A,message:U}=R(N);dt(A,U)||y.push({url:z,filePath:C}),c.add("scrape listing html error: "+(U??`error fetching listing for id, ${N}`))}})):c.add(`no results for file, ${b}`)}}catch(x){w.push(v);const{message:j}=R(x);c.add("scrape listing html error: "+(j??`error reading json data, ${b}, ${x}`))}else P&&u.warn(`file does not exist, ${b}, skipping`)}))}y.length>0&&(await gt(2e3),await Promise.all(y.map(async({url:g,filePath:m})=>{try{await E(t,g,m,{timeoutMs:n})}catch(v){const{message:b}=R(v);c.add("scrape listing html retry error: "+(b??`error fetching ${g}`))}})));const $=await T(),h=await I(),p=t==="zillow"?$:h;e.length>0?(d.stop("Listings HTML files have been saved to:"),u.message(o.join(p,t,"listings",o.basename(s)))):d.stop("No listings HTML files saved.");const _=o.join(p,t,"logs");if(await S(_,{recursive:!0}),c.get().filter(g=>!g.includes("rerun ")).length>l){const g=`${o.basename(s)}-html-errors-${a}.txt`,m=o.join(_,g);await c.write(m,[...new Set(c.get())].join(`
-`)),P&&u.error(`There were errors during processing, see ${o.resolve(m)}`)}},ee=async(t,e,s,i)=>{const{timeoutMs:r,run:n=1,reruns:a=0,skipBotCheck:l=!1}=i??{},f=new O;s||Z("inputDirectory is required"),t==="zillow"&&!l&&await q();const c=[];for(let h=1;h<=a+1;h++){a>0&&h>1&&f.add(`rerun ${h-1} of ${a}`);const p=c.length;(h===1||p)&&await Promise.all((p?c:e).map(async _=>{const g=`${s}/${_}`;if(await L(g))try{const m=await K(g,{extension:".json",prependDirectory:!0});await ht(m,{timeoutMs:r})}catch(m){c.push(_);const{message:v}=R(m);f.add("scrape listing html error: "+(v??`Error during fetch for ${_}, ${m}`))}else f.add(`listing directory does not exist, ${g}`)}))}const w=await T(),y=await I(),d=t==="zillow"?w:y,$=o.join(d,t,"logs");if(await S($,{recursive:!0}),f.get().filter(h=>!h.includes("rerun ")).length>a){const h=`${o.basename(s)}-html-errors-${n}.txt`,p=o.join($,h);await f.write(p,[...new Set(f.get())].join(`
-`)),P&&u.error(`There were errors during processing, see ${o.resolve(p)}`)}},se=async(t,e,s=e,i)=>{const{skipBotCheck:r=!1}=i??{},n=new O;if(t==="zillow"&&!r&&await q(),e||Z("inputDirectory is required"),await S(s,{recursive:!0}),await L(e))try{const d=await K(e,{extension:".json",prependDirectory:!0});await ht(d)}catch(d){const{message:$}=R(d);n.add("scrape listing html error: "+($??`Error during fetch for ${e}, ${d}`))}else n.add(`inputDirectory does not exist, ${e}`);const a=await T(),l=await I(),f=t==="zillow"?a:l,c=o.join(f,t,"logs");await S(c,{recursive:!0});const w=`${o.basename(e)}-html-errors.txt`,y=o.join(c,w);n.get().length>0&&(await n.write(y,[...new Set(n.get())].join(`
-`)),P&&u.error(`There were errors during processing, see ${o.resolve(y)}`))},ie=(t,e)=>t==="redfin"?`https://www.redfin.com/home/${e}`:t==="zillow"?`https://www.zillow.com/homedetails/${e}_zpid`:null,re=async(t,e,s,i)=>{const{skipBotCheck:r=!1}=i??{};t==="zillow"&&!r&&await q(),s||Z("outputDirectory is required"),await S(s,{recursive:!0}),await Promise.all(e.map(async n=>{const a=ie(t,n),l=`${s}/${n}.html`;a&&await E(t,a,l)}))},D=process.env.DEBUG,ft=async t=>{if(await L(t)){const e=t.replace(".html",".json");if(await L(e))D&&u.warning(`file already exists, ${e}`);else{D&&u.message(`scraping data for ${t}`);const s=(await W(t)).toString();(!s||s.trim()==="")&&(await X(t),Z(`empty file found at ${t}, deleted for retry`)),s.includes("px-captcha")&&(await X(t),Z(`captcha page found in ${t}, deleted for retry`));const i=await Ft(s);if(!i?.priceHistory&&i?.bestMatchedUnit?.hdpUrl){D&&u.warning(`rescraping ${t} - https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`),await L(t)&&(D&&u.warning(`deleting ${t}`),await X(t)),await L(e)&&(D&&u.warning(`deleting ${e}`),await X(e));const r=`https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`;await E("zillow",r,t),await ft(t)}else i?(D&&u.info(`writing ${e}`),await M(`${e}`,JSON.stringify(i))):Z(`problem scraping data for ${t}`)}}},ae=async t=>{if(await L(t)){const e=t.replace(".html",".json");if(await L(e))D&&u.warning(`file already exists, ${e}`);else{D&&u.message(`scraping data for ${t}`);const s=(await W(t)).toString(),i=Dt(s);i?(D&&u.warning(`writing ${e}`),await M(`${e}`,JSON.stringify(i))):Z(`problem scraping data for ${t}`)}}},yt=async(t,e,s)=>(await Promise.all(e.map(async i=>{try{t==="redfin"?await ae(i):await ft(i);const r=i.replace(".html",".json");return await L(r)?1:0}catch(r){const{message:n}=R(r);return s?.add(n),0}}))).reduce((i,r)=>i+r,0),rt=async(t,e,s)=>{const i=new O;let r=0;s||Z("inputDirectory is required");const n=Gt({style:"heavy",max:100,size:50});n.start("Scraping listings data");const a=e.length,l=a<20?a+1:20,f=a>l?st(a/l,1):a,c=Tt(e,f);for(const[h,p]of c.entries()){const _=st((Number(h)+1)/l*100,1);n.advance(st(1/l*100,1),`Scraping listings (${It(_)})`),await Promise.all(p.map(async g=>{D&&u.message(`processing files for ${g}`);const m=`${s}/${g}`;if(await L(m)){const v=await K(m,{extension:".html",prependDirectory:!0}),b=await yt(t,v,i);r=r+b}else i.add(`listing directory does not exist, ${m}`)}))}const w=await T(),y=await I(),d=t==="zillow"?w:y;r>0?(n.stop("Listings data has been saved to:"),u.message(o.join(d,t,"listings",o.basename(s)))):n.stop("No listings data saved.");const $=o.join(d,t,"logs");if(await S($,{recursive:!0}),i.get().length>0){const h=`${o.basename(s)}-listing-errors.txt`,p=o.join($,h);await i.write(p,[...new Set(i.get())].join(`
-`)),u.error(`There were errors during processing, see ${o.resolve(p)}`)}return{numListings:r}},at=process.env.DEBUG,ne=async(t,e,s,i,r)=>{const{daysOnZillow:n,timeoutMs:a}=r??{};if(await L(e))at&&u.warning(`${t} exists, skipping`),s.push(t);else{const l=await Rt({zipCode:t,daysOnZillow:n,mergePageResults:!0,timeoutMs:a});l?(at&&u.info(`writing ${e}`),await M(e,JSON.stringify(l)),s.push(t)):(at&&u.warning(`no results for ${t}`),i.push(t))}},nt=async(t,e,s)=>{const{daysListed:i,timeoutMs:r,run:n=1,reruns:a=0,fetchListings:l=!1,skipBotCheck:f=!1,silent:c=!1}=s??{},w=new O;f||await q({fetchListings:l});const y=[],d=[],$=[],h=[];await S(e,{recursive:!0});const p=pt();c||p.start("Scraping Zillow search results");for(let m=1;m<=a+1;m++){a>0&&m>1&&w.add(`rerun ${m-1} of ${a}`);const v=d.length;(m===1||v)&&await Promise.all((v?d:t).map(async b=>{const x=`${b}.json`,j=`${e}/${x}`;try{await ne(b,j,y,h,{daysOnZillow:i,timeoutMs:r})}catch(k){d.push(b);const{status:B,message:z}=R(k);dt(B,z)&&$.push(b),w.add("scrape listing results error: "+(z??`Error during fetch for ${b}, ${k}`))}}))}const _=await T();c||(y.length>0?(p.stop("Zillow search results have been saved to:"),u.message(o.join(_,"zillow","results",o.basename(e)))):p.stop("No results saved."));const g=o.join(_,"zillow","logs");if(await S(g,{recursive:!0}),w.get().filter(m=>!m.includes("rerun ")).length>a){const m=`${o.basename(e)}-results-errors-${n}.txt`,v=o.join(g,m);await w.write(v,[...new Set(w.get())].join(`
-`)),c||u.error(`There were errors during processing, see ${o.resolve(v)}`)}return{validZipCodes:y,botFilteredZipCodes:$,noResultsZipCodes:h}},oe=async(t,e,s,i)=>{const{daysListed:r,timeoutMs:n}=i??{};if(await L(e))console.log(`${t} exists, skipping`),s.push(t);else{const a=await Pt({zipCode:t,daysListed:r,timeoutMs:n});a?(console.log(`writing ${e}`),await M(e,JSON.stringify(a)),s.push(t)):console.log(`no results for ${t}`)}},$t=async(t,e,s)=>{const{daysListed:i,timeoutMs:r,run:n=1,reruns:a=0}=s??{},l=new O,f=[],c=[];await S(e,{recursive:!0});for(let d=1;d<=a+1;d++){a>0&&d>1&&l.add(`rerun ${d-1} of ${a}`);const $=c.length;(d===1||$)&&await Promise.all(($?c:t).map(async h=>{const p=`${h}.json`,_=`${e}/${p}`;try{await oe(h,_,f,{daysListed:i,timeoutMs:r})}catch(g){c.push(h);const{message:m}=R(g);l.add("scrape listing results error: "+(m??`Error during fetch for ${h}, ${g}`))}}))}const w=await I(),y=o.join(w,"redfin","logs");if(await S(y,{recursive:!0}),l.get().filter(d=>!d.includes("rerun ")).length>a){const d=`${o.basename(e)}-results-errors-${n}.txt`,$=o.join(y,d);await l.write($,[...new Set(l.get())].join(`
-`)),console.log(`\x1B[41m
-%s\x1B[0m`,`There were errors during processing, see ${o.resolve($)}`)}return{validZipCodes:f}},_t=["listing_id","listing_source","source_listing_id","parcel_number","listing_url","street_address","cleaned_address","cleaned_unit_number","city","state","zipcode","county","neighborhood_region","is_undisclosed_address","home_status","home_type","bedrooms","year_built","living_area","living_area_units","is_income_restricted","price_at_source","platform_rent_estimate","agent_name","agent_phone_number","broker_name","broker_phone_number","is_owned_by_listing_platform","date_last_updated_at_source","date_scraped","scrape_job_name"],le=t=>{if(t==null)return"";const e=typeof t=="object"?JSON.stringify(t):String(t);return e.includes(",")||e.includes('"')||e.includes(`
-`)?`"${e.replace(/"/g,'""')}"`:e},ce=t=>{const e=/^(.*?)\s+((?:#|APT|UNIT|STE|Suite|Apt|Unit)\s*.+)$/i.exec(t);if(e){const s=e[1].trim().toUpperCase();let i=e[2].trim();return i.startsWith("#")&&(i=`# ${i.replace(/^#\s*/,"").trim()}`),{cleaned:s,unit:i}}return{cleaned:t.toUpperCase(),unit:""}},de=(t,e)=>{const s=String(t.zpid),i=Xt("md5").update(s).digest("hex"),r=t.streetAddress||t.address?.streetAddress||"",{cleaned:n,unit:a}=ce(r),l=t.hdpUrl?`https://www.zillow.com${t.hdpUrl}`:`https://www.zillow.com/homedetails/${s}_zpid/`;return{listing_id:i,listing_source:"zillow",source_listing_id:s,parcel_number:t.resoFacts?.parcelNumber??"",listing_url:l,street_address:r,cleaned_address:n,cleaned_unit_number:a,city:t.city||t.address?.city||"",state:t.state||t.address?.state||"",zipcode:t.zipcode||t.address?.zipcode||"",county:t.county||"",neighborhood_region:t.neighborhoodRegion?.name||t.parentRegion?.name||"",is_undisclosed_address:t.isUndisclosedAddress??"",home_status:t.homeStatus||"",home_type:t.homeType||"",bedrooms:t.bedrooms??"",year_built:t.yearBuilt||t.resoFacts?.yearBuilt||"",living_area:t.livingAreaValue??"",living_area_units:t.livingAreaUnits||"Square Feet",is_income_restricted:t.isIncomeRestricted??"",price_at_source:t.price??"",platform_rent_estimate:t.rentZestimate??"",agent_name:t.attributionInfo?.agentName||"",agent_phone_number:t.attributionInfo?.agentPhoneNumber||"",broker_name:t.attributionInfo?.brokerName||"",broker_phone_number:t.attributionInfo?.brokerPhoneNumber||"",is_owned_by_listing_platform:t.attributionInfo?.mlsName==="Zillow Rentals",date_last_updated_at_source:t.attributionInfo?.lastUpdated||"",date_scraped:t.timestamp?lt(t.timestamp).format("YYYY-MM-DD HH:mm:ss"):"",scrape_job_name:e}},bt=async(t,e)=>{const s=o.basename(t),i=[_t.join(",")];let r;try{r=await et(t,{withFileTypes:!0})}catch{return 0}const n=r.filter(a=>a.isDirectory()).map(a=>a.name);for(const a of n){const l=o.join(t,a);if(!await L(l))continue;const f=await K(l,{extension:".json",prependDirectory:!0});for(const c of f)try{const w=JSON.parse(await W(c,"utf8"));if(!w.zpid)continue;const y=de(w,s);i.push(_t.map(d=>le(y[d])).join(","))}catch{}}return await S(o.dirname(e),{recursive:!0}),await M(e,i.join(`
-`)),i.length-1},ue=async t=>{try{const e=(await et(t,{withFileTypes:!0})).filter(s=>s.isDirectory()).map(s=>s.name).sort();return e.length>0?e[e.length-1]:null}catch{return null}},vt=async()=>{await ct.post("http://localhost:8082/browser/close")},we=async()=>{await ct.post("http://localhost:8082/server/shutdown")},pe=async(t,e,s,{daysListed:i,timeoutMs:r,run:n,reruns:a,preValidatedZipCodes:l=[],retry:f=!1})=>{let c=[],w=[],y=[];if(t.length>0){if(await(await Ht()?Ut():ut()),await vt(),{validZipCodes:c,botFilteredZipCodes:w,noResultsZipCodes:y}=await nt(t,e,{daysListed:i,timeoutMs:r,run:n,reruns:a,skipBotCheck:!0}),f&&w.length>0){u.warn(`Bot filtering hit ${w.length} of ${t.length} zip code(s) \u2014 will retry after listings are fetched`),await Nt()?(await ut(),await vt()):await gt(3e3);const h=w.length,p=await nt(w,e,{daysListed:i,timeoutMs:r,run:n,reruns:a,skipBotCheck:!0,silent:!0});c=[...c,...p.validZipCodes],w=p.botFilteredZipCodes,y=[...y,...p.noResultsZipCodes];const _=p.validZipCodes.length;u.info(`Inline retry: recovered ${_} of ${h} \xB7 ${w.length} still bot-filtered`)}c.length===0&&t.length>0&&u.warn("No results returned for any zip codes \u2014 possible soft bot filtering. Please try again shortly."),w.length>0&&u.warn(`${w.length} zip code(s) still bot-filtered \u2014 use --rerun to retry`)}const d=[...l,...c];await it("zillow",d,e,s,{timeoutMs:r,run:n,reruns:a,skipBotCheck:!0});const{numListings:$}=await rt("zillow",d,s);return{numListings:$,validZipCodes:d,noResultsZipCodes:y}},ge=async(t,e,s,{daysListed:i,timeoutMs:r,run:n,reruns:a})=>{const{validZipCodes:l}=await $t(t,e,{daysListed:i,timeoutMs:r,run:n,reruns:a});await it("redfin",l,e,s,{timeoutMs:r,run:n,reruns:a});const{numListings:f}=await rt("redfin",l,s);return{numListings:f,validZipCodes:l}};async function me(){Wt(Kt.inverse(" scrape listings "));const t=kt(process.argv.slice(2)),e=t.source??"zillow";e==="zillow"&&(await mt()||Z("Please launch the browser server before scraping."));const s=await T(),i=await I(),r=e==="zillow"?s:i,n=t["days-listed"]??(e==="zillow"?await Et():await At())??1,a=t.runs??1,l=t.reruns??0,f=t["timeout-ms"]??6e4,c=t.limit??await Yt()??void 0,w=t.offset??await qt()??0,y=t.retry??!1,d=t.rerun,$=typeof d=="string"?d:d?await ue(o.join(r,e,"results")):null;d&&!$&&Z("No previous run found to rerun."),$&&u.info(`Rerunning from: ${$}`);const h=$??lt().format("YYYY-MM-DD-HHmm"),p=t["results-directory"]??o.join(r,e,"results",h),_=t["listings-directory"]??o.join(r,e,"listings",h),g=t["logs-directory"]??o.join(r,e,"logs");let m=0;const v=async()=>{(e==="zillow"||e==="redfin")&&await mt()&&await we()};try{if(e==="redfin"){const x=await Jt();for(let j=1;j<=a;j++){x||Z("zip codes required, please run the createConfig script");const k=x,{numListings:B,validZipCodes:z}=await ge(k,p,_,{daysListed:n,timeoutMs:f,run:j,reruns:l});if(j===a){const F=Object.entries({numListings:B}).map(([Q,J])=>`${Q}: ${J}`).join(`
-`),C=wt(k,z).join(`
-`);await S(g,{recursive:!0});const N=`${o.basename(p)}-scraping-results.txt`,A=o.join(g,N),U=`${o.basename(p)}-invalid-zipcodes.txt`,H=o.join(g,U);await M(A,F),await M(H,C)}}}else if(e==="zillow"){const x=await Vt();x||Z("zip codes required, please run the createConfig script");const j=x.slice(w,c?w+c:void 0);let k=j,B=[];if($){const z=await et(p).catch(()=>[]),F=new Set(z.filter(C=>C.endsWith(".json")).map(C=>C.replace(".json","")));B=j.filter(C=>F.has(String(C))),k=j.filter(C=>!F.has(String(C))),k.length>0?u.info(`${B.length} already fetched \xB7 retrying ${k.length} missing`):u.info(`All ${B.length} zip codes already fetched \xB7 proceeding to HTML scraping`)}for(let z=1;z<=a;z++){const{numListings:F,validZipCodes:C,noResultsZipCodes:N}=await pe(k,p,_,{daysListed:n,timeoutMs:f,run:z,reruns:l,preValidatedZipCodes:B,retry:y});if(z===a){const A=o.basename(_),U=o.join(s,"zillow","csv",`${A}.csv`),H=await bt(_,U);m=H,H>0&&u.info(`CSV exported: ${H} listings \u2192 ${U}`);const Q=Object.entries({numListings:F,numCsvListings:H}).map(([Y,St])=>`${Y}: ${St}`).join(`
-`),J=wt(j,C),ot=new Set(N),V=J.filter(Y=>!ot.has(Y)),G=J.filter(Y=>ot.has(Y));await S(g,{recursive:!0});const jt=`${o.basename(p)}-scraping-results.txt`,Lt=o.join(g,jt),zt=`${o.basename(p)}-zipcodes-no-results.txt`,Ct=o.join(g,zt),Zt=`${o.basename(p)}-zipcodes-errored.txt`,xt=o.join(g,Zt);await M(Lt,Q),G.length>0&&await M(Ct,G.join(`
-`)),V.length>0&&await M(xt,V.join(`
-`));const tt=[`${C.length}/${j.length} with results`];G.length>0&&tt.push(`${G.length} no results`),V.length>0&&tt.push(`${V.length} errored`),u.info(`Zip codes: ${tt.join(" \xB7 ")}`),u.info(`Listings:  ${F} parsed \xB7 ${H} exported to CSV`)}}}const b=e==="zillow"?m>0:!0;return b?u.success("Scraping complete!"):u.warn("Scraping finished but no listings were exported \u2014 please try again shortly."),{success:b,totalCsvListings:m}}finally{await v()}}export{rt as a,it as b,ee as c,se as d,re as e,E as f,bt as g,$t as h,nt as i,me as r,yt as s};