@rent-scraper/scrape-listings 1.0.29 → 1.0.30

package/dist/bin/run-scrape-listings.mjs

@@ -1 +1 @@
-import{parseError as t}from"@rent-scraper/utils";import{r as i}from"../shared/scrape-listings.B-9u0WHc.mjs";import{log as m}from"@clack/prompts";import"minimist";import"dayjs";import"path";import"axios";import"fs/promises";import"@rent-scraper/api";import"@rent-scraper/api/config";import"crypto";import"node:timers/promises";import"@rent-scraper/utils/config";import"picocolors";i().then(()=>{process.exit(0)}).catch(r=>{const{message:o}=t(r);m.error(o),process.exit(1)});
+import{parseError as t}from"@rent-scraper/utils";import{r as i}from"../shared/scrape-listings.DPpklKj4.mjs";import{log as m}from"@clack/prompts";import"minimist";import"dayjs";import"path";import"axios";import"fs/promises";import"@rent-scraper/api";import"@rent-scraper/api/config";import"node:timers/promises";import"crypto";import"@rent-scraper/utils/config";import"picocolors";i().then(()=>{process.exit(0)}).catch(r=>{const{message:o}=t(r);m.error(o),process.exit(1)});

package/dist/index.d.mts

@@ -1,6 +1,7 @@
 import { ListingsSource, ZillowListingHtmlOptions } from '@rent-scraper/api';
+import { ErrorLog } from '@rent-scraper/utils';
 
-declare const scrapeListingDetailsFromHtmlByFilePaths: (source: ListingsSource, inputFilePaths: string[]) => Promise<void>;
+declare const scrapeListingDetailsFromHtmlByFilePaths: (source: ListingsSource, inputFilePaths: string[], errors?: ErrorLog) => Promise<number>;
 declare const scrapeListingDetailsFromHtmlByZipCodes: (source: ListingsSource, zipCodes: number[], inputDirectory: string) => Promise<{
     numListings: number;
 }>;
@@ -30,6 +31,9 @@ interface ScrapeZillowListingsByZipCodesOptions extends ScrapeListingsByZipCodes
     altFetch?: boolean;
     fetchListings?: boolean;
     skipBotCheck?: boolean;
+    silent?: boolean;
+    preValidatedZipCodes?: number[];
+    retry?: boolean;
 }
 
 declare const scrapeRedfinListingResultsByZipCodes: (zipCodes: number[], outputDirectory: string, options: ScrapeListingsByZipCodesOptions) => Promise<{
@@ -38,6 +42,8 @@ declare const scrapeRedfinListingResultsByZipCodes: (zipCodes: number[], outputD
 
 declare const scrapeZillowListingResultsByZipCodes: (zipCodes: number[], outputDirectory: string, options?: ScrapeZillowListingsByZipCodesOptions) => Promise<{
     validZipCodes: number[];
+    botFilteredZipCodes: number[];
+    noResultsZipCodes: number[];
 }>;
 
 export { fetchListingHtmlByUrlAndExport, runScrapeListings, scrapeListingDetailsFromHtmlByFilePaths, scrapeListingDetailsFromHtmlByZipCodes, scrapeListingHtmlByIds, scrapeListingHtmlByInputDirectory, scrapeListingHtmlByZipCodes, scrapeListingHtmlByZipCodesAndListingDetails, scrapeRedfinListingResultsByZipCodes, scrapeZillowListingResultsByZipCodes, scrapeZillowListingsToCsv };

package/dist/index.mjs

@@ -1 +1 @@
-export{f as fetchListingHtmlByUrlAndExport,r as runScrapeListings,s as scrapeListingDetailsFromHtmlByFilePaths,a as scrapeListingDetailsFromHtmlByZipCodes,e as scrapeListingHtmlByIds,d as scrapeListingHtmlByInputDirectory,b as scrapeListingHtmlByZipCodes,c as scrapeListingHtmlByZipCodesAndListingDetails,h as scrapeRedfinListingResultsByZipCodes,i as scrapeZillowListingResultsByZipCodes,g as scrapeZillowListingsToCsv}from"./shared/scrape-listings.B-9u0WHc.mjs";import"minimist";import"dayjs";import"path";import"axios";import"fs/promises";import"@rent-scraper/api";import"@rent-scraper/utils";import"@rent-scraper/api/config";import"@clack/prompts";import"crypto";import"node:timers/promises";import"@rent-scraper/utils/config";import"picocolors";
+export{f as fetchListingHtmlByUrlAndExport,r as runScrapeListings,s as scrapeListingDetailsFromHtmlByFilePaths,a as scrapeListingDetailsFromHtmlByZipCodes,e as scrapeListingHtmlByIds,d as scrapeListingHtmlByInputDirectory,b as scrapeListingHtmlByZipCodes,c as scrapeListingHtmlByZipCodesAndListingDetails,h as scrapeRedfinListingResultsByZipCodes,i as scrapeZillowListingResultsByZipCodes,g as scrapeZillowListingsToCsv}from"./shared/scrape-listings.DPpklKj4.mjs";import"minimist";import"dayjs";import"path";import"axios";import"fs/promises";import"@rent-scraper/api";import"@rent-scraper/utils";import"@rent-scraper/api/config";import"@clack/prompts";import"node:timers/promises";import"crypto";import"@rent-scraper/utils/config";import"picocolors";

package/dist/shared/scrape-listings.DPpklKj4.mjs

@@ -0,0 +1,15 @@
+import kt from"minimist";import ot from"dayjs";import o from"path";import lt from"axios";import{mkdir as k,writeFile as M,readFile as G,unlink as W,readdir as tt}from"fs/promises";import{checkForZillowBotFiltering as Y,isZillowBotFiltering as ct,fetchHtmlFromRedfinListingUrl as St,fetchHtmlFromZillowListingUrl as Bt,scrapeDataFromRedfinListingHtml as Mt,scrapeDataFromZillowListingHtml as Dt,getZillowListingResults as Ft,getRedfinListingResults as Pt,waitForSolvedZillowCaptcha as dt,isBrowserShowingCaptcha as Rt}from"@rent-scraper/api";import{ErrorLog as T,throwError as z,checkForFile as L,parseJsonFile as Ut,parseError as P,readFilesInDirectory as X,roundValue as et,chunkArray as Ht,parsePercentage as Nt,compareArrays as ut}from"@rent-scraper/utils";import{getZillowOutputPath as O,getRedfinOutputPath as I,getZillowDaysListed as Tt,getRedfinDaysListed as Ot,getZillowLimit as It,getZillowOffset as Et,getRedfinZipCodes as At,getZillowZipCodes as Yt}from"@rent-scraper/api/config";import{spinner as pt,log as u,progress as qt,intro as Jt}from"@clack/prompts";import{setTimeout as wt}from"node:timers/promises";import{createHash as Vt}from"crypto";import{checkBrowserServer as gt}from"@rent-scraper/utils/config";import Gt from"picocolors";const R=process.env.DEBUG,E=async(t,e,s,i)=>{const{timeoutMs:r}=i??{};if(await L(s))R&&u.warn(`${s} exists, skipping`);else try{R&&u.info(`writing ${s}`);const a=t==="redfin"?await St(e):await Bt(e,{timeoutMs:r});await M(`${s}`,a)}catch(a){z(`error fetching html for ${e}`,a)}},Wt=async(t,{timeoutMs:e})=>{if(await L(t)){const s=JSON.parse(await G(t,"utf8"))||{},{hdpUrl:i}=s||{};if(i){const r=`https://www.zillow.com${i}`,a=t.replace(".json",".html");await L(a)?R&&u.warn(`file already exists, ${a}`):await E("zillow",r,a,{timeoutMs:e})}else z(`file is empty, ${t}`)}else R&&u.warn(`file does not exist at this path, ${t}, skipping`)},mt=async(t,e)=>{const{timeoutMs:s}=e??{};await Promise.all(t.map(async i=>await Wt(i,{timeoutMs:s})))},Xt=(t,e="zillow")=>{if(e==="zillow"){const{zpid:s,detailUrl:i}=t||{},r=i&&!i.startsWith("http")?`https://www.zillow.com${i}`:i;return{id:s,url:r}}else if(e==="redfin"){const{propertyId:s,url:i}=t?.homeData||{};return{id:s,url:`https://www.redfin.com${i}`}}},st=async(t,e,s,i=s,r)=>{const{timeoutMs:a,run:n=1,reruns:l=0,skipBotCheck:h=!1}=r??{},c=new T;s||z("inputDirectory is required"),t==="zillow"&&!h&&await Y();const p=[],y=[],d=pt();d.start("Downloading listings html files");for(let g=1;g<=l+1;g++){l>0&&g>1&&c.add(`rerun ${g-1} of ${l}`);const m=p.length;(g===1||m)&&await Promise.all((m?p:e).map(async j=>{const _=`${s}/${j}.json`;if(await L(_))try{const{results:v}=await Ut(_)||{};if(!v)c.add(`empty file, ${_}`);else{const Z=i?`${i}/${j}`:`${s}/${j}`;await k(Z,{recursive:!0}),v?.length?await Promise.all(v.map(async B=>{const{id:S,url:C}=Xt(B,t)??{};if(!C)return c.add(`url missing for ${S}`);const x=`${S}.html`,U=`${Z}/${x}`;try{await E(t,C,U,{timeoutMs:a})}catch(H){const{status:N,message:D}=P(H);ct(N,D)||y.push({url:C,filePath:U}),c.add("scrape listing html error: "+(D??`error fetching listing for id, ${H}`))}})):c.add(`no results for file, ${_}`)}}catch(v){p.push(j);const{message:Z}=P(v);c.add("scrape listing html error: "+(Z??`error reading json data, ${_}, ${v}`))}else R&&u.warn(`file does not exist, ${_}, skipping`)}))}y.length>0&&(await wt(2e3),await Promise.all(y.map(async({url:g,filePath:m})=>{try{await E(t,g,m,{timeoutMs:a})}catch(j){const{message:_}=P(j);c.add("scrape listing html retry error: "+(_??`error fetching ${g}`))}})));const $=await O(),f=await I(),w=t==="zillow"?$:f;e.length>0?(d.stop("Listings HTML files have been saved to:"),u.message(o.join(w,t,"listings",o.basename(s)))):d.stop("No listings HTML files saved.");const b=o.join(w,t,"logs");if(await k(b,{recursive:!0}),c.get().filter(g=>!g.includes("rerun ")).length>l){const g=`${o.basename(s)}-html-errors-${n}.txt`,m=o.join(b,g);await c.write(m,[...new Set(c.get())].join(`
+`)),R&&u.error(`There were errors during processing, see ${o.resolve(m)}`)}},Qt=async(t,e,s,i)=>{const{timeoutMs:r,run:a=1,reruns:n=0,skipBotCheck:l=!1}=i??{},h=new T;s||z("inputDirectory is required"),t==="zillow"&&!l&&await Y();const c=[];for(let f=1;f<=n+1;f++){n>0&&f>1&&h.add(`rerun ${f-1} of ${n}`);const w=c.length;(f===1||w)&&await Promise.all((w?c:e).map(async b=>{const g=`${s}/${b}`;if(await L(g))try{const m=await X(g,{extension:".json",prependDirectory:!0});await mt(m,{timeoutMs:r})}catch(m){c.push(b);const{message:j}=P(m);h.add("scrape listing html error: "+(j??`Error during fetch for ${b}, ${m}`))}else h.add(`listing directory does not exist, ${g}`)}))}const p=await O(),y=await I(),d=t==="zillow"?p:y,$=o.join(d,t,"logs");if(await k($,{recursive:!0}),h.get().filter(f=>!f.includes("rerun ")).length>n){const f=`${o.basename(s)}-html-errors-${a}.txt`,w=o.join($,f);await h.write(w,[...new Set(h.get())].join(`
+`)),R&&u.error(`There were errors during processing, see ${o.resolve(w)}`)}},Kt=async(t,e,s=e,i)=>{const{skipBotCheck:r=!1}=i??{},a=new T;if(t==="zillow"&&!r&&await Y(),e||z("inputDirectory is required"),await k(s,{recursive:!0}),await L(e))try{const d=await X(e,{extension:".json",prependDirectory:!0});await mt(d)}catch(d){const{message:$}=P(d);a.add("scrape listing html error: "+($??`Error during fetch for ${e}, ${d}`))}else a.add(`inputDirectory does not exist, ${e}`);const n=await O(),l=await I(),h=t==="zillow"?n:l,c=o.join(h,t,"logs");await k(c,{recursive:!0});const p=`${o.basename(e)}-html-errors.txt`,y=o.join(c,p);a.get().length>0&&(await a.write(y,[...new Set(a.get())].join(`
+`)),R&&u.error(`There were errors during processing, see ${o.resolve(y)}`))},te=(t,e)=>t==="redfin"?`https://www.redfin.com/home/${e}`:t==="zillow"?`https://www.zillow.com/homedetails/${e}_zpid`:null,ee=async(t,e,s,i)=>{const{skipBotCheck:r=!1}=i??{};t==="zillow"&&!r&&await Y(),s||z("outputDirectory is required"),await k(s,{recursive:!0}),await Promise.all(e.map(async a=>{const n=te(t,a),l=`${s}/${a}.html`;n&&await E(t,n,l)}))},F=process.env.DEBUG,ft=async t=>{if(await L(t)){const e=t.replace(".html",".json");if(await L(e))F&&u.warning(`file already exists, ${e}`);else{F&&u.message(`scraping data for ${t}`);const s=(await G(t)).toString();(!s||s.trim()==="")&&(await W(t),z(`empty file found at ${t}, deleted for retry`)),s.includes("px-captcha")&&(await W(t),z(`captcha page found in ${t}, deleted for retry`));const i=await Dt(s);if(!i?.priceHistory&&i?.bestMatchedUnit?.hdpUrl){F&&u.warning(`rescraping ${t} - https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`),await L(t)&&(F&&u.warning(`deleting ${t}`),await W(t)),await L(e)&&(F&&u.warning(`deleting ${e}`),await W(e));const r=`https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`;await E("zillow",r,t),await ft(t)}else i?(F&&u.info(`writing ${e}`),await M(`${e}`,JSON.stringify(i))):z(`problem scraping data for ${t}`)}}},se=async t=>{if(await L(t)){const e=t.replace(".html",".json");if(await L(e))F&&u.warning(`file already exists, ${e}`);else{F&&u.message(`scraping data for ${t}`);const s=(await G(t)).toString(),i=Mt(s);i?(F&&u.warning(`writing ${e}`),await M(`${e}`,JSON.stringify(i))):z(`problem scraping data for ${t}`)}}},ht=async(t,e,s)=>(await Promise.all(e.map(async i=>{try{t==="redfin"?await se(i):await ft(i);const r=i.replace(".html",".json");return await L(r)?1:0}catch(r){const{message:a}=P(r);return s?.add(a),0}}))).reduce((i,r)=>i+r,0),it=async(t,e,s)=>{const i=new T;let r=0;s||z("inputDirectory is required");const a=qt({style:"heavy",max:100,size:50});a.start("Scraping listings data");const n=e.length,l=n<20?n+1:20,h=n>l?et(n/l,1):n,c=Ht(e,h);for(const[f,w]of c.entries()){const b=et((Number(f)+1)/l*100,1);a.advance(et(1/l*100,1),`Scraping listings (${Nt(b)})`),await Promise.all(w.map(async g=>{F&&u.message(`processing files for ${g}`);const m=`${s}/${g}`;if(await L(m)){const j=await X(m,{extension:".html",prependDirectory:!0}),_=await ht(t,j,i);r=r+_}else i.add(`listing directory does not exist, ${m}`)}))}const p=await O(),y=await I(),d=t==="zillow"?p:y;r>0?(a.stop("Listings data has been saved to:"),u.message(o.join(d,t,"listings",o.basename(s)))):a.stop("No listings data saved.");const $=o.join(d,t,"logs");if(await k($,{recursive:!0}),i.get().length>0){const f=`${o.basename(s)}-listing-errors.txt`,w=o.join($,f);await i.write(w,[...new Set(i.get())].join(`
+`)),u.error(`There were errors during processing, see ${o.resolve(w)}`)}return{numListings:r}},rt=process.env.DEBUG,ie=async(t,e,s,i,r)=>{const{daysOnZillow:a,timeoutMs:n}=r??{};if(await L(e))rt&&u.warning(`${t} exists, skipping`),s.push(t);else{const l=await Ft({zipCode:t,daysOnZillow:a,mergePageResults:!0,timeoutMs:n});l?(rt&&u.info(`writing ${e}`),await M(e,JSON.stringify(l)),s.push(t)):(rt&&u.warning(`no results for ${t}`),i.push(t))}},nt=async(t,e,s)=>{const{daysListed:i,timeoutMs:r,run:a=1,reruns:n=0,fetchListings:l=!1,skipBotCheck:h=!1,silent:c=!1}=s??{},p=new T;h||await Y({fetchListings:l});const y=[],d=[],$=[],f=[];await k(e,{recursive:!0});const w=pt();c||w.start("Scraping Zillow search results");for(let m=1;m<=n+1;m++){n>0&&m>1&&p.add(`rerun ${m-1} of ${n}`);const j=d.length;(m===1||j)&&await Promise.all((j?d:t).map(async _=>{const v=`${_}.json`,Z=`${e}/${v}`;try{await ie(_,Z,y,f,{daysOnZillow:i,timeoutMs:r})}catch(B){d.push(_);const{status:S,message:C}=P(B);ct(S,C)&&$.push(_),p.add("scrape listing results error: "+(C??`Error during fetch for ${_}, ${B}`))}}))}const b=await O();c||(y.length>0?(w.stop("Zillow search results have been saved to:"),u.message(o.join(b,"zillow","results",o.basename(e)))):w.stop("No results saved."));const g=o.join(b,"zillow","logs");if(await k(g,{recursive:!0}),p.get().filter(m=>!m.includes("rerun ")).length>n){const m=`${o.basename(e)}-results-errors-${a}.txt`,j=o.join(g,m);await p.write(j,[...new Set(p.get())].join(`
+`)),c||u.error(`There were errors during processing, see ${o.resolve(j)}`)}return{validZipCodes:y,botFilteredZipCodes:$,noResultsZipCodes:f}},re=async(t,e,s,i)=>{const{daysListed:r,timeoutMs:a}=i??{};if(await L(e))console.log(`${t} exists, skipping`),s.push(t);else{const n=await Pt({zipCode:t,daysListed:r,timeoutMs:a});n?(console.log(`writing ${e}`),await M(e,JSON.stringify(n)),s.push(t)):console.log(`no results for ${t}`)}},yt=async(t,e,s)=>{const{daysListed:i,timeoutMs:r,run:a=1,reruns:n=0}=s??{},l=new T,h=[],c=[];await k(e,{recursive:!0});for(let d=1;d<=n+1;d++){n>0&&d>1&&l.add(`rerun ${d-1} of ${n}`);const $=c.length;(d===1||$)&&await Promise.all(($?c:t).map(async f=>{const w=`${f}.json`,b=`${e}/${w}`;try{await re(f,b,h,{daysListed:i,timeoutMs:r})}catch(g){c.push(f);const{message:m}=P(g);l.add("scrape listing results error: "+(m??`Error during fetch for ${f}, ${g}`))}}))}const p=await I(),y=o.join(p,"redfin","logs");if(await k(y,{recursive:!0}),l.get().filter(d=>!d.includes("rerun ")).length>n){const d=`${o.basename(e)}-results-errors-${a}.txt`,$=o.join(y,d);await l.write($,[...new Set(l.get())].join(`
+`)),console.log(`\x1B[41m
+%s\x1B[0m`,`There were errors during processing, see ${o.resolve($)}`)}return{validZipCodes:h}},$t=["listing_id","listing_source","source_listing_id","parcel_number","listing_url","street_address","cleaned_address","cleaned_unit_number","city","state","zipcode","county","neighborhood_region","is_undisclosed_address","home_status","home_type","bedrooms","year_built","living_area","living_area_units","is_income_restricted","price_at_source","platform_rent_estimate","agent_name","agent_phone_number","broker_name","broker_phone_number","is_owned_by_listing_platform","date_last_updated_at_source","date_scraped","scrape_job_name"],ne=t=>{if(t==null)return"";const e=typeof t=="object"?JSON.stringify(t):String(t);return e.includes(",")||e.includes('"')||e.includes(`
+`)?`"${e.replace(/"/g,'""')}"`:e},ae=t=>{const e=/^(.*?)\s+((?:#|APT|UNIT|STE|Suite|Apt|Unit)\s*.+)$/i.exec(t);if(e){const s=e[1].trim().toUpperCase();let i=e[2].trim();return i.startsWith("#")&&(i=`# ${i.replace(/^#\s*/,"").trim()}`),{cleaned:s,unit:i}}return{cleaned:t.toUpperCase(),unit:""}},oe=(t,e)=>{const s=String(t.zpid),i=Vt("md5").update(s).digest("hex"),r=t.streetAddress||t.address?.streetAddress||"",{cleaned:a,unit:n}=ae(r),l=t.hdpUrl?`https://www.zillow.com${t.hdpUrl}`:`https://www.zillow.com/homedetails/${s}_zpid/`;return{listing_id:i,listing_source:"zillow",source_listing_id:s,parcel_number:t.resoFacts?.parcelNumber??"",listing_url:l,street_address:r,cleaned_address:a,cleaned_unit_number:n,city:t.city||t.address?.city||"",state:t.state||t.address?.state||"",zipcode:t.zipcode||t.address?.zipcode||"",county:t.county||"",neighborhood_region:t.neighborhoodRegion?.name||t.parentRegion?.name||"",is_undisclosed_address:t.isUndisclosedAddress??"",home_status:t.homeStatus||"",home_type:t.homeType||"",bedrooms:t.bedrooms??"",year_built:t.yearBuilt||t.resoFacts?.yearBuilt||"",living_area:t.livingAreaValue??"",living_area_units:t.livingAreaUnits||"Square Feet",is_income_restricted:t.isIncomeRestricted??"",price_at_source:t.price??"",platform_rent_estimate:t.rentZestimate??"",agent_name:t.attributionInfo?.agentName||"",agent_phone_number:t.attributionInfo?.agentPhoneNumber||"",broker_name:t.attributionInfo?.brokerName||"",broker_phone_number:t.attributionInfo?.brokerPhoneNumber||"",is_owned_by_listing_platform:t.attributionInfo?.mlsName==="Zillow Rentals",date_last_updated_at_source:t.attributionInfo?.lastUpdated||"",date_scraped:t.timestamp?ot(t.timestamp).format("YYYY-MM-DD HH:mm:ss"):"",scrape_job_name:e}},_t=async(t,e)=>{const s=o.basename(t),i=[$t.join(",")];let r;try{r=await tt(t,{withFileTypes:!0})}catch{return 0}const a=r.filter(n=>n.isDirectory()).map(n=>n.name);for(const n of a){const l=o.join(t,n);if(!await L(l))continue;const h=await X(l,{extension:".json",prependDirectory:!0});for(const c of h)try{const p=JSON.parse(await G(c,"utf8"));if(!p.zpid)continue;const y=oe(p,s);i.push($t.map(d=>ne(y[d])).join(","))}catch{}}return await k(o.dirname(e),{recursive:!0}),await M(e,i.join(`
+`)),i.length-1},le=async t=>{try{const e=(await tt(t,{withFileTypes:!0})).filter(s=>s.isDirectory()).map(s=>s.name).sort();return e.length>0?e[e.length-1]:null}catch{return null}},bt=async()=>{await lt.post("http://localhost:8082/browser/close")},ce=async()=>{await lt.post("http://localhost:8082/server/shutdown")},de=async(t,e,s,{daysListed:i,timeoutMs:r,run:a,reruns:n,preValidatedZipCodes:l=[],retry:h=!1})=>{let c=[],p=[],y=[];if(t.length>0){if(await dt(),await bt(),{validZipCodes:c,botFilteredZipCodes:p,noResultsZipCodes:y}=await nt(t,e,{daysListed:i,timeoutMs:r,run:a,reruns:n,skipBotCheck:!0}),h&&p.length>0){u.warn(`Bot filtering hit ${p.length} of ${t.length} zip code(s) \u2014 will retry after listings are fetched`),await Rt()?(await dt(),await bt()):await wt(3e3);const f=p.length,w=await nt(p,e,{daysListed:i,timeoutMs:r,run:a,reruns:n,skipBotCheck:!0,silent:!0});c=[...c,...w.validZipCodes],p=w.botFilteredZipCodes,y=[...y,...w.noResultsZipCodes];const b=w.validZipCodes.length;u.info(`Inline retry: recovered ${b} of ${f} \xB7 ${p.length} still bot-filtered`)}c.length===0&&t.length>0&&u.warn("No results returned for any zip codes \u2014 possible soft bot filtering. Please try again shortly."),p.length>0&&u.warn(`${p.length} zip code(s) still bot-filtered \u2014 use --rerun to retry`)}const d=[...l,...c];await st("zillow",d,e,s,{timeoutMs:r,run:a,reruns:n,skipBotCheck:!0});const{numListings:$}=await it("zillow",d,s);return{numListings:$,validZipCodes:d,noResultsZipCodes:y}},ue=async(t,e,s,{daysListed:i,timeoutMs:r,run:a,reruns:n})=>{const{validZipCodes:l}=await yt(t,e,{daysListed:i,timeoutMs:r,run:a,reruns:n});await st("redfin",l,e,s,{timeoutMs:r,run:a,reruns:n});const{numListings:h}=await it("redfin",l,s);return{numListings:h,validZipCodes:l}};async function pe(){Jt(Gt.inverse(" scrape listings "));const t=kt(process.argv.slice(2)),e=t.source??"zillow";e==="zillow"&&(await gt()||z("Please launch the browser server before scraping."));const s=await O(),i=await I(),r=e==="zillow"?s:i,a=t["days-listed"]??(e==="zillow"?await Tt():await Ot())??1,n=t.runs??1,l=t.reruns??0,h=t["timeout-ms"]??6e4,c=t.limit??await It()??void 0,p=t.offset??await Et()??0,y=t.retry??!1,d=t.rerun,$=typeof d=="string"?d:d?await le(o.join(r,e,"results")):null;d&&!$&&z("No previous run found to rerun."),$&&u.info(`Rerunning from: ${$}`);const f=$??ot().format("YYYY-MM-DD-HHmm"),w=t["results-directory"]??o.join(r,e,"results",f),b=t["listings-directory"]??o.join(r,e,"listings",f),g=t["logs-directory"]??o.join(r,e,"logs");let m=0;const j=async()=>{(e==="zillow"||e==="redfin")&&await gt()&&await ce()};try{if(e==="redfin"){const _=await At();for(let v=1;v<=n;v++){_||z("zip codes required, please run the createConfig script");const Z=_,{numListings:B,validZipCodes:S}=await ue(Z,w,b,{daysListed:a,timeoutMs:h,run:v,reruns:l});if(v===n){const C=Object.entries({numListings:B}).map(([Q,q])=>`${Q}: ${q}`).join(`
+`),x=ut(Z,S).join(`
+`);await k(g,{recursive:!0});const U=`${o.basename(w)}-scraping-results.txt`,H=o.join(g,U),N=`${o.basename(w)}-invalid-zipcodes.txt`,D=o.join(g,N);await M(H,C),await M(D,x)}}}else if(e==="zillow"){const _=await Yt();_||z("zip codes required, please run the createConfig script");const v=_.slice(p,c?p+c:void 0);let Z=v,B=[];if($){const S=await tt(w).catch(()=>[]),C=new Set(S.filter(x=>x.endsWith(".json")).map(x=>x.replace(".json","")));B=v.filter(x=>C.has(String(x))),Z=v.filter(x=>!C.has(String(x))),Z.length>0?u.info(`${B.length} already fetched \xB7 retrying ${Z.length} missing`):u.info(`All ${B.length} zip codes already fetched \xB7 proceeding to HTML scraping`)}for(let S=1;S<=n;S++){const{numListings:C,validZipCodes:x,noResultsZipCodes:U}=await de(Z,w,b,{daysListed:a,timeoutMs:h,run:S,reruns:l,preValidatedZipCodes:B,retry:y});if(S===n){const H=o.basename(b),N=o.join(s,"zillow","csv",`${H}.csv`),D=await _t(b,N);m=D,D>0&&u.info(`CSV exported: ${D} listings \u2192 ${N}`);const Q=Object.entries({numListings:C,numCsvListings:D}).map(([A,xt])=>`${A}: ${xt}`).join(`
+`),q=ut(v,x),at=new Set(U),J=q.filter(A=>!at.has(A)),V=q.filter(A=>at.has(A));await k(g,{recursive:!0});const vt=`${o.basename(w)}-scraping-results.txt`,jt=o.join(g,vt),Lt=`${o.basename(w)}-zipcodes-no-results.txt`,zt=o.join(g,Lt),Zt=`${o.basename(w)}-zipcodes-errored.txt`,Ct=o.join(g,Zt);await M(jt,Q),V.length>0&&await M(zt,V.join(`
+`)),J.length>0&&await M(Ct,J.join(`
+`));const K=[`${x.length}/${v.length} with results`];V.length>0&&K.push(`${V.length} no results`),J.length>0&&K.push(`${J.length} errored`),u.info(`Zip codes: ${K.join(" \xB7 ")}`),u.info(`Listings:  ${C} parsed \xB7 ${D} exported to CSV`)}}}e!=="zillow"||m>0?u.success("Scraping complete!"):u.warn("Scraping finished but no listings were exported \u2014 please try again shortly.")}finally{await j()}}export{it as a,st as b,Qt as c,Kt as d,ee as e,E as f,_t as g,yt as h,nt as i,pe as r,ht as s};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rent-scraper/scrape-listings",
-  "version": "1.0.29",
+  "version": "1.0.30",
   "type": "module",
   "exports": {
     ".": {
@@ -30,8 +30,8 @@
     "dayjs": "^1.11.13",
     "minimist": "^1.2.8",
     "picocolors": "^1.1.1",
-    "@rent-scraper/utils": "1.0.29",
-    "@rent-scraper/api": "1.0.29"
+    "@rent-scraper/utils": "1.0.30",
+    "@rent-scraper/api": "1.0.30"
   },
   "devDependencies": {
     "@types/minimist": "^1.2.5",

package/dist/shared/scrape-listings.B-9u0WHc.mjs

@@ -1,14 +0,0 @@
-import le from"minimist";import E from"dayjs";import n from"path";import G from"axios";import{mkdir as j,writeFile as Z,readFile as N,unlink as V,readdir as ce}from"fs/promises";import{checkForZillowBotFiltering as S,fetchHtmlFromRedfinListingUrl as de,fetchHtmlFromZillowListingUrl as ue,scrapeDataFromRedfinListingHtml as we,scrapeDataFromZillowListingHtml as me,getZillowListingResults as pe,getRedfinListingResults as ge,waitForSolvedZillowCaptcha as fe}from"@rent-scraper/api";import{ErrorLog as U,throwError as z,checkForFile as v,parseJsonFile as he,parseError as D,readFilesInDirectory as O,roundValue as Y,chunkArray as ye,parsePercentage as $e,compareArrays as K}from"@rent-scraper/utils";import{getZillowOutputPath as H,getRedfinOutputPath as P,getZillowDaysListed as _e,getRedfinDaysListed as be,getRedfinZipCodes as ve,getZillowZipCodes as je}from"@rent-scraper/api/config";import{spinner as W,log as w,progress as Le,intro as ze,confirm as Me,isCancel as Ze,cancel as xe,outro as Ce}from"@clack/prompts";import{createHash as ke}from"crypto";import{setTimeout as Q}from"node:timers/promises";import{checkBrowserServer as X}from"@rent-scraper/utils/config";import De from"picocolors";const B=process.env.DEBUG,R=async(e,t,s,i)=>{const{timeoutMs:o}=i??{};if(await v(s))B&&w.warn(`${s} exists, skipping`);else try{B&&w.info(`writing ${s}`);const a=e==="redfin"?await de(t):await ue(t,{timeoutMs:o});await Z(`${s}`,a)}catch(a){z(`error fetching html for ${t}`,a)}},Be=async(e,{timeoutMs:t})=>{if(await v(e)){const s=JSON.parse(await N(e,"utf8"))||{},{hdpUrl:i}=s||{};if(i){const o=`https://www.zillow.com${i}`,a=e.replace(".json",".html");await v(a)?B&&w.warn(`file already exists, ${a}`):await R("zillow",o,a,{timeoutMs:t})}else z(`file is empty, ${e}`)}else B&&w.warn(`file does not exist at this path, ${e}, skipping`)},ee=async(e,t)=>{const{timeoutMs:s}=t??{};await Promise.all(e.map(async i=>await Be(i,{timeoutMs:s})))},Se=(e,t="zillow")=>{if(t==="zillow"){const{zpid:s,detailUrl:i}=e||{};return{id:s,url:i}}else if(t==="redfin"){const{propertyId:s,url:i}=e?.homeData||{};return{id:s,url:`https://www.redfin.com${i}`}}},A=async(e,t,s,i=s,o)=>{const{timeoutMs:a,run:r=1,reruns:l=0,skipBotCheck:h=!1}=o??{},c=new U;s||z("inputDirectory is required"),e==="zillow"&&!h&&await S();const y=[],p=W();p.start("Downloading listings html files");for(let f=1;f<=l+1;f++){l>0&&f>1&&c.add(`rerun ${f-1} of ${l}`);const $=y.length;(f===1||$)&&await Promise.all(($?y:t).map(async _=>{const b=`${s}/${_}.json`;if(await v(b))try{const{results:L}=await he(b)||{};if(!L)c.add(`empty file, ${b}`);else{const M=i?`${i}/${_}`:`${s}/${_}`;await j(M,{recursive:!0}),L?.length?await Promise.all(L.map(async F=>{try{const{id:C,url:k}=Se(F,e)??{};if(!k)return c.add(`url missing for ${C}`);const I=`${C}.html`,T=`${M}/${I}`;await R(e,k,T,{timeoutMs:a})}catch(C){const{message:k}=D(C);c.add("scrape listing html error: "+(k??`error fetching listing for id, ${C}`))}})):c.add(`no results for file, ${b}`)}}catch(L){y.push(_);const{message:M}=D(L);c.add("scrape listing html error: "+(M??`error reading json data, ${b}, ${L}`))}else B&&w.warn(`file does not exist, ${b}, skipping`)}))}p.stop("Listings HTML files have been saved to:");const d=await H(),g=await P(),u=e==="zillow"?d:g;w.message(n.join(u,e,"listings",n.basename(s)));const m=n.join(u,e,"logs");if(await j(m,{recursive:!0}),c.get().filter(f=>!f.includes("rerun ")).length>l){const f=`${n.basename(s)}-html-errors-${r}.txt`,$=n.join(m,f);await c.write($,[...new Set(c.get())].join(`
-`)),B&&w.error(`There were errors during processing, see ${n.resolve($)}`)}},Ue=async(e,t,s,i)=>{const{timeoutMs:o,run:a=1,reruns:r=0,skipBotCheck:l=!1}=i??{},h=new U;s||z("inputDirectory is required"),e==="zillow"&&!l&&await S();const c=[];for(let u=1;u<=r+1;u++){r>0&&u>1&&h.add(`rerun ${u-1} of ${r}`);const m=c.length;(u===1||m)&&await Promise.all((m?c:t).map(async f=>{const $=`${s}/${f}`;if(await v($))try{const _=await O($,{extension:".json",prependDirectory:!0});await ee(_,{timeoutMs:o})}catch(_){c.push(f);const{message:b}=D(_);h.add("scrape listing html error: "+(b??`Error during fetch for ${f}, ${_}`))}else h.add(`listing directory does not exist, ${$}`)}))}const y=await H(),p=await P(),d=e==="zillow"?y:p,g=n.join(d,e,"logs");if(await j(g,{recursive:!0}),h.get().filter(u=>!u.includes("rerun ")).length>r){const u=`${n.basename(s)}-html-errors-${a}.txt`,m=n.join(g,u);await h.write(m,[...new Set(h.get())].join(`
-`)),B&&w.error(`There were errors during processing, see ${n.resolve(m)}`)}},He=async(e,t,s=t,i)=>{const{skipBotCheck:o=!1}=i??{},a=new U;if(e==="zillow"&&!o&&await S(),t||z("inputDirectory is required"),await j(s,{recursive:!0}),await v(t))try{const d=await O(t,{extension:".json",prependDirectory:!0});await ee(d)}catch(d){const{message:g}=D(d);a.add("scrape listing html error: "+(g??`Error during fetch for ${t}, ${d}`))}else a.add(`inputDirectory does not exist, ${t}`);const r=await H(),l=await P(),h=e==="zillow"?r:l,c=n.join(h,e,"logs");await j(c,{recursive:!0});const y=`${n.basename(t)}-html-errors.txt`,p=n.join(c,y);a.get().length>0&&(await a.write(p,[...new Set(a.get())].join(`
-`)),B&&w.error(`There were errors during processing, see ${n.resolve(p)}`))},Pe=(e,t)=>e==="redfin"?`https://www.redfin.com/home/${t}`:e==="zillow"?`https://www.zillow.com/homedetails/${t}_zpid`:null,Fe=async(e,t,s,i)=>{const{skipBotCheck:o=!1}=i??{};e==="zillow"&&!o&&await S(),s||z("outputDirectory is required"),await j(s,{recursive:!0}),await Promise.all(t.map(async a=>{const r=Pe(e,a),l=`${s}/${a}.html`;r&&await R(e,r,l)}))},x=process.env.DEBUG,te=async e=>{if(await v(e)){const t=e.replace(".html",".json");if(await v(t))x&&w.warning(`file already exists, ${t}`);else{x&&w.message(`scraping data for ${e}`);const s=(await N(e)).toString(),i=await me(s);if(!i?.priceHistory&&i?.bestMatchedUnit?.hdpUrl){x&&w.warning(`rescraping ${e} - https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`),await v(e)&&(x&&w.warning(`deleting ${e}`),await V(e)),await v(t)&&(x&&w.warning(`deleting ${t}`),await V(t));const o=`https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`;try{await R("zillow",o,e),await te(e)}catch(a){const{message:r}=D(a);w.error(r)}}else i?(x&&w.info(`writing ${t}`),await Z(`${t}`,JSON.stringify(i))):z(`problem scraping data for ${e}`)}}},Re=async e=>{if(await v(e)){const t=e.replace(".html",".json");if(await v(t))x&&w.warning(`file already exists, ${t}`);else{x&&w.message(`scraping data for ${e}`);const s=(await N(e)).toString(),i=we(s);i?(x&&w.warning(`writing ${t}`),await Z(`${t}`,JSON.stringify(i))):z(`problem scraping data for ${e}`)}}},se=async(e,t)=>{await Promise.all(t.map(async s=>e==="redfin"?await Re(s):await te(s)))},q=async(e,t,s)=>{const i=new U;let o=0;s||z("inputDirectory is required");const a=Le({style:"heavy",max:100,size:50});a.start("Scraping listings data");const r=t.length,l=r<20?r+1:20,h=r>l?Y(r/l,1):r,c=ye(t,h);for(const[u,m]of c.entries()){const f=Y((Number(u)+1)/l*100,1);a.advance(Y(1/l*100,1),`Scraping listings (${$e(f)})`),await Promise.all(m.map(async $=>{x&&w.message(`processing files for ${$}`);const _=`${s}/${$}`;if(await v(_)){const b=await O(_,{extension:".html",prependDirectory:!0});o=o+b.length,await se(e,b)}else i.add(`listing directory does not exist, ${_}`)}))}const y=await H(),p=await P(),d=e==="zillow"?y:p;a.stop("Listings data has been saved to:"),w.message(n.join(d,e,"listings",n.basename(s)));const g=n.join(d,e,"logs");if(await j(g,{recursive:!0}),i.get().length>0){const u=`${n.basename(s)}-listing-errors.txt`,m=n.join(g,u);await i.write(m,[...new Set(i.get())].join(`
-`)),w.error(`There were errors during processing, see ${n.resolve(m)}`)}return{numListings:o}},J=process.env.DEBUG,Ne=async(e,t,s,i)=>{const{daysOnZillow:o,timeoutMs:a}=i??{};if(await v(t))J&&w.warning(`${e} exists, skipping`),s.push(e);else{const r=await pe({zipCode:e,daysOnZillow:o,mergePageResults:!0,timeoutMs:a});r?(J&&w.info(`writing ${t}`),await Z(t,JSON.stringify(r)),s.push(e)):J&&w.warning(`no results for ${e}`)}},ie=async(e,t,s)=>{const{daysListed:i,timeoutMs:o,run:a=1,reruns:r=0,fetchListings:l=!1,skipBotCheck:h=!1}=s??{},c=new U;h||await S({fetchListings:l});const y=[],p=[];await j(t,{recursive:!0});const d=W();d.start("Scraping Zillow search results");for(let m=1;m<=r+1;m++){r>0&&m>1&&c.add(`rerun ${m-1} of ${r}`);const f=p.length;(m===1||f)&&await Promise.all((f?p:e).map(async $=>{const _=`${$}.json`,b=`${t}/${_}`;try{await Ne($,b,y,{daysOnZillow:i,timeoutMs:o})}catch(L){p.push($);const{message:M}=D(L);c.add("scrape listing results error: "+(M??`Error during fetch for ${$}, ${L}`))}}))}d.stop("Zillow search results have been saved to:");const g=await H();w.message(n.join(g,"zillow","results",n.basename(t)));const u=n.join(g,"zillow","logs");if(await j(u,{recursive:!0}),c.get().filter(m=>!m.includes("rerun ")).length>r){const m=`${n.basename(t)}-results-errors-${a}.txt`,f=n.join(u,m);await c.write(f,[...new Set(c.get())].join(`
-`)),w.error(`There were errors during processing, see ${n.resolve(f)}`)}return{validZipCodes:y}},Oe=async(e,t,s,i)=>{const{daysListed:o,timeoutMs:a}=i??{};if(await v(t))console.log(`${e} exists, skipping`),s.push(e);else{const r=await ge({zipCode:e,daysListed:o,timeoutMs:a});r?(console.log(`writing ${t}`),await Z(t,JSON.stringify(r)),s.push(e)):console.log(`no results for ${e}`)}},re=async(e,t,s)=>{const{daysListed:i,timeoutMs:o,run:a=1,reruns:r=0}=s??{},l=new U,h=[],c=[];await j(t,{recursive:!0});for(let d=1;d<=r+1;d++){r>0&&d>1&&l.add(`rerun ${d-1} of ${r}`);const g=c.length;(d===1||g)&&await Promise.all((g?c:e).map(async u=>{const m=`${u}.json`,f=`${t}/${m}`;try{await Oe(u,f,h,{daysListed:i,timeoutMs:o})}catch($){c.push(u);const{message:_}=D($);l.add("scrape listing results error: "+(_??`Error during fetch for ${u}, ${$}`))}}))}const y=await P(),p=n.join(y,"redfin","logs");if(await j(p,{recursive:!0}),l.get().filter(d=>!d.includes("rerun ")).length>r){const d=`${n.basename(t)}-results-errors-${a}.txt`,g=n.join(p,d);await l.write(g,[...new Set(l.get())].join(`
-`)),console.log(`\x1B[41m
-%s\x1B[0m`,`There were errors during processing, see ${n.resolve(g)}`)}return{validZipCodes:h}},ae=["listing_id","listing_source","source_listing_id","parcel_number","listing_url","street_address","cleaned_address","cleaned_unit_number","city","state","zipcode","county","neighborhood_region","is_undisclosed_address","home_status","home_type","bedrooms","year_built","living_area","living_area_units","is_income_restricted","price_at_source","platform_rent_estimate","agent_name","agent_phone_number","broker_name","broker_phone_number","is_owned_by_listing_platform","date_last_updated_at_source","date_scraped","scrape_job_name"],Ie=e=>{if(e==null)return"";const t=typeof e=="object"?JSON.stringify(e):String(e);return t.includes(",")||t.includes('"')||t.includes(`
-`)?`"${t.replace(/"/g,'""')}"`:t},Te=e=>{const t=/^(.*?)\s+((?:#|APT|UNIT|STE|Suite|Apt|Unit)\s*.+)$/i.exec(e);if(t){const s=t[1].trim().toUpperCase();let i=t[2].trim();return i.startsWith("#")&&(i=`# ${i.replace(/^#\s*/,"").trim()}`),{cleaned:s,unit:i}}return{cleaned:e.toUpperCase(),unit:""}},Ee=(e,t)=>{const s=String(e.zpid),i=ke("md5").update(s).digest("hex"),o=e.streetAddress||e.address?.streetAddress||"",{cleaned:a,unit:r}=Te(o),l=e.hdpUrl?`https://www.zillow.com${e.hdpUrl}`:`https://www.zillow.com/homedetails/${s}_zpid/`;return{listing_id:i,listing_source:"zillow",source_listing_id:s,parcel_number:e.resoFacts?.parcelNumber??"",listing_url:l,street_address:o,cleaned_address:a,cleaned_unit_number:r,city:e.city||e.address?.city||"",state:e.state||e.address?.state||"",zipcode:e.zipcode||e.address?.zipcode||"",county:e.county||"",neighborhood_region:e.neighborhoodRegion?.name||e.parentRegion?.name||"",is_undisclosed_address:e.isUndisclosedAddress??"",home_status:e.homeStatus||"",home_type:e.homeType||"",bedrooms:e.bedrooms??"",year_built:e.yearBuilt||e.resoFacts?.yearBuilt||"",living_area:e.livingAreaValue??"",living_area_units:e.livingAreaUnits||"Square Feet",is_income_restricted:e.isIncomeRestricted??"",price_at_source:e.price??"",platform_rent_estimate:e.rentZestimate??"",agent_name:e.attributionInfo?.agentName||"",agent_phone_number:e.attributionInfo?.agentPhoneNumber||"",broker_name:e.attributionInfo?.brokerName||"",broker_phone_number:e.attributionInfo?.brokerPhoneNumber||"",is_owned_by_listing_platform:e.attributionInfo?.mlsName==="Zillow Rentals",date_last_updated_at_source:e.attributionInfo?.lastUpdated||"",date_scraped:e.timestamp?E(e.timestamp).format("YYYY-MM-DD HH:mm:ss"):"",scrape_job_name:t}},ne=async(e,t)=>{const s=n.basename(e),i=[ae.join(",")];let o;try{o=await ce(e,{withFileTypes:!0})}catch{return 0}const a=o.filter(r=>r.isDirectory()).map(r=>r.name);for(const r of a){const l=n.join(e,r);if(!await v(l))continue;const h=await O(l,{extension:".json",prependDirectory:!0});for(const c of h)try{const y=JSON.parse(await N(c,"utf8"));if(!y.zpid)continue;const p=Ee(y,s);i.push(ae.map(d=>Ie(p[d])).join(","))}catch{}}return await j(n.dirname(t),{recursive:!0}),await Z(t,i.join(`
-`)),i.length-1},Ye=async()=>{await G.post("http://localhost:8082/browser/close")},Ae=async()=>{await G.post("http://localhost:8082/server/shutdown")},qe=async(e,t,s,{daysListed:i,timeoutMs:o,run:a,reruns:r})=>{try{await S()}catch(c){const{message:y}=D(c);w.error(y);const p=await Me({message:"You need to complete a captcha in your browser. Press Return to launch your browser and continue.",active:"OK",inactive:"Cancel"});if(Ze(p)||!p)return xe("Create config canceled. Please try again."),process.exit(1);await Q(1e3),Ce("Browser Launching..."),await Q(1e3),await fe()}await Ye();const{validZipCodes:l}=await ie(e,t,{daysListed:i,timeoutMs:o,run:a,reruns:r});await A("zillow",l,t,s,{timeoutMs:o,run:a,reruns:r});const{numListings:h}=await q("zillow",l,s);return{numListings:h,validZipCodes:l}},Je=async(e,t,s,{daysListed:i,timeoutMs:o,run:a,reruns:r})=>{const{validZipCodes:l}=await re(e,t,{daysListed:i,timeoutMs:o,run:a,reruns:r});await A("redfin",l,t,s,{timeoutMs:o,run:a,reruns:r});const{numListings:h}=await q("redfin",l,s);return{numListings:h,validZipCodes:l}};async function Ge(){ze(De.inverse(" scrape listings "));const e=le(process.argv.slice(2)),t=e.source??"zillow";t==="zillow"&&(await X()||z("Please launch the browser server before scraping."));const s=await H(),i=await P(),o=t==="zillow"?s:i,a=e["days-listed"]??(t==="zillow"?await _e():await be())??1,r=e.runs??1,l=e.reruns??0,h=e["timeout-ms"]??6e4,c=e["results-directory"]??n.join(o,t,"results",`${E().format("YYYY-MM-DD-HHmm")}`),y=e["listings-directory"]??n.join(o,t,"listings",`${E().format("YYYY-MM-DD-HHmm")}`),p=e["logs-directory"]??n.join(o,t,"logs");if(t==="redfin"){const d=await ve();for(let g=1;g<=r;g++){d||z("zip codes required, please run the createConfig script");const u=d,{numListings:m,validZipCodes:f}=await Je(u,c,y,{daysListed:a,timeoutMs:h,run:g,reruns:l});if(g===r){const $=Object.entries({numListings:m}).map(([C,k])=>`${C}: ${k}`).join(`
-`),_=K(u,f).join(`
-`);await j(p,{recursive:!0});const b=`${n.basename(c)}-scraping-results.txt`,L=n.join(p,b),M=`${n.basename(c)}-invalid-zipcodes.txt`,F=n.join(p,M);await Z(L,$),await Z(F,_)}}}else if(t==="zillow"){const d=await je();for(let g=1;g<=r;g++){d||z("zip codes required, please run the createConfig script");const u=d,{numListings:m,validZipCodes:f}=await qe(u,c,y,{daysListed:a,timeoutMs:h,run:g,reruns:l});if(g===r){const $=n.basename(y),_=n.join(s,"zillow","csv",`${$}.csv`),b=await ne(y,_);w.info(`CSV exported: ${b} listings \u2192 ${_}`);const L=Object.entries({numListings:m,numCsvListings:b}).map(([T,oe])=>`${T}: ${oe}`).join(`
-`),M=K(u,f).join(`
-`);await j(p,{recursive:!0});const F=`${n.basename(c)}-scraping-results.txt`,C=n.join(p,F),k=`${n.basename(c)}-invalid-zipcodes.txt`,I=n.join(p,k);await Z(C,L),await Z(I,M)}}}w.success("Scraping complete!"),(t==="zillow"||t==="redfin")&&await X()&&await Ae()}export{q as a,A as b,Ue as c,He as d,Fe as e,R as f,ne as g,re as h,ie as i,Ge as r,se as s};