@rent-scraper/scrape-listings 1.0.29 → 1.0.31

package/dist/bin/run-scrape-listings.mjs

@@ -1 +1 @@
-import{parseError as t}from"@rent-scraper/utils";import{r as i}from"../shared/scrape-listings.B-9u0WHc.mjs";import{log as m}from"@clack/prompts";import"minimist";import"dayjs";import"path";import"axios";import"fs/promises";import"@rent-scraper/api";import"@rent-scraper/api/config";import"crypto";import"node:timers/promises";import"@rent-scraper/utils/config";import"picocolors";i().then(()=>{process.exit(0)}).catch(r=>{const{message:o}=t(r);m.error(o),process.exit(1)});
+import{parseError as t}from"@rent-scraper/utils";import{r as i}from"../shared/scrape-listings.Cdf92NvA.mjs";import{log as m}from"@clack/prompts";import"minimist";import"dayjs";import"path";import"axios";import"fs/promises";import"@rent-scraper/api";import"@rent-scraper/api/config";import"node:timers/promises";import"crypto";import"@rent-scraper/utils/config";import"picocolors";i().then(()=>{process.exit(0)}).catch(r=>{const{message:o}=t(r);m.error(o),process.exit(1)});

package/dist/index.d.mts

@@ -1,6 +1,7 @@
 import { ListingsSource, ZillowListingHtmlOptions } from '@rent-scraper/api';
+import { ErrorLog } from '@rent-scraper/utils';
 
-declare const scrapeListingDetailsFromHtmlByFilePaths: (source: ListingsSource, inputFilePaths: string[]) => Promise<void>;
+declare const scrapeListingDetailsFromHtmlByFilePaths: (source: ListingsSource, inputFilePaths: string[], errors?: ErrorLog) => Promise<number>;
 declare const scrapeListingDetailsFromHtmlByZipCodes: (source: ListingsSource, zipCodes: number[], inputDirectory: string) => Promise<{
     numListings: number;
 }>;
@@ -16,7 +17,10 @@ declare const scrapeListingHtmlByZipCodesAndListingDetails: (source: ListingsSou
 declare const scrapeListingHtmlByInputDirectory: (source: ListingsSource, inputDirectory: string, outputDirectory?: string, options?: Pick<ScrapeListingHtmlOptions, "skipBotCheck">) => Promise<void>;
 declare const scrapeListingHtmlByIds: (source: ListingsSource, ids: string[], outputDirectory: string, options?: Pick<ScrapeListingHtmlOptions, "skipBotCheck">) => Promise<void>;
 
-declare function runScrapeListings(): Promise<void>;
+declare function runScrapeListings(): Promise<{
+    success: boolean;
+    totalCsvListings: number;
+}>;
 
 declare const scrapeZillowListingsToCsv: (listingsDirectory: string, outputFilePath: string) => Promise<number>;
 
@@ -30,6 +34,9 @@ interface ScrapeZillowListingsByZipCodesOptions extends ScrapeListingsByZipCodes
     altFetch?: boolean;
     fetchListings?: boolean;
     skipBotCheck?: boolean;
+    silent?: boolean;
+    preValidatedZipCodes?: number[];
+    retry?: boolean;
 }
 
 declare const scrapeRedfinListingResultsByZipCodes: (zipCodes: number[], outputDirectory: string, options: ScrapeListingsByZipCodesOptions) => Promise<{
@@ -38,6 +45,8 @@ declare const scrapeRedfinListingResultsByZipCodes: (zipCodes: number[], outputD
 
 declare const scrapeZillowListingResultsByZipCodes: (zipCodes: number[], outputDirectory: string, options?: ScrapeZillowListingsByZipCodesOptions) => Promise<{
     validZipCodes: number[];
+    botFilteredZipCodes: number[];
+    noResultsZipCodes: number[];
 }>;
 
 export { fetchListingHtmlByUrlAndExport, runScrapeListings, scrapeListingDetailsFromHtmlByFilePaths, scrapeListingDetailsFromHtmlByZipCodes, scrapeListingHtmlByIds, scrapeListingHtmlByInputDirectory, scrapeListingHtmlByZipCodes, scrapeListingHtmlByZipCodesAndListingDetails, scrapeRedfinListingResultsByZipCodes, scrapeZillowListingResultsByZipCodes, scrapeZillowListingsToCsv };

package/dist/index.mjs

@@ -1 +1 @@
-export{f as fetchListingHtmlByUrlAndExport,r as runScrapeListings,s as scrapeListingDetailsFromHtmlByFilePaths,a as scrapeListingDetailsFromHtmlByZipCodes,e as scrapeListingHtmlByIds,d as scrapeListingHtmlByInputDirectory,b as scrapeListingHtmlByZipCodes,c as scrapeListingHtmlByZipCodesAndListingDetails,h as scrapeRedfinListingResultsByZipCodes,i as scrapeZillowListingResultsByZipCodes,g as scrapeZillowListingsToCsv}from"./shared/scrape-listings.B-9u0WHc.mjs";import"minimist";import"dayjs";import"path";import"axios";import"fs/promises";import"@rent-scraper/api";import"@rent-scraper/utils";import"@rent-scraper/api/config";import"@clack/prompts";import"crypto";import"node:timers/promises";import"@rent-scraper/utils/config";import"picocolors";
+export{f as fetchListingHtmlByUrlAndExport,r as runScrapeListings,s as scrapeListingDetailsFromHtmlByFilePaths,a as scrapeListingDetailsFromHtmlByZipCodes,e as scrapeListingHtmlByIds,d as scrapeListingHtmlByInputDirectory,b as scrapeListingHtmlByZipCodes,c as scrapeListingHtmlByZipCodesAndListingDetails,h as scrapeRedfinListingResultsByZipCodes,i as scrapeZillowListingResultsByZipCodes,g as scrapeZillowListingsToCsv}from"./shared/scrape-listings.Cdf92NvA.mjs";import"minimist";import"dayjs";import"path";import"axios";import"fs/promises";import"@rent-scraper/api";import"@rent-scraper/utils";import"@rent-scraper/api/config";import"@clack/prompts";import"node:timers/promises";import"crypto";import"@rent-scraper/utils/config";import"picocolors";

package/dist/shared/scrape-listings.Cdf92NvA.mjs

@@ -0,0 +1,15 @@
+import kt from"minimist";import lt from"dayjs";import o from"path";import ct from"axios";import{mkdir as S,writeFile as M,readFile as W,unlink as X,readdir as et}from"fs/promises";import{checkForZillowBotFiltering as q,isZillowBotFiltering as dt,fetchHtmlFromRedfinListingUrl as Bt,fetchHtmlFromZillowListingUrl as Mt,scrapeDataFromRedfinListingHtml as Dt,scrapeDataFromZillowListingHtml as Ft,getZillowListingResults as Pt,getRedfinListingResults as Rt,waitForSolvedZillowCaptcha as ut,isBrowserShowingCaptcha as Ut}from"@rent-scraper/api";import{ErrorLog as O,throwError as Z,checkForFile as L,parseJsonFile as Ht,parseError as P,readFilesInDirectory as K,roundValue as st,chunkArray as Nt,parsePercentage as Ot,compareArrays as pt}from"@rent-scraper/utils";import{getZillowOutputPath as T,getRedfinOutputPath as I,getZillowDaysListed as Tt,getRedfinDaysListed as It,getZillowLimit as Et,getZillowOffset as At,getRedfinZipCodes as Yt,getZillowZipCodes as qt}from"@rent-scraper/api/config";import{spinner as wt,log as u,progress as Jt,intro as Vt}from"@clack/prompts";import{setTimeout as gt}from"node:timers/promises";import{createHash as Gt}from"crypto";import{checkBrowserServer as mt}from"@rent-scraper/utils/config";import Wt from"picocolors";const R=process.env.DEBUG,E=async(t,e,s,i)=>{const{timeoutMs:r}=i??{};if(await L(s))R&&u.warn(`${s} exists, skipping`);else try{R&&u.info(`writing ${s}`);const a=t==="redfin"?await Bt(e):await Mt(e,{timeoutMs:r});await M(`${s}`,a)}catch(a){Z(`error fetching html for ${e}`,a)}},Xt=async(t,{timeoutMs:e})=>{if(await L(t)){const s=JSON.parse(await W(t,"utf8"))||{},{hdpUrl:i}=s||{};if(i){const r=`https://www.zillow.com${i}`,a=t.replace(".json",".html");await L(a)?R&&u.warn(`file already exists, ${a}`):await E("zillow",r,a,{timeoutMs:e})}else Z(`file is empty, ${t}`)}else R&&u.warn(`file does not exist at this path, ${t}, skipping`)},ft=async(t,e)=>{const{timeoutMs:s}=e??{};await Promise.all(t.map(async i=>await Xt(i,{timeoutMs:s})))},Kt=(t,e="zillow")=>{if(e==="zillow"){const{zpid:s,detailUrl:i}=t||{},r=i&&!i.startsWith("http")?`https://www.zillow.com${i}`:i;return{id:s,url:r}}else if(e==="redfin"){const{propertyId:s,url:i}=t?.homeData||{};return{id:s,url:`https://www.redfin.com${i}`}}},it=async(t,e,s,i=s,r)=>{const{timeoutMs:a,run:n=1,reruns:l=0,skipBotCheck:h=!1}=r??{},c=new O;s||Z("inputDirectory is required"),t==="zillow"&&!h&&await q();const p=[],y=[],d=wt();d.start("Downloading listings html files");for(let g=1;g<=l+1;g++){l>0&&g>1&&c.add(`rerun ${g-1} of ${l}`);const m=p.length;(g===1||m)&&await Promise.all((m?p:e).map(async v=>{const b=`${s}/${v}.json`;if(await L(b))try{const{results:x}=await Ht(b)||{};if(!x)c.add(`empty file, ${b}`);else{const j=i?`${i}/${v}`:`${s}/${v}`;await S(j,{recursive:!0}),x?.length?await Promise.all(x.map(async k=>{const{id:B,url:z}=Kt(k,t)??{};if(!z)return c.add(`url missing for ${B}`);const F=`${B}.html`,C=`${j}/${F}`;try{await E(t,z,C,{timeoutMs:a})}catch(N){const{status:A,message:U}=P(N);dt(A,U)||y.push({url:z,filePath:C}),c.add("scrape listing html error: "+(U??`error fetching listing for id, ${N}`))}})):c.add(`no results for file, ${b}`)}}catch(x){p.push(v);const{message:j}=P(x);c.add("scrape listing html error: "+(j??`error reading json data, ${b}, ${x}`))}else R&&u.warn(`file does not exist, ${b}, skipping`)}))}y.length>0&&(await gt(2e3),await Promise.all(y.map(async({url:g,filePath:m})=>{try{await E(t,g,m,{timeoutMs:a})}catch(v){const{message:b}=P(v);c.add("scrape listing html retry error: "+(b??`error fetching ${g}`))}})));const $=await T(),f=await I(),w=t==="zillow"?$:f;e.length>0?(d.stop("Listings HTML files have been saved to:"),u.message(o.join(w,t,"listings",o.basename(s)))):d.stop("No listings HTML files saved.");const _=o.join(w,t,"logs");if(await S(_,{recursive:!0}),c.get().filter(g=>!g.includes("rerun ")).length>l){const g=`${o.basename(s)}-html-errors-${n}.txt`,m=o.join(_,g);await c.write(m,[...new Set(c.get())].join(`
+`)),R&&u.error(`There were errors during processing, see ${o.resolve(m)}`)}},Qt=async(t,e,s,i)=>{const{timeoutMs:r,run:a=1,reruns:n=0,skipBotCheck:l=!1}=i??{},h=new O;s||Z("inputDirectory is required"),t==="zillow"&&!l&&await q();const c=[];for(let f=1;f<=n+1;f++){n>0&&f>1&&h.add(`rerun ${f-1} of ${n}`);const w=c.length;(f===1||w)&&await Promise.all((w?c:e).map(async _=>{const g=`${s}/${_}`;if(await L(g))try{const m=await K(g,{extension:".json",prependDirectory:!0});await ft(m,{timeoutMs:r})}catch(m){c.push(_);const{message:v}=P(m);h.add("scrape listing html error: "+(v??`Error during fetch for ${_}, ${m}`))}else h.add(`listing directory does not exist, ${g}`)}))}const p=await T(),y=await I(),d=t==="zillow"?p:y,$=o.join(d,t,"logs");if(await S($,{recursive:!0}),h.get().filter(f=>!f.includes("rerun ")).length>n){const f=`${o.basename(s)}-html-errors-${a}.txt`,w=o.join($,f);await h.write(w,[...new Set(h.get())].join(`
+`)),R&&u.error(`There were errors during processing, see ${o.resolve(w)}`)}},te=async(t,e,s=e,i)=>{const{skipBotCheck:r=!1}=i??{},a=new O;if(t==="zillow"&&!r&&await q(),e||Z("inputDirectory is required"),await S(s,{recursive:!0}),await L(e))try{const d=await K(e,{extension:".json",prependDirectory:!0});await ft(d)}catch(d){const{message:$}=P(d);a.add("scrape listing html error: "+($??`Error during fetch for ${e}, ${d}`))}else a.add(`inputDirectory does not exist, ${e}`);const n=await T(),l=await I(),h=t==="zillow"?n:l,c=o.join(h,t,"logs");await S(c,{recursive:!0});const p=`${o.basename(e)}-html-errors.txt`,y=o.join(c,p);a.get().length>0&&(await a.write(y,[...new Set(a.get())].join(`
+`)),R&&u.error(`There were errors during processing, see ${o.resolve(y)}`))},ee=(t,e)=>t==="redfin"?`https://www.redfin.com/home/${e}`:t==="zillow"?`https://www.zillow.com/homedetails/${e}_zpid`:null,se=async(t,e,s,i)=>{const{skipBotCheck:r=!1}=i??{};t==="zillow"&&!r&&await q(),s||Z("outputDirectory is required"),await S(s,{recursive:!0}),await Promise.all(e.map(async a=>{const n=ee(t,a),l=`${s}/${a}.html`;n&&await E(t,n,l)}))},D=process.env.DEBUG,ht=async t=>{if(await L(t)){const e=t.replace(".html",".json");if(await L(e))D&&u.warning(`file already exists, ${e}`);else{D&&u.message(`scraping data for ${t}`);const s=(await W(t)).toString();(!s||s.trim()==="")&&(await X(t),Z(`empty file found at ${t}, deleted for retry`)),s.includes("px-captcha")&&(await X(t),Z(`captcha page found in ${t}, deleted for retry`));const i=await Ft(s);if(!i?.priceHistory&&i?.bestMatchedUnit?.hdpUrl){D&&u.warning(`rescraping ${t} - https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`),await L(t)&&(D&&u.warning(`deleting ${t}`),await X(t)),await L(e)&&(D&&u.warning(`deleting ${e}`),await X(e));const r=`https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`;await E("zillow",r,t),await ht(t)}else i?(D&&u.info(`writing ${e}`),await M(`${e}`,JSON.stringify(i))):Z(`problem scraping data for ${t}`)}}},ie=async t=>{if(await L(t)){const e=t.replace(".html",".json");if(await L(e))D&&u.warning(`file already exists, ${e}`);else{D&&u.message(`scraping data for ${t}`);const s=(await W(t)).toString(),i=Dt(s);i?(D&&u.warning(`writing ${e}`),await M(`${e}`,JSON.stringify(i))):Z(`problem scraping data for ${t}`)}}},yt=async(t,e,s)=>(await Promise.all(e.map(async i=>{try{t==="redfin"?await ie(i):await ht(i);const r=i.replace(".html",".json");return await L(r)?1:0}catch(r){const{message:a}=P(r);return s?.add(a),0}}))).reduce((i,r)=>i+r,0),rt=async(t,e,s)=>{const i=new O;let r=0;s||Z("inputDirectory is required");const a=Jt({style:"heavy",max:100,size:50});a.start("Scraping listings data");const n=e.length,l=n<20?n+1:20,h=n>l?st(n/l,1):n,c=Nt(e,h);for(const[f,w]of c.entries()){const _=st((Number(f)+1)/l*100,1);a.advance(st(1/l*100,1),`Scraping listings (${Ot(_)})`),await Promise.all(w.map(async g=>{D&&u.message(`processing files for ${g}`);const m=`${s}/${g}`;if(await L(m)){const v=await K(m,{extension:".html",prependDirectory:!0}),b=await yt(t,v,i);r=r+b}else i.add(`listing directory does not exist, ${m}`)}))}const p=await T(),y=await I(),d=t==="zillow"?p:y;r>0?(a.stop("Listings data has been saved to:"),u.message(o.join(d,t,"listings",o.basename(s)))):a.stop("No listings data saved.");const $=o.join(d,t,"logs");if(await S($,{recursive:!0}),i.get().length>0){const f=`${o.basename(s)}-listing-errors.txt`,w=o.join($,f);await i.write(w,[...new Set(i.get())].join(`
+`)),u.error(`There were errors during processing, see ${o.resolve(w)}`)}return{numListings:r}},nt=process.env.DEBUG,re=async(t,e,s,i,r)=>{const{daysOnZillow:a,timeoutMs:n}=r??{};if(await L(e))nt&&u.warning(`${t} exists, skipping`),s.push(t);else{const l=await Pt({zipCode:t,daysOnZillow:a,mergePageResults:!0,timeoutMs:n});l?(nt&&u.info(`writing ${e}`),await M(e,JSON.stringify(l)),s.push(t)):(nt&&u.warning(`no results for ${t}`),i.push(t))}},at=async(t,e,s)=>{const{daysListed:i,timeoutMs:r,run:a=1,reruns:n=0,fetchListings:l=!1,skipBotCheck:h=!1,silent:c=!1}=s??{},p=new O;h||await q({fetchListings:l});const y=[],d=[],$=[],f=[];await S(e,{recursive:!0});const w=wt();c||w.start("Scraping Zillow search results");for(let m=1;m<=n+1;m++){n>0&&m>1&&p.add(`rerun ${m-1} of ${n}`);const v=d.length;(m===1||v)&&await Promise.all((v?d:t).map(async b=>{const x=`${b}.json`,j=`${e}/${x}`;try{await re(b,j,y,f,{daysOnZillow:i,timeoutMs:r})}catch(k){d.push(b);const{status:B,message:z}=P(k);dt(B,z)&&$.push(b),p.add("scrape listing results error: "+(z??`Error during fetch for ${b}, ${k}`))}}))}const _=await T();c||(y.length>0?(w.stop("Zillow search results have been saved to:"),u.message(o.join(_,"zillow","results",o.basename(e)))):w.stop("No results saved."));const g=o.join(_,"zillow","logs");if(await S(g,{recursive:!0}),p.get().filter(m=>!m.includes("rerun ")).length>n){const m=`${o.basename(e)}-results-errors-${a}.txt`,v=o.join(g,m);await p.write(v,[...new Set(p.get())].join(`
+`)),c||u.error(`There were errors during processing, see ${o.resolve(v)}`)}return{validZipCodes:y,botFilteredZipCodes:$,noResultsZipCodes:f}},ne=async(t,e,s,i)=>{const{daysListed:r,timeoutMs:a}=i??{};if(await L(e))console.log(`${t} exists, skipping`),s.push(t);else{const n=await Rt({zipCode:t,daysListed:r,timeoutMs:a});n?(console.log(`writing ${e}`),await M(e,JSON.stringify(n)),s.push(t)):console.log(`no results for ${t}`)}},$t=async(t,e,s)=>{const{daysListed:i,timeoutMs:r,run:a=1,reruns:n=0}=s??{},l=new O,h=[],c=[];await S(e,{recursive:!0});for(let d=1;d<=n+1;d++){n>0&&d>1&&l.add(`rerun ${d-1} of ${n}`);const $=c.length;(d===1||$)&&await Promise.all(($?c:t).map(async f=>{const w=`${f}.json`,_=`${e}/${w}`;try{await ne(f,_,h,{daysListed:i,timeoutMs:r})}catch(g){c.push(f);const{message:m}=P(g);l.add("scrape listing results error: "+(m??`Error during fetch for ${f}, ${g}`))}}))}const p=await I(),y=o.join(p,"redfin","logs");if(await S(y,{recursive:!0}),l.get().filter(d=>!d.includes("rerun ")).length>n){const d=`${o.basename(e)}-results-errors-${a}.txt`,$=o.join(y,d);await l.write($,[...new Set(l.get())].join(`
+`)),console.log(`\x1B[41m
+%s\x1B[0m`,`There were errors during processing, see ${o.resolve($)}`)}return{validZipCodes:h}},_t=["listing_id","listing_source","source_listing_id","parcel_number","listing_url","street_address","cleaned_address","cleaned_unit_number","city","state","zipcode","county","neighborhood_region","is_undisclosed_address","home_status","home_type","bedrooms","year_built","living_area","living_area_units","is_income_restricted","price_at_source","platform_rent_estimate","agent_name","agent_phone_number","broker_name","broker_phone_number","is_owned_by_listing_platform","date_last_updated_at_source","date_scraped","scrape_job_name"],ae=t=>{if(t==null)return"";const e=typeof t=="object"?JSON.stringify(t):String(t);return e.includes(",")||e.includes('"')||e.includes(`
+`)?`"${e.replace(/"/g,'""')}"`:e},oe=t=>{const e=/^(.*?)\s+((?:#|APT|UNIT|STE|Suite|Apt|Unit)\s*.+)$/i.exec(t);if(e){const s=e[1].trim().toUpperCase();let i=e[2].trim();return i.startsWith("#")&&(i=`# ${i.replace(/^#\s*/,"").trim()}`),{cleaned:s,unit:i}}return{cleaned:t.toUpperCase(),unit:""}},le=(t,e)=>{const s=String(t.zpid),i=Gt("md5").update(s).digest("hex"),r=t.streetAddress||t.address?.streetAddress||"",{cleaned:a,unit:n}=oe(r),l=t.hdpUrl?`https://www.zillow.com${t.hdpUrl}`:`https://www.zillow.com/homedetails/${s}_zpid/`;return{listing_id:i,listing_source:"zillow",source_listing_id:s,parcel_number:t.resoFacts?.parcelNumber??"",listing_url:l,street_address:r,cleaned_address:a,cleaned_unit_number:n,city:t.city||t.address?.city||"",state:t.state||t.address?.state||"",zipcode:t.zipcode||t.address?.zipcode||"",county:t.county||"",neighborhood_region:t.neighborhoodRegion?.name||t.parentRegion?.name||"",is_undisclosed_address:t.isUndisclosedAddress??"",home_status:t.homeStatus||"",home_type:t.homeType||"",bedrooms:t.bedrooms??"",year_built:t.yearBuilt||t.resoFacts?.yearBuilt||"",living_area:t.livingAreaValue??"",living_area_units:t.livingAreaUnits||"Square Feet",is_income_restricted:t.isIncomeRestricted??"",price_at_source:t.price??"",platform_rent_estimate:t.rentZestimate??"",agent_name:t.attributionInfo?.agentName||"",agent_phone_number:t.attributionInfo?.agentPhoneNumber||"",broker_name:t.attributionInfo?.brokerName||"",broker_phone_number:t.attributionInfo?.brokerPhoneNumber||"",is_owned_by_listing_platform:t.attributionInfo?.mlsName==="Zillow Rentals",date_last_updated_at_source:t.attributionInfo?.lastUpdated||"",date_scraped:t.timestamp?lt(t.timestamp).format("YYYY-MM-DD HH:mm:ss"):"",scrape_job_name:e}},bt=async(t,e)=>{const s=o.basename(t),i=[_t.join(",")];let r;try{r=await et(t,{withFileTypes:!0})}catch{return 0}const a=r.filter(n=>n.isDirectory()).map(n=>n.name);for(const n of a){const l=o.join(t,n);if(!await L(l))continue;const h=await K(l,{extension:".json",prependDirectory:!0});for(const c of h)try{const p=JSON.parse(await W(c,"utf8"));if(!p.zpid)continue;const y=le(p,s);i.push(_t.map(d=>ae(y[d])).join(","))}catch{}}return await S(o.dirname(e),{recursive:!0}),await M(e,i.join(`
+`)),i.length-1},ce=async t=>{try{const e=(await et(t,{withFileTypes:!0})).filter(s=>s.isDirectory()).map(s=>s.name).sort();return e.length>0?e[e.length-1]:null}catch{return null}},vt=async()=>{await ct.post("http://localhost:8082/browser/close")},de=async()=>{await ct.post("http://localhost:8082/server/shutdown")},ue=async(t,e,s,{daysListed:i,timeoutMs:r,run:a,reruns:n,preValidatedZipCodes:l=[],retry:h=!1})=>{let c=[],p=[],y=[];if(t.length>0){if(await ut(),await vt(),{validZipCodes:c,botFilteredZipCodes:p,noResultsZipCodes:y}=await at(t,e,{daysListed:i,timeoutMs:r,run:a,reruns:n,skipBotCheck:!0}),h&&p.length>0){u.warn(`Bot filtering hit ${p.length} of ${t.length} zip code(s) \u2014 will retry after listings are fetched`),await Ut()?(await ut(),await vt()):await gt(3e3);const f=p.length,w=await at(p,e,{daysListed:i,timeoutMs:r,run:a,reruns:n,skipBotCheck:!0,silent:!0});c=[...c,...w.validZipCodes],p=w.botFilteredZipCodes,y=[...y,...w.noResultsZipCodes];const _=w.validZipCodes.length;u.info(`Inline retry: recovered ${_} of ${f} \xB7 ${p.length} still bot-filtered`)}c.length===0&&t.length>0&&u.warn("No results returned for any zip codes \u2014 possible soft bot filtering. Please try again shortly."),p.length>0&&u.warn(`${p.length} zip code(s) still bot-filtered \u2014 use --rerun to retry`)}const d=[...l,...c];await it("zillow",d,e,s,{timeoutMs:r,run:a,reruns:n,skipBotCheck:!0});const{numListings:$}=await rt("zillow",d,s);return{numListings:$,validZipCodes:d,noResultsZipCodes:y}},pe=async(t,e,s,{daysListed:i,timeoutMs:r,run:a,reruns:n})=>{const{validZipCodes:l}=await $t(t,e,{daysListed:i,timeoutMs:r,run:a,reruns:n});await it("redfin",l,e,s,{timeoutMs:r,run:a,reruns:n});const{numListings:h}=await rt("redfin",l,s);return{numListings:h,validZipCodes:l}};async function we(){Vt(Wt.inverse(" scrape listings "));const t=kt(process.argv.slice(2)),e=t.source??"zillow";e==="zillow"&&(await mt()||Z("Please launch the browser server before scraping."));const s=await T(),i=await I(),r=e==="zillow"?s:i,a=t["days-listed"]??(e==="zillow"?await Tt():await It())??1,n=t.runs??1,l=t.reruns??0,h=t["timeout-ms"]??6e4,c=t.limit??await Et()??void 0,p=t.offset??await At()??0,y=t.retry??!1,d=t.rerun,$=typeof d=="string"?d:d?await ce(o.join(r,e,"results")):null;d&&!$&&Z("No previous run found to rerun."),$&&u.info(`Rerunning from: ${$}`);const f=$??lt().format("YYYY-MM-DD-HHmm"),w=t["results-directory"]??o.join(r,e,"results",f),_=t["listings-directory"]??o.join(r,e,"listings",f),g=t["logs-directory"]??o.join(r,e,"logs");let m=0;const v=async()=>{(e==="zillow"||e==="redfin")&&await mt()&&await de()};try{if(e==="redfin"){const x=await Yt();for(let j=1;j<=n;j++){x||Z("zip codes required, please run the createConfig script");const k=x,{numListings:B,validZipCodes:z}=await pe(k,w,_,{daysListed:a,timeoutMs:h,run:j,reruns:l});if(j===n){const F=Object.entries({numListings:B}).map(([Q,J])=>`${Q}: ${J}`).join(`
+`),C=pt(k,z).join(`
+`);await S(g,{recursive:!0});const N=`${o.basename(w)}-scraping-results.txt`,A=o.join(g,N),U=`${o.basename(w)}-invalid-zipcodes.txt`,H=o.join(g,U);await M(A,F),await M(H,C)}}}else if(e==="zillow"){const x=await qt();x||Z("zip codes required, please run the createConfig script");const j=x.slice(p,c?p+c:void 0);let k=j,B=[];if($){const z=await et(w).catch(()=>[]),F=new Set(z.filter(C=>C.endsWith(".json")).map(C=>C.replace(".json","")));B=j.filter(C=>F.has(String(C))),k=j.filter(C=>!F.has(String(C))),k.length>0?u.info(`${B.length} already fetched \xB7 retrying ${k.length} missing`):u.info(`All ${B.length} zip codes already fetched \xB7 proceeding to HTML scraping`)}for(let z=1;z<=n;z++){const{numListings:F,validZipCodes:C,noResultsZipCodes:N}=await ue(k,w,_,{daysListed:a,timeoutMs:h,run:z,reruns:l,preValidatedZipCodes:B,retry:y});if(z===n){const A=o.basename(_),U=o.join(s,"zillow","csv",`${A}.csv`),H=await bt(_,U);m=H,H>0&&u.info(`CSV exported: ${H} listings \u2192 ${U}`);const Q=Object.entries({numListings:F,numCsvListings:H}).map(([Y,St])=>`${Y}: ${St}`).join(`
+`),J=pt(j,C),ot=new Set(N),V=J.filter(Y=>!ot.has(Y)),G=J.filter(Y=>ot.has(Y));await S(g,{recursive:!0});const jt=`${o.basename(w)}-scraping-results.txt`,Lt=o.join(g,jt),zt=`${o.basename(w)}-zipcodes-no-results.txt`,Ct=o.join(g,zt),Zt=`${o.basename(w)}-zipcodes-errored.txt`,xt=o.join(g,Zt);await M(Lt,Q),G.length>0&&await M(Ct,G.join(`
+`)),V.length>0&&await M(xt,V.join(`
+`));const tt=[`${C.length}/${j.length} with results`];G.length>0&&tt.push(`${G.length} no results`),V.length>0&&tt.push(`${V.length} errored`),u.info(`Zip codes: ${tt.join(" \xB7 ")}`),u.info(`Listings:  ${F} parsed \xB7 ${H} exported to CSV`)}}}const b=e==="zillow"?m>0:!0;return b?u.success("Scraping complete!"):u.warn("Scraping finished but no listings were exported \u2014 please try again shortly."),{success:b,totalCsvListings:m}}finally{await v()}}export{rt as a,it as b,Qt as c,te as d,se as e,E as f,bt as g,$t as h,at as i,we as r,yt as s};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rent-scraper/scrape-listings",
-  "version": "1.0.29",
+  "version": "1.0.31",
   "type": "module",
   "exports": {
     ".": {
@@ -30,8 +30,8 @@
     "dayjs": "^1.11.13",
     "minimist": "^1.2.8",
     "picocolors": "^1.1.1",
-    "@rent-scraper/utils": "1.0.29",
-    "@rent-scraper/api": "1.0.29"
+    "@rent-scraper/api": "1.0.31",
+    "@rent-scraper/utils": "1.0.31"
   },
   "devDependencies": {
     "@types/minimist": "^1.2.5",

package/dist/shared/scrape-listings.B-9u0WHc.mjs

@@ -1,14 +0,0 @@
-import le from"minimist";import E from"dayjs";import n from"path";import G from"axios";import{mkdir as j,writeFile as Z,readFile as N,unlink as V,readdir as ce}from"fs/promises";import{checkForZillowBotFiltering as S,fetchHtmlFromRedfinListingUrl as de,fetchHtmlFromZillowListingUrl as ue,scrapeDataFromRedfinListingHtml as we,scrapeDataFromZillowListingHtml as me,getZillowListingResults as pe,getRedfinListingResults as ge,waitForSolvedZillowCaptcha as fe}from"@rent-scraper/api";import{ErrorLog as U,throwError as z,checkForFile as v,parseJsonFile as he,parseError as D,readFilesInDirectory as O,roundValue as Y,chunkArray as ye,parsePercentage as $e,compareArrays as K}from"@rent-scraper/utils";import{getZillowOutputPath as H,getRedfinOutputPath as P,getZillowDaysListed as _e,getRedfinDaysListed as be,getRedfinZipCodes as ve,getZillowZipCodes as je}from"@rent-scraper/api/config";import{spinner as W,log as w,progress as Le,intro as ze,confirm as Me,isCancel as Ze,cancel as xe,outro as Ce}from"@clack/prompts";import{createHash as ke}from"crypto";import{setTimeout as Q}from"node:timers/promises";import{checkBrowserServer as X}from"@rent-scraper/utils/config";import De from"picocolors";const B=process.env.DEBUG,R=async(e,t,s,i)=>{const{timeoutMs:o}=i??{};if(await v(s))B&&w.warn(`${s} exists, skipping`);else try{B&&w.info(`writing ${s}`);const a=e==="redfin"?await de(t):await ue(t,{timeoutMs:o});await Z(`${s}`,a)}catch(a){z(`error fetching html for ${t}`,a)}},Be=async(e,{timeoutMs:t})=>{if(await v(e)){const s=JSON.parse(await N(e,"utf8"))||{},{hdpUrl:i}=s||{};if(i){const o=`https://www.zillow.com${i}`,a=e.replace(".json",".html");await v(a)?B&&w.warn(`file already exists, ${a}`):await R("zillow",o,a,{timeoutMs:t})}else z(`file is empty, ${e}`)}else B&&w.warn(`file does not exist at this path, ${e}, skipping`)},ee=async(e,t)=>{const{timeoutMs:s}=t??{};await Promise.all(e.map(async i=>await Be(i,{timeoutMs:s})))},Se=(e,t="zillow")=>{if(t==="zillow"){const{zpid:s,detailUrl:i}=e||{};return{id:s,url:i}}else if(t==="redfin"){const{propertyId:s,url:i}=e?.homeData||{};return{id:s,url:`https://www.redfin.com${i}`}}},A=async(e,t,s,i=s,o)=>{const{timeoutMs:a,run:r=1,reruns:l=0,skipBotCheck:h=!1}=o??{},c=new U;s||z("inputDirectory is required"),e==="zillow"&&!h&&await S();const y=[],p=W();p.start("Downloading listings html files");for(let f=1;f<=l+1;f++){l>0&&f>1&&c.add(`rerun ${f-1} of ${l}`);const $=y.length;(f===1||$)&&await Promise.all(($?y:t).map(async _=>{const b=`${s}/${_}.json`;if(await v(b))try{const{results:L}=await he(b)||{};if(!L)c.add(`empty file, ${b}`);else{const M=i?`${i}/${_}`:`${s}/${_}`;await j(M,{recursive:!0}),L?.length?await Promise.all(L.map(async F=>{try{const{id:C,url:k}=Se(F,e)??{};if(!k)return c.add(`url missing for ${C}`);const I=`${C}.html`,T=`${M}/${I}`;await R(e,k,T,{timeoutMs:a})}catch(C){const{message:k}=D(C);c.add("scrape listing html error: "+(k??`error fetching listing for id, ${C}`))}})):c.add(`no results for file, ${b}`)}}catch(L){y.push(_);const{message:M}=D(L);c.add("scrape listing html error: "+(M??`error reading json data, ${b}, ${L}`))}else B&&w.warn(`file does not exist, ${b}, skipping`)}))}p.stop("Listings HTML files have been saved to:");const d=await H(),g=await P(),u=e==="zillow"?d:g;w.message(n.join(u,e,"listings",n.basename(s)));const m=n.join(u,e,"logs");if(await j(m,{recursive:!0}),c.get().filter(f=>!f.includes("rerun ")).length>l){const f=`${n.basename(s)}-html-errors-${r}.txt`,$=n.join(m,f);await c.write($,[...new Set(c.get())].join(`
-`)),B&&w.error(`There were errors during processing, see ${n.resolve($)}`)}},Ue=async(e,t,s,i)=>{const{timeoutMs:o,run:a=1,reruns:r=0,skipBotCheck:l=!1}=i??{},h=new U;s||z("inputDirectory is required"),e==="zillow"&&!l&&await S();const c=[];for(let u=1;u<=r+1;u++){r>0&&u>1&&h.add(`rerun ${u-1} of ${r}`);const m=c.length;(u===1||m)&&await Promise.all((m?c:t).map(async f=>{const $=`${s}/${f}`;if(await v($))try{const _=await O($,{extension:".json",prependDirectory:!0});await ee(_,{timeoutMs:o})}catch(_){c.push(f);const{message:b}=D(_);h.add("scrape listing html error: "+(b??`Error during fetch for ${f}, ${_}`))}else h.add(`listing directory does not exist, ${$}`)}))}const y=await H(),p=await P(),d=e==="zillow"?y:p,g=n.join(d,e,"logs");if(await j(g,{recursive:!0}),h.get().filter(u=>!u.includes("rerun ")).length>r){const u=`${n.basename(s)}-html-errors-${a}.txt`,m=n.join(g,u);await h.write(m,[...new Set(h.get())].join(`
-`)),B&&w.error(`There were errors during processing, see ${n.resolve(m)}`)}},He=async(e,t,s=t,i)=>{const{skipBotCheck:o=!1}=i??{},a=new U;if(e==="zillow"&&!o&&await S(),t||z("inputDirectory is required"),await j(s,{recursive:!0}),await v(t))try{const d=await O(t,{extension:".json",prependDirectory:!0});await ee(d)}catch(d){const{message:g}=D(d);a.add("scrape listing html error: "+(g??`Error during fetch for ${t}, ${d}`))}else a.add(`inputDirectory does not exist, ${t}`);const r=await H(),l=await P(),h=e==="zillow"?r:l,c=n.join(h,e,"logs");await j(c,{recursive:!0});const y=`${n.basename(t)}-html-errors.txt`,p=n.join(c,y);a.get().length>0&&(await a.write(p,[...new Set(a.get())].join(`
-`)),B&&w.error(`There were errors during processing, see ${n.resolve(p)}`))},Pe=(e,t)=>e==="redfin"?`https://www.redfin.com/home/${t}`:e==="zillow"?`https://www.zillow.com/homedetails/${t}_zpid`:null,Fe=async(e,t,s,i)=>{const{skipBotCheck:o=!1}=i??{};e==="zillow"&&!o&&await S(),s||z("outputDirectory is required"),await j(s,{recursive:!0}),await Promise.all(t.map(async a=>{const r=Pe(e,a),l=`${s}/${a}.html`;r&&await R(e,r,l)}))},x=process.env.DEBUG,te=async e=>{if(await v(e)){const t=e.replace(".html",".json");if(await v(t))x&&w.warning(`file already exists, ${t}`);else{x&&w.message(`scraping data for ${e}`);const s=(await N(e)).toString(),i=await me(s);if(!i?.priceHistory&&i?.bestMatchedUnit?.hdpUrl){x&&w.warning(`rescraping ${e} - https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`),await v(e)&&(x&&w.warning(`deleting ${e}`),await V(e)),await v(t)&&(x&&w.warning(`deleting ${t}`),await V(t));const o=`https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`;try{await R("zillow",o,e),await te(e)}catch(a){const{message:r}=D(a);w.error(r)}}else i?(x&&w.info(`writing ${t}`),await Z(`${t}`,JSON.stringify(i))):z(`problem scraping data for ${e}`)}}},Re=async e=>{if(await v(e)){const t=e.replace(".html",".json");if(await v(t))x&&w.warning(`file already exists, ${t}`);else{x&&w.message(`scraping data for ${e}`);const s=(await N(e)).toString(),i=we(s);i?(x&&w.warning(`writing ${t}`),await Z(`${t}`,JSON.stringify(i))):z(`problem scraping data for ${e}`)}}},se=async(e,t)=>{await Promise.all(t.map(async s=>e==="redfin"?await Re(s):await te(s)))},q=async(e,t,s)=>{const i=new U;let o=0;s||z("inputDirectory is required");const a=Le({style:"heavy",max:100,size:50});a.start("Scraping listings data");const r=t.length,l=r<20?r+1:20,h=r>l?Y(r/l,1):r,c=ye(t,h);for(const[u,m]of c.entries()){const f=Y((Number(u)+1)/l*100,1);a.advance(Y(1/l*100,1),`Scraping listings (${$e(f)})`),await Promise.all(m.map(async $=>{x&&w.message(`processing files for ${$}`);const _=`${s}/${$}`;if(await v(_)){const b=await O(_,{extension:".html",prependDirectory:!0});o=o+b.length,await se(e,b)}else i.add(`listing directory does not exist, ${_}`)}))}const y=await H(),p=await P(),d=e==="zillow"?y:p;a.stop("Listings data has been saved to:"),w.message(n.join(d,e,"listings",n.basename(s)));const g=n.join(d,e,"logs");if(await j(g,{recursive:!0}),i.get().length>0){const u=`${n.basename(s)}-listing-errors.txt`,m=n.join(g,u);await i.write(m,[...new Set(i.get())].join(`
-`)),w.error(`There were errors during processing, see ${n.resolve(m)}`)}return{numListings:o}},J=process.env.DEBUG,Ne=async(e,t,s,i)=>{const{daysOnZillow:o,timeoutMs:a}=i??{};if(await v(t))J&&w.warning(`${e} exists, skipping`),s.push(e);else{const r=await pe({zipCode:e,daysOnZillow:o,mergePageResults:!0,timeoutMs:a});r?(J&&w.info(`writing ${t}`),await Z(t,JSON.stringify(r)),s.push(e)):J&&w.warning(`no results for ${e}`)}},ie=async(e,t,s)=>{const{daysListed:i,timeoutMs:o,run:a=1,reruns:r=0,fetchListings:l=!1,skipBotCheck:h=!1}=s??{},c=new U;h||await S({fetchListings:l});const y=[],p=[];await j(t,{recursive:!0});const d=W();d.start("Scraping Zillow search results");for(let m=1;m<=r+1;m++){r>0&&m>1&&c.add(`rerun ${m-1} of ${r}`);const f=p.length;(m===1||f)&&await Promise.all((f?p:e).map(async $=>{const _=`${$}.json`,b=`${t}/${_}`;try{await Ne($,b,y,{daysOnZillow:i,timeoutMs:o})}catch(L){p.push($);const{message:M}=D(L);c.add("scrape listing results error: "+(M??`Error during fetch for ${$}, ${L}`))}}))}d.stop("Zillow search results have been saved to:");const g=await H();w.message(n.join(g,"zillow","results",n.basename(t)));const u=n.join(g,"zillow","logs");if(await j(u,{recursive:!0}),c.get().filter(m=>!m.includes("rerun ")).length>r){const m=`${n.basename(t)}-results-errors-${a}.txt`,f=n.join(u,m);await c.write(f,[...new Set(c.get())].join(`
-`)),w.error(`There were errors during processing, see ${n.resolve(f)}`)}return{validZipCodes:y}},Oe=async(e,t,s,i)=>{const{daysListed:o,timeoutMs:a}=i??{};if(await v(t))console.log(`${e} exists, skipping`),s.push(e);else{const r=await ge({zipCode:e,daysListed:o,timeoutMs:a});r?(console.log(`writing ${t}`),await Z(t,JSON.stringify(r)),s.push(e)):console.log(`no results for ${e}`)}},re=async(e,t,s)=>{const{daysListed:i,timeoutMs:o,run:a=1,reruns:r=0}=s??{},l=new U,h=[],c=[];await j(t,{recursive:!0});for(let d=1;d<=r+1;d++){r>0&&d>1&&l.add(`rerun ${d-1} of ${r}`);const g=c.length;(d===1||g)&&await Promise.all((g?c:e).map(async u=>{const m=`${u}.json`,f=`${t}/${m}`;try{await Oe(u,f,h,{daysListed:i,timeoutMs:o})}catch($){c.push(u);const{message:_}=D($);l.add("scrape listing results error: "+(_??`Error during fetch for ${u}, ${$}`))}}))}const y=await P(),p=n.join(y,"redfin","logs");if(await j(p,{recursive:!0}),l.get().filter(d=>!d.includes("rerun ")).length>r){const d=`${n.basename(t)}-results-errors-${a}.txt`,g=n.join(p,d);await l.write(g,[...new Set(l.get())].join(`
-`)),console.log(`\x1B[41m
-%s\x1B[0m`,`There were errors during processing, see ${n.resolve(g)}`)}return{validZipCodes:h}},ae=["listing_id","listing_source","source_listing_id","parcel_number","listing_url","street_address","cleaned_address","cleaned_unit_number","city","state","zipcode","county","neighborhood_region","is_undisclosed_address","home_status","home_type","bedrooms","year_built","living_area","living_area_units","is_income_restricted","price_at_source","platform_rent_estimate","agent_name","agent_phone_number","broker_name","broker_phone_number","is_owned_by_listing_platform","date_last_updated_at_source","date_scraped","scrape_job_name"],Ie=e=>{if(e==null)return"";const t=typeof e=="object"?JSON.stringify(e):String(e);return t.includes(",")||t.includes('"')||t.includes(`
-`)?`"${t.replace(/"/g,'""')}"`:t},Te=e=>{const t=/^(.*?)\s+((?:#|APT|UNIT|STE|Suite|Apt|Unit)\s*.+)$/i.exec(e);if(t){const s=t[1].trim().toUpperCase();let i=t[2].trim();return i.startsWith("#")&&(i=`# ${i.replace(/^#\s*/,"").trim()}`),{cleaned:s,unit:i}}return{cleaned:e.toUpperCase(),unit:""}},Ee=(e,t)=>{const s=String(e.zpid),i=ke("md5").update(s).digest("hex"),o=e.streetAddress||e.address?.streetAddress||"",{cleaned:a,unit:r}=Te(o),l=e.hdpUrl?`https://www.zillow.com${e.hdpUrl}`:`https://www.zillow.com/homedetails/${s}_zpid/`;return{listing_id:i,listing_source:"zillow",source_listing_id:s,parcel_number:e.resoFacts?.parcelNumber??"",listing_url:l,street_address:o,cleaned_address:a,cleaned_unit_number:r,city:e.city||e.address?.city||"",state:e.state||e.address?.state||"",zipcode:e.zipcode||e.address?.zipcode||"",county:e.county||"",neighborhood_region:e.neighborhoodRegion?.name||e.parentRegion?.name||"",is_undisclosed_address:e.isUndisclosedAddress??"",home_status:e.homeStatus||"",home_type:e.homeType||"",bedrooms:e.bedrooms??"",year_built:e.yearBuilt||e.resoFacts?.yearBuilt||"",living_area:e.livingAreaValue??"",living_area_units:e.livingAreaUnits||"Square Feet",is_income_restricted:e.isIncomeRestricted??"",price_at_source:e.price??"",platform_rent_estimate:e.rentZestimate??"",agent_name:e.attributionInfo?.agentName||"",agent_phone_number:e.attributionInfo?.agentPhoneNumber||"",broker_name:e.attributionInfo?.brokerName||"",broker_phone_number:e.attributionInfo?.brokerPhoneNumber||"",is_owned_by_listing_platform:e.attributionInfo?.mlsName==="Zillow Rentals",date_last_updated_at_source:e.attributionInfo?.lastUpdated||"",date_scraped:e.timestamp?E(e.timestamp).format("YYYY-MM-DD HH:mm:ss"):"",scrape_job_name:t}},ne=async(e,t)=>{const s=n.basename(e),i=[ae.join(",")];let o;try{o=await ce(e,{withFileTypes:!0})}catch{return 0}const a=o.filter(r=>r.isDirectory()).map(r=>r.name);for(const r of a){const l=n.join(e,r);if(!await v(l))continue;const h=await O(l,{extension:".json",prependDirectory:!0});for(const c of h)try{const y=JSON.parse(await N(c,"utf8"));if(!y.zpid)continue;const p=Ee(y,s);i.push(ae.map(d=>Ie(p[d])).join(","))}catch{}}return await j(n.dirname(t),{recursive:!0}),await Z(t,i.join(`
-`)),i.length-1},Ye=async()=>{await G.post("http://localhost:8082/browser/close")},Ae=async()=>{await G.post("http://localhost:8082/server/shutdown")},qe=async(e,t,s,{daysListed:i,timeoutMs:o,run:a,reruns:r})=>{try{await S()}catch(c){const{message:y}=D(c);w.error(y);const p=await Me({message:"You need to complete a captcha in your browser. Press Return to launch your browser and continue.",active:"OK",inactive:"Cancel"});if(Ze(p)||!p)return xe("Create config canceled. Please try again."),process.exit(1);await Q(1e3),Ce("Browser Launching..."),await Q(1e3),await fe()}await Ye();const{validZipCodes:l}=await ie(e,t,{daysListed:i,timeoutMs:o,run:a,reruns:r});await A("zillow",l,t,s,{timeoutMs:o,run:a,reruns:r});const{numListings:h}=await q("zillow",l,s);return{numListings:h,validZipCodes:l}},Je=async(e,t,s,{daysListed:i,timeoutMs:o,run:a,reruns:r})=>{const{validZipCodes:l}=await re(e,t,{daysListed:i,timeoutMs:o,run:a,reruns:r});await A("redfin",l,t,s,{timeoutMs:o,run:a,reruns:r});const{numListings:h}=await q("redfin",l,s);return{numListings:h,validZipCodes:l}};async function Ge(){ze(De.inverse(" scrape listings "));const e=le(process.argv.slice(2)),t=e.source??"zillow";t==="zillow"&&(await X()||z("Please launch the browser server before scraping."));const s=await H(),i=await P(),o=t==="zillow"?s:i,a=e["days-listed"]??(t==="zillow"?await _e():await be())??1,r=e.runs??1,l=e.reruns??0,h=e["timeout-ms"]??6e4,c=e["results-directory"]??n.join(o,t,"results",`${E().format("YYYY-MM-DD-HHmm")}`),y=e["listings-directory"]??n.join(o,t,"listings",`${E().format("YYYY-MM-DD-HHmm")}`),p=e["logs-directory"]??n.join(o,t,"logs");if(t==="redfin"){const d=await ve();for(let g=1;g<=r;g++){d||z("zip codes required, please run the createConfig script");const u=d,{numListings:m,validZipCodes:f}=await Je(u,c,y,{daysListed:a,timeoutMs:h,run:g,reruns:l});if(g===r){const $=Object.entries({numListings:m}).map(([C,k])=>`${C}: ${k}`).join(`
-`),_=K(u,f).join(`
-`);await j(p,{recursive:!0});const b=`${n.basename(c)}-scraping-results.txt`,L=n.join(p,b),M=`${n.basename(c)}-invalid-zipcodes.txt`,F=n.join(p,M);await Z(L,$),await Z(F,_)}}}else if(t==="zillow"){const d=await je();for(let g=1;g<=r;g++){d||z("zip codes required, please run the createConfig script");const u=d,{numListings:m,validZipCodes:f}=await qe(u,c,y,{daysListed:a,timeoutMs:h,run:g,reruns:l});if(g===r){const $=n.basename(y),_=n.join(s,"zillow","csv",`${$}.csv`),b=await ne(y,_);w.info(`CSV exported: ${b} listings \u2192 ${_}`);const L=Object.entries({numListings:m,numCsvListings:b}).map(([T,oe])=>`${T}: ${oe}`).join(`
-`),M=K(u,f).join(`
-`);await j(p,{recursive:!0});const F=`${n.basename(c)}-scraping-results.txt`,C=n.join(p,F),k=`${n.basename(c)}-invalid-zipcodes.txt`,I=n.join(p,k);await Z(C,L),await Z(I,M)}}}w.success("Scraping complete!"),(t==="zillow"||t==="redfin")&&await X()&&await Ae()}export{q as a,A as b,Ue as c,He as d,Fe as e,R as f,ne as g,re as h,ie as i,Ge as r,se as s};