@rent-scraper/scrape-listings 1.0.30 → 1.0.31

package/dist/bin/run-scrape-listings.mjs

@@ -1 +1 @@
-import{parseError as t}from"@rent-scraper/utils";import{r as i}from"../shared/scrape-listings.DPpklKj4.mjs";import{log as m}from"@clack/prompts";import"minimist";import"dayjs";import"path";import"axios";import"fs/promises";import"@rent-scraper/api";import"@rent-scraper/api/config";import"node:timers/promises";import"crypto";import"@rent-scraper/utils/config";import"picocolors";i().then(()=>{process.exit(0)}).catch(r=>{const{message:o}=t(r);m.error(o),process.exit(1)});
+import{parseError as t}from"@rent-scraper/utils";import{r as i}from"../shared/scrape-listings.Cdf92NvA.mjs";import{log as m}from"@clack/prompts";import"minimist";import"dayjs";import"path";import"axios";import"fs/promises";import"@rent-scraper/api";import"@rent-scraper/api/config";import"node:timers/promises";import"crypto";import"@rent-scraper/utils/config";import"picocolors";i().then(()=>{process.exit(0)}).catch(r=>{const{message:o}=t(r);m.error(o),process.exit(1)});

package/dist/index.d.mts

@@ -17,7 +17,10 @@ declare const scrapeListingHtmlByZipCodesAndListingDetails: (source: ListingsSou
 declare const scrapeListingHtmlByInputDirectory: (source: ListingsSource, inputDirectory: string, outputDirectory?: string, options?: Pick<ScrapeListingHtmlOptions, "skipBotCheck">) => Promise<void>;
 declare const scrapeListingHtmlByIds: (source: ListingsSource, ids: string[], outputDirectory: string, options?: Pick<ScrapeListingHtmlOptions, "skipBotCheck">) => Promise<void>;
 
-declare function runScrapeListings(): Promise<void>;
+declare function runScrapeListings(): Promise<{
+    success: boolean;
+    totalCsvListings: number;
+}>;
 
 declare const scrapeZillowListingsToCsv: (listingsDirectory: string, outputFilePath: string) => Promise<number>;
 

package/dist/index.mjs

@@ -1 +1 @@
-export{f as fetchListingHtmlByUrlAndExport,r as runScrapeListings,s as scrapeListingDetailsFromHtmlByFilePaths,a as scrapeListingDetailsFromHtmlByZipCodes,e as scrapeListingHtmlByIds,d as scrapeListingHtmlByInputDirectory,b as scrapeListingHtmlByZipCodes,c as scrapeListingHtmlByZipCodesAndListingDetails,h as scrapeRedfinListingResultsByZipCodes,i as scrapeZillowListingResultsByZipCodes,g as scrapeZillowListingsToCsv}from"./shared/scrape-listings.DPpklKj4.mjs";import"minimist";import"dayjs";import"path";import"axios";import"fs/promises";import"@rent-scraper/api";import"@rent-scraper/utils";import"@rent-scraper/api/config";import"@clack/prompts";import"node:timers/promises";import"crypto";import"@rent-scraper/utils/config";import"picocolors";
+export{f as fetchListingHtmlByUrlAndExport,r as runScrapeListings,s as scrapeListingDetailsFromHtmlByFilePaths,a as scrapeListingDetailsFromHtmlByZipCodes,e as scrapeListingHtmlByIds,d as scrapeListingHtmlByInputDirectory,b as scrapeListingHtmlByZipCodes,c as scrapeListingHtmlByZipCodesAndListingDetails,h as scrapeRedfinListingResultsByZipCodes,i as scrapeZillowListingResultsByZipCodes,g as scrapeZillowListingsToCsv}from"./shared/scrape-listings.Cdf92NvA.mjs";import"minimist";import"dayjs";import"path";import"axios";import"fs/promises";import"@rent-scraper/api";import"@rent-scraper/utils";import"@rent-scraper/api/config";import"@clack/prompts";import"node:timers/promises";import"crypto";import"@rent-scraper/utils/config";import"picocolors";

package/dist/shared/scrape-listings.Cdf92NvA.mjs

@@ -0,0 +1,15 @@
+import kt from"minimist";import lt from"dayjs";import o from"path";import ct from"axios";import{mkdir as S,writeFile as M,readFile as W,unlink as X,readdir as et}from"fs/promises";import{checkForZillowBotFiltering as q,isZillowBotFiltering as dt,fetchHtmlFromRedfinListingUrl as Bt,fetchHtmlFromZillowListingUrl as Mt,scrapeDataFromRedfinListingHtml as Dt,scrapeDataFromZillowListingHtml as Ft,getZillowListingResults as Pt,getRedfinListingResults as Rt,waitForSolvedZillowCaptcha as ut,isBrowserShowingCaptcha as Ut}from"@rent-scraper/api";import{ErrorLog as O,throwError as Z,checkForFile as L,parseJsonFile as Ht,parseError as P,readFilesInDirectory as K,roundValue as st,chunkArray as Nt,parsePercentage as Ot,compareArrays as pt}from"@rent-scraper/utils";import{getZillowOutputPath as T,getRedfinOutputPath as I,getZillowDaysListed as Tt,getRedfinDaysListed as It,getZillowLimit as Et,getZillowOffset as At,getRedfinZipCodes as Yt,getZillowZipCodes as qt}from"@rent-scraper/api/config";import{spinner as wt,log as u,progress as Jt,intro as Vt}from"@clack/prompts";import{setTimeout as gt}from"node:timers/promises";import{createHash as Gt}from"crypto";import{checkBrowserServer as mt}from"@rent-scraper/utils/config";import Wt from"picocolors";const R=process.env.DEBUG,E=async(t,e,s,i)=>{const{timeoutMs:r}=i??{};if(await L(s))R&&u.warn(`${s} exists, skipping`);else try{R&&u.info(`writing ${s}`);const a=t==="redfin"?await Bt(e):await Mt(e,{timeoutMs:r});await M(`${s}`,a)}catch(a){Z(`error fetching html for ${e}`,a)}},Xt=async(t,{timeoutMs:e})=>{if(await L(t)){const s=JSON.parse(await W(t,"utf8"))||{},{hdpUrl:i}=s||{};if(i){const r=`https://www.zillow.com${i}`,a=t.replace(".json",".html");await L(a)?R&&u.warn(`file already exists, ${a}`):await E("zillow",r,a,{timeoutMs:e})}else Z(`file is empty, ${t}`)}else R&&u.warn(`file does not exist at this path, ${t}, skipping`)},ft=async(t,e)=>{const{timeoutMs:s}=e??{};await Promise.all(t.map(async i=>await Xt(i,{timeoutMs:s})))},Kt=(t,e="zillow")=>{if(e==="zillow"){const{zpid:s,detailUrl:i}=t||{},r=i&&!i.startsWith("http")?`https://www.zillow.com${i}`:i;return{id:s,url:r}}else if(e==="redfin"){const{propertyId:s,url:i}=t?.homeData||{};return{id:s,url:`https://www.redfin.com${i}`}}},it=async(t,e,s,i=s,r)=>{const{timeoutMs:a,run:n=1,reruns:l=0,skipBotCheck:h=!1}=r??{},c=new O;s||Z("inputDirectory is required"),t==="zillow"&&!h&&await q();const p=[],y=[],d=wt();d.start("Downloading listings html files");for(let g=1;g<=l+1;g++){l>0&&g>1&&c.add(`rerun ${g-1} of ${l}`);const m=p.length;(g===1||m)&&await Promise.all((m?p:e).map(async v=>{const b=`${s}/${v}.json`;if(await L(b))try{const{results:x}=await Ht(b)||{};if(!x)c.add(`empty file, ${b}`);else{const j=i?`${i}/${v}`:`${s}/${v}`;await S(j,{recursive:!0}),x?.length?await Promise.all(x.map(async k=>{const{id:B,url:z}=Kt(k,t)??{};if(!z)return c.add(`url missing for ${B}`);const F=`${B}.html`,C=`${j}/${F}`;try{await E(t,z,C,{timeoutMs:a})}catch(N){const{status:A,message:U}=P(N);dt(A,U)||y.push({url:z,filePath:C}),c.add("scrape listing html error: "+(U??`error fetching listing for id, ${N}`))}})):c.add(`no results for file, ${b}`)}}catch(x){p.push(v);const{message:j}=P(x);c.add("scrape listing html error: "+(j??`error reading json data, ${b}, ${x}`))}else R&&u.warn(`file does not exist, ${b}, skipping`)}))}y.length>0&&(await gt(2e3),await Promise.all(y.map(async({url:g,filePath:m})=>{try{await E(t,g,m,{timeoutMs:a})}catch(v){const{message:b}=P(v);c.add("scrape listing html retry error: "+(b??`error fetching ${g}`))}})));const $=await T(),f=await I(),w=t==="zillow"?$:f;e.length>0?(d.stop("Listings HTML files have been saved to:"),u.message(o.join(w,t,"listings",o.basename(s)))):d.stop("No listings HTML files saved.");const _=o.join(w,t,"logs");if(await S(_,{recursive:!0}),c.get().filter(g=>!g.includes("rerun ")).length>l){const g=`${o.basename(s)}-html-errors-${n}.txt`,m=o.join(_,g);await c.write(m,[...new Set(c.get())].join(`
+`)),R&&u.error(`There were errors during processing, see ${o.resolve(m)}`)}},Qt=async(t,e,s,i)=>{const{timeoutMs:r,run:a=1,reruns:n=0,skipBotCheck:l=!1}=i??{},h=new O;s||Z("inputDirectory is required"),t==="zillow"&&!l&&await q();const c=[];for(let f=1;f<=n+1;f++){n>0&&f>1&&h.add(`rerun ${f-1} of ${n}`);const w=c.length;(f===1||w)&&await Promise.all((w?c:e).map(async _=>{const g=`${s}/${_}`;if(await L(g))try{const m=await K(g,{extension:".json",prependDirectory:!0});await ft(m,{timeoutMs:r})}catch(m){c.push(_);const{message:v}=P(m);h.add("scrape listing html error: "+(v??`Error during fetch for ${_}, ${m}`))}else h.add(`listing directory does not exist, ${g}`)}))}const p=await T(),y=await I(),d=t==="zillow"?p:y,$=o.join(d,t,"logs");if(await S($,{recursive:!0}),h.get().filter(f=>!f.includes("rerun ")).length>n){const f=`${o.basename(s)}-html-errors-${a}.txt`,w=o.join($,f);await h.write(w,[...new Set(h.get())].join(`
+`)),R&&u.error(`There were errors during processing, see ${o.resolve(w)}`)}},te=async(t,e,s=e,i)=>{const{skipBotCheck:r=!1}=i??{},a=new O;if(t==="zillow"&&!r&&await q(),e||Z("inputDirectory is required"),await S(s,{recursive:!0}),await L(e))try{const d=await K(e,{extension:".json",prependDirectory:!0});await ft(d)}catch(d){const{message:$}=P(d);a.add("scrape listing html error: "+($??`Error during fetch for ${e}, ${d}`))}else a.add(`inputDirectory does not exist, ${e}`);const n=await T(),l=await I(),h=t==="zillow"?n:l,c=o.join(h,t,"logs");await S(c,{recursive:!0});const p=`${o.basename(e)}-html-errors.txt`,y=o.join(c,p);a.get().length>0&&(await a.write(y,[...new Set(a.get())].join(`
+`)),R&&u.error(`There were errors during processing, see ${o.resolve(y)}`))},ee=(t,e)=>t==="redfin"?`https://www.redfin.com/home/${e}`:t==="zillow"?`https://www.zillow.com/homedetails/${e}_zpid`:null,se=async(t,e,s,i)=>{const{skipBotCheck:r=!1}=i??{};t==="zillow"&&!r&&await q(),s||Z("outputDirectory is required"),await S(s,{recursive:!0}),await Promise.all(e.map(async a=>{const n=ee(t,a),l=`${s}/${a}.html`;n&&await E(t,n,l)}))},D=process.env.DEBUG,ht=async t=>{if(await L(t)){const e=t.replace(".html",".json");if(await L(e))D&&u.warning(`file already exists, ${e}`);else{D&&u.message(`scraping data for ${t}`);const s=(await W(t)).toString();(!s||s.trim()==="")&&(await X(t),Z(`empty file found at ${t}, deleted for retry`)),s.includes("px-captcha")&&(await X(t),Z(`captcha page found in ${t}, deleted for retry`));const i=await Ft(s);if(!i?.priceHistory&&i?.bestMatchedUnit?.hdpUrl){D&&u.warning(`rescraping ${t} - https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`),await L(t)&&(D&&u.warning(`deleting ${t}`),await X(t)),await L(e)&&(D&&u.warning(`deleting ${e}`),await X(e));const r=`https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`;await E("zillow",r,t),await ht(t)}else i?(D&&u.info(`writing ${e}`),await M(`${e}`,JSON.stringify(i))):Z(`problem scraping data for ${t}`)}}},ie=async t=>{if(await L(t)){const e=t.replace(".html",".json");if(await L(e))D&&u.warning(`file already exists, ${e}`);else{D&&u.message(`scraping data for ${t}`);const s=(await W(t)).toString(),i=Dt(s);i?(D&&u.warning(`writing ${e}`),await M(`${e}`,JSON.stringify(i))):Z(`problem scraping data for ${t}`)}}},yt=async(t,e,s)=>(await Promise.all(e.map(async i=>{try{t==="redfin"?await ie(i):await ht(i);const r=i.replace(".html",".json");return await L(r)?1:0}catch(r){const{message:a}=P(r);return s?.add(a),0}}))).reduce((i,r)=>i+r,0),rt=async(t,e,s)=>{const i=new O;let r=0;s||Z("inputDirectory is required");const a=Jt({style:"heavy",max:100,size:50});a.start("Scraping listings data");const n=e.length,l=n<20?n+1:20,h=n>l?st(n/l,1):n,c=Nt(e,h);for(const[f,w]of c.entries()){const _=st((Number(f)+1)/l*100,1);a.advance(st(1/l*100,1),`Scraping listings (${Ot(_)})`),await Promise.all(w.map(async g=>{D&&u.message(`processing files for ${g}`);const m=`${s}/${g}`;if(await L(m)){const v=await K(m,{extension:".html",prependDirectory:!0}),b=await yt(t,v,i);r=r+b}else i.add(`listing directory does not exist, ${m}`)}))}const p=await T(),y=await I(),d=t==="zillow"?p:y;r>0?(a.stop("Listings data has been saved to:"),u.message(o.join(d,t,"listings",o.basename(s)))):a.stop("No listings data saved.");const $=o.join(d,t,"logs");if(await S($,{recursive:!0}),i.get().length>0){const f=`${o.basename(s)}-listing-errors.txt`,w=o.join($,f);await i.write(w,[...new Set(i.get())].join(`
+`)),u.error(`There were errors during processing, see ${o.resolve(w)}`)}return{numListings:r}},nt=process.env.DEBUG,re=async(t,e,s,i,r)=>{const{daysOnZillow:a,timeoutMs:n}=r??{};if(await L(e))nt&&u.warning(`${t} exists, skipping`),s.push(t);else{const l=await Pt({zipCode:t,daysOnZillow:a,mergePageResults:!0,timeoutMs:n});l?(nt&&u.info(`writing ${e}`),await M(e,JSON.stringify(l)),s.push(t)):(nt&&u.warning(`no results for ${t}`),i.push(t))}},at=async(t,e,s)=>{const{daysListed:i,timeoutMs:r,run:a=1,reruns:n=0,fetchListings:l=!1,skipBotCheck:h=!1,silent:c=!1}=s??{},p=new O;h||await q({fetchListings:l});const y=[],d=[],$=[],f=[];await S(e,{recursive:!0});const w=wt();c||w.start("Scraping Zillow search results");for(let m=1;m<=n+1;m++){n>0&&m>1&&p.add(`rerun ${m-1} of ${n}`);const v=d.length;(m===1||v)&&await Promise.all((v?d:t).map(async b=>{const x=`${b}.json`,j=`${e}/${x}`;try{await re(b,j,y,f,{daysOnZillow:i,timeoutMs:r})}catch(k){d.push(b);const{status:B,message:z}=P(k);dt(B,z)&&$.push(b),p.add("scrape listing results error: "+(z??`Error during fetch for ${b}, ${k}`))}}))}const _=await T();c||(y.length>0?(w.stop("Zillow search results have been saved to:"),u.message(o.join(_,"zillow","results",o.basename(e)))):w.stop("No results saved."));const g=o.join(_,"zillow","logs");if(await S(g,{recursive:!0}),p.get().filter(m=>!m.includes("rerun ")).length>n){const m=`${o.basename(e)}-results-errors-${a}.txt`,v=o.join(g,m);await p.write(v,[...new Set(p.get())].join(`
+`)),c||u.error(`There were errors during processing, see ${o.resolve(v)}`)}return{validZipCodes:y,botFilteredZipCodes:$,noResultsZipCodes:f}},ne=async(t,e,s,i)=>{const{daysListed:r,timeoutMs:a}=i??{};if(await L(e))console.log(`${t} exists, skipping`),s.push(t);else{const n=await Rt({zipCode:t,daysListed:r,timeoutMs:a});n?(console.log(`writing ${e}`),await M(e,JSON.stringify(n)),s.push(t)):console.log(`no results for ${t}`)}},$t=async(t,e,s)=>{const{daysListed:i,timeoutMs:r,run:a=1,reruns:n=0}=s??{},l=new O,h=[],c=[];await S(e,{recursive:!0});for(let d=1;d<=n+1;d++){n>0&&d>1&&l.add(`rerun ${d-1} of ${n}`);const $=c.length;(d===1||$)&&await Promise.all(($?c:t).map(async f=>{const w=`${f}.json`,_=`${e}/${w}`;try{await ne(f,_,h,{daysListed:i,timeoutMs:r})}catch(g){c.push(f);const{message:m}=P(g);l.add("scrape listing results error: "+(m??`Error during fetch for ${f}, ${g}`))}}))}const p=await I(),y=o.join(p,"redfin","logs");if(await S(y,{recursive:!0}),l.get().filter(d=>!d.includes("rerun ")).length>n){const d=`${o.basename(e)}-results-errors-${a}.txt`,$=o.join(y,d);await l.write($,[...new Set(l.get())].join(`
+`)),console.log(`\x1B[41m
+%s\x1B[0m`,`There were errors during processing, see ${o.resolve($)}`)}return{validZipCodes:h}},_t=["listing_id","listing_source","source_listing_id","parcel_number","listing_url","street_address","cleaned_address","cleaned_unit_number","city","state","zipcode","county","neighborhood_region","is_undisclosed_address","home_status","home_type","bedrooms","year_built","living_area","living_area_units","is_income_restricted","price_at_source","platform_rent_estimate","agent_name","agent_phone_number","broker_name","broker_phone_number","is_owned_by_listing_platform","date_last_updated_at_source","date_scraped","scrape_job_name"],ae=t=>{if(t==null)return"";const e=typeof t=="object"?JSON.stringify(t):String(t);return e.includes(",")||e.includes('"')||e.includes(`
+`)?`"${e.replace(/"/g,'""')}"`:e},oe=t=>{const e=/^(.*?)\s+((?:#|APT|UNIT|STE|Suite|Apt|Unit)\s*.+)$/i.exec(t);if(e){const s=e[1].trim().toUpperCase();let i=e[2].trim();return i.startsWith("#")&&(i=`# ${i.replace(/^#\s*/,"").trim()}`),{cleaned:s,unit:i}}return{cleaned:t.toUpperCase(),unit:""}},le=(t,e)=>{const s=String(t.zpid),i=Gt("md5").update(s).digest("hex"),r=t.streetAddress||t.address?.streetAddress||"",{cleaned:a,unit:n}=oe(r),l=t.hdpUrl?`https://www.zillow.com${t.hdpUrl}`:`https://www.zillow.com/homedetails/${s}_zpid/`;return{listing_id:i,listing_source:"zillow",source_listing_id:s,parcel_number:t.resoFacts?.parcelNumber??"",listing_url:l,street_address:r,cleaned_address:a,cleaned_unit_number:n,city:t.city||t.address?.city||"",state:t.state||t.address?.state||"",zipcode:t.zipcode||t.address?.zipcode||"",county:t.county||"",neighborhood_region:t.neighborhoodRegion?.name||t.parentRegion?.name||"",is_undisclosed_address:t.isUndisclosedAddress??"",home_status:t.homeStatus||"",home_type:t.homeType||"",bedrooms:t.bedrooms??"",year_built:t.yearBuilt||t.resoFacts?.yearBuilt||"",living_area:t.livingAreaValue??"",living_area_units:t.livingAreaUnits||"Square Feet",is_income_restricted:t.isIncomeRestricted??"",price_at_source:t.price??"",platform_rent_estimate:t.rentZestimate??"",agent_name:t.attributionInfo?.agentName||"",agent_phone_number:t.attributionInfo?.agentPhoneNumber||"",broker_name:t.attributionInfo?.brokerName||"",broker_phone_number:t.attributionInfo?.brokerPhoneNumber||"",is_owned_by_listing_platform:t.attributionInfo?.mlsName==="Zillow Rentals",date_last_updated_at_source:t.attributionInfo?.lastUpdated||"",date_scraped:t.timestamp?lt(t.timestamp).format("YYYY-MM-DD HH:mm:ss"):"",scrape_job_name:e}},bt=async(t,e)=>{const s=o.basename(t),i=[_t.join(",")];let r;try{r=await et(t,{withFileTypes:!0})}catch{return 0}const a=r.filter(n=>n.isDirectory()).map(n=>n.name);for(const n of a){const l=o.join(t,n);if(!await L(l))continue;const h=await K(l,{extension:".json",prependDirectory:!0});for(const c of h)try{const p=JSON.parse(await W(c,"utf8"));if(!p.zpid)continue;const y=le(p,s);i.push(_t.map(d=>ae(y[d])).join(","))}catch{}}return await S(o.dirname(e),{recursive:!0}),await M(e,i.join(`
+`)),i.length-1},ce=async t=>{try{const e=(await et(t,{withFileTypes:!0})).filter(s=>s.isDirectory()).map(s=>s.name).sort();return e.length>0?e[e.length-1]:null}catch{return null}},vt=async()=>{await ct.post("http://localhost:8082/browser/close")},de=async()=>{await ct.post("http://localhost:8082/server/shutdown")},ue=async(t,e,s,{daysListed:i,timeoutMs:r,run:a,reruns:n,preValidatedZipCodes:l=[],retry:h=!1})=>{let c=[],p=[],y=[];if(t.length>0){if(await ut(),await vt(),{validZipCodes:c,botFilteredZipCodes:p,noResultsZipCodes:y}=await at(t,e,{daysListed:i,timeoutMs:r,run:a,reruns:n,skipBotCheck:!0}),h&&p.length>0){u.warn(`Bot filtering hit ${p.length} of ${t.length} zip code(s) \u2014 will retry after listings are fetched`),await Ut()?(await ut(),await vt()):await gt(3e3);const f=p.length,w=await at(p,e,{daysListed:i,timeoutMs:r,run:a,reruns:n,skipBotCheck:!0,silent:!0});c=[...c,...w.validZipCodes],p=w.botFilteredZipCodes,y=[...y,...w.noResultsZipCodes];const _=w.validZipCodes.length;u.info(`Inline retry: recovered ${_} of ${f} \xB7 ${p.length} still bot-filtered`)}c.length===0&&t.length>0&&u.warn("No results returned for any zip codes \u2014 possible soft bot filtering. Please try again shortly."),p.length>0&&u.warn(`${p.length} zip code(s) still bot-filtered \u2014 use --rerun to retry`)}const d=[...l,...c];await it("zillow",d,e,s,{timeoutMs:r,run:a,reruns:n,skipBotCheck:!0});const{numListings:$}=await rt("zillow",d,s);return{numListings:$,validZipCodes:d,noResultsZipCodes:y}},pe=async(t,e,s,{daysListed:i,timeoutMs:r,run:a,reruns:n})=>{const{validZipCodes:l}=await $t(t,e,{daysListed:i,timeoutMs:r,run:a,reruns:n});await it("redfin",l,e,s,{timeoutMs:r,run:a,reruns:n});const{numListings:h}=await rt("redfin",l,s);return{numListings:h,validZipCodes:l}};async function we(){Vt(Wt.inverse(" scrape listings "));const t=kt(process.argv.slice(2)),e=t.source??"zillow";e==="zillow"&&(await mt()||Z("Please launch the browser server before scraping."));const s=await T(),i=await I(),r=e==="zillow"?s:i,a=t["days-listed"]??(e==="zillow"?await Tt():await It())??1,n=t.runs??1,l=t.reruns??0,h=t["timeout-ms"]??6e4,c=t.limit??await Et()??void 0,p=t.offset??await At()??0,y=t.retry??!1,d=t.rerun,$=typeof d=="string"?d:d?await ce(o.join(r,e,"results")):null;d&&!$&&Z("No previous run found to rerun."),$&&u.info(`Rerunning from: ${$}`);const f=$??lt().format("YYYY-MM-DD-HHmm"),w=t["results-directory"]??o.join(r,e,"results",f),_=t["listings-directory"]??o.join(r,e,"listings",f),g=t["logs-directory"]??o.join(r,e,"logs");let m=0;const v=async()=>{(e==="zillow"||e==="redfin")&&await mt()&&await de()};try{if(e==="redfin"){const x=await Yt();for(let j=1;j<=n;j++){x||Z("zip codes required, please run the createConfig script");const k=x,{numListings:B,validZipCodes:z}=await pe(k,w,_,{daysListed:a,timeoutMs:h,run:j,reruns:l});if(j===n){const F=Object.entries({numListings:B}).map(([Q,J])=>`${Q}: ${J}`).join(`
+`),C=pt(k,z).join(`
+`);await S(g,{recursive:!0});const N=`${o.basename(w)}-scraping-results.txt`,A=o.join(g,N),U=`${o.basename(w)}-invalid-zipcodes.txt`,H=o.join(g,U);await M(A,F),await M(H,C)}}}else if(e==="zillow"){const x=await qt();x||Z("zip codes required, please run the createConfig script");const j=x.slice(p,c?p+c:void 0);let k=j,B=[];if($){const z=await et(w).catch(()=>[]),F=new Set(z.filter(C=>C.endsWith(".json")).map(C=>C.replace(".json","")));B=j.filter(C=>F.has(String(C))),k=j.filter(C=>!F.has(String(C))),k.length>0?u.info(`${B.length} already fetched \xB7 retrying ${k.length} missing`):u.info(`All ${B.length} zip codes already fetched \xB7 proceeding to HTML scraping`)}for(let z=1;z<=n;z++){const{numListings:F,validZipCodes:C,noResultsZipCodes:N}=await ue(k,w,_,{daysListed:a,timeoutMs:h,run:z,reruns:l,preValidatedZipCodes:B,retry:y});if(z===n){const A=o.basename(_),U=o.join(s,"zillow","csv",`${A}.csv`),H=await bt(_,U);m=H,H>0&&u.info(`CSV exported: ${H} listings \u2192 ${U}`);const Q=Object.entries({numListings:F,numCsvListings:H}).map(([Y,St])=>`${Y}: ${St}`).join(`
+`),J=pt(j,C),ot=new Set(N),V=J.filter(Y=>!ot.has(Y)),G=J.filter(Y=>ot.has(Y));await S(g,{recursive:!0});const jt=`${o.basename(w)}-scraping-results.txt`,Lt=o.join(g,jt),zt=`${o.basename(w)}-zipcodes-no-results.txt`,Ct=o.join(g,zt),Zt=`${o.basename(w)}-zipcodes-errored.txt`,xt=o.join(g,Zt);await M(Lt,Q),G.length>0&&await M(Ct,G.join(`
+`)),V.length>0&&await M(xt,V.join(`
+`));const tt=[`${C.length}/${j.length} with results`];G.length>0&&tt.push(`${G.length} no results`),V.length>0&&tt.push(`${V.length} errored`),u.info(`Zip codes: ${tt.join(" \xB7 ")}`),u.info(`Listings:  ${F} parsed \xB7 ${H} exported to CSV`)}}}const b=e==="zillow"?m>0:!0;return b?u.success("Scraping complete!"):u.warn("Scraping finished but no listings were exported \u2014 please try again shortly."),{success:b,totalCsvListings:m}}finally{await v()}}export{rt as a,it as b,Qt as c,te as d,se as e,E as f,bt as g,$t as h,at as i,we as r,yt as s};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rent-scraper/scrape-listings",
-  "version": "1.0.30",
+  "version": "1.0.31",
   "type": "module",
   "exports": {
     ".": {
@@ -30,8 +30,8 @@
     "dayjs": "^1.11.13",
     "minimist": "^1.2.8",
     "picocolors": "^1.1.1",
-    "@rent-scraper/utils": "1.0.30",
-    "@rent-scraper/api": "1.0.30"
+    "@rent-scraper/api": "1.0.31",
+    "@rent-scraper/utils": "1.0.31"
   },
   "devDependencies": {
     "@types/minimist": "^1.2.5",

package/dist/shared/scrape-listings.DPpklKj4.mjs

@@ -1,15 +0,0 @@
-import kt from"minimist";import ot from"dayjs";import o from"path";import lt from"axios";import{mkdir as k,writeFile as M,readFile as G,unlink as W,readdir as tt}from"fs/promises";import{checkForZillowBotFiltering as Y,isZillowBotFiltering as ct,fetchHtmlFromRedfinListingUrl as St,fetchHtmlFromZillowListingUrl as Bt,scrapeDataFromRedfinListingHtml as Mt,scrapeDataFromZillowListingHtml as Dt,getZillowListingResults as Ft,getRedfinListingResults as Pt,waitForSolvedZillowCaptcha as dt,isBrowserShowingCaptcha as Rt}from"@rent-scraper/api";import{ErrorLog as T,throwError as z,checkForFile as L,parseJsonFile as Ut,parseError as P,readFilesInDirectory as X,roundValue as et,chunkArray as Ht,parsePercentage as Nt,compareArrays as ut}from"@rent-scraper/utils";import{getZillowOutputPath as O,getRedfinOutputPath as I,getZillowDaysListed as Tt,getRedfinDaysListed as Ot,getZillowLimit as It,getZillowOffset as Et,getRedfinZipCodes as At,getZillowZipCodes as Yt}from"@rent-scraper/api/config";import{spinner as pt,log as u,progress as qt,intro as Jt}from"@clack/prompts";import{setTimeout as wt}from"node:timers/promises";import{createHash as Vt}from"crypto";import{checkBrowserServer as gt}from"@rent-scraper/utils/config";import Gt from"picocolors";const R=process.env.DEBUG,E=async(t,e,s,i)=>{const{timeoutMs:r}=i??{};if(await L(s))R&&u.warn(`${s} exists, skipping`);else try{R&&u.info(`writing ${s}`);const a=t==="redfin"?await St(e):await Bt(e,{timeoutMs:r});await M(`${s}`,a)}catch(a){z(`error fetching html for ${e}`,a)}},Wt=async(t,{timeoutMs:e})=>{if(await L(t)){const s=JSON.parse(await G(t,"utf8"))||{},{hdpUrl:i}=s||{};if(i){const r=`https://www.zillow.com${i}`,a=t.replace(".json",".html");await L(a)?R&&u.warn(`file already exists, ${a}`):await E("zillow",r,a,{timeoutMs:e})}else z(`file is empty, ${t}`)}else R&&u.warn(`file does not exist at this path, ${t}, skipping`)},mt=async(t,e)=>{const{timeoutMs:s}=e??{};await Promise.all(t.map(async i=>await Wt(i,{timeoutMs:s})))},Xt=(t,e="zillow")=>{if(e==="zillow"){const{zpid:s,detailUrl:i}=t||{},r=i&&!i.startsWith("http")?`https://www.zillow.com${i}`:i;return{id:s,url:r}}else if(e==="redfin"){const{propertyId:s,url:i}=t?.homeData||{};return{id:s,url:`https://www.redfin.com${i}`}}},st=async(t,e,s,i=s,r)=>{const{timeoutMs:a,run:n=1,reruns:l=0,skipBotCheck:h=!1}=r??{},c=new T;s||z("inputDirectory is required"),t==="zillow"&&!h&&await Y();const p=[],y=[],d=pt();d.start("Downloading listings html files");for(let g=1;g<=l+1;g++){l>0&&g>1&&c.add(`rerun ${g-1} of ${l}`);const m=p.length;(g===1||m)&&await Promise.all((m?p:e).map(async j=>{const _=`${s}/${j}.json`;if(await L(_))try{const{results:v}=await Ut(_)||{};if(!v)c.add(`empty file, ${_}`);else{const Z=i?`${i}/${j}`:`${s}/${j}`;await k(Z,{recursive:!0}),v?.length?await Promise.all(v.map(async B=>{const{id:S,url:C}=Xt(B,t)??{};if(!C)return c.add(`url missing for ${S}`);const x=`${S}.html`,U=`${Z}/${x}`;try{await E(t,C,U,{timeoutMs:a})}catch(H){const{status:N,message:D}=P(H);ct(N,D)||y.push({url:C,filePath:U}),c.add("scrape listing html error: "+(D??`error fetching listing for id, ${H}`))}})):c.add(`no results for file, ${_}`)}}catch(v){p.push(j);const{message:Z}=P(v);c.add("scrape listing html error: "+(Z??`error reading json data, ${_}, ${v}`))}else R&&u.warn(`file does not exist, ${_}, skipping`)}))}y.length>0&&(await wt(2e3),await Promise.all(y.map(async({url:g,filePath:m})=>{try{await E(t,g,m,{timeoutMs:a})}catch(j){const{message:_}=P(j);c.add("scrape listing html retry error: "+(_??`error fetching ${g}`))}})));const $=await O(),f=await I(),w=t==="zillow"?$:f;e.length>0?(d.stop("Listings HTML files have been saved to:"),u.message(o.join(w,t,"listings",o.basename(s)))):d.stop("No listings HTML files saved.");const b=o.join(w,t,"logs");if(await k(b,{recursive:!0}),c.get().filter(g=>!g.includes("rerun ")).length>l){const g=`${o.basename(s)}-html-errors-${n}.txt`,m=o.join(b,g);await c.write(m,[...new Set(c.get())].join(`
-`)),R&&u.error(`There were errors during processing, see ${o.resolve(m)}`)}},Qt=async(t,e,s,i)=>{const{timeoutMs:r,run:a=1,reruns:n=0,skipBotCheck:l=!1}=i??{},h=new T;s||z("inputDirectory is required"),t==="zillow"&&!l&&await Y();const c=[];for(let f=1;f<=n+1;f++){n>0&&f>1&&h.add(`rerun ${f-1} of ${n}`);const w=c.length;(f===1||w)&&await Promise.all((w?c:e).map(async b=>{const g=`${s}/${b}`;if(await L(g))try{const m=await X(g,{extension:".json",prependDirectory:!0});await mt(m,{timeoutMs:r})}catch(m){c.push(b);const{message:j}=P(m);h.add("scrape listing html error: "+(j??`Error during fetch for ${b}, ${m}`))}else h.add(`listing directory does not exist, ${g}`)}))}const p=await O(),y=await I(),d=t==="zillow"?p:y,$=o.join(d,t,"logs");if(await k($,{recursive:!0}),h.get().filter(f=>!f.includes("rerun ")).length>n){const f=`${o.basename(s)}-html-errors-${a}.txt`,w=o.join($,f);await h.write(w,[...new Set(h.get())].join(`
-`)),R&&u.error(`There were errors during processing, see ${o.resolve(w)}`)}},Kt=async(t,e,s=e,i)=>{const{skipBotCheck:r=!1}=i??{},a=new T;if(t==="zillow"&&!r&&await Y(),e||z("inputDirectory is required"),await k(s,{recursive:!0}),await L(e))try{const d=await X(e,{extension:".json",prependDirectory:!0});await mt(d)}catch(d){const{message:$}=P(d);a.add("scrape listing html error: "+($??`Error during fetch for ${e}, ${d}`))}else a.add(`inputDirectory does not exist, ${e}`);const n=await O(),l=await I(),h=t==="zillow"?n:l,c=o.join(h,t,"logs");await k(c,{recursive:!0});const p=`${o.basename(e)}-html-errors.txt`,y=o.join(c,p);a.get().length>0&&(await a.write(y,[...new Set(a.get())].join(`
-`)),R&&u.error(`There were errors during processing, see ${o.resolve(y)}`))},te=(t,e)=>t==="redfin"?`https://www.redfin.com/home/${e}`:t==="zillow"?`https://www.zillow.com/homedetails/${e}_zpid`:null,ee=async(t,e,s,i)=>{const{skipBotCheck:r=!1}=i??{};t==="zillow"&&!r&&await Y(),s||z("outputDirectory is required"),await k(s,{recursive:!0}),await Promise.all(e.map(async a=>{const n=te(t,a),l=`${s}/${a}.html`;n&&await E(t,n,l)}))},F=process.env.DEBUG,ft=async t=>{if(await L(t)){const e=t.replace(".html",".json");if(await L(e))F&&u.warning(`file already exists, ${e}`);else{F&&u.message(`scraping data for ${t}`);const s=(await G(t)).toString();(!s||s.trim()==="")&&(await W(t),z(`empty file found at ${t}, deleted for retry`)),s.includes("px-captcha")&&(await W(t),z(`captcha page found in ${t}, deleted for retry`));const i=await Dt(s);if(!i?.priceHistory&&i?.bestMatchedUnit?.hdpUrl){F&&u.warning(`rescraping ${t} - https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`),await L(t)&&(F&&u.warning(`deleting ${t}`),await W(t)),await L(e)&&(F&&u.warning(`deleting ${e}`),await W(e));const r=`https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`;await E("zillow",r,t),await ft(t)}else i?(F&&u.info(`writing ${e}`),await M(`${e}`,JSON.stringify(i))):z(`problem scraping data for ${t}`)}}},se=async t=>{if(await L(t)){const e=t.replace(".html",".json");if(await L(e))F&&u.warning(`file already exists, ${e}`);else{F&&u.message(`scraping data for ${t}`);const s=(await G(t)).toString(),i=Mt(s);i?(F&&u.warning(`writing ${e}`),await M(`${e}`,JSON.stringify(i))):z(`problem scraping data for ${t}`)}}},ht=async(t,e,s)=>(await Promise.all(e.map(async i=>{try{t==="redfin"?await se(i):await ft(i);const r=i.replace(".html",".json");return await L(r)?1:0}catch(r){const{message:a}=P(r);return s?.add(a),0}}))).reduce((i,r)=>i+r,0),it=async(t,e,s)=>{const i=new T;let r=0;s||z("inputDirectory is required");const a=qt({style:"heavy",max:100,size:50});a.start("Scraping listings data");const n=e.length,l=n<20?n+1:20,h=n>l?et(n/l,1):n,c=Ht(e,h);for(const[f,w]of c.entries()){const b=et((Number(f)+1)/l*100,1);a.advance(et(1/l*100,1),`Scraping listings (${Nt(b)})`),await Promise.all(w.map(async g=>{F&&u.message(`processing files for ${g}`);const m=`${s}/${g}`;if(await L(m)){const j=await X(m,{extension:".html",prependDirectory:!0}),_=await ht(t,j,i);r=r+_}else i.add(`listing directory does not exist, ${m}`)}))}const p=await O(),y=await I(),d=t==="zillow"?p:y;r>0?(a.stop("Listings data has been saved to:"),u.message(o.join(d,t,"listings",o.basename(s)))):a.stop("No listings data saved.");const $=o.join(d,t,"logs");if(await k($,{recursive:!0}),i.get().length>0){const f=`${o.basename(s)}-listing-errors.txt`,w=o.join($,f);await i.write(w,[...new Set(i.get())].join(`
-`)),u.error(`There were errors during processing, see ${o.resolve(w)}`)}return{numListings:r}},rt=process.env.DEBUG,ie=async(t,e,s,i,r)=>{const{daysOnZillow:a,timeoutMs:n}=r??{};if(await L(e))rt&&u.warning(`${t} exists, skipping`),s.push(t);else{const l=await Ft({zipCode:t,daysOnZillow:a,mergePageResults:!0,timeoutMs:n});l?(rt&&u.info(`writing ${e}`),await M(e,JSON.stringify(l)),s.push(t)):(rt&&u.warning(`no results for ${t}`),i.push(t))}},nt=async(t,e,s)=>{const{daysListed:i,timeoutMs:r,run:a=1,reruns:n=0,fetchListings:l=!1,skipBotCheck:h=!1,silent:c=!1}=s??{},p=new T;h||await Y({fetchListings:l});const y=[],d=[],$=[],f=[];await k(e,{recursive:!0});const w=pt();c||w.start("Scraping Zillow search results");for(let m=1;m<=n+1;m++){n>0&&m>1&&p.add(`rerun ${m-1} of ${n}`);const j=d.length;(m===1||j)&&await Promise.all((j?d:t).map(async _=>{const v=`${_}.json`,Z=`${e}/${v}`;try{await ie(_,Z,y,f,{daysOnZillow:i,timeoutMs:r})}catch(B){d.push(_);const{status:S,message:C}=P(B);ct(S,C)&&$.push(_),p.add("scrape listing results error: "+(C??`Error during fetch for ${_}, ${B}`))}}))}const b=await O();c||(y.length>0?(w.stop("Zillow search results have been saved to:"),u.message(o.join(b,"zillow","results",o.basename(e)))):w.stop("No results saved."));const g=o.join(b,"zillow","logs");if(await k(g,{recursive:!0}),p.get().filter(m=>!m.includes("rerun ")).length>n){const m=`${o.basename(e)}-results-errors-${a}.txt`,j=o.join(g,m);await p.write(j,[...new Set(p.get())].join(`
-`)),c||u.error(`There were errors during processing, see ${o.resolve(j)}`)}return{validZipCodes:y,botFilteredZipCodes:$,noResultsZipCodes:f}},re=async(t,e,s,i)=>{const{daysListed:r,timeoutMs:a}=i??{};if(await L(e))console.log(`${t} exists, skipping`),s.push(t);else{const n=await Pt({zipCode:t,daysListed:r,timeoutMs:a});n?(console.log(`writing ${e}`),await M(e,JSON.stringify(n)),s.push(t)):console.log(`no results for ${t}`)}},yt=async(t,e,s)=>{const{daysListed:i,timeoutMs:r,run:a=1,reruns:n=0}=s??{},l=new T,h=[],c=[];await k(e,{recursive:!0});for(let d=1;d<=n+1;d++){n>0&&d>1&&l.add(`rerun ${d-1} of ${n}`);const $=c.length;(d===1||$)&&await Promise.all(($?c:t).map(async f=>{const w=`${f}.json`,b=`${e}/${w}`;try{await re(f,b,h,{daysListed:i,timeoutMs:r})}catch(g){c.push(f);const{message:m}=P(g);l.add("scrape listing results error: "+(m??`Error during fetch for ${f}, ${g}`))}}))}const p=await I(),y=o.join(p,"redfin","logs");if(await k(y,{recursive:!0}),l.get().filter(d=>!d.includes("rerun ")).length>n){const d=`${o.basename(e)}-results-errors-${a}.txt`,$=o.join(y,d);await l.write($,[...new Set(l.get())].join(`
-`)),console.log(`\x1B[41m
-%s\x1B[0m`,`There were errors during processing, see ${o.resolve($)}`)}return{validZipCodes:h}},$t=["listing_id","listing_source","source_listing_id","parcel_number","listing_url","street_address","cleaned_address","cleaned_unit_number","city","state","zipcode","county","neighborhood_region","is_undisclosed_address","home_status","home_type","bedrooms","year_built","living_area","living_area_units","is_income_restricted","price_at_source","platform_rent_estimate","agent_name","agent_phone_number","broker_name","broker_phone_number","is_owned_by_listing_platform","date_last_updated_at_source","date_scraped","scrape_job_name"],ne=t=>{if(t==null)return"";const e=typeof t=="object"?JSON.stringify(t):String(t);return e.includes(",")||e.includes('"')||e.includes(`
-`)?`"${e.replace(/"/g,'""')}"`:e},ae=t=>{const e=/^(.*?)\s+((?:#|APT|UNIT|STE|Suite|Apt|Unit)\s*.+)$/i.exec(t);if(e){const s=e[1].trim().toUpperCase();let i=e[2].trim();return i.startsWith("#")&&(i=`# ${i.replace(/^#\s*/,"").trim()}`),{cleaned:s,unit:i}}return{cleaned:t.toUpperCase(),unit:""}},oe=(t,e)=>{const s=String(t.zpid),i=Vt("md5").update(s).digest("hex"),r=t.streetAddress||t.address?.streetAddress||"",{cleaned:a,unit:n}=ae(r),l=t.hdpUrl?`https://www.zillow.com${t.hdpUrl}`:`https://www.zillow.com/homedetails/${s}_zpid/`;return{listing_id:i,listing_source:"zillow",source_listing_id:s,parcel_number:t.resoFacts?.parcelNumber??"",listing_url:l,street_address:r,cleaned_address:a,cleaned_unit_number:n,city:t.city||t.address?.city||"",state:t.state||t.address?.state||"",zipcode:t.zipcode||t.address?.zipcode||"",county:t.county||"",neighborhood_region:t.neighborhoodRegion?.name||t.parentRegion?.name||"",is_undisclosed_address:t.isUndisclosedAddress??"",home_status:t.homeStatus||"",home_type:t.homeType||"",bedrooms:t.bedrooms??"",year_built:t.yearBuilt||t.resoFacts?.yearBuilt||"",living_area:t.livingAreaValue??"",living_area_units:t.livingAreaUnits||"Square Feet",is_income_restricted:t.isIncomeRestricted??"",price_at_source:t.price??"",platform_rent_estimate:t.rentZestimate??"",agent_name:t.attributionInfo?.agentName||"",agent_phone_number:t.attributionInfo?.agentPhoneNumber||"",broker_name:t.attributionInfo?.brokerName||"",broker_phone_number:t.attributionInfo?.brokerPhoneNumber||"",is_owned_by_listing_platform:t.attributionInfo?.mlsName==="Zillow Rentals",date_last_updated_at_source:t.attributionInfo?.lastUpdated||"",date_scraped:t.timestamp?ot(t.timestamp).format("YYYY-MM-DD HH:mm:ss"):"",scrape_job_name:e}},_t=async(t,e)=>{const s=o.basename(t),i=[$t.join(",")];let r;try{r=await tt(t,{withFileTypes:!0})}catch{return 0}const a=r.filter(n=>n.isDirectory()).map(n=>n.name);for(const n of a){const l=o.join(t,n);if(!await L(l))continue;const h=await X(l,{extension:".json",prependDirectory:!0});for(const c of h)try{const p=JSON.parse(await G(c,"utf8"));if(!p.zpid)continue;const y=oe(p,s);i.push($t.map(d=>ne(y[d])).join(","))}catch{}}return await k(o.dirname(e),{recursive:!0}),await M(e,i.join(`
-`)),i.length-1},le=async t=>{try{const e=(await tt(t,{withFileTypes:!0})).filter(s=>s.isDirectory()).map(s=>s.name).sort();return e.length>0?e[e.length-1]:null}catch{return null}},bt=async()=>{await lt.post("http://localhost:8082/browser/close")},ce=async()=>{await lt.post("http://localhost:8082/server/shutdown")},de=async(t,e,s,{daysListed:i,timeoutMs:r,run:a,reruns:n,preValidatedZipCodes:l=[],retry:h=!1})=>{let c=[],p=[],y=[];if(t.length>0){if(await dt(),await bt(),{validZipCodes:c,botFilteredZipCodes:p,noResultsZipCodes:y}=await nt(t,e,{daysListed:i,timeoutMs:r,run:a,reruns:n,skipBotCheck:!0}),h&&p.length>0){u.warn(`Bot filtering hit ${p.length} of ${t.length} zip code(s) \u2014 will retry after listings are fetched`),await Rt()?(await dt(),await bt()):await wt(3e3);const f=p.length,w=await nt(p,e,{daysListed:i,timeoutMs:r,run:a,reruns:n,skipBotCheck:!0,silent:!0});c=[...c,...w.validZipCodes],p=w.botFilteredZipCodes,y=[...y,...w.noResultsZipCodes];const b=w.validZipCodes.length;u.info(`Inline retry: recovered ${b} of ${f} \xB7 ${p.length} still bot-filtered`)}c.length===0&&t.length>0&&u.warn("No results returned for any zip codes \u2014 possible soft bot filtering. Please try again shortly."),p.length>0&&u.warn(`${p.length} zip code(s) still bot-filtered \u2014 use --rerun to retry`)}const d=[...l,...c];await st("zillow",d,e,s,{timeoutMs:r,run:a,reruns:n,skipBotCheck:!0});const{numListings:$}=await it("zillow",d,s);return{numListings:$,validZipCodes:d,noResultsZipCodes:y}},ue=async(t,e,s,{daysListed:i,timeoutMs:r,run:a,reruns:n})=>{const{validZipCodes:l}=await yt(t,e,{daysListed:i,timeoutMs:r,run:a,reruns:n});await st("redfin",l,e,s,{timeoutMs:r,run:a,reruns:n});const{numListings:h}=await it("redfin",l,s);return{numListings:h,validZipCodes:l}};async function pe(){Jt(Gt.inverse(" scrape listings "));const t=kt(process.argv.slice(2)),e=t.source??"zillow";e==="zillow"&&(await gt()||z("Please launch the browser server before scraping."));const s=await O(),i=await I(),r=e==="zillow"?s:i,a=t["days-listed"]??(e==="zillow"?await Tt():await Ot())??1,n=t.runs??1,l=t.reruns??0,h=t["timeout-ms"]??6e4,c=t.limit??await It()??void 0,p=t.offset??await Et()??0,y=t.retry??!1,d=t.rerun,$=typeof d=="string"?d:d?await le(o.join(r,e,"results")):null;d&&!$&&z("No previous run found to rerun."),$&&u.info(`Rerunning from: ${$}`);const f=$??ot().format("YYYY-MM-DD-HHmm"),w=t["results-directory"]??o.join(r,e,"results",f),b=t["listings-directory"]??o.join(r,e,"listings",f),g=t["logs-directory"]??o.join(r,e,"logs");let m=0;const j=async()=>{(e==="zillow"||e==="redfin")&&await gt()&&await ce()};try{if(e==="redfin"){const _=await At();for(let v=1;v<=n;v++){_||z("zip codes required, please run the createConfig script");const Z=_,{numListings:B,validZipCodes:S}=await ue(Z,w,b,{daysListed:a,timeoutMs:h,run:v,reruns:l});if(v===n){const C=Object.entries({numListings:B}).map(([Q,q])=>`${Q}: ${q}`).join(`
-`),x=ut(Z,S).join(`
-`);await k(g,{recursive:!0});const U=`${o.basename(w)}-scraping-results.txt`,H=o.join(g,U),N=`${o.basename(w)}-invalid-zipcodes.txt`,D=o.join(g,N);await M(H,C),await M(D,x)}}}else if(e==="zillow"){const _=await Yt();_||z("zip codes required, please run the createConfig script");const v=_.slice(p,c?p+c:void 0);let Z=v,B=[];if($){const S=await tt(w).catch(()=>[]),C=new Set(S.filter(x=>x.endsWith(".json")).map(x=>x.replace(".json","")));B=v.filter(x=>C.has(String(x))),Z=v.filter(x=>!C.has(String(x))),Z.length>0?u.info(`${B.length} already fetched \xB7 retrying ${Z.length} missing`):u.info(`All ${B.length} zip codes already fetched \xB7 proceeding to HTML scraping`)}for(let S=1;S<=n;S++){const{numListings:C,validZipCodes:x,noResultsZipCodes:U}=await de(Z,w,b,{daysListed:a,timeoutMs:h,run:S,reruns:l,preValidatedZipCodes:B,retry:y});if(S===n){const H=o.basename(b),N=o.join(s,"zillow","csv",`${H}.csv`),D=await _t(b,N);m=D,D>0&&u.info(`CSV exported: ${D} listings \u2192 ${N}`);const Q=Object.entries({numListings:C,numCsvListings:D}).map(([A,xt])=>`${A}: ${xt}`).join(`
-`),q=ut(v,x),at=new Set(U),J=q.filter(A=>!at.has(A)),V=q.filter(A=>at.has(A));await k(g,{recursive:!0});const vt=`${o.basename(w)}-scraping-results.txt`,jt=o.join(g,vt),Lt=`${o.basename(w)}-zipcodes-no-results.txt`,zt=o.join(g,Lt),Zt=`${o.basename(w)}-zipcodes-errored.txt`,Ct=o.join(g,Zt);await M(jt,Q),V.length>0&&await M(zt,V.join(`
-`)),J.length>0&&await M(Ct,J.join(`
-`));const K=[`${x.length}/${v.length} with results`];V.length>0&&K.push(`${V.length} no results`),J.length>0&&K.push(`${J.length} errored`),u.info(`Zip codes: ${K.join(" \xB7 ")}`),u.info(`Listings:  ${C} parsed \xB7 ${D} exported to CSV`)}}}e!=="zillow"||m>0?u.success("Scraping complete!"):u.warn("Scraping finished but no listings were exported \u2014 please try again shortly.")}finally{await j()}}export{it as a,st as b,Qt as c,Kt as d,ee as e,E as f,_t as g,yt as h,nt as i,pe as r,ht as s};