@rent-scraper/scrape-listings 1.0.35 → 1.0.37

package/dist/bin/run-scrape-listing-details.d.mts

@@ -0,0 +1,2 @@
+
+export { };

package/dist/bin/run-scrape-listing-details.mjs

@@ -0,0 +1 @@
+import v from"minimist";import t from"path";import{readdir as m,unlink as F,readFile as S,writeFile as N}from"fs/promises";import{intro as P,log as r}from"@clack/prompts";import z from"picocolors";import{parseError as C}from"@rent-scraper/utils";import{getZillowOutputPath as D}from"@rent-scraper/api/config";import{scrapeDataFromZillowListingHtml as L}from"@rent-scraper/api";import{s as O}from"../shared/scrape-listings.Qpf6zDxA.mjs";import"crypto";import"dayjs";P(z.inverse(" scrape listing details "));const w=v(process.argv.slice(2)),T=async n=>{try{const i=(await m(n,{withFileTypes:!0})).filter(o=>o.isDirectory()).map(o=>o.name).sort();return i.length>0?i[i.length-1]:null}catch{return null}};try{const n=await D(),i=t.join(n,"zillow","listings"),o=w["listings-directory"]?t.basename(w["listings-directory"]):await T(i);o||(r.error("No listings directory found. Pass --listings-directory <path>."),process.exit(1));const a=w["listings-directory"]??t.join(i,o);r.info(`Using listings directory: ${a}`);const d=(await m(a,{withFileTypes:!0})).filter(s=>s.isDirectory()).map(s=>s.name);let f=0;for(const s of d){const l=t.join(a,s),g=await m(l);for(const e of g)e.endsWith(".json")&&(await F(t.join(l,e)),f++)}f>0&&r.info(`Cleared ${f} existing JSON files`);let y=0,c=0;for(const s of d){const l=t.join(a,s),g=(await m(l)).filter(e=>e.endsWith(".html"));for(const e of g){const j=t.join(l,e),x=j.replace(".html",".json");try{const p=(await S(j)).toString();if(!p||p.trim()===""||p.includes("px-captcha")){c++;continue}const $=await L(p);$?(await N(x,JSON.stringify($)),y++):c++}catch{c++}}}r.info(`Parsed ${y} listings${c>0?` (${c} skipped)`:""}`);const h=t.join(n,"zillow","csv",`${o}.csv`),u=await O(a,h);u>0?r.success(`${u} listings \u2192 ${h}`):r.warning("No listings exported to CSV"),process.exit(0)}catch(n){const{message:i}=C(n);r.error(i),process.exit(1)}

package/dist/bin/run-scrape-listings.mjs

@@ -1 +1 @@
-import{parseError as t}from"@rent-scraper/utils";import{r as i}from"../shared/scrape-listings.D0kKRGJj.mjs";import{log as m}from"@clack/prompts";import"minimist";import"dayjs";import"path";import"axios";import"fs/promises";import"@rent-scraper/api";import"@rent-scraper/api/config";import"node:timers/promises";import"crypto";import"@rent-scraper/utils/config";import"picocolors";i().then(()=>{process.exit(0)}).catch(r=>{const{message:o}=t(r);m.error(o),process.exit(1)});
+import{parseError as t}from"@rent-scraper/utils";import{r as i}from"../shared/scrape-listings.BJUtdD4K.mjs";import{log as m}from"@clack/prompts";import"minimist";import"dayjs";import"path";import"axios";import"fs/promises";import"@rent-scraper/api";import"@rent-scraper/api/config";import"node:timers/promises";import"../shared/scrape-listings.Qpf6zDxA.mjs";import"crypto";import"@rent-scraper/utils/config";import"picocolors";i().then(()=>{process.exit(0)}).catch(r=>{const{message:o}=t(r);m.error(o),process.exit(1)});

package/dist/index.mjs

@@ -1 +1 @@
-export{f as fetchListingHtmlByUrlAndExport,r as runScrapeListings,s as scrapeListingDetailsFromHtmlByFilePaths,a as scrapeListingDetailsFromHtmlByZipCodes,e as scrapeListingHtmlByIds,d as scrapeListingHtmlByInputDirectory,b as scrapeListingHtmlByZipCodes,c as scrapeListingHtmlByZipCodesAndListingDetails,h as scrapeRedfinListingResultsByZipCodes,i as scrapeZillowListingResultsByZipCodes,g as scrapeZillowListingsToCsv}from"./shared/scrape-listings.D0kKRGJj.mjs";import"minimist";import"dayjs";import"path";import"axios";import"fs/promises";import"@rent-scraper/api";import"@rent-scraper/utils";import"@rent-scraper/api/config";import"@clack/prompts";import"node:timers/promises";import"crypto";import"@rent-scraper/utils/config";import"picocolors";
+export{f as fetchListingHtmlByUrlAndExport,r as runScrapeListings,s as scrapeListingDetailsFromHtmlByFilePaths,a as scrapeListingDetailsFromHtmlByZipCodes,e as scrapeListingHtmlByIds,d as scrapeListingHtmlByInputDirectory,b as scrapeListingHtmlByZipCodes,c as scrapeListingHtmlByZipCodesAndListingDetails,g as scrapeRedfinListingResultsByZipCodes,h as scrapeZillowListingResultsByZipCodes}from"./shared/scrape-listings.BJUtdD4K.mjs";export{s as scrapeZillowListingsToCsv}from"./shared/scrape-listings.Qpf6zDxA.mjs";import"minimist";import"dayjs";import"path";import"axios";import"fs/promises";import"@rent-scraper/api";import"@rent-scraper/utils";import"@rent-scraper/api/config";import"@clack/prompts";import"node:timers/promises";import"@rent-scraper/utils/config";import"picocolors";import"crypto";

package/dist/shared/scrape-listings.BJUtdD4K.mjs

@@ -0,0 +1,13 @@
+import Mt from"minimist";import Bt from"dayjs";import n from"path";import lt from"axios";import{mkdir as M,writeFile as P,readFile as tt,unlink as _,readdir as ct}from"fs/promises";import{checkForZillowBotFiltering as V,isZillowBotFiltering as wt,fetchHtmlFromRedfinListingUrl as kt,fetchHtmlFromZillowListingUrl as St,scrapeDataFromRedfinListingHtml as Dt,scrapeDataFromZillowListingHtml as Ft,getZillowListingResults as Pt,getRedfinListingResults as Rt,waitForZillowCaptchaSolve as Ht,waitForSolvedZillowCaptcha as dt,getZillowForceCaptcha as Ot,isBrowserShowingCaptcha as Ut}from"@rent-scraper/api";import{ErrorLog as T,throwError as b,checkForFile as C,parseJsonFile as Et,parseError as H,readFilesInDirectory as et,roundValue as st,chunkArray as Tt,parsePercentage as Nt,compareArrays as ut}from"@rent-scraper/utils";import{getZillowOutputPath as N,getRedfinOutputPath as q,getZillowDaysListed as qt,getRedfinDaysListed as It,getZillowLimit as Jt,getZillowOffset as Yt,getRedfinZipCodes as At,getZillowZipCodes as Vt}from"@rent-scraper/api/config";import{spinner as gt,log as l,progress as Gt,intro as Wt}from"@clack/prompts";import{setTimeout as pt}from"node:timers/promises";import{s as Xt}from"./scrape-listings.Qpf6zDxA.mjs";import{checkBrowserServer as mt}from"@rent-scraper/utils/config";import _t from"picocolors";const O=process.env.DEBUG,I=async(t,e,s,i)=>{const{timeoutMs:r}=i??{};if(await C(s))O&&l.warn(`${s} exists, skipping`);else try{O&&l.info(`writing ${s}`);const a=t==="redfin"?await kt(e):await St(e,{timeoutMs:r});await P(`${s}`,a)}catch(a){b(`error fetching html for ${e}`,a)}},Kt=async(t,{timeoutMs:e})=>{if(await C(t)){const s=JSON.parse(await tt(t,"utf8"))||{},{hdpUrl:i}=s||{};if(i){const r=`https://www.zillow.com${i}`,a=t.replace(".json",".html");await C(a)?O&&l.warn(`file already exists, ${a}`):await I("zillow",r,a,{timeoutMs:e})}else b(`file is empty, ${t}`)}else O&&l.warn(`file does not exist at this path, ${t}, skipping`)},ht=async(t,e)=>{const{timeoutMs:s}=e??{};await Promise.all(t.map(async i=>await Kt(i,{timeoutMs:s})))},Qt=(t,e="zillow")=>{if(e==="zillow"){const{zpid:s,detailUrl:i}=t||{},r=i&&!i.startsWith("http")?`https://www.zillow.com${i}`:i;return{id:s,url:r}}else if(e==="redfin"){const{propertyId:s,url:i}=t?.homeData||{};return{id:s,url:`https://www.redfin.com${i}`}}},it=async(t,e,s,i=s,r)=>{const{timeoutMs:a,run:o=1,reruns:d=0,skipBotCheck:y=!1}=r??{},c=new T;s||b("inputDirectory is required"),t==="zillow"&&!y&&await V();const m=[],$=[],w=gt();w.start("Downloading listings html files");for(let g=1;g<=d+1;g++){d>0&&g>1&&c.add(`rerun ${g-1} of ${d}`);const p=m.length;(g===1||p)&&await Promise.all((p?m:e).map(async L=>{const j=`${s}/${L}.json`;if(await C(j))try{const{results:S}=await Et(j)||{};if(!S)c.add(`empty file, ${j}`);else{const B=i?`${i}/${L}`:`${s}/${L}`;await M(B,{recursive:!0}),S?.length?await Promise.all(S.map(async Z=>{const{id:k,url:x}=Qt(Z,t)??{};if(!x)return c.add(`url missing for ${k}`);const D=`${k}.html`,F=`${B}/${D}`;try{await I(t,x,F,{timeoutMs:a})}catch(z){const{status:J,message:E}=H(z);wt(J,E)||$.push({url:x,filePath:F}),c.add("scrape listing html error: "+(E??`error fetching listing for id, ${z}`))}})):c.add(`no results for file, ${j}`)}}catch(S){m.push(L);const{message:B}=H(S);c.add("scrape listing html error: "+(B??`error reading json data, ${j}, ${S}`))}else O&&l.warn(`file does not exist, ${j}, skipping`)}))}$.length>0&&(await pt(2e3),await Promise.all($.map(async({url:g,filePath:p})=>{try{await I(t,g,p,{timeoutMs:a})}catch(L){const{message:j}=H(L);c.add("scrape listing html retry error: "+(j??`error fetching ${g}`))}})));const f=await N(),h=await q(),u=t==="zillow"?f:h;e.length>0?(w.stop("Listings HTML files have been saved to:"),l.message(n.join(u,t,"listings",n.basename(s)))):w.stop("No listings HTML files saved.");const v=n.join(u,t,"logs");if(await M(v,{recursive:!0}),c.get().filter(g=>!g.includes("rerun ")).length>d){const g=`${n.basename(s)}-html-errors-${o}.txt`,p=n.join(v,g);await c.write(p,[...new Set(c.get())].join(`
+`)),O&&l.error(`There were errors during processing, see ${n.resolve(p)}`)}},te=async(t,e,s,i)=>{const{timeoutMs:r,run:a=1,reruns:o=0,skipBotCheck:d=!1}=i??{},y=new T;s||b("inputDirectory is required"),t==="zillow"&&!d&&await V();const c=[];for(let h=1;h<=o+1;h++){o>0&&h>1&&y.add(`rerun ${h-1} of ${o}`);const u=c.length;(h===1||u)&&await Promise.all((u?c:e).map(async v=>{const g=`${s}/${v}`;if(await C(g))try{const p=await et(g,{extension:".json",prependDirectory:!0});await ht(p,{timeoutMs:r})}catch(p){c.push(v);const{message:L}=H(p);y.add("scrape listing html error: "+(L??`Error during fetch for ${v}, ${p}`))}else y.add(`listing directory does not exist, ${g}`)}))}const m=await N(),$=await q(),w=t==="zillow"?m:$,f=n.join(w,t,"logs");if(await M(f,{recursive:!0}),y.get().filter(h=>!h.includes("rerun ")).length>o){const h=`${n.basename(s)}-html-errors-${a}.txt`,u=n.join(f,h);await y.write(u,[...new Set(y.get())].join(`
+`)),O&&l.error(`There were errors during processing, see ${n.resolve(u)}`)}},ee=async(t,e,s=e,i)=>{const{skipBotCheck:r=!1}=i??{},a=new T;if(t==="zillow"&&!r&&await V(),e||b("inputDirectory is required"),await M(s,{recursive:!0}),await C(e))try{const w=await et(e,{extension:".json",prependDirectory:!0});await ht(w)}catch(w){const{message:f}=H(w);a.add("scrape listing html error: "+(f??`Error during fetch for ${e}, ${w}`))}else a.add(`inputDirectory does not exist, ${e}`);const o=await N(),d=await q(),y=t==="zillow"?o:d,c=n.join(y,t,"logs");await M(c,{recursive:!0});const m=`${n.basename(e)}-html-errors.txt`,$=n.join(c,m);a.get().length>0&&(await a.write($,[...new Set(a.get())].join(`
+`)),O&&l.error(`There were errors during processing, see ${n.resolve($)}`))},se=(t,e)=>t==="redfin"?`https://www.redfin.com/home/${e}`:t==="zillow"?`https://www.zillow.com/homedetails/${e}_zpid`:null,ie=async(t,e,s,i)=>{const{skipBotCheck:r=!1}=i??{};t==="zillow"&&!r&&await V(),s||b("outputDirectory is required"),await M(s,{recursive:!0}),await Promise.all(e.map(async a=>{const o=se(t,a),d=`${s}/${a}.html`;o&&await I(t,o,d)}))},R=process.env.DEBUG,ft=async t=>{if(await C(t)){const e=t.replace(".html",".json");if(await C(e))R&&l.warning(`file already exists, ${e}`);else{R&&l.message(`scraping data for ${t}`);const s=(await tt(t)).toString();(!s||s.trim()==="")&&(await _(t),b(`empty file found at ${t}, deleted for retry`)),s.includes("px-captcha")&&(await _(t),b(`captcha page found in ${t}, deleted for retry`));const i=await Ft(s);if(!i?.priceHistory&&i?.bestMatchedUnit?.hdpUrl){R&&l.warning(`rescraping ${t} - https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`),await C(t)&&(R&&l.warning(`deleting ${t}`),await _(t)),await C(e)&&(R&&l.warning(`deleting ${e}`),await _(e));const r=`https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`;await I("zillow",r,t),await ft(t)}else i?(R&&l.info(`writing ${e}`),await P(`${e}`,JSON.stringify(i))):b(`problem scraping data for ${t}`)}}},re=async t=>{if(await C(t)){const e=t.replace(".html",".json");if(await C(e))R&&l.warning(`file already exists, ${e}`);else{R&&l.message(`scraping data for ${t}`);const s=(await tt(t)).toString(),i=Dt(s);i?(R&&l.warning(`writing ${e}`),await P(`${e}`,JSON.stringify(i))):b(`problem scraping data for ${t}`)}}},yt=async(t,e,s)=>(await Promise.all(e.map(async i=>{try{t==="redfin"?await re(i):await ft(i);const r=i.replace(".html",".json");return await C(r)?1:0}catch(r){const{message:a}=H(r);return s?.add(a),0}}))).reduce((i,r)=>i+r,0),rt=async(t,e,s)=>{const i=new T;let r=0;s||b("inputDirectory is required");const a=Gt({style:"heavy",max:100,size:50});a.start("Scraping listings data");const o=e.length,d=o<20?o+1:20,y=o>d?st(o/d,1):o,c=Tt(e,y);for(const[h,u]of c.entries()){const v=st((Number(h)+1)/c.length*100,1);a.advance(st(1/c.length*100,1),`Scraping listings (${Nt(v)})`),await Promise.all(u.map(async g=>{R&&l.message(`processing files for ${g}`);const p=`${s}/${g}`;if(await C(p)){const L=await et(p,{extension:".html",prependDirectory:!0}),j=await yt(t,L,i);r=r+j}else i.add(`listing directory does not exist, ${p}`)}))}const m=await N(),$=await q(),w=t==="zillow"?m:$;r>0?(a.stop("Listings data has been saved to:"),l.message(n.join(w,t,"listings",n.basename(s)))):a.stop("No listings data saved.");const f=n.join(w,t,"logs");if(await M(f,{recursive:!0}),i.get().length>0){const h=`${n.basename(s)}-listing-errors.txt`,u=n.join(f,h);await i.write(u,[...new Set(i.get())].join(`
+`)),l.error(`There were errors during processing, see ${n.resolve(u)}`)}return{numListings:r}},at=process.env.DEBUG,ae=async(t,e,s,i,r)=>{const{daysOnZillow:a,timeoutMs:o}=r??{};if(await C(e))at&&l.warning(`${t} exists, skipping`),s.push(t);else{const d=await Pt({zipCode:t,daysOnZillow:a,mergePageResults:!0,timeoutMs:o});d?(at&&l.info(`writing ${e}`),await P(e,JSON.stringify(d)),s.push(t)):(at&&l.warning(`no results for ${t}`),i.push(t))}},nt=async(t,e,s)=>{const{daysListed:i,timeoutMs:r,run:a=1,reruns:o=0,fetchListings:d=!1,skipBotCheck:y=!1,silent:c=!1}=s??{},m=new T;y||await V({fetchListings:d});const $=[],w=[],f=[],h=[];await M(e,{recursive:!0});const u=gt();c||u.start("Scraping Zillow search results");for(let p=1;p<=o+1;p++){o>0&&p>1&&m.add(`rerun ${p-1} of ${o}`);const L=w.length;(p===1||L)&&await Promise.all((L?w:t).map(async j=>{const S=`${j}.json`,B=`${e}/${S}`;try{await ae(j,B,$,h,{daysOnZillow:i,timeoutMs:r})}catch(Z){w.push(j);const{status:k,message:x}=H(Z);wt(k,x)&&f.push(j),m.add("scrape listing results error: "+(x??`Error during fetch for ${j}, ${Z}`))}}))}const v=await N();c||($.length>0?(u.stop("Zillow search results have been saved to:"),l.message(n.join(v,"zillow","results",n.basename(e)))):u.stop("No results saved."));const g=n.join(v,"zillow","logs");if(await M(g,{recursive:!0}),m.get().filter(p=>!p.includes("rerun ")).length>o){const p=`${n.basename(e)}-results-errors-${a}.txt`,L=n.join(g,p);await m.write(L,[...new Set(m.get())].join(`
+`)),c||l.error(`There were errors during processing, see ${n.resolve(L)}`)}return{validZipCodes:$,botFilteredZipCodes:f,noResultsZipCodes:h}},ne=async(t,e,s,i)=>{const{daysListed:r,timeoutMs:a}=i??{};if(await C(e))console.log(`${t} exists, skipping`),s.push(t);else{const o=await Rt({zipCode:t,daysListed:r,timeoutMs:a});o?(console.log(`writing ${e}`),await P(e,JSON.stringify(o)),s.push(t)):console.log(`no results for ${t}`)}},$t=async(t,e,s)=>{const{daysListed:i,timeoutMs:r,run:a=1,reruns:o=0}=s??{},d=new T,y=[],c=[];await M(e,{recursive:!0});for(let w=1;w<=o+1;w++){o>0&&w>1&&d.add(`rerun ${w-1} of ${o}`);const f=c.length;(w===1||f)&&await Promise.all((f?c:t).map(async h=>{const u=`${h}.json`,v=`${e}/${u}`;try{await ne(h,v,y,{daysListed:i,timeoutMs:r})}catch(g){c.push(h);const{message:p}=H(g);d.add("scrape listing results error: "+(p??`Error during fetch for ${h}, ${g}`))}}))}const m=await q(),$=n.join(m,"redfin","logs");if(await M($,{recursive:!0}),d.get().filter(w=>!w.includes("rerun ")).length>o){const w=`${n.basename(e)}-results-errors-${a}.txt`,f=n.join($,w);await d.write(f,[...new Set(d.get())].join(`
+`)),console.log(`\x1B[41m
+%s\x1B[0m`,`There were errors during processing, see ${n.resolve(f)}`)}return{validZipCodes:y}},oe=async t=>{try{const e=(await ct(t,{withFileTypes:!0})).filter(s=>s.isDirectory()).map(s=>s.name).sort();return e.length>0?e[e.length-1]:null}catch{return null}},vt=async()=>{await lt.post("http://localhost:8082/browser/close")},le=async()=>{await lt.post("http://localhost:8082/server/shutdown")},ce=async(t,e,s,{daysListed:i,timeoutMs:r,run:a,reruns:o,preValidatedZipCodes:d=[],retry:y=!1})=>{let c=[],m=[],$=[];if(t.length>0){if(await(await Ot()?Ht():dt()),await vt(),{validZipCodes:c,botFilteredZipCodes:m,noResultsZipCodes:$}=await nt(t,e,{daysListed:i,timeoutMs:r,run:a,reruns:o,skipBotCheck:!0}),y&&m.length>0){l.warn(`Bot filtering hit ${m.length} of ${t.length} zip code(s) \u2014 will retry after listings are fetched`),await Ut()?(await dt(),await vt()):await pt(3e3);const h=m.length,u=await nt(m,e,{daysListed:i,timeoutMs:r,run:a,reruns:o,skipBotCheck:!0,silent:!0});c=[...c,...u.validZipCodes],m=u.botFilteredZipCodes,$=[...$,...u.noResultsZipCodes];const v=u.validZipCodes.length;l.info(`Inline retry: recovered ${v} of ${h} \xB7 ${m.length} still bot-filtered`)}c.length===0&&t.length>0&&(d.length>0?l.warn(`No new results for re-fetched zip codes \u2014 proceeding with ${d.length} previously fetched`):l.warn("No results returned for any zip codes \u2014 possible soft bot filtering. Please try again shortly.")),m.length>0&&l.warn(`${m.length} zip code(s) still bot-filtered \u2014 use --rerun to retry`)}const w=[...d,...c];await it("zillow",w,e,s,{timeoutMs:r,run:a,reruns:o,skipBotCheck:!0});const{numListings:f}=await rt("zillow",w,s);return{numListings:f,validZipCodes:w,noResultsZipCodes:$}},we=async(t,e,s,{daysListed:i,timeoutMs:r,run:a,reruns:o})=>{const{validZipCodes:d}=await $t(t,e,{daysListed:i,timeoutMs:r,run:a,reruns:o});await it("redfin",d,e,s,{timeoutMs:r,run:a,reruns:o});const{numListings:y}=await rt("redfin",d,s);return{numListings:y,validZipCodes:d}};async function de(){Wt(_t.inverse(" scrape listings "));const t=Mt(process.argv.slice(2)),e=t.source??"zillow";e==="zillow"&&(await mt()||b("Please launch the browser server before scraping."));const s=await N(),i=await q(),r=e==="zillow"?s:i,a=t["days-listed"]??(e==="zillow"?await qt():await It())??1,o=t.runs??1,d=t.reruns??0,y=t["timeout-ms"]??6e4,c=t.limit??await Jt()??void 0,m=t.offset??await Yt()??0,$=t.retry??!1,w=t.rerun,f=typeof w=="string"?w:w?await oe(n.join(r,e,"results")):null;w&&!f&&b("No previous run found to rerun."),f&&l.info(`Rerunning from: ${f}`);const h=f??Bt().format("YYYY-MM-DD-HHmm"),u=t["results-directory"]??n.join(r,e,"results",h),v=t["listings-directory"]??n.join(r,e,"listings",h),g=t["logs-directory"]??n.join(r,e,"logs");let p=0,L=0;const j=async()=>{(e==="zillow"||e==="redfin")&&await mt()&&await le()};try{if(e==="redfin"){const B=await At();for(let Z=1;Z<=o;Z++){B||b("zip codes required, please run the createConfig script");const k=B,{numListings:x,validZipCodes:D}=await we(k,u,v,{daysListed:a,timeoutMs:y,run:Z,reruns:d});if(Z===o){const F=Object.entries({numListings:x}).map(([K,W])=>`${K}: ${W}`).join(`
+`),z=ut(k,D).join(`
+`);await M(g,{recursive:!0});const J=`${n.basename(u)}-scraping-results.txt`,E=n.join(g,J),G=`${n.basename(u)}-invalid-zipcodes.txt`,U=n.join(g,G);await P(E,F),await P(U,z)}}}else if(e==="zillow"){const B=await Vt();B||b("zip codes required, please run the createConfig script");const Z=B.slice(m,c?m+c:void 0);let k=Z,x=[];if(f){const D=await ct(u).catch(()=>[]),F=new Set(D.filter(z=>z.endsWith(".json")).map(z=>z.replace(".json","")));x=Z.filter(z=>F.has(String(z))),k=Z.filter(z=>!F.has(String(z))),k.length>0?l.info(`${x.length} already fetched \xB7 retrying ${k.length} missing`):l.info(`All ${x.length} zip codes already fetched \xB7 proceeding to HTML scraping`)}for(let D=1;D<=o;D++){const{numListings:F,validZipCodes:z,noResultsZipCodes:J}=await ce(k,u,v,{daysListed:a,timeoutMs:y,run:D,reruns:d,preValidatedZipCodes:x,retry:$});if(D===o){const E=n.basename(v),G=n.join(s,"zillow","csv",`${E}.csv`),U=await Xt(v,G);p=U,U>0&&l.info(`CSV exported: ${U} listings \u2192 ${G}`);const K=Object.entries({numListings:F,numCsvListings:U}).map(([A,xt])=>`${A}: ${xt}`).join(`
+`),W=ut(Z,z),ot=new Set(J),Y=W.filter(A=>!ot.has(A)),X=W.filter(A=>ot.has(A));await M(g,{recursive:!0});const Lt=`${n.basename(u)}-scraping-results.txt`,jt=n.join(g,Lt),Ct=`${n.basename(u)}-zipcodes-no-results.txt`,Zt=n.join(g,Ct),zt=`${n.basename(u)}-zipcodes-errored.txt`,bt=n.join(g,zt);await P(jt,K),X.length>0&&await P(Zt,X.join(`
+`)),Y.length>0&&await P(bt,Y.join(`
+`)),L=Y.length;const Q=[`${z.length}/${Z.length} with results`];X.length>0&&Q.push(`${X.length} no results`),Y.length>0&&Q.push(`${Y.length} errored`),l.info(`Zip codes: ${Q.join(" \xB7 ")}`),l.info(`Listings:  ${F} parsed \xB7 ${U} exported to CSV`)}}}const S=e==="zillow"?p>0:!0;return S?l.success("Scraping complete!"):l.warn("Scraping finished but no listings were exported \u2014 please try again shortly."),{success:S,totalCsvListings:p,missedZipCodes:L,resultsDirectory:u,listingsDirectory:v}}finally{await j()}}export{rt as a,it as b,te as c,ee as d,ie as e,I as f,$t as g,nt as h,de as r,yt as s};

package/dist/shared/scrape-listings.Qpf6zDxA.mjs

@@ -0,0 +1,3 @@
+import{createHash as b}from"crypto";import{readdir as g,readFile as h,mkdir as f,writeFile as y}from"fs/promises";import d from"path";import w from"dayjs";import{checkForFile as z,readFilesInDirectory as I}from"@rent-scraper/utils";const _=["listing_id","listing_source","source_listing_id","parcel_number","listing_url","street_address","cleaned_address","cleaned_unit_number","city","state","zipcode","county","neighborhood_region","is_undisclosed_address","home_status","home_type","bedrooms","year_built","living_area","living_area_units","is_income_restricted","price_at_source","platform_rent_estimate","agent_name","agent_phone_number","broker_name","broker_phone_number","is_owned_by_listing_platform","date_last_updated_at_source","date_scraped","scrape_job_name"],N=t=>{if(t==null)return"";const e=typeof t=="object"?JSON.stringify(t):String(t);return e.includes(",")||e.includes('"')||e.includes(`
+`)?`"${e.replace(/"/g,'""')}"`:e},k=t=>{const e=/^(.*?)\s+((?:#|APT|UNIT|STE|Suite|Apt|Unit)\s*.+)$/i.exec(t);if(e){const i=e[1].trim().toUpperCase();let r=e[2].trim();return r.startsWith("#")&&(r=`# ${r.replace(/^#\s*/,"").trim()}`),{cleaned:i,unit:r}}return{cleaned:t.toUpperCase(),unit:""}},F=(t,e)=>{const i=String(t.zpid),r=b("md5").update(i).digest("hex"),n=t.streetAddress||t.address?.streetAddress||"",{cleaned:a,unit:s}=k(n),o=t.hdpUrl?`https://www.zillow.com${t.hdpUrl}`:`https://www.zillow.com/homedetails/${i}_zpid/`;return{listing_id:r,listing_source:"zillow",source_listing_id:i,parcel_number:t.resoFacts?.parcelNumber??"",listing_url:o,street_address:n,cleaned_address:a,cleaned_unit_number:s,city:t.city||t.address?.city||"",state:t.state||t.address?.state||"",zipcode:t.zipcode||t.address?.zipcode||"",county:t.county||"",neighborhood_region:t.neighborhoodRegion?.name||t.parentRegion?.name||"",is_undisclosed_address:t.isUndisclosedAddress??"",home_status:t.homeStatus||"",home_type:t.homeType||"",bedrooms:t.bedrooms??"",year_built:t.yearBuilt||t.resoFacts?.yearBuilt||"",living_area:t.livingAreaValue??"",living_area_units:t.livingAreaUnits||"Square Feet",is_income_restricted:t.isIncomeRestricted??"",price_at_source:t.price??"",platform_rent_estimate:t.rentZestimate??"",agent_name:t.attributionInfo?.agentName||"",agent_phone_number:t.attributionInfo?.agentPhoneNumber||"",broker_name:t.attributionInfo?.brokerName||"",broker_phone_number:t.attributionInfo?.brokerPhoneNumber||"",is_owned_by_listing_platform:t.attributionInfo?.mlsName==="Zillow Rentals",date_last_updated_at_source:t.attributionInfo?.lastUpdated||"",date_scraped:t.timestamp?w(t.timestamp).format("YYYY-MM-DD HH:mm:ss"):"",scrape_job_name:e}},U=async(t,e)=>{const i=d.basename(t),r=[_.join(",")];let n;try{n=await g(t,{withFileTypes:!0})}catch{return 0}const a=n.filter(s=>s.isDirectory()).map(s=>s.name);for(const s of a){const o=d.join(t,s);if(!await z(o))continue;const l=await I(o,{extension:".json",prependDirectory:!0});for(const m of l)try{const c=JSON.parse(await h(m,"utf8"));if(!c.zpid)continue;const u=F(c,i);r.push(_.map(p=>N(u[p])).join(","))}catch{}}return await f(d.dirname(e),{recursive:!0}),await y(e,r.join(`
+`)),r.length-1};export{U as s};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rent-scraper/scrape-listings",
-  "version": "1.0.35",
+  "version": "1.0.37",
   "type": "module",
   "exports": {
     ".": {
@@ -10,6 +10,10 @@
     "./bin/run-scrape-listings": {
       "types": "./dist/bin/run-scrape-listings.d.mts",
       "default": "./dist/bin/run-scrape-listings.mjs"
+    },
+    "./bin/run-scrape-listing-details": {
+      "types": "./dist/bin/run-scrape-listing-details.d.mts",
+      "default": "./dist/bin/run-scrape-listing-details.mjs"
     }
   },
   "author": "Max Stein <maxwell.stein@gmail.com> (https://maxstein.net)",
@@ -30,8 +34,8 @@
     "dayjs": "^1.11.13",
     "minimist": "^1.2.8",
     "picocolors": "^1.1.1",
-    "@rent-scraper/utils": "1.0.35",
-    "@rent-scraper/api": "1.0.35"
+    "@rent-scraper/utils": "1.0.37",
+    "@rent-scraper/api": "1.0.37"
   },
   "devDependencies": {
     "@types/minimist": "^1.2.5",
@@ -45,6 +49,8 @@
     "build": "unbuild",
     "start": "node --env-file-if-exists=.env dist/bin/run-scrape-listings.mjs",
     "dev": "tsx --env-file-if-exists=.env src/bin/run-scrape-listings.ts",
+    "details:dev": "tsx --env-file-if-exists=.env src/bin/run-scrape-listing-details.ts",
+    "details": "node --env-file-if-exists=.env dist/bin/run-scrape-listing-details.mjs",
     "cli:dev": "tsx --env-file-if-exists=.env src/cli.ts",
     "cli": "node --env-file-if-exists=.env dist/cli.mjs",
     "typecheck": "tsc --noEmit",

package/dist/shared/scrape-listings.D0kKRGJj.mjs

@@ -1,15 +0,0 @@
-import Mt from"minimist";import ct from"dayjs";import o from"path";import dt from"axios";import{mkdir as Z,writeFile as B,readFile as X,unlink as K,readdir as st}from"fs/promises";import{checkForZillowBotFiltering as J,isZillowBotFiltering as ut,fetchHtmlFromRedfinListingUrl as Bt,fetchHtmlFromZillowListingUrl as Dt,scrapeDataFromRedfinListingHtml as Ft,scrapeDataFromZillowListingHtml as Pt,getZillowListingResults as Rt,getRedfinListingResults as Ut,waitForZillowCaptchaSolve as Nt,waitForSolvedZillowCaptcha as pt,getZillowForceCaptcha as Ht,isBrowserShowingCaptcha as Ot}from"@rent-scraper/api";import{ErrorLog as O,throwError as C,checkForFile as j,parseJsonFile as Tt,parseError as R,readFilesInDirectory as Q,roundValue as it,chunkArray as It,parsePercentage as Et,compareArrays as wt}from"@rent-scraper/utils";import{getZillowOutputPath as T,getRedfinOutputPath as I,getZillowDaysListed as At,getRedfinDaysListed as Yt,getZillowLimit as qt,getZillowOffset as Jt,getRedfinZipCodes as Vt,getZillowZipCodes as Gt}from"@rent-scraper/api/config";import{spinner as gt,log as d,progress as Wt,intro as Xt}from"@clack/prompts";import{setTimeout as mt}from"node:timers/promises";import{createHash as Kt}from"crypto";import{checkBrowserServer as ht}from"@rent-scraper/utils/config";import Qt from"picocolors";const U=process.env.DEBUG,E=async(t,e,s,i)=>{const{timeoutMs:r}=i??{};if(await j(s))U&&d.warn(`${s} exists, skipping`);else try{U&&d.info(`writing ${s}`);const a=t==="redfin"?await Bt(e):await Dt(e,{timeoutMs:r});await B(`${s}`,a)}catch(a){C(`error fetching html for ${e}`,a)}},te=async(t,{timeoutMs:e})=>{if(await j(t)){const s=JSON.parse(await X(t,"utf8"))||{},{hdpUrl:i}=s||{};if(i){const r=`https://www.zillow.com${i}`,a=t.replace(".json",".html");await j(a)?U&&d.warn(`file already exists, ${a}`):await E("zillow",r,a,{timeoutMs:e})}else C(`file is empty, ${t}`)}else U&&d.warn(`file does not exist at this path, ${t}, skipping`)},ft=async(t,e)=>{const{timeoutMs:s}=e??{};await Promise.all(t.map(async i=>await te(i,{timeoutMs:s})))},ee=(t,e="zillow")=>{if(e==="zillow"){const{zpid:s,detailUrl:i}=t||{},r=i&&!i.startsWith("http")?`https://www.zillow.com${i}`:i;return{id:s,url:r}}else if(e==="redfin"){const{propertyId:s,url:i}=t?.homeData||{};return{id:s,url:`https://www.redfin.com${i}`}}},rt=async(t,e,s,i=s,r)=>{const{timeoutMs:a,run:n=1,reruns:l=0,skipBotCheck:f=!1}=r??{},c=new O;s||C("inputDirectory is required"),t==="zillow"&&!f&&await J();const w=[],y=[],u=gt();u.start("Downloading listings html files");for(let g=1;g<=l+1;g++){l>0&&g>1&&c.add(`rerun ${g-1} of ${l}`);const m=w.length;(g===1||m)&&await Promise.all((m?w:e).map(async b=>{const v=`${s}/${b}.json`;if(await j(v))try{const{results:M}=await Tt(v)||{};if(!M)c.add(`empty file, ${v}`);else{const S=i?`${i}/${b}`:`${s}/${b}`;await Z(S,{recursive:!0}),M?.length?await Promise.all(M.map(async L=>{const{id:k,url:x}=ee(L,t)??{};if(!x)return c.add(`url missing for ${k}`);const D=`${k}.html`,F=`${S}/${D}`;try{await E(t,x,F,{timeoutMs:a})}catch(z){const{status:A,message:H}=R(z);ut(A,H)||y.push({url:x,filePath:F}),c.add("scrape listing html error: "+(H??`error fetching listing for id, ${z}`))}})):c.add(`no results for file, ${v}`)}}catch(M){w.push(b);const{message:S}=R(M);c.add("scrape listing html error: "+(S??`error reading json data, ${v}, ${M}`))}else U&&d.warn(`file does not exist, ${v}, skipping`)}))}y.length>0&&(await mt(2e3),await Promise.all(y.map(async({url:g,filePath:m})=>{try{await E(t,g,m,{timeoutMs:a})}catch(b){const{message:v}=R(b);c.add("scrape listing html retry error: "+(v??`error fetching ${g}`))}})));const $=await T(),h=await I(),p=t==="zillow"?$:h;e.length>0?(u.stop("Listings HTML files have been saved to:"),d.message(o.join(p,t,"listings",o.basename(s)))):u.stop("No listings HTML files saved.");const _=o.join(p,t,"logs");if(await Z(_,{recursive:!0}),c.get().filter(g=>!g.includes("rerun ")).length>l){const g=`${o.basename(s)}-html-errors-${n}.txt`,m=o.join(_,g);await c.write(m,[...new Set(c.get())].join(`
-`)),U&&d.error(`There were errors during processing, see ${o.resolve(m)}`)}},se=async(t,e,s,i)=>{const{timeoutMs:r,run:a=1,reruns:n=0,skipBotCheck:l=!1}=i??{},f=new O;s||C("inputDirectory is required"),t==="zillow"&&!l&&await J();const c=[];for(let h=1;h<=n+1;h++){n>0&&h>1&&f.add(`rerun ${h-1} of ${n}`);const p=c.length;(h===1||p)&&await Promise.all((p?c:e).map(async _=>{const g=`${s}/${_}`;if(await j(g))try{const m=await Q(g,{extension:".json",prependDirectory:!0});await ft(m,{timeoutMs:r})}catch(m){c.push(_);const{message:b}=R(m);f.add("scrape listing html error: "+(b??`Error during fetch for ${_}, ${m}`))}else f.add(`listing directory does not exist, ${g}`)}))}const w=await T(),y=await I(),u=t==="zillow"?w:y,$=o.join(u,t,"logs");if(await Z($,{recursive:!0}),f.get().filter(h=>!h.includes("rerun ")).length>n){const h=`${o.basename(s)}-html-errors-${a}.txt`,p=o.join($,h);await f.write(p,[...new Set(f.get())].join(`
-`)),U&&d.error(`There were errors during processing, see ${o.resolve(p)}`)}},ie=async(t,e,s=e,i)=>{const{skipBotCheck:r=!1}=i??{},a=new O;if(t==="zillow"&&!r&&await J(),e||C("inputDirectory is required"),await Z(s,{recursive:!0}),await j(e))try{const u=await Q(e,{extension:".json",prependDirectory:!0});await ft(u)}catch(u){const{message:$}=R(u);a.add("scrape listing html error: "+($??`Error during fetch for ${e}, ${u}`))}else a.add(`inputDirectory does not exist, ${e}`);const n=await T(),l=await I(),f=t==="zillow"?n:l,c=o.join(f,t,"logs");await Z(c,{recursive:!0});const w=`${o.basename(e)}-html-errors.txt`,y=o.join(c,w);a.get().length>0&&(await a.write(y,[...new Set(a.get())].join(`
-`)),U&&d.error(`There were errors during processing, see ${o.resolve(y)}`))},re=(t,e)=>t==="redfin"?`https://www.redfin.com/home/${e}`:t==="zillow"?`https://www.zillow.com/homedetails/${e}_zpid`:null,ne=async(t,e,s,i)=>{const{skipBotCheck:r=!1}=i??{};t==="zillow"&&!r&&await J(),s||C("outputDirectory is required"),await Z(s,{recursive:!0}),await Promise.all(e.map(async a=>{const n=re(t,a),l=`${s}/${a}.html`;n&&await E(t,n,l)}))},P=process.env.DEBUG,yt=async t=>{if(await j(t)){const e=t.replace(".html",".json");if(await j(e))P&&d.warning(`file already exists, ${e}`);else{P&&d.message(`scraping data for ${t}`);const s=(await X(t)).toString();(!s||s.trim()==="")&&(await K(t),C(`empty file found at ${t}, deleted for retry`)),s.includes("px-captcha")&&(await K(t),C(`captcha page found in ${t}, deleted for retry`));const i=await Pt(s);if(!i?.priceHistory&&i?.bestMatchedUnit?.hdpUrl){P&&d.warning(`rescraping ${t} - https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`),await j(t)&&(P&&d.warning(`deleting ${t}`),await K(t)),await j(e)&&(P&&d.warning(`deleting ${e}`),await K(e));const r=`https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`;await E("zillow",r,t),await yt(t)}else i?(P&&d.info(`writing ${e}`),await B(`${e}`,JSON.stringify(i))):C(`problem scraping data for ${t}`)}}},ae=async t=>{if(await j(t)){const e=t.replace(".html",".json");if(await j(e))P&&d.warning(`file already exists, ${e}`);else{P&&d.message(`scraping data for ${t}`);const s=(await X(t)).toString(),i=Ft(s);i?(P&&d.warning(`writing ${e}`),await B(`${e}`,JSON.stringify(i))):C(`problem scraping data for ${t}`)}}},$t=async(t,e,s)=>(await Promise.all(e.map(async i=>{try{t==="redfin"?await ae(i):await yt(i);const r=i.replace(".html",".json");return await j(r)?1:0}catch(r){const{message:a}=R(r);return s?.add(a),0}}))).reduce((i,r)=>i+r,0),nt=async(t,e,s)=>{const i=new O;let r=0;s||C("inputDirectory is required");const a=Wt({style:"heavy",max:100,size:50});a.start("Scraping listings data");const n=e.length,l=n<20?n+1:20,f=n>l?it(n/l,1):n,c=It(e,f);for(const[h,p]of c.entries()){const _=it((Number(h)+1)/c.length*100,1);a.advance(it(1/c.length*100,1),`Scraping listings (${Et(_)})`),await Promise.all(p.map(async g=>{P&&d.message(`processing files for ${g}`);const m=`${s}/${g}`;if(await j(m)){const b=await Q(m,{extension:".html",prependDirectory:!0}),v=await $t(t,b,i);r=r+v}else i.add(`listing directory does not exist, ${m}`)}))}const w=await T(),y=await I(),u=t==="zillow"?w:y;r>0?(a.stop("Listings data has been saved to:"),d.message(o.join(u,t,"listings",o.basename(s)))):a.stop("No listings data saved.");const $=o.join(u,t,"logs");if(await Z($,{recursive:!0}),i.get().length>0){const h=`${o.basename(s)}-listing-errors.txt`,p=o.join($,h);await i.write(p,[...new Set(i.get())].join(`
-`)),d.error(`There were errors during processing, see ${o.resolve(p)}`)}return{numListings:r}},at=process.env.DEBUG,oe=async(t,e,s,i,r)=>{const{daysOnZillow:a,timeoutMs:n}=r??{};if(await j(e))at&&d.warning(`${t} exists, skipping`),s.push(t);else{const l=await Rt({zipCode:t,daysOnZillow:a,mergePageResults:!0,timeoutMs:n});l?(at&&d.info(`writing ${e}`),await B(e,JSON.stringify(l)),s.push(t)):(at&&d.warning(`no results for ${t}`),i.push(t))}},ot=async(t,e,s)=>{const{daysListed:i,timeoutMs:r,run:a=1,reruns:n=0,fetchListings:l=!1,skipBotCheck:f=!1,silent:c=!1}=s??{},w=new O;f||await J({fetchListings:l});const y=[],u=[],$=[],h=[];await Z(e,{recursive:!0});const p=gt();c||p.start("Scraping Zillow search results");for(let m=1;m<=n+1;m++){n>0&&m>1&&w.add(`rerun ${m-1} of ${n}`);const b=u.length;(m===1||b)&&await Promise.all((b?u:t).map(async v=>{const M=`${v}.json`,S=`${e}/${M}`;try{await oe(v,S,y,h,{daysOnZillow:i,timeoutMs:r})}catch(L){u.push(v);const{status:k,message:x}=R(L);ut(k,x)&&$.push(v),w.add("scrape listing results error: "+(x??`Error during fetch for ${v}, ${L}`))}}))}const _=await T();c||(y.length>0?(p.stop("Zillow search results have been saved to:"),d.message(o.join(_,"zillow","results",o.basename(e)))):p.stop("No results saved."));const g=o.join(_,"zillow","logs");if(await Z(g,{recursive:!0}),w.get().filter(m=>!m.includes("rerun ")).length>n){const m=`${o.basename(e)}-results-errors-${a}.txt`,b=o.join(g,m);await w.write(b,[...new Set(w.get())].join(`
-`)),c||d.error(`There were errors during processing, see ${o.resolve(b)}`)}return{validZipCodes:y,botFilteredZipCodes:$,noResultsZipCodes:h}},le=async(t,e,s,i)=>{const{daysListed:r,timeoutMs:a}=i??{};if(await j(e))console.log(`${t} exists, skipping`),s.push(t);else{const n=await Ut({zipCode:t,daysListed:r,timeoutMs:a});n?(console.log(`writing ${e}`),await B(e,JSON.stringify(n)),s.push(t)):console.log(`no results for ${t}`)}},_t=async(t,e,s)=>{const{daysListed:i,timeoutMs:r,run:a=1,reruns:n=0}=s??{},l=new O,f=[],c=[];await Z(e,{recursive:!0});for(let u=1;u<=n+1;u++){n>0&&u>1&&l.add(`rerun ${u-1} of ${n}`);const $=c.length;(u===1||$)&&await Promise.all(($?c:t).map(async h=>{const p=`${h}.json`,_=`${e}/${p}`;try{await le(h,_,f,{daysListed:i,timeoutMs:r})}catch(g){c.push(h);const{message:m}=R(g);l.add("scrape listing results error: "+(m??`Error during fetch for ${h}, ${g}`))}}))}const w=await I(),y=o.join(w,"redfin","logs");if(await Z(y,{recursive:!0}),l.get().filter(u=>!u.includes("rerun ")).length>n){const u=`${o.basename(e)}-results-errors-${a}.txt`,$=o.join(y,u);await l.write($,[...new Set(l.get())].join(`
-`)),console.log(`\x1B[41m
-%s\x1B[0m`,`There were errors during processing, see ${o.resolve($)}`)}return{validZipCodes:f}},bt=["listing_id","listing_source","source_listing_id","parcel_number","listing_url","street_address","cleaned_address","cleaned_unit_number","city","state","zipcode","county","neighborhood_region","is_undisclosed_address","home_status","home_type","bedrooms","year_built","living_area","living_area_units","is_income_restricted","price_at_source","platform_rent_estimate","agent_name","agent_phone_number","broker_name","broker_phone_number","is_owned_by_listing_platform","date_last_updated_at_source","date_scraped","scrape_job_name"],ce=t=>{if(t==null)return"";const e=typeof t=="object"?JSON.stringify(t):String(t);return e.includes(",")||e.includes('"')||e.includes(`
-`)?`"${e.replace(/"/g,'""')}"`:e},de=t=>{const e=/^(.*?)\s+((?:#|APT|UNIT|STE|Suite|Apt|Unit)\s*.+)$/i.exec(t);if(e){const s=e[1].trim().toUpperCase();let i=e[2].trim();return i.startsWith("#")&&(i=`# ${i.replace(/^#\s*/,"").trim()}`),{cleaned:s,unit:i}}return{cleaned:t.toUpperCase(),unit:""}},ue=(t,e)=>{const s=String(t.zpid),i=Kt("md5").update(s).digest("hex"),r=t.streetAddress||t.address?.streetAddress||"",{cleaned:a,unit:n}=de(r),l=t.hdpUrl?`https://www.zillow.com${t.hdpUrl}`:`https://www.zillow.com/homedetails/${s}_zpid/`;return{listing_id:i,listing_source:"zillow",source_listing_id:s,parcel_number:t.resoFacts?.parcelNumber??"",listing_url:l,street_address:r,cleaned_address:a,cleaned_unit_number:n,city:t.city||t.address?.city||"",state:t.state||t.address?.state||"",zipcode:t.zipcode||t.address?.zipcode||"",county:t.county||"",neighborhood_region:t.neighborhoodRegion?.name||t.parentRegion?.name||"",is_undisclosed_address:t.isUndisclosedAddress??"",home_status:t.homeStatus||"",home_type:t.homeType||"",bedrooms:t.bedrooms??"",year_built:t.yearBuilt||t.resoFacts?.yearBuilt||"",living_area:t.livingAreaValue??"",living_area_units:t.livingAreaUnits||"Square Feet",is_income_restricted:t.isIncomeRestricted??"",price_at_source:t.price??"",platform_rent_estimate:t.rentZestimate??"",agent_name:t.attributionInfo?.agentName||"",agent_phone_number:t.attributionInfo?.agentPhoneNumber||"",broker_name:t.attributionInfo?.brokerName||"",broker_phone_number:t.attributionInfo?.brokerPhoneNumber||"",is_owned_by_listing_platform:t.attributionInfo?.mlsName==="Zillow Rentals",date_last_updated_at_source:t.attributionInfo?.lastUpdated||"",date_scraped:t.timestamp?ct(t.timestamp).format("YYYY-MM-DD HH:mm:ss"):"",scrape_job_name:e}},vt=async(t,e)=>{const s=o.basename(t),i=[bt.join(",")];let r;try{r=await st(t,{withFileTypes:!0})}catch{return 0}const a=r.filter(n=>n.isDirectory()).map(n=>n.name);for(const n of a){const l=o.join(t,n);if(!await j(l))continue;const f=await Q(l,{extension:".json",prependDirectory:!0});for(const c of f)try{const w=JSON.parse(await X(c,"utf8"));if(!w.zpid)continue;const y=ue(w,s);i.push(bt.map(u=>ce(y[u])).join(","))}catch{}}return await Z(o.dirname(e),{recursive:!0}),await B(e,i.join(`
-`)),i.length-1},pe=async t=>{try{const e=(await st(t,{withFileTypes:!0})).filter(s=>s.isDirectory()).map(s=>s.name).sort();return e.length>0?e[e.length-1]:null}catch{return null}},jt=async()=>{await dt.post("http://localhost:8082/browser/close")},we=async()=>{await dt.post("http://localhost:8082/server/shutdown")},ge=async(t,e,s,{daysListed:i,timeoutMs:r,run:a,reruns:n,preValidatedZipCodes:l=[],retry:f=!1})=>{let c=[],w=[],y=[];if(t.length>0){if(await(await Ht()?Nt():pt()),await jt(),{validZipCodes:c,botFilteredZipCodes:w,noResultsZipCodes:y}=await ot(t,e,{daysListed:i,timeoutMs:r,run:a,reruns:n,skipBotCheck:!0}),f&&w.length>0){d.warn(`Bot filtering hit ${w.length} of ${t.length} zip code(s) \u2014 will retry after listings are fetched`),await Ot()?(await pt(),await jt()):await mt(3e3);const h=w.length,p=await ot(w,e,{daysListed:i,timeoutMs:r,run:a,reruns:n,skipBotCheck:!0,silent:!0});c=[...c,...p.validZipCodes],w=p.botFilteredZipCodes,y=[...y,...p.noResultsZipCodes];const _=p.validZipCodes.length;d.info(`Inline retry: recovered ${_} of ${h} \xB7 ${w.length} still bot-filtered`)}c.length===0&&t.length>0&&(l.length>0?d.warn(`No new results for re-fetched zip codes \u2014 proceeding with ${l.length} previously fetched`):d.warn("No results returned for any zip codes \u2014 possible soft bot filtering. Please try again shortly.")),w.length>0&&d.warn(`${w.length} zip code(s) still bot-filtered \u2014 use --rerun to retry`)}const u=[...l,...c];await rt("zillow",u,e,s,{timeoutMs:r,run:a,reruns:n,skipBotCheck:!0});const{numListings:$}=await nt("zillow",u,s);return{numListings:$,validZipCodes:u,noResultsZipCodes:y}},me=async(t,e,s,{daysListed:i,timeoutMs:r,run:a,reruns:n})=>{const{validZipCodes:l}=await _t(t,e,{daysListed:i,timeoutMs:r,run:a,reruns:n});await rt("redfin",l,e,s,{timeoutMs:r,run:a,reruns:n});const{numListings:f}=await nt("redfin",l,s);return{numListings:f,validZipCodes:l}};async function he(){Xt(Qt.inverse(" scrape listings "));const t=Mt(process.argv.slice(2)),e=t.source??"zillow";e==="zillow"&&(await ht()||C("Please launch the browser server before scraping."));const s=await T(),i=await I(),r=e==="zillow"?s:i,a=t["days-listed"]??(e==="zillow"?await At():await Yt())??1,n=t.runs??1,l=t.reruns??0,f=t["timeout-ms"]??6e4,c=t.limit??await qt()??void 0,w=t.offset??await Jt()??0,y=t.retry??!1,u=t.rerun,$=typeof u=="string"?u:u?await pe(o.join(r,e,"results")):null;u&&!$&&C("No previous run found to rerun."),$&&d.info(`Rerunning from: ${$}`);const h=$??ct().format("YYYY-MM-DD-HHmm"),p=t["results-directory"]??o.join(r,e,"results",h),_=t["listings-directory"]??o.join(r,e,"listings",h),g=t["logs-directory"]??o.join(r,e,"logs");let m=0,b=0;const v=async()=>{(e==="zillow"||e==="redfin")&&await ht()&&await we()};try{if(e==="redfin"){const S=await Vt();for(let L=1;L<=n;L++){S||C("zip codes required, please run the createConfig script");const k=S,{numListings:x,validZipCodes:D}=await me(k,p,_,{daysListed:a,timeoutMs:f,run:L,reruns:l});if(L===n){const F=Object.entries({numListings:x}).map(([tt,G])=>`${tt}: ${G}`).join(`
-`),z=wt(k,D).join(`
-`);await Z(g,{recursive:!0});const A=`${o.basename(p)}-scraping-results.txt`,H=o.join(g,A),V=`${o.basename(p)}-invalid-zipcodes.txt`,N=o.join(g,V);await B(H,F),await B(N,z)}}}else if(e==="zillow"){const S=await Gt();S||C("zip codes required, please run the createConfig script");const L=S.slice(w,c?w+c:void 0);let k=L,x=[];if($){const D=await st(p).catch(()=>[]),F=new Set(D.filter(z=>z.endsWith(".json")).map(z=>z.replace(".json","")));x=L.filter(z=>F.has(String(z))),k=L.filter(z=>!F.has(String(z))),k.length>0?d.info(`${x.length} already fetched \xB7 retrying ${k.length} missing`):d.info(`All ${x.length} zip codes already fetched \xB7 proceeding to HTML scraping`)}for(let D=1;D<=n;D++){const{numListings:F,validZipCodes:z,noResultsZipCodes:A}=await ge(k,p,_,{daysListed:a,timeoutMs:f,run:D,reruns:l,preValidatedZipCodes:x,retry:y});if(D===n){const H=o.basename(_),V=o.join(s,"zillow","csv",`${H}.csv`),N=await vt(_,V);m=N,N>0&&d.info(`CSV exported: ${N} listings \u2192 ${V}`);const tt=Object.entries({numListings:F,numCsvListings:N}).map(([q,kt])=>`${q}: ${kt}`).join(`
-`),G=wt(L,z),lt=new Set(A),Y=G.filter(q=>!lt.has(q)),W=G.filter(q=>lt.has(q));await Z(g,{recursive:!0});const Lt=`${o.basename(p)}-scraping-results.txt`,zt=o.join(g,Lt),Ct=`${o.basename(p)}-zipcodes-no-results.txt`,Zt=o.join(g,Ct),xt=`${o.basename(p)}-zipcodes-errored.txt`,St=o.join(g,xt);await B(zt,tt),W.length>0&&await B(Zt,W.join(`
-`)),Y.length>0&&await B(St,Y.join(`
-`)),b=Y.length;const et=[`${z.length}/${L.length} with results`];W.length>0&&et.push(`${W.length} no results`),Y.length>0&&et.push(`${Y.length} errored`),d.info(`Zip codes: ${et.join(" \xB7 ")}`),d.info(`Listings:  ${F} parsed \xB7 ${N} exported to CSV`)}}}const M=e==="zillow"?m>0:!0;return M?d.success("Scraping complete!"):d.warn("Scraping finished but no listings were exported \u2014 please try again shortly."),{success:M,totalCsvListings:m,missedZipCodes:b,resultsDirectory:p,listingsDirectory:_}}finally{await v()}}export{nt as a,rt as b,se as c,ie as d,ne as e,E as f,vt as g,_t as h,ot as i,he as r,$t as s};