@rent-scraper/scrape-listings 1.0.36 → 1.0.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
import{parseError as t}from"@rent-scraper/utils";import{r as i}from"../shared/scrape-listings.
|
|
1
|
+
import{parseError as t}from"@rent-scraper/utils";import{r as i}from"../shared/scrape-listings.DbFGLyo-.mjs";import{log as m}from"@clack/prompts";import"minimist";import"dayjs";import"path";import"axios";import"fs/promises";import"@rent-scraper/api";import"@rent-scraper/api/config";import"node:timers/promises";import"../shared/scrape-listings.Qpf6zDxA.mjs";import"crypto";import"@rent-scraper/utils/config";import"picocolors";i().then(()=>{process.exit(0)}).catch(r=>{const{message:o}=t(r);m.error(o),process.exit(1)});
|
package/dist/index.mjs
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export{f as fetchListingHtmlByUrlAndExport,r as runScrapeListings,s as scrapeListingDetailsFromHtmlByFilePaths,a as scrapeListingDetailsFromHtmlByZipCodes,e as scrapeListingHtmlByIds,d as scrapeListingHtmlByInputDirectory,b as scrapeListingHtmlByZipCodes,c as scrapeListingHtmlByZipCodesAndListingDetails,g as scrapeRedfinListingResultsByZipCodes,h as scrapeZillowListingResultsByZipCodes}from"./shared/scrape-listings.
|
|
1
|
+
export{f as fetchListingHtmlByUrlAndExport,r as runScrapeListings,s as scrapeListingDetailsFromHtmlByFilePaths,a as scrapeListingDetailsFromHtmlByZipCodes,e as scrapeListingHtmlByIds,d as scrapeListingHtmlByInputDirectory,b as scrapeListingHtmlByZipCodes,c as scrapeListingHtmlByZipCodesAndListingDetails,g as scrapeRedfinListingResultsByZipCodes,h as scrapeZillowListingResultsByZipCodes}from"./shared/scrape-listings.DbFGLyo-.mjs";export{s as scrapeZillowListingsToCsv}from"./shared/scrape-listings.Qpf6zDxA.mjs";import"minimist";import"dayjs";import"path";import"axios";import"fs/promises";import"@rent-scraper/api";import"@rent-scraper/utils";import"@rent-scraper/api/config";import"@clack/prompts";import"node:timers/promises";import"@rent-scraper/utils/config";import"picocolors";import"crypto";
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import Bt from"minimist";import kt from"dayjs";import n from"path";import lt from"axios";import{mkdir as M,writeFile as P,readFile as tt,unlink as X,readdir as ct}from"fs/promises";import{checkForZillowBotFiltering as V,isZillowBotFiltering as wt,fetchHtmlFromRedfinListingUrl as St,fetchHtmlFromZillowListingUrl as Dt,scrapeDataFromRedfinListingHtml as Ft,scrapeDataFromZillowListingHtml as Pt,getZillowListingResults as Rt,getRedfinListingResults as Ht,waitForZillowCaptchaSolve as Ot,waitForSolvedZillowCaptcha as dt,getZillowForceCaptcha as Ut,isBrowserShowingCaptcha as Tt}from"@rent-scraper/api";import{ErrorLog as E,throwError as b,checkForFile as C,parseJsonFile as Et,parseError as H,readFilesInDirectory as et,roundValue as st,chunkArray as ut,parsePercentage as Nt,compareArrays as gt}from"@rent-scraper/utils";import{getZillowOutputPath as N,getRedfinOutputPath as q,getZillowDaysListed as qt,getRedfinDaysListed as It,getZillowLimit as Jt,getZillowOffset as Yt,getRedfinZipCodes as At,getZillowZipCodes as Vt}from"@rent-scraper/api/config";import{spinner as pt,log as c,progress as Gt,intro as _t}from"@clack/prompts";import{setTimeout as mt}from"node:timers/promises";import{s as Wt}from"./scrape-listings.Qpf6zDxA.mjs";import{checkBrowserServer as ft}from"@rent-scraper/utils/config";import Xt from"picocolors";const O=process.env.DEBUG,I=async(t,e,s,i)=>{const{timeoutMs:o}=i??{};if(await C(s))O&&c.warn(`${s} exists, skipping`);else try{O&&c.info(`writing ${s}`);const a=t==="redfin"?await St(e):await Dt(e,{timeoutMs:o});await P(`${s}`,a)}catch(a){b(`error fetching html for ${e}`,a)}},Kt=async(t,{timeoutMs:e})=>{if(await C(t)){const s=JSON.parse(await tt(t,"utf8"))||{},{hdpUrl:i}=s||{};if(i){const o=`https://www.zillow.com${i}`,a=t.replace(".json",".html");await C(a)?O&&c.warn(`file already exists, ${a}`):await I("zillow",o,a,{timeoutMs:e})}else b(`file is empty, ${t}`)}else O&&c.warn(`file does not exist at this path, ${t}, skipping`)},ht=async(t,e)=>{const{timeoutMs:s}=e??{};await Promise.all(t.map(async i=>await Kt(i,{timeoutMs:s})))},Qt=(t,e="zillow")=>{if(e==="zillow"){const{zpid:s,detailUrl:i}=t||{},o=i&&!i.startsWith("http")?`https://www.zillow.com${i}`:i;return{id:s,url:o}}else if(e==="redfin"){const{propertyId:s,url:i}=t?.homeData||{};return{id:s,url:`https://www.redfin.com${i}`}}},it=async(t,e,s,i=s,o)=>{const{timeoutMs:a,run:r=1,reruns:l=0,skipBotCheck:h=!1}=o??{},w=new E;s||b("inputDirectory is required"),t==="zillow"&&!h&&await V();const m=[],$=[],d=pt();d.start("Downloading listings html files");for(let g=1;g<=l+1;g++){l>0&&g>1&&w.add(`rerun ${g-1} of ${l}`);const p=m.length;(g===1||p)&&await Promise.all((p?m:e).map(async L=>{const j=`${s}/${L}.json`;if(await C(j))try{const{results:S}=await Et(j)||{};if(!S)w.add(`empty file, ${j}`);else{const B=i?`${i}/${L}`:`${s}/${L}`;await M(B,{recursive:!0}),S?.length?await Promise.all(S.map(async Z=>{const{id:k,url:x}=Qt(Z,t)??{};if(!x)return w.add(`url missing for ${k}`);const D=`${k}.html`,F=`${B}/${D}`;try{await I(t,x,F,{timeoutMs:a})}catch(z){const{status:J,message:T}=H(z);wt(J,T)||$.push({url:x,filePath:F}),w.add("scrape listing html error: "+(T??`error fetching listing for id, ${z}`))}})):w.add(`no results for file, ${j}`)}}catch(S){m.push(L);const{message:B}=H(S);w.add("scrape listing html error: "+(B??`error reading json data, ${j}, ${S}`))}else O&&c.warn(`file does not exist, ${j}, skipping`)}))}$.length>0&&(await mt(2e3),await Promise.all($.map(async({url:g,filePath:p})=>{try{await I(t,g,p,{timeoutMs:a})}catch(L){const{message:j}=H(L);w.add("scrape listing html retry error: "+(j??`error fetching ${g}`))}})));const y=await N(),f=await q(),u=t==="zillow"?y:f;e.length>0?(d.stop("Listings HTML files have been saved to:"),c.message(n.join(u,t,"listings",n.basename(s)))):d.stop("No listings HTML files saved.");const v=n.join(u,t,"logs");if(await M(v,{recursive:!0}),w.get().filter(g=>!g.includes("rerun ")).length>l){const g=`${n.basename(s)}-html-errors-${r}.txt`,p=n.join(v,g);await w.write(p,[...new Set(w.get())].join(`
|
|
2
|
+
`)),O&&c.error(`There were errors during processing, see ${n.resolve(p)}`)}},te=async(t,e,s,i)=>{const{timeoutMs:o,run:a=1,reruns:r=0,skipBotCheck:l=!1}=i??{},h=new E;s||b("inputDirectory is required"),t==="zillow"&&!l&&await V();const w=[];for(let f=1;f<=r+1;f++){r>0&&f>1&&h.add(`rerun ${f-1} of ${r}`);const u=w.length;(f===1||u)&&await Promise.all((u?w:e).map(async v=>{const g=`${s}/${v}`;if(await C(g))try{const p=await et(g,{extension:".json",prependDirectory:!0});await ht(p,{timeoutMs:o})}catch(p){w.push(v);const{message:L}=H(p);h.add("scrape listing html error: "+(L??`Error during fetch for ${v}, ${p}`))}else h.add(`listing directory does not exist, ${g}`)}))}const m=await N(),$=await q(),d=t==="zillow"?m:$,y=n.join(d,t,"logs");if(await M(y,{recursive:!0}),h.get().filter(f=>!f.includes("rerun ")).length>r){const f=`${n.basename(s)}-html-errors-${a}.txt`,u=n.join(y,f);await h.write(u,[...new Set(h.get())].join(`
|
|
3
|
+
`)),O&&c.error(`There were errors during processing, see ${n.resolve(u)}`)}},ee=async(t,e,s=e,i)=>{const{skipBotCheck:o=!1}=i??{},a=new E;if(t==="zillow"&&!o&&await V(),e||b("inputDirectory is required"),await M(s,{recursive:!0}),await C(e))try{const d=await et(e,{extension:".json",prependDirectory:!0});await ht(d)}catch(d){const{message:y}=H(d);a.add("scrape listing html error: "+(y??`Error during fetch for ${e}, ${d}`))}else a.add(`inputDirectory does not exist, ${e}`);const r=await N(),l=await q(),h=t==="zillow"?r:l,w=n.join(h,t,"logs");await M(w,{recursive:!0});const m=`${n.basename(e)}-html-errors.txt`,$=n.join(w,m);a.get().length>0&&(await a.write($,[...new Set(a.get())].join(`
|
|
4
|
+
`)),O&&c.error(`There were errors during processing, see ${n.resolve($)}`))},se=(t,e)=>t==="redfin"?`https://www.redfin.com/home/${e}`:t==="zillow"?`https://www.zillow.com/homedetails/${e}_zpid`:null,ie=async(t,e,s,i)=>{const{skipBotCheck:o=!1}=i??{};t==="zillow"&&!o&&await V(),s||b("outputDirectory is required"),await M(s,{recursive:!0}),await Promise.all(e.map(async a=>{const r=se(t,a),l=`${s}/${a}.html`;r&&await I(t,r,l)}))},R=process.env.DEBUG,yt=async t=>{if(await C(t)){const e=t.replace(".html",".json");if(await C(e))R&&c.warning(`file already exists, ${e}`);else{R&&c.message(`scraping data for ${t}`);const s=(await tt(t)).toString();(!s||s.trim()==="")&&(await X(t),b(`empty file found at ${t}, deleted for retry`)),s.includes("px-captcha")&&(await X(t),b(`captcha page found in ${t}, deleted for retry`));const i=await Pt(s);if(!i?.priceHistory&&i?.bestMatchedUnit?.hdpUrl){R&&c.warning(`rescraping ${t} - https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`),await C(t)&&(R&&c.warning(`deleting ${t}`),await X(t)),await C(e)&&(R&&c.warning(`deleting ${e}`),await X(e));const o=`https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`;await I("zillow",o,t),await yt(t)}else i?(R&&c.info(`writing ${e}`),await P(`${e}`,JSON.stringify(i))):b(`problem scraping data for ${t}`)}}},re=async t=>{if(await C(t)){const e=t.replace(".html",".json");if(await C(e))R&&c.warning(`file already exists, ${e}`);else{R&&c.message(`scraping data for ${t}`);const s=(await tt(t)).toString(),i=Ft(s);i?(R&&c.warning(`writing ${e}`),await P(`${e}`,JSON.stringify(i))):b(`problem scraping data for ${t}`)}}},$t=async(t,e,s)=>{let i=0;for(const o of ut(e,5)){const a=await Promise.all(o.map(async r=>{try{t==="redfin"?await re(r):await yt(r);const l=r.replace(".html",".json");return await C(l)?1:0}catch(l){const{message:h}=H(l);return s?.add(h),0}}));i+=a.reduce((r,l)=>r+l,0)}return i},rt=async(t,e,s)=>{const i=new E;let o=0;s||b("inputDirectory is required");const a=Gt({style:"heavy",max:100,size:50});a.start("Scraping listings data");const r=e.length,l=r<20?r+1:20,h=r>l?st(r/l,1):r,w=ut(e,h);for(const[f,u]of w.entries()){const v=st((Number(f)+1)/w.length*100,1);a.advance(st(1/w.length*100,1),`Scraping listings (${Nt(v)})`),await Promise.all(u.map(async g=>{R&&c.message(`processing files for ${g}`);const p=`${s}/${g}`;if(await C(p)){const L=await et(p,{extension:".html",prependDirectory:!0}),j=await $t(t,L,i);o=o+j}else i.add(`listing directory does not exist, ${p}`)}))}const m=await N(),$=await q(),d=t==="zillow"?m:$;o>0?(a.stop("Listings data has been saved to:"),c.message(n.join(d,t,"listings",n.basename(s)))):a.stop("No listings data saved.");const y=n.join(d,t,"logs");if(await M(y,{recursive:!0}),i.get().length>0){const f=`${n.basename(s)}-listing-errors.txt`,u=n.join(y,f);await i.write(u,[...new Set(i.get())].join(`
|
|
5
|
+
`)),c.error(`There were errors during processing, see ${n.resolve(u)}`)}return{numListings:o}},at=process.env.DEBUG,ae=async(t,e,s,i,o)=>{const{daysOnZillow:a,timeoutMs:r}=o??{};if(await C(e))at&&c.warning(`${t} exists, skipping`),s.push(t);else{const l=await Rt({zipCode:t,daysOnZillow:a,mergePageResults:!0,timeoutMs:r});l?(at&&c.info(`writing ${e}`),await P(e,JSON.stringify(l)),s.push(t)):(at&&c.warning(`no results for ${t}`),i.push(t))}},nt=async(t,e,s)=>{const{daysListed:i,timeoutMs:o,run:a=1,reruns:r=0,fetchListings:l=!1,skipBotCheck:h=!1,silent:w=!1}=s??{},m=new E;h||await V({fetchListings:l});const $=[],d=[],y=[],f=[];await M(e,{recursive:!0});const u=pt();w||u.start("Scraping Zillow search results");for(let p=1;p<=r+1;p++){r>0&&p>1&&m.add(`rerun ${p-1} of ${r}`);const L=d.length;(p===1||L)&&await Promise.all((L?d:t).map(async j=>{const S=`${j}.json`,B=`${e}/${S}`;try{await ae(j,B,$,f,{daysOnZillow:i,timeoutMs:o})}catch(Z){d.push(j);const{status:k,message:x}=H(Z);wt(k,x)&&y.push(j),m.add("scrape listing results error: "+(x??`Error during fetch for ${j}, ${Z}`))}}))}const v=await N();w||($.length>0?(u.stop("Zillow search results have been saved to:"),c.message(n.join(v,"zillow","results",n.basename(e)))):u.stop("No results saved."));const g=n.join(v,"zillow","logs");if(await M(g,{recursive:!0}),m.get().filter(p=>!p.includes("rerun ")).length>r){const p=`${n.basename(e)}-results-errors-${a}.txt`,L=n.join(g,p);await m.write(L,[...new Set(m.get())].join(`
|
|
6
|
+
`)),w||c.error(`There were errors during processing, see ${n.resolve(L)}`)}return{validZipCodes:$,botFilteredZipCodes:y,noResultsZipCodes:f}},ne=async(t,e,s,i)=>{const{daysListed:o,timeoutMs:a}=i??{};if(await C(e))console.log(`${t} exists, skipping`),s.push(t);else{const r=await Ht({zipCode:t,daysListed:o,timeoutMs:a});r?(console.log(`writing ${e}`),await P(e,JSON.stringify(r)),s.push(t)):console.log(`no results for ${t}`)}},vt=async(t,e,s)=>{const{daysListed:i,timeoutMs:o,run:a=1,reruns:r=0}=s??{},l=new E,h=[],w=[];await M(e,{recursive:!0});for(let d=1;d<=r+1;d++){r>0&&d>1&&l.add(`rerun ${d-1} of ${r}`);const y=w.length;(d===1||y)&&await Promise.all((y?w:t).map(async f=>{const u=`${f}.json`,v=`${e}/${u}`;try{await ne(f,v,h,{daysListed:i,timeoutMs:o})}catch(g){w.push(f);const{message:p}=H(g);l.add("scrape listing results error: "+(p??`Error during fetch for ${f}, ${g}`))}}))}const m=await q(),$=n.join(m,"redfin","logs");if(await M($,{recursive:!0}),l.get().filter(d=>!d.includes("rerun ")).length>r){const d=`${n.basename(e)}-results-errors-${a}.txt`,y=n.join($,d);await l.write(y,[...new Set(l.get())].join(`
|
|
7
|
+
`)),console.log(`\x1B[41m
|
|
8
|
+
%s\x1B[0m`,`There were errors during processing, see ${n.resolve(y)}`)}return{validZipCodes:h}},oe=async t=>{try{const e=(await ct(t,{withFileTypes:!0})).filter(s=>s.isDirectory()).map(s=>s.name).sort();return e.length>0?e[e.length-1]:null}catch{return null}},Lt=async()=>{await lt.post("http://localhost:8082/browser/close")},le=async()=>{await lt.post("http://localhost:8082/server/shutdown")},ce=async(t,e,s,{daysListed:i,timeoutMs:o,run:a,reruns:r,preValidatedZipCodes:l=[],retry:h=!1})=>{let w=[],m=[],$=[];if(t.length>0){if(await(await Ut()?Ot():dt()),await Lt(),{validZipCodes:w,botFilteredZipCodes:m,noResultsZipCodes:$}=await nt(t,e,{daysListed:i,timeoutMs:o,run:a,reruns:r,skipBotCheck:!0}),h&&m.length>0){c.warn(`Bot filtering hit ${m.length} of ${t.length} zip code(s) \u2014 will retry after listings are fetched`),await Tt()?(await dt(),await Lt()):await mt(3e3);const f=m.length,u=await nt(m,e,{daysListed:i,timeoutMs:o,run:a,reruns:r,skipBotCheck:!0,silent:!0});w=[...w,...u.validZipCodes],m=u.botFilteredZipCodes,$=[...$,...u.noResultsZipCodes];const v=u.validZipCodes.length;c.info(`Inline retry: recovered ${v} of ${f} \xB7 ${m.length} still bot-filtered`)}w.length===0&&t.length>0&&(l.length>0?c.warn(`No new results for re-fetched zip codes \u2014 proceeding with ${l.length} previously fetched`):c.warn("No results returned for any zip codes \u2014 possible soft bot filtering. Please try again shortly.")),m.length>0&&c.warn(`${m.length} zip code(s) still bot-filtered \u2014 use --rerun to retry`)}const d=[...l,...w];await it("zillow",d,e,s,{timeoutMs:o,run:a,reruns:r,skipBotCheck:!0});const{numListings:y}=await rt("zillow",d,s);return{numListings:y,validZipCodes:d,noResultsZipCodes:$}},we=async(t,e,s,{daysListed:i,timeoutMs:o,run:a,reruns:r})=>{const{validZipCodes:l}=await vt(t,e,{daysListed:i,timeoutMs:o,run:a,reruns:r});await it("redfin",l,e,s,{timeoutMs:o,run:a,reruns:r});const{numListings:h}=await rt("redfin",l,s);return{numListings:h,validZipCodes:l}};async function de(){_t(Xt.inverse(" scrape listings "));const t=Bt(process.argv.slice(2)),e=t.source??"zillow";e==="zillow"&&(await ft()||b("Please launch the browser server before scraping."));const s=await N(),i=await q(),o=e==="zillow"?s:i,a=t["days-listed"]??(e==="zillow"?await qt():await It())??1,r=t.runs??1,l=t.reruns??0,h=t["timeout-ms"]??6e4,w=t.limit??await Jt()??void 0,m=t.offset??await Yt()??0,$=t.retry??!1,d=t.rerun,y=typeof d=="string"?d:d?await oe(n.join(o,e,"results")):null;d&&!y&&b("No previous run found to rerun."),y&&c.info(`Rerunning from: ${y}`);const f=y??kt().format("YYYY-MM-DD-HHmm"),u=t["results-directory"]??n.join(o,e,"results",f),v=t["listings-directory"]??n.join(o,e,"listings",f),g=t["logs-directory"]??n.join(o,e,"logs");let p=0,L=0;const j=async()=>{(e==="zillow"||e==="redfin")&&await ft()&&await le()};try{if(e==="redfin"){const B=await At();for(let Z=1;Z<=r;Z++){B||b("zip codes required, please run the createConfig script");const k=B,{numListings:x,validZipCodes:D}=await we(k,u,v,{daysListed:a,timeoutMs:h,run:Z,reruns:l});if(Z===r){const F=Object.entries({numListings:x}).map(([K,_])=>`${K}: ${_}`).join(`
|
|
9
|
+
`),z=gt(k,D).join(`
|
|
10
|
+
`);await M(g,{recursive:!0});const J=`${n.basename(u)}-scraping-results.txt`,T=n.join(g,J),G=`${n.basename(u)}-invalid-zipcodes.txt`,U=n.join(g,G);await P(T,F),await P(U,z)}}}else if(e==="zillow"){const B=await Vt();B||b("zip codes required, please run the createConfig script");const Z=B.slice(m,w?m+w:void 0);let k=Z,x=[];if(y){const D=await ct(u).catch(()=>[]),F=new Set(D.filter(z=>z.endsWith(".json")).map(z=>z.replace(".json","")));x=Z.filter(z=>F.has(String(z))),k=Z.filter(z=>!F.has(String(z))),k.length>0?c.info(`${x.length} already fetched \xB7 retrying ${k.length} missing`):c.info(`All ${x.length} zip codes already fetched \xB7 proceeding to HTML scraping`)}for(let D=1;D<=r;D++){const{numListings:F,validZipCodes:z,noResultsZipCodes:J}=await ce(k,u,v,{daysListed:a,timeoutMs:h,run:D,reruns:l,preValidatedZipCodes:x,retry:$});if(D===r){const T=n.basename(v),G=n.join(s,"zillow","csv",`${T}.csv`),U=await Wt(v,G);p=U,U>0&&c.info(`CSV exported: ${U} listings \u2192 ${G}`);const K=Object.entries({numListings:F,numCsvListings:U}).map(([A,Mt])=>`${A}: ${Mt}`).join(`
|
|
11
|
+
`),_=gt(Z,z),ot=new Set(J),Y=_.filter(A=>!ot.has(A)),W=_.filter(A=>ot.has(A));await M(g,{recursive:!0});const jt=`${n.basename(u)}-scraping-results.txt`,Ct=n.join(g,jt),Zt=`${n.basename(u)}-zipcodes-no-results.txt`,zt=n.join(g,Zt),bt=`${n.basename(u)}-zipcodes-errored.txt`,xt=n.join(g,bt);await P(Ct,K),W.length>0&&await P(zt,W.join(`
|
|
12
|
+
`)),Y.length>0&&await P(xt,Y.join(`
|
|
13
|
+
`)),L=Y.length;const Q=[`${z.length}/${Z.length} with results`];W.length>0&&Q.push(`${W.length} no results`),Y.length>0&&Q.push(`${Y.length} errored`),c.info(`Zip codes: ${Q.join(" \xB7 ")}`),c.info(`Listings: ${F} parsed \xB7 ${U} exported to CSV`)}}}const S=e==="zillow"?p>0:!0;return S?c.success("Scraping complete!"):c.warn("Scraping finished but no listings were exported \u2014 please try again shortly."),{success:S,totalCsvListings:p,missedZipCodes:L,resultsDirectory:u,listingsDirectory:v}}finally{await j()}}export{rt as a,it as b,te as c,ee as d,ie as e,I as f,vt as g,nt as h,de as r,$t as s};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rent-scraper/scrape-listings",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.38",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": {
|
|
@@ -34,8 +34,8 @@
|
|
|
34
34
|
"dayjs": "^1.11.13",
|
|
35
35
|
"minimist": "^1.2.8",
|
|
36
36
|
"picocolors": "^1.1.1",
|
|
37
|
-
"@rent-scraper/
|
|
38
|
-
"@rent-scraper/
|
|
37
|
+
"@rent-scraper/utils": "1.0.38",
|
|
38
|
+
"@rent-scraper/api": "1.0.38"
|
|
39
39
|
},
|
|
40
40
|
"devDependencies": {
|
|
41
41
|
"@types/minimist": "^1.2.5",
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
import Mt from"minimist";import Bt from"dayjs";import n from"path";import lt from"axios";import{mkdir as M,writeFile as P,readFile as tt,unlink as _,readdir as ct}from"fs/promises";import{checkForZillowBotFiltering as V,isZillowBotFiltering as wt,fetchHtmlFromRedfinListingUrl as kt,fetchHtmlFromZillowListingUrl as St,scrapeDataFromRedfinListingHtml as Dt,scrapeDataFromZillowListingHtml as Ft,getZillowListingResults as Pt,getRedfinListingResults as Rt,waitForZillowCaptchaSolve as Ht,waitForSolvedZillowCaptcha as dt,getZillowForceCaptcha as Ot,isBrowserShowingCaptcha as Ut}from"@rent-scraper/api";import{ErrorLog as T,throwError as b,checkForFile as C,parseJsonFile as Et,parseError as H,readFilesInDirectory as et,roundValue as st,chunkArray as Tt,parsePercentage as Nt,compareArrays as ut}from"@rent-scraper/utils";import{getZillowOutputPath as N,getRedfinOutputPath as q,getZillowDaysListed as qt,getRedfinDaysListed as It,getZillowLimit as Jt,getZillowOffset as Yt,getRedfinZipCodes as At,getZillowZipCodes as Vt}from"@rent-scraper/api/config";import{spinner as gt,log as l,progress as Gt,intro as Wt}from"@clack/prompts";import{setTimeout as pt}from"node:timers/promises";import{s as Xt}from"./scrape-listings.Qpf6zDxA.mjs";import{checkBrowserServer as mt}from"@rent-scraper/utils/config";import _t from"picocolors";const O=process.env.DEBUG,I=async(t,e,s,i)=>{const{timeoutMs:r}=i??{};if(await C(s))O&&l.warn(`${s} exists, skipping`);else try{O&&l.info(`writing ${s}`);const a=t==="redfin"?await kt(e):await St(e,{timeoutMs:r});await P(`${s}`,a)}catch(a){b(`error fetching html for ${e}`,a)}},Kt=async(t,{timeoutMs:e})=>{if(await C(t)){const s=JSON.parse(await tt(t,"utf8"))||{},{hdpUrl:i}=s||{};if(i){const r=`https://www.zillow.com${i}`,a=t.replace(".json",".html");await C(a)?O&&l.warn(`file already exists, ${a}`):await I("zillow",r,a,{timeoutMs:e})}else b(`file is empty, ${t}`)}else O&&l.warn(`file does not exist at this path, ${t}, skipping`)},ht=async(t,e)=>{const{timeoutMs:s}=e??{};await Promise.all(t.map(async i=>await Kt(i,{timeoutMs:s})))},Qt=(t,e="zillow")=>{if(e==="zillow"){const{zpid:s,detailUrl:i}=t||{},r=i&&!i.startsWith("http")?`https://www.zillow.com${i}`:i;return{id:s,url:r}}else if(e==="redfin"){const{propertyId:s,url:i}=t?.homeData||{};return{id:s,url:`https://www.redfin.com${i}`}}},it=async(t,e,s,i=s,r)=>{const{timeoutMs:a,run:o=1,reruns:d=0,skipBotCheck:y=!1}=r??{},c=new T;s||b("inputDirectory is required"),t==="zillow"&&!y&&await V();const m=[],$=[],w=gt();w.start("Downloading listings html files");for(let g=1;g<=d+1;g++){d>0&&g>1&&c.add(`rerun ${g-1} of ${d}`);const p=m.length;(g===1||p)&&await Promise.all((p?m:e).map(async L=>{const j=`${s}/${L}.json`;if(await C(j))try{const{results:S}=await Et(j)||{};if(!S)c.add(`empty file, ${j}`);else{const B=i?`${i}/${L}`:`${s}/${L}`;await M(B,{recursive:!0}),S?.length?await Promise.all(S.map(async Z=>{const{id:k,url:x}=Qt(Z,t)??{};if(!x)return c.add(`url missing for ${k}`);const D=`${k}.html`,F=`${B}/${D}`;try{await I(t,x,F,{timeoutMs:a})}catch(z){const{status:J,message:E}=H(z);wt(J,E)||$.push({url:x,filePath:F}),c.add("scrape listing html error: "+(E??`error fetching listing for id, ${z}`))}})):c.add(`no results for file, ${j}`)}}catch(S){m.push(L);const{message:B}=H(S);c.add("scrape listing html error: "+(B??`error reading json data, ${j}, ${S}`))}else O&&l.warn(`file does not exist, ${j}, skipping`)}))}$.length>0&&(await pt(2e3),await Promise.all($.map(async({url:g,filePath:p})=>{try{await I(t,g,p,{timeoutMs:a})}catch(L){const{message:j}=H(L);c.add("scrape listing html retry error: "+(j??`error fetching ${g}`))}})));const f=await N(),h=await q(),u=t==="zillow"?f:h;e.length>0?(w.stop("Listings HTML files have been saved to:"),l.message(n.join(u,t,"listings",n.basename(s)))):w.stop("No listings HTML files saved.");const v=n.join(u,t,"logs");if(await M(v,{recursive:!0}),c.get().filter(g=>!g.includes("rerun ")).length>d){const g=`${n.basename(s)}-html-errors-${o}.txt`,p=n.join(v,g);await c.write(p,[...new Set(c.get())].join(`
|
|
2
|
-
`)),O&&l.error(`There were errors during processing, see ${n.resolve(p)}`)}},te=async(t,e,s,i)=>{const{timeoutMs:r,run:a=1,reruns:o=0,skipBotCheck:d=!1}=i??{},y=new T;s||b("inputDirectory is required"),t==="zillow"&&!d&&await V();const c=[];for(let h=1;h<=o+1;h++){o>0&&h>1&&y.add(`rerun ${h-1} of ${o}`);const u=c.length;(h===1||u)&&await Promise.all((u?c:e).map(async v=>{const g=`${s}/${v}`;if(await C(g))try{const p=await et(g,{extension:".json",prependDirectory:!0});await ht(p,{timeoutMs:r})}catch(p){c.push(v);const{message:L}=H(p);y.add("scrape listing html error: "+(L??`Error during fetch for ${v}, ${p}`))}else y.add(`listing directory does not exist, ${g}`)}))}const m=await N(),$=await q(),w=t==="zillow"?m:$,f=n.join(w,t,"logs");if(await M(f,{recursive:!0}),y.get().filter(h=>!h.includes("rerun ")).length>o){const h=`${n.basename(s)}-html-errors-${a}.txt`,u=n.join(f,h);await y.write(u,[...new Set(y.get())].join(`
|
|
3
|
-
`)),O&&l.error(`There were errors during processing, see ${n.resolve(u)}`)}},ee=async(t,e,s=e,i)=>{const{skipBotCheck:r=!1}=i??{},a=new T;if(t==="zillow"&&!r&&await V(),e||b("inputDirectory is required"),await M(s,{recursive:!0}),await C(e))try{const w=await et(e,{extension:".json",prependDirectory:!0});await ht(w)}catch(w){const{message:f}=H(w);a.add("scrape listing html error: "+(f??`Error during fetch for ${e}, ${w}`))}else a.add(`inputDirectory does not exist, ${e}`);const o=await N(),d=await q(),y=t==="zillow"?o:d,c=n.join(y,t,"logs");await M(c,{recursive:!0});const m=`${n.basename(e)}-html-errors.txt`,$=n.join(c,m);a.get().length>0&&(await a.write($,[...new Set(a.get())].join(`
|
|
4
|
-
`)),O&&l.error(`There were errors during processing, see ${n.resolve($)}`))},se=(t,e)=>t==="redfin"?`https://www.redfin.com/home/${e}`:t==="zillow"?`https://www.zillow.com/homedetails/${e}_zpid`:null,ie=async(t,e,s,i)=>{const{skipBotCheck:r=!1}=i??{};t==="zillow"&&!r&&await V(),s||b("outputDirectory is required"),await M(s,{recursive:!0}),await Promise.all(e.map(async a=>{const o=se(t,a),d=`${s}/${a}.html`;o&&await I(t,o,d)}))},R=process.env.DEBUG,ft=async t=>{if(await C(t)){const e=t.replace(".html",".json");if(await C(e))R&&l.warning(`file already exists, ${e}`);else{R&&l.message(`scraping data for ${t}`);const s=(await tt(t)).toString();(!s||s.trim()==="")&&(await _(t),b(`empty file found at ${t}, deleted for retry`)),s.includes("px-captcha")&&(await _(t),b(`captcha page found in ${t}, deleted for retry`));const i=await Ft(s);if(!i?.priceHistory&&i?.bestMatchedUnit?.hdpUrl){R&&l.warning(`rescraping ${t} - https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`),await C(t)&&(R&&l.warning(`deleting ${t}`),await _(t)),await C(e)&&(R&&l.warning(`deleting ${e}`),await _(e));const r=`https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`;await I("zillow",r,t),await ft(t)}else i?(R&&l.info(`writing ${e}`),await P(`${e}`,JSON.stringify(i))):b(`problem scraping data for ${t}`)}}},re=async t=>{if(await C(t)){const e=t.replace(".html",".json");if(await C(e))R&&l.warning(`file already exists, ${e}`);else{R&&l.message(`scraping data for ${t}`);const s=(await tt(t)).toString(),i=Dt(s);i?(R&&l.warning(`writing ${e}`),await P(`${e}`,JSON.stringify(i))):b(`problem scraping data for ${t}`)}}},yt=async(t,e,s)=>(await Promise.all(e.map(async i=>{try{t==="redfin"?await re(i):await ft(i);const r=i.replace(".html",".json");return await C(r)?1:0}catch(r){const{message:a}=H(r);return s?.add(a),0}}))).reduce((i,r)=>i+r,0),rt=async(t,e,s)=>{const i=new T;let r=0;s||b("inputDirectory is required");const a=Gt({style:"heavy",max:100,size:50});a.start("Scraping listings data");const o=e.length,d=o<20?o+1:20,y=o>d?st(o/d,1):o,c=Tt(e,y);for(const[h,u]of c.entries()){const v=st((Number(h)+1)/c.length*100,1);a.advance(st(1/c.length*100,1),`Scraping listings (${Nt(v)})`),await Promise.all(u.map(async g=>{R&&l.message(`processing files for ${g}`);const p=`${s}/${g}`;if(await C(p)){const L=await et(p,{extension:".html",prependDirectory:!0}),j=await yt(t,L,i);r=r+j}else i.add(`listing directory does not exist, ${p}`)}))}const m=await N(),$=await q(),w=t==="zillow"?m:$;r>0?(a.stop("Listings data has been saved to:"),l.message(n.join(w,t,"listings",n.basename(s)))):a.stop("No listings data saved.");const f=n.join(w,t,"logs");if(await M(f,{recursive:!0}),i.get().length>0){const h=`${n.basename(s)}-listing-errors.txt`,u=n.join(f,h);await i.write(u,[...new Set(i.get())].join(`
|
|
5
|
-
`)),l.error(`There were errors during processing, see ${n.resolve(u)}`)}return{numListings:r}},at=process.env.DEBUG,ae=async(t,e,s,i,r)=>{const{daysOnZillow:a,timeoutMs:o}=r??{};if(await C(e))at&&l.warning(`${t} exists, skipping`),s.push(t);else{const d=await Pt({zipCode:t,daysOnZillow:a,mergePageResults:!0,timeoutMs:o});d?(at&&l.info(`writing ${e}`),await P(e,JSON.stringify(d)),s.push(t)):(at&&l.warning(`no results for ${t}`),i.push(t))}},nt=async(t,e,s)=>{const{daysListed:i,timeoutMs:r,run:a=1,reruns:o=0,fetchListings:d=!1,skipBotCheck:y=!1,silent:c=!1}=s??{},m=new T;y||await V({fetchListings:d});const $=[],w=[],f=[],h=[];await M(e,{recursive:!0});const u=gt();c||u.start("Scraping Zillow search results");for(let p=1;p<=o+1;p++){o>0&&p>1&&m.add(`rerun ${p-1} of ${o}`);const L=w.length;(p===1||L)&&await Promise.all((L?w:t).map(async j=>{const S=`${j}.json`,B=`${e}/${S}`;try{await ae(j,B,$,h,{daysOnZillow:i,timeoutMs:r})}catch(Z){w.push(j);const{status:k,message:x}=H(Z);wt(k,x)&&f.push(j),m.add("scrape listing results error: "+(x??`Error during fetch for ${j}, ${Z}`))}}))}const v=await N();c||($.length>0?(u.stop("Zillow search results have been saved to:"),l.message(n.join(v,"zillow","results",n.basename(e)))):u.stop("No results saved."));const g=n.join(v,"zillow","logs");if(await M(g,{recursive:!0}),m.get().filter(p=>!p.includes("rerun ")).length>o){const p=`${n.basename(e)}-results-errors-${a}.txt`,L=n.join(g,p);await m.write(L,[...new Set(m.get())].join(`
|
|
6
|
-
`)),c||l.error(`There were errors during processing, see ${n.resolve(L)}`)}return{validZipCodes:$,botFilteredZipCodes:f,noResultsZipCodes:h}},ne=async(t,e,s,i)=>{const{daysListed:r,timeoutMs:a}=i??{};if(await C(e))console.log(`${t} exists, skipping`),s.push(t);else{const o=await Rt({zipCode:t,daysListed:r,timeoutMs:a});o?(console.log(`writing ${e}`),await P(e,JSON.stringify(o)),s.push(t)):console.log(`no results for ${t}`)}},$t=async(t,e,s)=>{const{daysListed:i,timeoutMs:r,run:a=1,reruns:o=0}=s??{},d=new T,y=[],c=[];await M(e,{recursive:!0});for(let w=1;w<=o+1;w++){o>0&&w>1&&d.add(`rerun ${w-1} of ${o}`);const f=c.length;(w===1||f)&&await Promise.all((f?c:t).map(async h=>{const u=`${h}.json`,v=`${e}/${u}`;try{await ne(h,v,y,{daysListed:i,timeoutMs:r})}catch(g){c.push(h);const{message:p}=H(g);d.add("scrape listing results error: "+(p??`Error during fetch for ${h}, ${g}`))}}))}const m=await q(),$=n.join(m,"redfin","logs");if(await M($,{recursive:!0}),d.get().filter(w=>!w.includes("rerun ")).length>o){const w=`${n.basename(e)}-results-errors-${a}.txt`,f=n.join($,w);await d.write(f,[...new Set(d.get())].join(`
|
|
7
|
-
`)),console.log(`\x1B[41m
|
|
8
|
-
%s\x1B[0m`,`There were errors during processing, see ${n.resolve(f)}`)}return{validZipCodes:y}},oe=async t=>{try{const e=(await ct(t,{withFileTypes:!0})).filter(s=>s.isDirectory()).map(s=>s.name).sort();return e.length>0?e[e.length-1]:null}catch{return null}},vt=async()=>{await lt.post("http://localhost:8082/browser/close")},le=async()=>{await lt.post("http://localhost:8082/server/shutdown")},ce=async(t,e,s,{daysListed:i,timeoutMs:r,run:a,reruns:o,preValidatedZipCodes:d=[],retry:y=!1})=>{let c=[],m=[],$=[];if(t.length>0){if(await(await Ot()?Ht():dt()),await vt(),{validZipCodes:c,botFilteredZipCodes:m,noResultsZipCodes:$}=await nt(t,e,{daysListed:i,timeoutMs:r,run:a,reruns:o,skipBotCheck:!0}),y&&m.length>0){l.warn(`Bot filtering hit ${m.length} of ${t.length} zip code(s) \u2014 will retry after listings are fetched`),await Ut()?(await dt(),await vt()):await pt(3e3);const h=m.length,u=await nt(m,e,{daysListed:i,timeoutMs:r,run:a,reruns:o,skipBotCheck:!0,silent:!0});c=[...c,...u.validZipCodes],m=u.botFilteredZipCodes,$=[...$,...u.noResultsZipCodes];const v=u.validZipCodes.length;l.info(`Inline retry: recovered ${v} of ${h} \xB7 ${m.length} still bot-filtered`)}c.length===0&&t.length>0&&(d.length>0?l.warn(`No new results for re-fetched zip codes \u2014 proceeding with ${d.length} previously fetched`):l.warn("No results returned for any zip codes \u2014 possible soft bot filtering. Please try again shortly.")),m.length>0&&l.warn(`${m.length} zip code(s) still bot-filtered \u2014 use --rerun to retry`)}const w=[...d,...c];await it("zillow",w,e,s,{timeoutMs:r,run:a,reruns:o,skipBotCheck:!0});const{numListings:f}=await rt("zillow",w,s);return{numListings:f,validZipCodes:w,noResultsZipCodes:$}},we=async(t,e,s,{daysListed:i,timeoutMs:r,run:a,reruns:o})=>{const{validZipCodes:d}=await $t(t,e,{daysListed:i,timeoutMs:r,run:a,reruns:o});await it("redfin",d,e,s,{timeoutMs:r,run:a,reruns:o});const{numListings:y}=await rt("redfin",d,s);return{numListings:y,validZipCodes:d}};async function de(){Wt(_t.inverse(" scrape listings "));const t=Mt(process.argv.slice(2)),e=t.source??"zillow";e==="zillow"&&(await mt()||b("Please launch the browser server before scraping."));const s=await N(),i=await q(),r=e==="zillow"?s:i,a=t["days-listed"]??(e==="zillow"?await qt():await It())??1,o=t.runs??1,d=t.reruns??0,y=t["timeout-ms"]??6e4,c=t.limit??await Jt()??void 0,m=t.offset??await Yt()??0,$=t.retry??!1,w=t.rerun,f=typeof w=="string"?w:w?await oe(n.join(r,e,"results")):null;w&&!f&&b("No previous run found to rerun."),f&&l.info(`Rerunning from: ${f}`);const h=f??Bt().format("YYYY-MM-DD-HHmm"),u=t["results-directory"]??n.join(r,e,"results",h),v=t["listings-directory"]??n.join(r,e,"listings",h),g=t["logs-directory"]??n.join(r,e,"logs");let p=0,L=0;const j=async()=>{(e==="zillow"||e==="redfin")&&await mt()&&await le()};try{if(e==="redfin"){const B=await At();for(let Z=1;Z<=o;Z++){B||b("zip codes required, please run the createConfig script");const k=B,{numListings:x,validZipCodes:D}=await we(k,u,v,{daysListed:a,timeoutMs:y,run:Z,reruns:d});if(Z===o){const F=Object.entries({numListings:x}).map(([K,W])=>`${K}: ${W}`).join(`
|
|
9
|
-
`),z=ut(k,D).join(`
|
|
10
|
-
`);await M(g,{recursive:!0});const J=`${n.basename(u)}-scraping-results.txt`,E=n.join(g,J),G=`${n.basename(u)}-invalid-zipcodes.txt`,U=n.join(g,G);await P(E,F),await P(U,z)}}}else if(e==="zillow"){const B=await Vt();B||b("zip codes required, please run the createConfig script");const Z=B.slice(m,c?m+c:void 0);let k=Z,x=[];if(f){const D=await ct(u).catch(()=>[]),F=new Set(D.filter(z=>z.endsWith(".json")).map(z=>z.replace(".json","")));x=Z.filter(z=>F.has(String(z))),k=Z.filter(z=>!F.has(String(z))),k.length>0?l.info(`${x.length} already fetched \xB7 retrying ${k.length} missing`):l.info(`All ${x.length} zip codes already fetched \xB7 proceeding to HTML scraping`)}for(let D=1;D<=o;D++){const{numListings:F,validZipCodes:z,noResultsZipCodes:J}=await ce(k,u,v,{daysListed:a,timeoutMs:y,run:D,reruns:d,preValidatedZipCodes:x,retry:$});if(D===o){const E=n.basename(v),G=n.join(s,"zillow","csv",`${E}.csv`),U=await Xt(v,G);p=U,U>0&&l.info(`CSV exported: ${U} listings \u2192 ${G}`);const K=Object.entries({numListings:F,numCsvListings:U}).map(([A,xt])=>`${A}: ${xt}`).join(`
|
|
11
|
-
`),W=ut(Z,z),ot=new Set(J),Y=W.filter(A=>!ot.has(A)),X=W.filter(A=>ot.has(A));await M(g,{recursive:!0});const Lt=`${n.basename(u)}-scraping-results.txt`,jt=n.join(g,Lt),Ct=`${n.basename(u)}-zipcodes-no-results.txt`,Zt=n.join(g,Ct),zt=`${n.basename(u)}-zipcodes-errored.txt`,bt=n.join(g,zt);await P(jt,K),X.length>0&&await P(Zt,X.join(`
|
|
12
|
-
`)),Y.length>0&&await P(bt,Y.join(`
|
|
13
|
-
`)),L=Y.length;const Q=[`${z.length}/${Z.length} with results`];X.length>0&&Q.push(`${X.length} no results`),Y.length>0&&Q.push(`${Y.length} errored`),l.info(`Zip codes: ${Q.join(" \xB7 ")}`),l.info(`Listings: ${F} parsed \xB7 ${U} exported to CSV`)}}}const S=e==="zillow"?p>0:!0;return S?l.success("Scraping complete!"):l.warn("Scraping finished but no listings were exported \u2014 please try again shortly."),{success:S,totalCsvListings:p,missedZipCodes:L,resultsDirectory:u,listingsDirectory:v}}finally{await j()}}export{rt as a,it as b,te as c,ee as d,ie as e,I as f,$t as g,nt as h,de as r,yt as s};
|