@rent-scraper/scrape-listings 1.0.34 → 1.0.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
import{parseError as t}from"@rent-scraper/utils";import{r as i}from"../shared/scrape-listings.
|
|
1
|
+
import{parseError as t}from"@rent-scraper/utils";import{r as i}from"../shared/scrape-listings.D0kKRGJj.mjs";import{log as m}from"@clack/prompts";import"minimist";import"dayjs";import"path";import"axios";import"fs/promises";import"@rent-scraper/api";import"@rent-scraper/api/config";import"node:timers/promises";import"crypto";import"@rent-scraper/utils/config";import"picocolors";i().then(()=>{process.exit(0)}).catch(r=>{const{message:o}=t(r);m.error(o),process.exit(1)});
|
package/dist/index.mjs
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export{f as fetchListingHtmlByUrlAndExport,r as runScrapeListings,s as scrapeListingDetailsFromHtmlByFilePaths,a as scrapeListingDetailsFromHtmlByZipCodes,e as scrapeListingHtmlByIds,d as scrapeListingHtmlByInputDirectory,b as scrapeListingHtmlByZipCodes,c as scrapeListingHtmlByZipCodesAndListingDetails,h as scrapeRedfinListingResultsByZipCodes,i as scrapeZillowListingResultsByZipCodes,g as scrapeZillowListingsToCsv}from"./shared/scrape-listings.
|
|
1
|
+
export{f as fetchListingHtmlByUrlAndExport,r as runScrapeListings,s as scrapeListingDetailsFromHtmlByFilePaths,a as scrapeListingDetailsFromHtmlByZipCodes,e as scrapeListingHtmlByIds,d as scrapeListingHtmlByInputDirectory,b as scrapeListingHtmlByZipCodes,c as scrapeListingHtmlByZipCodesAndListingDetails,h as scrapeRedfinListingResultsByZipCodes,i as scrapeZillowListingResultsByZipCodes,g as scrapeZillowListingsToCsv}from"./shared/scrape-listings.D0kKRGJj.mjs";import"minimist";import"dayjs";import"path";import"axios";import"fs/promises";import"@rent-scraper/api";import"@rent-scraper/utils";import"@rent-scraper/api/config";import"@clack/prompts";import"node:timers/promises";import"crypto";import"@rent-scraper/utils/config";import"picocolors";
|
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import Mt from"minimist";import ct from"dayjs";import o from"path";import dt from"axios";import{mkdir as Z,writeFile as B,readFile as X,unlink as K,readdir as st}from"fs/promises";import{checkForZillowBotFiltering as J,isZillowBotFiltering as ut,fetchHtmlFromRedfinListingUrl as Bt,fetchHtmlFromZillowListingUrl as Dt,scrapeDataFromRedfinListingHtml as Ft,scrapeDataFromZillowListingHtml as Pt,getZillowListingResults as Rt,getRedfinListingResults as Ut,waitForZillowCaptchaSolve as Nt,waitForSolvedZillowCaptcha as
|
|
2
|
-
`)),U&&
|
|
3
|
-
`)),U&&
|
|
4
|
-
`)),U&&
|
|
5
|
-
`)),
|
|
6
|
-
`)),d
|
|
1
|
+
import Mt from"minimist";import ct from"dayjs";import o from"path";import dt from"axios";import{mkdir as Z,writeFile as B,readFile as X,unlink as K,readdir as st}from"fs/promises";import{checkForZillowBotFiltering as J,isZillowBotFiltering as ut,fetchHtmlFromRedfinListingUrl as Bt,fetchHtmlFromZillowListingUrl as Dt,scrapeDataFromRedfinListingHtml as Ft,scrapeDataFromZillowListingHtml as Pt,getZillowListingResults as Rt,getRedfinListingResults as Ut,waitForZillowCaptchaSolve as Nt,waitForSolvedZillowCaptcha as pt,getZillowForceCaptcha as Ht,isBrowserShowingCaptcha as Ot}from"@rent-scraper/api";import{ErrorLog as O,throwError as C,checkForFile as j,parseJsonFile as Tt,parseError as R,readFilesInDirectory as Q,roundValue as it,chunkArray as It,parsePercentage as Et,compareArrays as wt}from"@rent-scraper/utils";import{getZillowOutputPath as T,getRedfinOutputPath as I,getZillowDaysListed as At,getRedfinDaysListed as Yt,getZillowLimit as qt,getZillowOffset as Jt,getRedfinZipCodes as Vt,getZillowZipCodes as Gt}from"@rent-scraper/api/config";import{spinner as gt,log as d,progress as Wt,intro as Xt}from"@clack/prompts";import{setTimeout as mt}from"node:timers/promises";import{createHash as Kt}from"crypto";import{checkBrowserServer as ht}from"@rent-scraper/utils/config";import Qt from"picocolors";const U=process.env.DEBUG,E=async(t,e,s,i)=>{const{timeoutMs:r}=i??{};if(await j(s))U&&d.warn(`${s} exists, skipping`);else try{U&&d.info(`writing ${s}`);const a=t==="redfin"?await Bt(e):await Dt(e,{timeoutMs:r});await B(`${s}`,a)}catch(a){C(`error fetching html for ${e}`,a)}},te=async(t,{timeoutMs:e})=>{if(await j(t)){const s=JSON.parse(await X(t,"utf8"))||{},{hdpUrl:i}=s||{};if(i){const r=`https://www.zillow.com${i}`,a=t.replace(".json",".html");await j(a)?U&&d.warn(`file already exists, ${a}`):await E("zillow",r,a,{timeoutMs:e})}else C(`file is empty, ${t}`)}else U&&d.warn(`file does not exist at this path, ${t}, skipping`)},ft=async(t,e)=>{const{timeoutMs:s}=e??{};await Promise.all(t.map(async i=>await te(i,{timeoutMs:s})))},ee=(t,e="zillow")=>{if(e==="zillow"){const{zpid:s,detailUrl:i}=t||{},r=i&&!i.startsWith("http")?`https://www.zillow.com${i}`:i;return{id:s,url:r}}else if(e==="redfin"){const{propertyId:s,url:i}=t?.homeData||{};return{id:s,url:`https://www.redfin.com${i}`}}},rt=async(t,e,s,i=s,r)=>{const{timeoutMs:a,run:n=1,reruns:l=0,skipBotCheck:f=!1}=r??{},c=new O;s||C("inputDirectory is required"),t==="zillow"&&!f&&await J();const w=[],y=[],u=gt();u.start("Downloading listings html files");for(let g=1;g<=l+1;g++){l>0&&g>1&&c.add(`rerun ${g-1} of ${l}`);const m=w.length;(g===1||m)&&await Promise.all((m?w:e).map(async b=>{const v=`${s}/${b}.json`;if(await j(v))try{const{results:M}=await Tt(v)||{};if(!M)c.add(`empty file, ${v}`);else{const S=i?`${i}/${b}`:`${s}/${b}`;await Z(S,{recursive:!0}),M?.length?await Promise.all(M.map(async L=>{const{id:k,url:x}=ee(L,t)??{};if(!x)return c.add(`url missing for ${k}`);const D=`${k}.html`,F=`${S}/${D}`;try{await E(t,x,F,{timeoutMs:a})}catch(z){const{status:A,message:H}=R(z);ut(A,H)||y.push({url:x,filePath:F}),c.add("scrape listing html error: "+(H??`error fetching listing for id, ${z}`))}})):c.add(`no results for file, ${v}`)}}catch(M){w.push(b);const{message:S}=R(M);c.add("scrape listing html error: "+(S??`error reading json data, ${v}, ${M}`))}else U&&d.warn(`file does not exist, ${v}, skipping`)}))}y.length>0&&(await mt(2e3),await Promise.all(y.map(async({url:g,filePath:m})=>{try{await E(t,g,m,{timeoutMs:a})}catch(b){const{message:v}=R(b);c.add("scrape listing html retry error: "+(v??`error fetching ${g}`))}})));const $=await T(),h=await I(),p=t==="zillow"?$:h;e.length>0?(u.stop("Listings HTML files have been saved to:"),d.message(o.join(p,t,"listings",o.basename(s)))):u.stop("No listings HTML files saved.");const _=o.join(p,t,"logs");if(await Z(_,{recursive:!0}),c.get().filter(g=>!g.includes("rerun ")).length>l){const g=`${o.basename(s)}-html-errors-${n}.txt`,m=o.join(_,g);await c.write(m,[...new Set(c.get())].join(`
|
|
2
|
+
`)),U&&d.error(`There were errors during processing, see ${o.resolve(m)}`)}},se=async(t,e,s,i)=>{const{timeoutMs:r,run:a=1,reruns:n=0,skipBotCheck:l=!1}=i??{},f=new O;s||C("inputDirectory is required"),t==="zillow"&&!l&&await J();const c=[];for(let h=1;h<=n+1;h++){n>0&&h>1&&f.add(`rerun ${h-1} of ${n}`);const p=c.length;(h===1||p)&&await Promise.all((p?c:e).map(async _=>{const g=`${s}/${_}`;if(await j(g))try{const m=await Q(g,{extension:".json",prependDirectory:!0});await ft(m,{timeoutMs:r})}catch(m){c.push(_);const{message:b}=R(m);f.add("scrape listing html error: "+(b??`Error during fetch for ${_}, ${m}`))}else f.add(`listing directory does not exist, ${g}`)}))}const w=await T(),y=await I(),u=t==="zillow"?w:y,$=o.join(u,t,"logs");if(await Z($,{recursive:!0}),f.get().filter(h=>!h.includes("rerun ")).length>n){const h=`${o.basename(s)}-html-errors-${a}.txt`,p=o.join($,h);await f.write(p,[...new Set(f.get())].join(`
|
|
3
|
+
`)),U&&d.error(`There were errors during processing, see ${o.resolve(p)}`)}},ie=async(t,e,s=e,i)=>{const{skipBotCheck:r=!1}=i??{},a=new O;if(t==="zillow"&&!r&&await J(),e||C("inputDirectory is required"),await Z(s,{recursive:!0}),await j(e))try{const u=await Q(e,{extension:".json",prependDirectory:!0});await ft(u)}catch(u){const{message:$}=R(u);a.add("scrape listing html error: "+($??`Error during fetch for ${e}, ${u}`))}else a.add(`inputDirectory does not exist, ${e}`);const n=await T(),l=await I(),f=t==="zillow"?n:l,c=o.join(f,t,"logs");await Z(c,{recursive:!0});const w=`${o.basename(e)}-html-errors.txt`,y=o.join(c,w);a.get().length>0&&(await a.write(y,[...new Set(a.get())].join(`
|
|
4
|
+
`)),U&&d.error(`There were errors during processing, see ${o.resolve(y)}`))},re=(t,e)=>t==="redfin"?`https://www.redfin.com/home/${e}`:t==="zillow"?`https://www.zillow.com/homedetails/${e}_zpid`:null,ne=async(t,e,s,i)=>{const{skipBotCheck:r=!1}=i??{};t==="zillow"&&!r&&await J(),s||C("outputDirectory is required"),await Z(s,{recursive:!0}),await Promise.all(e.map(async a=>{const n=re(t,a),l=`${s}/${a}.html`;n&&await E(t,n,l)}))},P=process.env.DEBUG,yt=async t=>{if(await j(t)){const e=t.replace(".html",".json");if(await j(e))P&&d.warning(`file already exists, ${e}`);else{P&&d.message(`scraping data for ${t}`);const s=(await X(t)).toString();(!s||s.trim()==="")&&(await K(t),C(`empty file found at ${t}, deleted for retry`)),s.includes("px-captcha")&&(await K(t),C(`captcha page found in ${t}, deleted for retry`));const i=await Pt(s);if(!i?.priceHistory&&i?.bestMatchedUnit?.hdpUrl){P&&d.warning(`rescraping ${t} - https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`),await j(t)&&(P&&d.warning(`deleting ${t}`),await K(t)),await j(e)&&(P&&d.warning(`deleting ${e}`),await K(e));const r=`https://www.zillow.com${i?.bestMatchedUnit?.hdpUrl}`;await E("zillow",r,t),await yt(t)}else i?(P&&d.info(`writing ${e}`),await B(`${e}`,JSON.stringify(i))):C(`problem scraping data for ${t}`)}}},ae=async t=>{if(await j(t)){const e=t.replace(".html",".json");if(await j(e))P&&d.warning(`file already exists, ${e}`);else{P&&d.message(`scraping data for ${t}`);const s=(await X(t)).toString(),i=Ft(s);i?(P&&d.warning(`writing ${e}`),await B(`${e}`,JSON.stringify(i))):C(`problem scraping data for ${t}`)}}},$t=async(t,e,s)=>(await Promise.all(e.map(async i=>{try{t==="redfin"?await ae(i):await yt(i);const r=i.replace(".html",".json");return await j(r)?1:0}catch(r){const{message:a}=R(r);return s?.add(a),0}}))).reduce((i,r)=>i+r,0),nt=async(t,e,s)=>{const i=new O;let r=0;s||C("inputDirectory is required");const a=Wt({style:"heavy",max:100,size:50});a.start("Scraping listings data");const n=e.length,l=n<20?n+1:20,f=n>l?it(n/l,1):n,c=It(e,f);for(const[h,p]of c.entries()){const _=it((Number(h)+1)/c.length*100,1);a.advance(it(1/c.length*100,1),`Scraping listings (${Et(_)})`),await Promise.all(p.map(async g=>{P&&d.message(`processing files for ${g}`);const m=`${s}/${g}`;if(await j(m)){const b=await Q(m,{extension:".html",prependDirectory:!0}),v=await $t(t,b,i);r=r+v}else i.add(`listing directory does not exist, ${m}`)}))}const w=await T(),y=await I(),u=t==="zillow"?w:y;r>0?(a.stop("Listings data has been saved to:"),d.message(o.join(u,t,"listings",o.basename(s)))):a.stop("No listings data saved.");const $=o.join(u,t,"logs");if(await Z($,{recursive:!0}),i.get().length>0){const h=`${o.basename(s)}-listing-errors.txt`,p=o.join($,h);await i.write(p,[...new Set(i.get())].join(`
|
|
5
|
+
`)),d.error(`There were errors during processing, see ${o.resolve(p)}`)}return{numListings:r}},at=process.env.DEBUG,oe=async(t,e,s,i,r)=>{const{daysOnZillow:a,timeoutMs:n}=r??{};if(await j(e))at&&d.warning(`${t} exists, skipping`),s.push(t);else{const l=await Rt({zipCode:t,daysOnZillow:a,mergePageResults:!0,timeoutMs:n});l?(at&&d.info(`writing ${e}`),await B(e,JSON.stringify(l)),s.push(t)):(at&&d.warning(`no results for ${t}`),i.push(t))}},ot=async(t,e,s)=>{const{daysListed:i,timeoutMs:r,run:a=1,reruns:n=0,fetchListings:l=!1,skipBotCheck:f=!1,silent:c=!1}=s??{},w=new O;f||await J({fetchListings:l});const y=[],u=[],$=[],h=[];await Z(e,{recursive:!0});const p=gt();c||p.start("Scraping Zillow search results");for(let m=1;m<=n+1;m++){n>0&&m>1&&w.add(`rerun ${m-1} of ${n}`);const b=u.length;(m===1||b)&&await Promise.all((b?u:t).map(async v=>{const M=`${v}.json`,S=`${e}/${M}`;try{await oe(v,S,y,h,{daysOnZillow:i,timeoutMs:r})}catch(L){u.push(v);const{status:k,message:x}=R(L);ut(k,x)&&$.push(v),w.add("scrape listing results error: "+(x??`Error during fetch for ${v}, ${L}`))}}))}const _=await T();c||(y.length>0?(p.stop("Zillow search results have been saved to:"),d.message(o.join(_,"zillow","results",o.basename(e)))):p.stop("No results saved."));const g=o.join(_,"zillow","logs");if(await Z(g,{recursive:!0}),w.get().filter(m=>!m.includes("rerun ")).length>n){const m=`${o.basename(e)}-results-errors-${a}.txt`,b=o.join(g,m);await w.write(b,[...new Set(w.get())].join(`
|
|
6
|
+
`)),c||d.error(`There were errors during processing, see ${o.resolve(b)}`)}return{validZipCodes:y,botFilteredZipCodes:$,noResultsZipCodes:h}},le=async(t,e,s,i)=>{const{daysListed:r,timeoutMs:a}=i??{};if(await j(e))console.log(`${t} exists, skipping`),s.push(t);else{const n=await Ut({zipCode:t,daysListed:r,timeoutMs:a});n?(console.log(`writing ${e}`),await B(e,JSON.stringify(n)),s.push(t)):console.log(`no results for ${t}`)}},_t=async(t,e,s)=>{const{daysListed:i,timeoutMs:r,run:a=1,reruns:n=0}=s??{},l=new O,f=[],c=[];await Z(e,{recursive:!0});for(let u=1;u<=n+1;u++){n>0&&u>1&&l.add(`rerun ${u-1} of ${n}`);const $=c.length;(u===1||$)&&await Promise.all(($?c:t).map(async h=>{const p=`${h}.json`,_=`${e}/${p}`;try{await le(h,_,f,{daysListed:i,timeoutMs:r})}catch(g){c.push(h);const{message:m}=R(g);l.add("scrape listing results error: "+(m??`Error during fetch for ${h}, ${g}`))}}))}const w=await I(),y=o.join(w,"redfin","logs");if(await Z(y,{recursive:!0}),l.get().filter(u=>!u.includes("rerun ")).length>n){const u=`${o.basename(e)}-results-errors-${a}.txt`,$=o.join(y,u);await l.write($,[...new Set(l.get())].join(`
|
|
7
7
|
`)),console.log(`\x1B[41m
|
|
8
8
|
%s\x1B[0m`,`There were errors during processing, see ${o.resolve($)}`)}return{validZipCodes:f}},bt=["listing_id","listing_source","source_listing_id","parcel_number","listing_url","street_address","cleaned_address","cleaned_unit_number","city","state","zipcode","county","neighborhood_region","is_undisclosed_address","home_status","home_type","bedrooms","year_built","living_area","living_area_units","is_income_restricted","price_at_source","platform_rent_estimate","agent_name","agent_phone_number","broker_name","broker_phone_number","is_owned_by_listing_platform","date_last_updated_at_source","date_scraped","scrape_job_name"],ce=t=>{if(t==null)return"";const e=typeof t=="object"?JSON.stringify(t):String(t);return e.includes(",")||e.includes('"')||e.includes(`
|
|
9
|
-
`)?`"${e.replace(/"/g,'""')}"`:e},de=t=>{const e=/^(.*?)\s+((?:#|APT|UNIT|STE|Suite|Apt|Unit)\s*.+)$/i.exec(t);if(e){const s=e[1].trim().toUpperCase();let i=e[2].trim();return i.startsWith("#")&&(i=`# ${i.replace(/^#\s*/,"").trim()}`),{cleaned:s,unit:i}}return{cleaned:t.toUpperCase(),unit:""}},ue=(t,e)=>{const s=String(t.zpid),i=Kt("md5").update(s).digest("hex"),r=t.streetAddress||t.address?.streetAddress||"",{cleaned:a,unit:n}=de(r),l=t.hdpUrl?`https://www.zillow.com${t.hdpUrl}`:`https://www.zillow.com/homedetails/${s}_zpid/`;return{listing_id:i,listing_source:"zillow",source_listing_id:s,parcel_number:t.resoFacts?.parcelNumber??"",listing_url:l,street_address:r,cleaned_address:a,cleaned_unit_number:n,city:t.city||t.address?.city||"",state:t.state||t.address?.state||"",zipcode:t.zipcode||t.address?.zipcode||"",county:t.county||"",neighborhood_region:t.neighborhoodRegion?.name||t.parentRegion?.name||"",is_undisclosed_address:t.isUndisclosedAddress??"",home_status:t.homeStatus||"",home_type:t.homeType||"",bedrooms:t.bedrooms??"",year_built:t.yearBuilt||t.resoFacts?.yearBuilt||"",living_area:t.livingAreaValue??"",living_area_units:t.livingAreaUnits||"Square Feet",is_income_restricted:t.isIncomeRestricted??"",price_at_source:t.price??"",platform_rent_estimate:t.rentZestimate??"",agent_name:t.attributionInfo?.agentName||"",agent_phone_number:t.attributionInfo?.agentPhoneNumber||"",broker_name:t.attributionInfo?.brokerName||"",broker_phone_number:t.attributionInfo?.brokerPhoneNumber||"",is_owned_by_listing_platform:t.attributionInfo?.mlsName==="Zillow Rentals",date_last_updated_at_source:t.attributionInfo?.lastUpdated||"",date_scraped:t.timestamp?ct(t.timestamp).format("YYYY-MM-DD HH:mm:ss"):"",scrape_job_name:e}},vt=async(t,e)=>{const s=o.basename(t),i=[bt.join(",")];let r;try{r=await st(t,{withFileTypes:!0})}catch{return 0}const a=r.filter(n=>n.isDirectory()).map(n=>n.name);for(const n of a){const l=o.join(t,n);if(!await j(l))continue;const f=await Q(l,{extension:".json",prependDirectory:!0});for(const
|
|
10
|
-
`)),i.length-1},
|
|
11
|
-
`),
|
|
12
|
-
`);await Z(g,{recursive:!0});const A=`${o.basename(
|
|
13
|
-
`),G=
|
|
9
|
+
`)?`"${e.replace(/"/g,'""')}"`:e},de=t=>{const e=/^(.*?)\s+((?:#|APT|UNIT|STE|Suite|Apt|Unit)\s*.+)$/i.exec(t);if(e){const s=e[1].trim().toUpperCase();let i=e[2].trim();return i.startsWith("#")&&(i=`# ${i.replace(/^#\s*/,"").trim()}`),{cleaned:s,unit:i}}return{cleaned:t.toUpperCase(),unit:""}},ue=(t,e)=>{const s=String(t.zpid),i=Kt("md5").update(s).digest("hex"),r=t.streetAddress||t.address?.streetAddress||"",{cleaned:a,unit:n}=de(r),l=t.hdpUrl?`https://www.zillow.com${t.hdpUrl}`:`https://www.zillow.com/homedetails/${s}_zpid/`;return{listing_id:i,listing_source:"zillow",source_listing_id:s,parcel_number:t.resoFacts?.parcelNumber??"",listing_url:l,street_address:r,cleaned_address:a,cleaned_unit_number:n,city:t.city||t.address?.city||"",state:t.state||t.address?.state||"",zipcode:t.zipcode||t.address?.zipcode||"",county:t.county||"",neighborhood_region:t.neighborhoodRegion?.name||t.parentRegion?.name||"",is_undisclosed_address:t.isUndisclosedAddress??"",home_status:t.homeStatus||"",home_type:t.homeType||"",bedrooms:t.bedrooms??"",year_built:t.yearBuilt||t.resoFacts?.yearBuilt||"",living_area:t.livingAreaValue??"",living_area_units:t.livingAreaUnits||"Square Feet",is_income_restricted:t.isIncomeRestricted??"",price_at_source:t.price??"",platform_rent_estimate:t.rentZestimate??"",agent_name:t.attributionInfo?.agentName||"",agent_phone_number:t.attributionInfo?.agentPhoneNumber||"",broker_name:t.attributionInfo?.brokerName||"",broker_phone_number:t.attributionInfo?.brokerPhoneNumber||"",is_owned_by_listing_platform:t.attributionInfo?.mlsName==="Zillow Rentals",date_last_updated_at_source:t.attributionInfo?.lastUpdated||"",date_scraped:t.timestamp?ct(t.timestamp).format("YYYY-MM-DD HH:mm:ss"):"",scrape_job_name:e}},vt=async(t,e)=>{const s=o.basename(t),i=[bt.join(",")];let r;try{r=await st(t,{withFileTypes:!0})}catch{return 0}const a=r.filter(n=>n.isDirectory()).map(n=>n.name);for(const n of a){const l=o.join(t,n);if(!await j(l))continue;const f=await Q(l,{extension:".json",prependDirectory:!0});for(const c of f)try{const w=JSON.parse(await X(c,"utf8"));if(!w.zpid)continue;const y=ue(w,s);i.push(bt.map(u=>ce(y[u])).join(","))}catch{}}return await Z(o.dirname(e),{recursive:!0}),await B(e,i.join(`
|
|
10
|
+
`)),i.length-1},pe=async t=>{try{const e=(await st(t,{withFileTypes:!0})).filter(s=>s.isDirectory()).map(s=>s.name).sort();return e.length>0?e[e.length-1]:null}catch{return null}},jt=async()=>{await dt.post("http://localhost:8082/browser/close")},we=async()=>{await dt.post("http://localhost:8082/server/shutdown")},ge=async(t,e,s,{daysListed:i,timeoutMs:r,run:a,reruns:n,preValidatedZipCodes:l=[],retry:f=!1})=>{let c=[],w=[],y=[];if(t.length>0){if(await(await Ht()?Nt():pt()),await jt(),{validZipCodes:c,botFilteredZipCodes:w,noResultsZipCodes:y}=await ot(t,e,{daysListed:i,timeoutMs:r,run:a,reruns:n,skipBotCheck:!0}),f&&w.length>0){d.warn(`Bot filtering hit ${w.length} of ${t.length} zip code(s) \u2014 will retry after listings are fetched`),await Ot()?(await pt(),await jt()):await mt(3e3);const h=w.length,p=await ot(w,e,{daysListed:i,timeoutMs:r,run:a,reruns:n,skipBotCheck:!0,silent:!0});c=[...c,...p.validZipCodes],w=p.botFilteredZipCodes,y=[...y,...p.noResultsZipCodes];const _=p.validZipCodes.length;d.info(`Inline retry: recovered ${_} of ${h} \xB7 ${w.length} still bot-filtered`)}c.length===0&&t.length>0&&(l.length>0?d.warn(`No new results for re-fetched zip codes \u2014 proceeding with ${l.length} previously fetched`):d.warn("No results returned for any zip codes \u2014 possible soft bot filtering. Please try again shortly.")),w.length>0&&d.warn(`${w.length} zip code(s) still bot-filtered \u2014 use --rerun to retry`)}const u=[...l,...c];await rt("zillow",u,e,s,{timeoutMs:r,run:a,reruns:n,skipBotCheck:!0});const{numListings:$}=await nt("zillow",u,s);return{numListings:$,validZipCodes:u,noResultsZipCodes:y}},me=async(t,e,s,{daysListed:i,timeoutMs:r,run:a,reruns:n})=>{const{validZipCodes:l}=await _t(t,e,{daysListed:i,timeoutMs:r,run:a,reruns:n});await rt("redfin",l,e,s,{timeoutMs:r,run:a,reruns:n});const{numListings:f}=await nt("redfin",l,s);return{numListings:f,validZipCodes:l}};async function he(){Xt(Qt.inverse(" scrape listings "));const t=Mt(process.argv.slice(2)),e=t.source??"zillow";e==="zillow"&&(await ht()||C("Please launch the browser server before scraping."));const s=await T(),i=await I(),r=e==="zillow"?s:i,a=t["days-listed"]??(e==="zillow"?await At():await Yt())??1,n=t.runs??1,l=t.reruns??0,f=t["timeout-ms"]??6e4,c=t.limit??await qt()??void 0,w=t.offset??await Jt()??0,y=t.retry??!1,u=t.rerun,$=typeof u=="string"?u:u?await pe(o.join(r,e,"results")):null;u&&!$&&C("No previous run found to rerun."),$&&d.info(`Rerunning from: ${$}`);const h=$??ct().format("YYYY-MM-DD-HHmm"),p=t["results-directory"]??o.join(r,e,"results",h),_=t["listings-directory"]??o.join(r,e,"listings",h),g=t["logs-directory"]??o.join(r,e,"logs");let m=0,b=0;const v=async()=>{(e==="zillow"||e==="redfin")&&await ht()&&await we()};try{if(e==="redfin"){const S=await Vt();for(let L=1;L<=n;L++){S||C("zip codes required, please run the createConfig script");const k=S,{numListings:x,validZipCodes:D}=await me(k,p,_,{daysListed:a,timeoutMs:f,run:L,reruns:l});if(L===n){const F=Object.entries({numListings:x}).map(([tt,G])=>`${tt}: ${G}`).join(`
|
|
11
|
+
`),z=wt(k,D).join(`
|
|
12
|
+
`);await Z(g,{recursive:!0});const A=`${o.basename(p)}-scraping-results.txt`,H=o.join(g,A),V=`${o.basename(p)}-invalid-zipcodes.txt`,N=o.join(g,V);await B(H,F),await B(N,z)}}}else if(e==="zillow"){const S=await Gt();S||C("zip codes required, please run the createConfig script");const L=S.slice(w,c?w+c:void 0);let k=L,x=[];if($){const D=await st(p).catch(()=>[]),F=new Set(D.filter(z=>z.endsWith(".json")).map(z=>z.replace(".json","")));x=L.filter(z=>F.has(String(z))),k=L.filter(z=>!F.has(String(z))),k.length>0?d.info(`${x.length} already fetched \xB7 retrying ${k.length} missing`):d.info(`All ${x.length} zip codes already fetched \xB7 proceeding to HTML scraping`)}for(let D=1;D<=n;D++){const{numListings:F,validZipCodes:z,noResultsZipCodes:A}=await ge(k,p,_,{daysListed:a,timeoutMs:f,run:D,reruns:l,preValidatedZipCodes:x,retry:y});if(D===n){const H=o.basename(_),V=o.join(s,"zillow","csv",`${H}.csv`),N=await vt(_,V);m=N,N>0&&d.info(`CSV exported: ${N} listings \u2192 ${V}`);const tt=Object.entries({numListings:F,numCsvListings:N}).map(([q,kt])=>`${q}: ${kt}`).join(`
|
|
13
|
+
`),G=wt(L,z),lt=new Set(A),Y=G.filter(q=>!lt.has(q)),W=G.filter(q=>lt.has(q));await Z(g,{recursive:!0});const Lt=`${o.basename(p)}-scraping-results.txt`,zt=o.join(g,Lt),Ct=`${o.basename(p)}-zipcodes-no-results.txt`,Zt=o.join(g,Ct),xt=`${o.basename(p)}-zipcodes-errored.txt`,St=o.join(g,xt);await B(zt,tt),W.length>0&&await B(Zt,W.join(`
|
|
14
14
|
`)),Y.length>0&&await B(St,Y.join(`
|
|
15
|
-
`)),b=Y.length;const et=[`${
|
|
15
|
+
`)),b=Y.length;const et=[`${z.length}/${L.length} with results`];W.length>0&&et.push(`${W.length} no results`),Y.length>0&&et.push(`${Y.length} errored`),d.info(`Zip codes: ${et.join(" \xB7 ")}`),d.info(`Listings: ${F} parsed \xB7 ${N} exported to CSV`)}}}const M=e==="zillow"?m>0:!0;return M?d.success("Scraping complete!"):d.warn("Scraping finished but no listings were exported \u2014 please try again shortly."),{success:M,totalCsvListings:m,missedZipCodes:b,resultsDirectory:p,listingsDirectory:_}}finally{await v()}}export{nt as a,rt as b,se as c,ie as d,ne as e,E as f,vt as g,_t as h,ot as i,he as r,$t as s};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rent-scraper/scrape-listings",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.35",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": {
|
|
@@ -30,8 +30,8 @@
|
|
|
30
30
|
"dayjs": "^1.11.13",
|
|
31
31
|
"minimist": "^1.2.8",
|
|
32
32
|
"picocolors": "^1.1.1",
|
|
33
|
-
"@rent-scraper/utils": "1.0.
|
|
34
|
-
"@rent-scraper/api": "1.0.
|
|
33
|
+
"@rent-scraper/utils": "1.0.35",
|
|
34
|
+
"@rent-scraper/api": "1.0.35"
|
|
35
35
|
},
|
|
36
36
|
"devDependencies": {
|
|
37
37
|
"@types/minimist": "^1.2.5",
|