@rent-scraper/api 1.0.35 → 1.0.36

package/dist/index.mjs

@@ -1,8 +1,8 @@
-import h from"dayjs";import*as M from"cheerio";import{getValueFromConfigFile as F}from"@rent-scraper/utils/config";import{throwError as T,parseError as O,getRandomArrayValue as K}from"@rent-scraper/utils";import{l as Q,g as A,h as N,b as P}from"./shared/api.Bouw8I_6.mjs";export{a as getZillowDaysListed,c as getZillowForceCaptcha,d as getZillowLimit,e as getZillowOffset,f as getZillowOutputPath}from"./shared/api.Bouw8I_6.mjs";import c from"axios";import j from"puppeteer";const H=async(e,t)=>{const{fromFile:r=!0}=t??{};return r?(await F("redfin","regionIds"))[String(e)]??await R(e):await R(e)},g=e=>JSON.parse(e.replace("{}&&","")),R=async e=>{const t="https://www.redfin.com/stingray/do/rental-location-autocomplete",r={"Content-Type":"application/json; charset=utf-8","User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"},o=`?location=${e}&v=2`;try{const a=`${t}${o}`,i=await(await fetch(a,{method:"GET",headers:r})).text(),s=g(i),{exactMatch:n}=s.payload||{},l=Number(n.id.replace(n.type,"").replace("_",""));return l||console.log("no region id found for zip"),l}catch{return console.log("zip code not found"),null}},J=async e=>({data:await f(e)}),v=async e=>{const t=await f(e);return V(t)},V=e=>{const t=M.load(e)('script:contains("ReactServerAgent.cache")').text()?.replace(`_tLAB.wait(function(){
+import h from"dayjs";import*as O from"cheerio";import{getValueFromConfigFile as N}from"@rent-scraper/utils/config";import{throwError as T,parseError as H,getRandomArrayValue as K}from"@rent-scraper/utils";import{l as Q,g as A,h as v,b as P}from"./shared/api.Bouw8I_6.mjs";export{a as getZillowDaysListed,c as getZillowForceCaptcha,d as getZillowLimit,e as getZillowOffset,f as getZillowOutputPath}from"./shared/api.Bouw8I_6.mjs";import c from"axios";import j from"puppeteer";const V=async(e,t)=>{const{fromFile:r=!0}=t??{};return r?(await N("redfin","regionIds"))[String(e)]??await R(e):await R(e)},g=e=>JSON.parse(e.replace("{}&&","")),R=async e=>{const t="https://www.redfin.com/stingray/do/rental-location-autocomplete",r={"Content-Type":"application/json; charset=utf-8","User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"},o=`?location=${e}&v=2`;try{const a=`${t}${o}`,i=await(await fetch(a,{method:"GET",headers:r})).text(),s=g(i),{exactMatch:n}=s.payload||{},l=Number(n.id.replace(n.type,"").replace("_",""));return l||console.log("no region id found for zip"),l}catch{return console.log("zip code not found"),null}},J=async e=>({data:await f(e)}),B=async e=>{const t=await f(e);return U(t)},U=e=>{const t=O.load(e)('script:contains("ReactServerAgent.cache")').text()?.replace(`_tLAB.wait(function(){
 (function (root) {
 /* -- Data -- */
 root.__reactServerState || (root.__reactServerState = {});
-root.__reactServerState.InitialContext = {"ReactServerAgent.cache":`,"")?.split(',"deviceType":"desktop"')[0],r=(t&&JSON.parse(t))?.dataCache,o=r?.["/stingray/api/home/details/initialInfo"]?.res?.text&&g(r["/stingray/api/home/details/initialInfo"].res.text)?.payload,a=o?.propertyId??o?.redirectUrl?.split("/").pop(),i=r?.[`/stingray/api/v1/rentals/homecards?propertyIds=${a}`]?.res?.text,s=i&&g(i),n=r?.["/stingray/api/home/details/belowTheFold"]?.res?.text,l=n&&g(n)?.payload,p=r?.[`/stingray/api/home/details/propertyParcelInfo?propertyId=${a}&pageType=6`]?.res?.text,d=p&&g(p)?.payload,_=s?.homes?.[0]?.rentalExtension?.rentalId,m=r?.[`/stingray/api/v1/rentals/${_}/floorPlans?`]?.res?.text,y=m&&g(m),u={hasPropertyHistory:!!l?.propertyHistoryInfo,propertyId:a,rentalId:_,initialInfo:o,homeCards:s,floorPlans:y,belowTheFold:l,propertyParcelInfo:d};return{timestamp:h(),...u}},f=async(e,t)=>{const{timeoutMs:r}=t??{},o=await Q(),a={headers:{"Content-Type":"text/html; charset=utf-8","User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",...o&&{Cookie:o}},...r&&{signal:(s=>{const n=new AbortController;return setTimeout(()=>n.abort(),s||0),n.signal})(r)}},i=await fetch(e,a);if(i.status===202&&T(`Redfin bot filtering (202) for ${e} \u2014 restart the browser server to refresh the cookie.`),i.ok)return await i.text();{const s=await i.text().catch(()=>""),n=(s&&JSON.parse(s)?.error)??`${i.status} ${i.statusText}`;T(n)}},ee=async(e,t)=>await v(t??`https://www.redfin.com/us/home/${e}`),te=async({regionId:e,zipCode:t,timeoutMs:r,daysListed:o})=>{t&&(e=await H(t)??e);try{const a=await B({regionId:e,timeoutMs:r}),i=a?.numMatchedHomes,s=o?a?.homes?.filter(p=>h().diff(p?.rentalExtension?.freshnessTimestamp,"day")<o):a?.homes,n=s?.map(p=>p?.detailUrl),l=s?.length;return l?{timestamp:h(),...t&&{zipCode:t},regionId:e,numResults:l,totalResultCount:i,results:s,urls:n}:null}catch(a){throw console.log(a),a}},B=async({regionId:e,timeoutMs:t})=>{const r="https://www.redfin.com/stingray/api/v1/search/rentals",o={"Content-Type":"application/json; charset=utf-8","User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"},a=`?al=1&isRentals=true&market=socal&num_homes=350&ord=days-on-redfin-asc&page_number=1&region_id=${e}&region_type=2&sf=1,2,3,5,6,7&start=0&status=9&uipt=1,2,3,4&use_max_pins=true&v=8`,i={headers:o,...t&&{signal:(n=>{const l=new AbortController;return setTimeout(()=>l.abort(),n||0),l.signal})(t)}},s=`${r}${a}`;return await(await fetch(s,i)).json()},U=async(e,t)=>{const{fromFile:r=!0}=t??{};return r?(await F("zillow","regionIds")??{})[String(e)]??null:await re(e)},re=async e=>{const t="https://www.zillow.com/graphql",r={operationName:"getAutocompleteResults",variables:{query:String(e),queryOptions:{maxResults:10,userIdentity:{abKey:"3176854b-962d-4266-8bb8-f9e8f2377158"},userSearchContext:"FOR_RENT",userLocation:{latitude:34.11,longitude:-118.23}},resultType:["REGIONS","FORSALE","RENTALS","SOLD","COMMUNITIES","BUILDER_COMMUNITIES"]},query:`query getAutocompleteResults($query: String!, $queryOptions: SearchAssistanceQueryOptions, $resultType: [SearchAssistanceResultType]) {
+root.__reactServerState.InitialContext = {"ReactServerAgent.cache":`,"")?.split(',"deviceType":"desktop"')[0],r=(t&&JSON.parse(t))?.dataCache,o=r?.["/stingray/api/home/details/initialInfo"]?.res?.text&&g(r["/stingray/api/home/details/initialInfo"].res.text)?.payload,a=o?.propertyId??o?.redirectUrl?.split("/").pop(),i=r?.[`/stingray/api/v1/rentals/homecards?propertyIds=${a}`]?.res?.text,s=i&&g(i),n=r?.["/stingray/api/home/details/belowTheFold"]?.res?.text,l=n&&g(n)?.payload,p=r?.[`/stingray/api/home/details/propertyParcelInfo?propertyId=${a}&pageType=6`]?.res?.text,d=p&&g(p)?.payload,_=s?.homes?.[0]?.rentalExtension?.rentalId,m=r?.[`/stingray/api/v1/rentals/${_}/floorPlans?`]?.res?.text,y=m&&g(m),u={hasPropertyHistory:!!l?.propertyHistoryInfo,propertyId:a,rentalId:_,initialInfo:o,homeCards:s,floorPlans:y,belowTheFold:l,propertyParcelInfo:d};return{timestamp:h(),...u}},f=async(e,t)=>{const{timeoutMs:r}=t??{},o=await Q(),a={headers:{"Content-Type":"text/html; charset=utf-8","User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",...o&&{Cookie:o}},...r&&{signal:(s=>{const n=new AbortController;return setTimeout(()=>n.abort(),s||0),n.signal})(r)}},i=await fetch(e,a);if(i.status===202&&T(`Redfin bot filtering (202) for ${e} \u2014 restart the browser server to refresh the cookie.`),i.ok)return await i.text();{const s=await i.text().catch(()=>""),n=(s&&JSON.parse(s)?.error)??`${i.status} ${i.statusText}`;T(n)}},ee=async(e,t)=>await B(t??`https://www.redfin.com/us/home/${e}`),te=async({regionId:e,zipCode:t,timeoutMs:r,daysListed:o})=>{t&&(e=await V(t)??e);try{const a=await G({regionId:e,timeoutMs:r}),i=a?.numMatchedHomes,s=o?a?.homes?.filter(p=>h().diff(p?.rentalExtension?.freshnessTimestamp,"day")<o):a?.homes,n=s?.map(p=>p?.detailUrl),l=s?.length;return l?{timestamp:h(),...t&&{zipCode:t},regionId:e,numResults:l,totalResultCount:i,results:s,urls:n}:null}catch(a){throw console.log(a),a}},G=async({regionId:e,timeoutMs:t})=>{const r="https://www.redfin.com/stingray/api/v1/search/rentals",o={"Content-Type":"application/json; charset=utf-8","User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"},a=`?al=1&isRentals=true&market=socal&num_homes=350&ord=days-on-redfin-asc&page_number=1&region_id=${e}&region_type=2&sf=1,2,3,5,6,7&start=0&status=9&uipt=1,2,3,4&use_max_pins=true&v=8`,i={headers:o,...t&&{signal:(n=>{const l=new AbortController;return setTimeout(()=>l.abort(),n||0),l.signal})(t)}},s=`${r}${a}`;return await(await fetch(s,i)).json()},x=async(e,t)=>{const{fromFile:r=!0}=t??{};return r?(await N("zillow","regionIds")??{})[String(e)]??null:await re(e)},re=async e=>{const t="https://www.zillow.com/graphql",r={operationName:"getAutocompleteResults",variables:{query:String(e),queryOptions:{maxResults:10,userIdentity:{abKey:"3176854b-962d-4266-8bb8-f9e8f2377158"},userSearchContext:"FOR_RENT",userLocation:{latitude:34.11,longitude:-118.23}},resultType:["REGIONS","FORSALE","RENTALS","SOLD","COMMUNITIES","BUILDER_COMMUNITIES"]},query:`query getAutocompleteResults($query: String!, $queryOptions: SearchAssistanceQueryOptions, $resultType: [SearchAssistanceResultType]) {
   searchAssistanceResult: zgsAutocompleteRequest(
     query: $query
     queryOptions: $queryOptions
@@ -129,7 +129,7 @@ fragment RentalCommunityResultFields on SearchAssistanceRentalCommunityResult {
 fragment BuilderCommunityResultFields on SearchAssistanceBuilderCommunityResult {
   plid
   __typename
-}`},o={accept:"*/*","accept-encoding":"gzip, deflate, br, zstd","accept-language":"en-US,en;q=0.7","client-id":"hops-homepage","content-type":"application/json",origin:"https://www.zillow.com",priority:"u=1, i",referer:"https://www.zillow.com/","sec-ch-ua":'"Brave";v="135", "Not-A.Brand";v="8", "Chromium";v="135"',"sec-ch-ua-mobile":"?0","sec-ch-ua-platform":'"macOS"',"user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36","sec-fetch-site":"same-origin","sec-fetch-dest":"empty","sec-fetch-mode":"cors",cookie:await A()};try{const{data:a}=await c.post(t,r,{headers:o})||{},{regionId:i}=a.data.searchAssistanceResult.results.filter(s=>s?.regionId)?.[0]||{};return i||console.log("no region id found for zip"),i}catch(a){const{status:i,message:s}=O(a)??{};if(C(i,s))T(s,i);else return console.log("zip code not found"),null}},G={Accept:"*/*","Accept-Language":"en-US,en;q=0.5","Content-Type":"application/json",Origin:"https://www.zillow.com",Referer:"https://www.zillow.com/",Priority:"u=1, i","Sec-Ch-Ua":'"Brave";v="147", "Not.A/Brand";v="8", "Chromium";v="147"',"Sec-Ch-Ua-Mobile":"?0","Sec-Ch-Ua-Platform":'"macOS"',"Sec-Fetch-Dest":"empty","Sec-Fetch-Mode":"cors","Sec-Fetch-Site":"same-origin","Sec-Gpc":"1","User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"},x=async({page:e,daysOnZillow:t,regionId:r,zipCode:o,mergePageResults:a,timeoutMs:i})=>{o&&(r=await U(o)??r),e=(a?1:e)??1;const s=await w({page:e,daysOnZillow:t,regionId:r,zipCode:o,timeoutMs:i}),n=s?.cat1?.searchList?.totalResultCount,l=s?.cat1?.searchList?.resultsPerPage,p=s?.cat1?.searchList?.totalPages;if(e&&e>p)return null;const d=a&&Array.from({length:p},(u,I)=>I+1),_=a&&p>1&&d?(await Promise.all(d?.map(async u=>await w({page:u,daysOnZillow:t,regionId:r,zipCode:o},!0)))).flat():s?.cat1?.searchResults?.listResults,m=_?.map(u=>u?.detailUrl),y=_?.length;return y?{timestamp:h(),...o&&{zipCode:o},regionId:r,numResults:y,totalResultCount:n,resultsPerPage:l,...a&&{mergePageResults:a},...d?{pagesCrawled:d?.length}:{page:e},totalPages:p,results:_,urls:m}:null},w=async({page:e,daysOnZillow:t,regionId:r,zipCode:o,timeoutMs:a},i)=>{const s="https://www.zillow.com/async-create-search-page-state",n={searchQueryState:{pagination:{currentPage:e??1},isMapVisible:!1,mapBounds:{east:-116.2057669921875,north:35.424914784130365,south:32.12915408246069,west:-120.3915580078125},regionSelection:[{regionId:r,regionType:7}],filterState:{sortSelection:{value:"days"},doz:{value:String(t??5)},isNewConstruction:{value:!1},isForSaleForeclosure:{value:!1},isForSaleByOwner:{value:!1},isForSaleByAgent:{value:!1},isForRent:{value:!0},isComingSoon:{value:!1},isAuction:{value:!1},isAllHomes:{value:!0}},isListVisible:!0,...o&&{usersSearchTerm:String(o)}},wants:{cat1:["listResults"]},requestId:6,isDebugRequest:!1},l=m=>{const y=new AbortController;return setTimeout(()=>y.abort(),m||0),y.signal},p=await A(),d={headers:{...G,cookie:p},...a&&{signal:l(a)}},{data:_}=await c.put(s,n,d)||{};return i?_?.cat1?.searchResults?.listResults:_},S=async(e,t)=>{const{timeoutMs:r}=t??{},o={headers:{Host:"www.zillow.com",accept:"*/*","content-type":"text/html; charset=utf-8","user-agent":"insomnia/10.3.0"},...r&&{signal:(i=>{const s=new AbortController;return setTimeout(()=>s.abort(),i||0),s.signal})(r)}},{data:a}=await c.options(e,o)||{};if(!a||a===""){const i=new Error("empty response");throw i.response={status:0,data:"empty response"},i}if(typeof a=="string"&&a.includes("px-captcha")){const i=new Error("captcha");throw i.response={status:403,data:"captcha"},i}return a},oe=async e=>({data:await S(e)}),ae=async e=>{const t=await S(e);return z(t)},ie=(e,t)=>{if(t==="building")return e?.initialReduxState?.gdp?.building??null;{const r=e&&JSON.parse(e?.gdpClientCache);return r?Object.values(r).find(o=>o?.property)?.property??null:null}},z=e=>{const t=M.load(e)("#__NEXT_DATA__").text(),r=t&&JSON.parse(t)?.props?.pageProps?.componentProps,o=r?.initialReduxState?.gdp?"building":"property",a=ie(r,o);return{timestamp:h(),...a}},E={accept:"*/*","accept-encoding":"gzip, deflate, br, zstd","accept-language":"en-US,en;q=0.9","client-id":"for-sale-sub-app-browser-client","content-type":"application/json","Cache-Control":"no-cache",Pragma:"no-cache",priority:"u=1, i","sec-ch-ua":'"Google Chrome";v="136", "Not(A:Brand";v="8", "Chromium";v="136"',"sec-ch-ua-mobile":"?0","sec-ch-ua-platform":'"macOS"',"sec-fetch-dest":"empty","sec-fetch-mode":"cors","sec-fetch-site":"same-origin","user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36","x-z-enable-oauth-conversion":"true"},k=async(e,t)=>{const{altFetch:r,onlyRemovedListings:o=!1}=t??{},a=(r?await ne(e,t):await se(e,t))||{},{property:i,...s}=a;if(i){const{buyAbilityData:n,collections:l,nearbyCities:p,nearbyNeighborhoods:d,nearbyZipcodes:_,schools:m,nearbyHomes:y,listingMetadata:u,homeInsights:I,responsivePhotos:ue,submitFlow:ge,contactFormRenderData:he,tourEligibility:Te,onsiteMessage:Ae,ZoDsFsUpsellTop:be,streetViewMetadataUrlMapLightboxAddress:Se,responsivePhotosOriginalRatio:Pe,thirdPartyVirtualTour:Re,vrModel:fe,topNavJson:we,...b}=i||{};if(!b?.priceHistory)return null;if(o){const{homeStatus:D}=b??{};if(!D)return console.log("no home status",e),null;if(D==="FOR_RENT"){const[L]=b?.priceHistory??{};return L?.event?console.log("recent price history not removed",e,L?.event):console.log("no recent price history"),null}}return{timestamp:h(),...b,...s,...r&&{altFetch:r}}}},se=async(e,t)=>{const{timeoutMs:r}=t??{},o="https://www.zillow.com/graphql",a=`?extensions=%7B%22persistedQuery%22%3A%7B%22version%22%3A1%2C%22sha256Hash%22%3A%222a50ebbffc7b6fb6e70fad24b7738df5db7f2ed5b61c2b980542749accff1f17%22%7D%7D&variables=%7B%22zpid%22%3A${e}%2C%22platform%22%3A%22DESKTOP_WEB%22%2C%22formType%22%3A%22OPAQUE%22%2C%22contactFormRenderParameter%22%3A%7B%22zpid%22%3A${e}%2C%22platform%22%3A%22desktop%22%2C%22isDoubleScroll%22%3Atrue%7D%2C%22skipCFRD%22%3Afalse%2C%22ompPlatform%22%3A%22web%22%7D`,i=`${o}${a}`,s=d=>{const _=new AbortController;return setTimeout(()=>_.abort(),d||0),_.signal},n=await A(),l={headers:{...E,cookie:n},...r&&{signal:s(r)}},{data:p}=await c.get(i,l)||{};return p?.data},ne=async(e,{timeoutMs:t})=>{const r="https://www.zillow.com/graphql",o={operationName:"ForRentShopperPlatformFullRenderQuery",variables:{zpid:e,altId:null},query:`query ForRentShopperPlatformFullRenderQuery($zpid: ID!, $altId: ID, $skipPropertyPrice: Boolean = false, $skipPropertyAddress: Boolean = false) {
+}`},o={accept:"*/*","accept-encoding":"gzip, deflate, br, zstd","accept-language":"en-US,en;q=0.7","client-id":"hops-homepage","content-type":"application/json",origin:"https://www.zillow.com",priority:"u=1, i",referer:"https://www.zillow.com/","sec-ch-ua":'"Brave";v="135", "Not-A.Brand";v="8", "Chromium";v="135"',"sec-ch-ua-mobile":"?0","sec-ch-ua-platform":'"macOS"',"user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36","sec-fetch-site":"same-origin","sec-fetch-dest":"empty","sec-fetch-mode":"cors",cookie:await A()};try{const{data:a}=await c.post(t,r,{headers:o})||{},{regionId:i}=a.data.searchAssistanceResult.results.filter(s=>s?.regionId)?.[0]||{};return i||console.log("no region id found for zip"),i}catch(a){const{status:i,message:s}=H(a)??{};if(C(i,s))T(s,i);else return console.log("zip code not found"),null}},z={Accept:"*/*","Accept-Language":"en-US,en;q=0.5","Content-Type":"application/json",Origin:"https://www.zillow.com",Referer:"https://www.zillow.com/",Priority:"u=1, i","Sec-Ch-Ua":'"Brave";v="147", "Not.A/Brand";v="8", "Chromium";v="147"',"Sec-Ch-Ua-Mobile":"?0","Sec-Ch-Ua-Platform":'"macOS"',"Sec-Fetch-Dest":"empty","Sec-Fetch-Mode":"cors","Sec-Fetch-Site":"same-origin","Sec-Gpc":"1","User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"},k=async({page:e,daysOnZillow:t,regionId:r,zipCode:o,mergePageResults:a,timeoutMs:i})=>{o&&(r=await x(o)??r),e=(a?1:e)??1;const s=await w({page:e,daysOnZillow:t,regionId:r,zipCode:o,timeoutMs:i}),n=s?.cat1?.searchList?.totalResultCount,l=s?.cat1?.searchList?.resultsPerPage,p=s?.cat1?.searchList?.totalPages;if(e&&e>p)return null;const d=a&&Array.from({length:p},(u,L)=>L+1),_=a&&p>1&&d?(await Promise.all(d?.map(async u=>await w({page:u,daysOnZillow:t,regionId:r,zipCode:o},!0)))).flat():s?.cat1?.searchResults?.listResults,m=_?.map(u=>u?.detailUrl),y=_?.length;return y?{timestamp:h(),...o&&{zipCode:o},regionId:r,numResults:y,totalResultCount:n,resultsPerPage:l,...a&&{mergePageResults:a},...d?{pagesCrawled:d?.length}:{page:e},totalPages:p,results:_,urls:m}:null},w=async({page:e,daysOnZillow:t,regionId:r,zipCode:o,timeoutMs:a},i)=>{const s="https://www.zillow.com/async-create-search-page-state",n={searchQueryState:{pagination:{currentPage:e??1},isMapVisible:!1,mapBounds:{east:-116.2057669921875,north:35.424914784130365,south:32.12915408246069,west:-120.3915580078125},regionSelection:[{regionId:r,regionType:7}],filterState:{sortSelection:{value:"days"},doz:{value:String(t??5)},isNewConstruction:{value:!1},isForSaleForeclosure:{value:!1},isForSaleByOwner:{value:!1},isForSaleByAgent:{value:!1},isForRent:{value:!0},isComingSoon:{value:!1},isAuction:{value:!1},isAllHomes:{value:!0}},isListVisible:!0,...o&&{usersSearchTerm:String(o)}},wants:{cat1:["listResults"]},requestId:6,isDebugRequest:!1},l=m=>{const y=new AbortController;return setTimeout(()=>y.abort(),m||0),y.signal},p=await A(),d={headers:{...z,cookie:p},...a&&{signal:l(a)}},{data:_}=await c.put(s,n,d)||{};return i?_?.cat1?.searchResults?.listResults:_},S=async(e,t)=>{const{timeoutMs:r}=t??{},o={headers:{Host:"www.zillow.com",accept:"*/*","content-type":"text/html; charset=utf-8","user-agent":"insomnia/10.3.0"},...r&&{signal:(i=>{const s=new AbortController;return setTimeout(()=>s.abort(),i||0),s.signal})(r)}},{data:a}=await c.options(e,o)||{};if(!a||a===""){const i=new Error("empty response");throw i.response={status:0,data:"empty response"},i}if(typeof a=="string"&&a.includes("px-captcha")){const i=new Error("captcha");throw i.response={status:403,data:"captcha"},i}return a},oe=async e=>({data:await S(e)}),ae=async e=>{const t=await S(e);return W(t)},ie=(e,t)=>{if(t==="building")return e?.initialReduxState?.gdp?.building??null;{const r=e&&JSON.parse(e?.gdpClientCache);return r?Object.values(r).find(o=>o?.property)?.property??null:null}},W=e=>{const t=O.load(e)("#__NEXT_DATA__").text(),r=t&&JSON.parse(t)?.props?.pageProps?.componentProps,o=r?.initialReduxState?.gdp?"building":"property",a=ie(r,o);return{timestamp:h(),...a}},E={accept:"*/*","accept-encoding":"gzip, deflate, br, zstd","accept-language":"en-US,en;q=0.9","client-id":"for-sale-sub-app-browser-client","content-type":"application/json","Cache-Control":"no-cache",Pragma:"no-cache",priority:"u=1, i","sec-ch-ua":'"Google Chrome";v="136", "Not(A:Brand";v="8", "Chromium";v="136"',"sec-ch-ua-mobile":"?0","sec-ch-ua-platform":'"macOS"',"sec-fetch-dest":"empty","sec-fetch-mode":"cors","sec-fetch-site":"same-origin","user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36","x-z-enable-oauth-conversion":"true"},Z=async(e,t)=>{const{altFetch:r,onlyRemovedListings:o=!1}=t??{},a=(r?await ne(e,t):await se(e,t))||{},{property:i,...s}=a;if(i){const{buyAbilityData:n,collections:l,nearbyCities:p,nearbyNeighborhoods:d,nearbyZipcodes:_,schools:m,nearbyHomes:y,listingMetadata:u,homeInsights:L,responsivePhotos:ue,submitFlow:ge,contactFormRenderData:he,tourEligibility:Te,onsiteMessage:Ae,ZoDsFsUpsellTop:be,streetViewMetadataUrlMapLightboxAddress:Se,responsivePhotosOriginalRatio:Pe,thirdPartyVirtualTour:Re,vrModel:fe,topNavJson:we,...b}=i||{};if(!b?.priceHistory)return null;if(o){const{homeStatus:M}=b??{};if(!M)return console.log("no home status",e),null;if(M==="FOR_RENT"){const[F]=b?.priceHistory??{};return F?.event?console.log("recent price history not removed",e,F?.event):console.log("no recent price history"),null}}return{timestamp:h(),...b,...s,...r&&{altFetch:r}}}},se=async(e,t)=>{const{timeoutMs:r}=t??{},o="https://www.zillow.com/graphql",a=`?extensions=%7B%22persistedQuery%22%3A%7B%22version%22%3A1%2C%22sha256Hash%22%3A%222a50ebbffc7b6fb6e70fad24b7738df5db7f2ed5b61c2b980542749accff1f17%22%7D%7D&variables=%7B%22zpid%22%3A${e}%2C%22platform%22%3A%22DESKTOP_WEB%22%2C%22formType%22%3A%22OPAQUE%22%2C%22contactFormRenderParameter%22%3A%7B%22zpid%22%3A${e}%2C%22platform%22%3A%22desktop%22%2C%22isDoubleScroll%22%3Atrue%7D%2C%22skipCFRD%22%3Afalse%2C%22ompPlatform%22%3A%22web%22%7D`,i=`${o}${a}`,s=d=>{const _=new AbortController;return setTimeout(()=>_.abort(),d||0),_.signal},n=await A(),l={headers:{...E,cookie:n},...r&&{signal:s(r)}},{data:p}=await c.get(i,l)||{};return p?.data},ne=async(e,{timeoutMs:t})=>{const r="https://www.zillow.com/graphql",o={operationName:"ForRentShopperPlatformFullRenderQuery",variables:{zpid:e,altId:null},query:`query ForRentShopperPlatformFullRenderQuery($zpid: ID!, $altId: ID, $skipPropertyPrice: Boolean = false, $skipPropertyAddress: Boolean = false) {
   property(zpid: $zpid, palsId: $altId) {
     ...FrActionBarContent_property
     ...FrMediaColumnContent_property
@@ -2458,4 +2458,4 @@ fragment RentalPrequalificationCard on ABTests {
     }
   }
 }
-`},{data:o}=await c.post(t,r,{headers:E})||{};return o?.data?.property},C=(e,t)=>e===403&&t?.includes("captcha"),pe=e=>K(e),W=async e=>{const{data:t}=await c.post("http://localhost:8082/browser/open",{url:e});return t?.browser},Z=async()=>{const{data:e}=await c.post("http://localhost:8082/browser/status");return e?.browser},_e=async()=>{try{return(await Z())?.status==="captcha"}catch{return!1}},X=async()=>{await c.post("http://localhost:8082/cookie/refresh")},$=async(e,t=0)=>{try{await N()||T("zip codes required, please run the createConfig script");const{fetchListings:r=!1}=e??{},o=3101,{results:a}=await x({regionId:o,daysOnZillow:1,mergePageResults:!1})??{},{zpid:i,detailUrl:s}=a?.[0]??{};return i&&r?await k(i):a?.length&&!r||(s&&!s.startsWith("/apartments")?await S(s):t<5?await $(e,t+1):T("Could not find a valid listing URL to check for bot filtering.")),!0}catch(r){const{status:o,message:a}=O(r);C(o,a)&&T("Zillow bot filtering! Please open https://zillow.com in your browser and complete the challenge to continue.")}},Y=async()=>{const{data:e}=await c.post("http://localhost:8082/captcha/solve");return e?.solved===!0},de=async()=>{let e=!1;for(;;){try{const t=await W("https://www.zillow.com/homes/for_rent/");if(t?.status==="captcha"){if(!e&&await P()){e=!0,await Y(),e=!1;continue}}else if(t?.status==="navigated")return await X(),!0}catch{}await new Promise(t=>setTimeout(t,500))}},ce=async()=>{let e=!1,t=null,r=!1;for(;;){try{if(!e)(await W("https://www.zillow.com/homes/for_rent/"))?.status==="captcha"&&(e=!0,t=Date.now());else{if((await Z())?.status==="navigated"){if((t?Date.now()-t:0)>=3e3)return await X(),!0;e=!1,t=null,r=!1}if(!r&&await P()){r=!0,await Y();continue}}}catch{}await new Promise(o=>setTimeout(o,e?500:100))}},q=()=>{const e=["Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36","Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.1 Safari/605.1.15","Mozilla/5.0 (Macintosh; Intel Mac OS X 13_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.1 Safari/605.1.15"],t=Math.floor(Math.random()*e.length);return e[t]},ye=async(e,t)=>{const r=await j.launch({headless:!0,timeout:1e5,defaultViewport:null}),o=await r.newPage();if(e.includes("file://"))await o.setJavaScriptEnabled(!1);else{const i=q();await o.setUserAgent(i)}await o.goto(e,{waitUntil:"networkidle2"});const a=await o.evaluate(()=>document.querySelector(".layout-content-container")?.clientHeight);a&&await o.setViewport({width:1024,height:a}),await o.screenshot({path:t,fullPage:!0,captureBeyondViewport:!0}),await o.close(),await r.close(),console.log(`screenshot complete, ${t}`)},me=e=>{if(e==null||typeof e!="object")return!1;const t=e;return!(t.address===void 0||typeof t.bedrooms!="number"||typeof t.bedrooms!="number"||typeof t.daysOnZillow!="number"||typeof t.hdpUrl!="string"||typeof t.homeStatus!="string"||typeof t.homeType!="string"||typeof t.isRentalListingOffMarket!="boolean"||typeof t.price!="number"||typeof t.timeOnZillow!="string"||typeof t.zipcode!="string"||typeof t.zpid!="number")};export{$ as checkForZillowBotFiltering,f as fetchHtmlFromRedfinListingUrl,S as fetchHtmlFromZillowListingUrl,B as fetchRedfinListingResults,R as fetchRedfinRegionIdByZipCode,w as fetchZillowListingResults,q as generateRandomUA,J as getHtmlFromRedfinListingUrl,oe as getHtmlFromZillowListingUrl,pe as getRandomZipCode,ee as getRedfinListingDetailsById,te as getRedfinListingResults,H as getRedfinRegionIdByZipCode,P as getZillowAutoCaptcha,A as getZillowCookie,le as getZillowListingContactDetailsByZpid,k as getZillowListingDetailsByZpid,x as getZillowListingResults,U as getZillowRegionIdByZipCode,N as getZillowZipCodes,_e as isBrowserShowingCaptcha,C as isZillowBotFiltering,me as isZillowListingForRow,g as parseRedfinResponse,V as scrapeDataFromRedfinListingHtml,v as scrapeDataFromRedfinListingUrl,z as scrapeDataFromZillowListingHtml,ae as scrapeDataFromZillowListingUrl,ye as screenshotZillowListing,de as waitForSolvedZillowCaptcha,ce as waitForZillowCaptchaSolve,G as zillowListingResultsHeaders};
+`},{data:o}=await c.post(t,r,{headers:E})||{};return o?.data?.property},C=(e,t)=>e===403&&t?.includes("captcha"),pe=e=>K(e),X=async e=>{const{data:t}=await c.post("http://localhost:8082/browser/open",{url:e});return t?.browser},I=async()=>{const{data:e}=await c.post("http://localhost:8082/browser/status");return e?.browser},_e=async()=>{try{return(await I())?.status==="captcha"}catch{return!1}},D=async()=>{await c.post("http://localhost:8082/cookie/refresh")},$=async(e,t=0)=>{try{await v()||T("zip codes required, please run the createConfig script");const{fetchListings:r=!1}=e??{},o=3101,{results:a}=await k({regionId:o,daysOnZillow:1,mergePageResults:!1})??{},{zpid:i,detailUrl:s}=a?.[0]??{};return i&&r?await Z(i):a?.length&&!r||(s&&!s.startsWith("/apartments")?await S(s):t<5?await $(e,t+1):T("Could not find a valid listing URL to check for bot filtering.")),!0}catch(r){const{status:o,message:a}=H(r);C(o,a)&&T("Zillow bot filtering! Please open https://zillow.com in your browser and complete the challenge to continue.")}},Y=async()=>{const{data:e}=await c.post("http://localhost:8082/captcha/solve");return e?.solved===!0},de=async()=>{let e=!1,t=!1;for(;;){try{if(e){if((await I())?.status==="navigated")return await D(),!0;!t&&await P()&&(t=!0,await Y(),t=!1)}else{const r=await X("https://www.zillow.com/homes/for_rent/");if(r?.status==="captcha")e=!0;else if(r?.status==="navigated")return await D(),!0}}catch{}await new Promise(r=>setTimeout(r,e?500:100))}},ce=async()=>{let e=!1,t=null,r=!1;for(;;){try{if(!e)(await X("https://www.zillow.com/homes/for_rent/"))?.status==="captcha"&&(e=!0,t=Date.now());else{if((await I())?.status==="navigated"&&(t?Date.now()-t:0)>=3e3)return await D(),!0;if(!r&&await P()){r=!0,await Y();continue}}}catch{}await new Promise(o=>setTimeout(o,e?500:100))}},q=()=>{const e=["Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36","Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.1 Safari/605.1.15","Mozilla/5.0 (Macintosh; Intel Mac OS X 13_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.1 Safari/605.1.15"],t=Math.floor(Math.random()*e.length);return e[t]},ye=async(e,t)=>{const r=await j.launch({headless:!0,timeout:1e5,defaultViewport:null}),o=await r.newPage();if(e.includes("file://"))await o.setJavaScriptEnabled(!1);else{const i=q();await o.setUserAgent(i)}await o.goto(e,{waitUntil:"networkidle2"});const a=await o.evaluate(()=>document.querySelector(".layout-content-container")?.clientHeight);a&&await o.setViewport({width:1024,height:a}),await o.screenshot({path:t,fullPage:!0,captureBeyondViewport:!0}),await o.close(),await r.close(),console.log(`screenshot complete, ${t}`)},me=e=>{if(e==null||typeof e!="object")return!1;const t=e;return!(t.address===void 0||typeof t.bedrooms!="number"||typeof t.bedrooms!="number"||typeof t.daysOnZillow!="number"||typeof t.hdpUrl!="string"||typeof t.homeStatus!="string"||typeof t.homeType!="string"||typeof t.isRentalListingOffMarket!="boolean"||typeof t.price!="number"||typeof t.timeOnZillow!="string"||typeof t.zipcode!="string"||typeof t.zpid!="number")};export{$ as checkForZillowBotFiltering,f as fetchHtmlFromRedfinListingUrl,S as fetchHtmlFromZillowListingUrl,G as fetchRedfinListingResults,R as fetchRedfinRegionIdByZipCode,w as fetchZillowListingResults,q as generateRandomUA,J as getHtmlFromRedfinListingUrl,oe as getHtmlFromZillowListingUrl,pe as getRandomZipCode,ee as getRedfinListingDetailsById,te as getRedfinListingResults,V as getRedfinRegionIdByZipCode,P as getZillowAutoCaptcha,A as getZillowCookie,le as getZillowListingContactDetailsByZpid,Z as getZillowListingDetailsByZpid,k as getZillowListingResults,x as getZillowRegionIdByZipCode,v as getZillowZipCodes,_e as isBrowserShowingCaptcha,C as isZillowBotFiltering,me as isZillowListingForRow,g as parseRedfinResponse,U as scrapeDataFromRedfinListingHtml,B as scrapeDataFromRedfinListingUrl,W as scrapeDataFromZillowListingHtml,ae as scrapeDataFromZillowListingUrl,ye as screenshotZillowListing,de as waitForSolvedZillowCaptcha,ce as waitForZillowCaptchaSolve,z as zillowListingResultsHeaders};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rent-scraper/api",
-  "version": "1.0.35",
+  "version": "1.0.36",
   "type": "module",
   "main": "./dist/index.mjs",
   "module": "./dist/index.mjs",
@@ -31,7 +31,7 @@
     "cheerio": "^1.1.0",
     "dayjs": "^1.11.13",
     "puppeteer": "^24.11.2",
-    "@rent-scraper/utils": "1.0.35"
+    "@rent-scraper/utils": "1.0.36"
   },
   "devDependencies": {
     "unbuild": "^3.5.0",