@rent-scraper/api 1.0.28 → 1.0.30

package/dist/config.d.mts

@@ -1,5 +1,8 @@
 declare const getZillowCookie: () => Promise<string | null>;
 declare const getZillowDaysListed: () => Promise<string | null>;
+declare const getZillowAutoCaptcha: () => Promise<boolean | null>;
+declare const getZillowLimit: () => Promise<number | null>;
+declare const getZillowOffset: () => Promise<number | null>;
 declare const getZillowOutputPath: () => Promise<string>;
 declare const getZillowZipCodes: () => Promise<number[]>;
 type Zpid = string;
@@ -9,5 +12,5 @@ declare const getRedfinZipCodes: () => Promise<number[]>;
 declare const getRedfinDaysListed: () => Promise<string | null>;
 declare const getRedfinCookie: () => Promise<string | null>;
 
-export { getRedfinCookie, getRedfinDaysListed, getRedfinOutputPath, getRedfinZipCodes, getZillowCookie, getZillowDaysListed, getZillowOutputPath, getZillowZipCodes };
+export { getRedfinCookie, getRedfinDaysListed, getRedfinOutputPath, getRedfinZipCodes, getZillowAutoCaptcha, getZillowCookie, getZillowDaysListed, getZillowLimit, getZillowOffset, getZillowOutputPath, getZillowZipCodes };
 export type { Zpid };

package/dist/config.mjs

@@ -1 +1 @@
-export{h as getRedfinCookie,f as getRedfinDaysListed,d as getRedfinOutputPath,e as getRedfinZipCodes,g as getZillowCookie,a as getZillowDaysListed,b as getZillowOutputPath,c as getZillowZipCodes}from"./shared/api.bGXs5gok.mjs";import"@rent-scraper/utils/config";
+export{k as getRedfinCookie,j as getRedfinDaysListed,h as getRedfinOutputPath,i as getRedfinZipCodes,b as getZillowAutoCaptcha,g as getZillowCookie,a as getZillowDaysListed,c as getZillowLimit,d as getZillowOffset,e as getZillowOutputPath,f as getZillowZipCodes}from"./shared/api.DfAvAXsZ.mjs";import"@rent-scraper/utils/config";

package/dist/index.d.mts

@@ -67,8 +67,10 @@ interface CheckForZillowBotFilteringOptions {
     fetchListings?: boolean;
     regionId?: string;
 }
+declare const isBrowserShowingCaptcha: () => Promise<boolean>;
 declare const checkForZillowBotFiltering: (options?: CheckForZillowBotFilteringOptions, attempt?: number) => Promise<true | undefined>;
-declare const waitForSolvedZillowCaptcha: () => Promise<unknown>;
+declare const waitForSolvedZillowCaptcha: () => Promise<boolean>;
+declare const waitForZillowCaptchaSolve: () => Promise<boolean>;
 
 type ListingEventType = 'Listed for rent' | 'Price change';
 type NonListingEventType = 'Contingent' | 'Listed for sale' | 'Listing removed' | 'Pending sale' | 'Sold';
@@ -765,5 +767,5 @@ declare const generateRandomUA: () => string;
  */
 declare const screenshotZillowListing: (url: string, filePath: ScreenshotOptions["path"]) => Promise<void>;
 
-export { checkForZillowBotFiltering, fetchHtmlFromRedfinListingUrl, fetchHtmlFromZillowListingUrl, fetchRedfinListingResults, fetchRedfinRegionIdByZipCode, fetchZillowListingResults, generateRandomUA, getHtmlFromRedfinListingUrl, getHtmlFromZillowListingUrl, getRandomZipCode, getRedfinListingDetailsById, getRedfinListingResults, getRedfinRegionIdByZipCode, getZillowListingContactDetailsByZpid, getZillowListingDetailsByZpid, getZillowListingResults, getZillowRegionIdByZipCode, isZillowBotFiltering, isZillowListingForRow, parseRedfinResponse, scrapeDataFromRedfinListingHtml, scrapeDataFromRedfinListingUrl, scrapeDataFromZillowListingHtml, scrapeDataFromZillowListingUrl, screenshotZillowListing, waitForSolvedZillowCaptcha, zillowListingResultsHeaders };
+export { checkForZillowBotFiltering, fetchHtmlFromRedfinListingUrl, fetchHtmlFromZillowListingUrl, fetchRedfinListingResults, fetchRedfinRegionIdByZipCode, fetchZillowListingResults, generateRandomUA, getHtmlFromRedfinListingUrl, getHtmlFromZillowListingUrl, getRandomZipCode, getRedfinListingDetailsById, getRedfinListingResults, getRedfinRegionIdByZipCode, getZillowListingContactDetailsByZpid, getZillowListingDetailsByZpid, getZillowListingResults, getZillowRegionIdByZipCode, isBrowserShowingCaptcha, isZillowBotFiltering, isZillowListingForRow, parseRedfinResponse, scrapeDataFromRedfinListingHtml, scrapeDataFromRedfinListingUrl, scrapeDataFromZillowListingHtml, scrapeDataFromZillowListingUrl, screenshotZillowListing, waitForSolvedZillowCaptcha, waitForZillowCaptchaSolve, zillowListingResultsHeaders };
 export type { ListingsSource, PriceHistoryItem, PriceHistoryItemBase, PriceHistoryItemForCheck, PriceHistoryItemOther, PriceHistoryItemOtherForCheck, PriceHistoryItemRental, PriceHistoryItemRentalForCheck, RedfinListingHtmlOptions, RedfinListingResultsOptions, RegionId, TaxHistoryItem, ZillowListing, ZillowListingDetailsOptions, ZillowListingForCheck, ZillowListingForRow, ZillowListingHtmlOptions, ZillowListingResults, ZillowListingResultsOptions, ZipCode };

package/dist/index.mjs

@@ -1,8 +1,8 @@
-import h from"dayjs";import*as M from"cheerio";import{getValueFromConfigFile as F}from"@rent-scraper/utils/config";import{throwError as T,parseError as O,getRandomArrayValue as W}from"@rent-scraper/utils";import{h as Z,g as b,c as X}from"./shared/api.bGXs5gok.mjs";import u from"axios";import Y from"puppeteer";const N=async(e,t)=>{const{fromFile:r=!0}=t??{};return r?(await F("redfin","regionIds"))[String(e)]??await P(e):await P(e)},g=e=>JSON.parse(e.replace("{}&&","")),P=async e=>{const t="https://www.redfin.com/stingray/do/rental-location-autocomplete",r={"Content-Type":"application/json; charset=utf-8","User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"},o=`?location=${e}&v=2`;try{const a=`${t}${o}`,n=await(await fetch(a,{method:"GET",headers:r})).text(),i=g(n),{exactMatch:s}=i.payload||{},p=Number(s.id.replace(s.type,"").replace("_",""));return p||console.log("no region id found for zip"),p}catch{return console.log("zip code not found"),null}},$=async e=>({data:await R(e)}),H=async e=>{const t=await R(e);return v(t)},v=e=>{const t=M.load(e)('script:contains("ReactServerAgent.cache")').text()?.replace(`_tLAB.wait(function(){
+import h from"dayjs";import*as L from"cheerio";import{getValueFromConfigFile as M}from"@rent-scraper/utils/config";import{throwError as T,parseError as F,getRandomArrayValue as q}from"@rent-scraper/utils";import{k as K,g as b,f as Q,b as O}from"./shared/api.DfAvAXsZ.mjs";import c from"axios";import j from"puppeteer";const N=async(e,t)=>{const{fromFile:r=!0}=t??{};return r?(await M("redfin","regionIds"))[String(e)]??await P(e):await P(e)},g=e=>JSON.parse(e.replace("{}&&","")),P=async e=>{const t="https://www.redfin.com/stingray/do/rental-location-autocomplete",r={"Content-Type":"application/json; charset=utf-8","User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"},o=`?location=${e}&v=2`;try{const a=`${t}${o}`,i=await(await fetch(a,{method:"GET",headers:r})).text(),s=g(i),{exactMatch:n}=s.payload||{},p=Number(n.id.replace(n.type,"").replace("_",""));return p||console.log("no region id found for zip"),p}catch{return console.log("zip code not found"),null}},J=async e=>({data:await R(e)}),H=async e=>{const t=await R(e);return v(t)},v=e=>{const t=L.load(e)('script:contains("ReactServerAgent.cache")').text()?.replace(`_tLAB.wait(function(){
 (function (root) {
 /* -- Data -- */
 root.__reactServerState || (root.__reactServerState = {});
-root.__reactServerState.InitialContext = {"ReactServerAgent.cache":`,"")?.split(',"deviceType":"desktop"')[0],r=(t&&JSON.parse(t))?.dataCache,o=r?.["/stingray/api/home/details/initialInfo"]?.res?.text&&g(r["/stingray/api/home/details/initialInfo"].res.text)?.payload,a=o?.propertyId??o?.redirectUrl?.split("/").pop(),n=r?.[`/stingray/api/v1/rentals/homecards?propertyIds=${a}`]?.res?.text,i=n&&g(n),s=r?.["/stingray/api/home/details/belowTheFold"]?.res?.text,p=s&&g(s)?.payload,l=r?.[`/stingray/api/home/details/propertyParcelInfo?propertyId=${a}&pageType=6`]?.res?.text,d=l&&g(l)?.payload,_=i?.homes?.[0]?.rentalExtension?.rentalId,y=r?.[`/stingray/api/v1/rentals/${_}/floorPlans?`]?.res?.text,c=y&&g(y),m={hasPropertyHistory:!!p?.propertyHistoryInfo,propertyId:a,rentalId:_,initialInfo:o,homeCards:i,floorPlans:c,belowTheFold:p,propertyParcelInfo:d};return{timestamp:h(),...m}},R=async(e,t)=>{const{timeoutMs:r}=t??{},o=await Z(),a={headers:{"Content-Type":"text/html; charset=utf-8","User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",...o&&{Cookie:o}},...r&&{signal:(i=>{const s=new AbortController;return setTimeout(()=>s.abort(),i||0),s.signal})(r)}},n=await fetch(e,a);if(n.status===202&&T(`Redfin bot filtering (202) for ${e} \u2014 restart the browser server to refresh the cookie.`),n.ok)return await n.text();{const i=await n.text().catch(()=>""),s=(i&&JSON.parse(i)?.error)??`${n.status} ${n.statusText}`;T(s)}},q=async(e,t)=>await H(t??`https://www.redfin.com/us/home/${e}`),K=async({regionId:e,zipCode:t,timeoutMs:r,daysListed:o})=>{t&&(e=await N(t)??e);try{const a=await V({regionId:e,timeoutMs:r}),n=a?.numMatchedHomes,i=o?a?.homes?.filter(l=>h().diff(l?.rentalExtension?.freshnessTimestamp,"day")<o):a?.homes,s=i?.map(l=>l?.detailUrl),p=i?.length;return p?{timestamp:h(),...t&&{zipCode:t},regionId:e,numResults:p,totalResultCount:n,results:i,urls:s}:null}catch(a){throw console.log(a),a}},V=async({regionId:e,timeoutMs:t})=>{const r="https://www.redfin.com/stingray/api/v1/search/rentals",o={"Content-Type":"application/json; charset=utf-8","User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"},a=`?al=1&isRentals=true&market=socal&num_homes=350&ord=days-on-redfin-asc&page_number=1&region_id=${e}&region_type=2&sf=1,2,3,5,6,7&start=0&status=9&uipt=1,2,3,4&use_max_pins=true&v=8`,n={headers:o,...t&&{signal:(s=>{const p=new AbortController;return setTimeout(()=>p.abort(),s||0),p.signal})(t)}},i=`${r}${a}`;return await(await fetch(i,n)).json()},B=async(e,t)=>{const{fromFile:r=!0}=t??{};return r?(await F("zillow","regionIds")??{})[String(e)]??null:await Q(e)},Q=async e=>{const t="https://www.zillow.com/graphql",r={operationName:"getAutocompleteResults",variables:{query:String(e),queryOptions:{maxResults:10,userIdentity:{abKey:"3176854b-962d-4266-8bb8-f9e8f2377158"},userSearchContext:"FOR_RENT",userLocation:{latitude:34.11,longitude:-118.23}},resultType:["REGIONS","FORSALE","RENTALS","SOLD","COMMUNITIES","BUILDER_COMMUNITIES"]},query:`query getAutocompleteResults($query: String!, $queryOptions: SearchAssistanceQueryOptions, $resultType: [SearchAssistanceResultType]) {
+root.__reactServerState.InitialContext = {"ReactServerAgent.cache":`,"")?.split(',"deviceType":"desktop"')[0],r=(t&&JSON.parse(t))?.dataCache,o=r?.["/stingray/api/home/details/initialInfo"]?.res?.text&&g(r["/stingray/api/home/details/initialInfo"].res.text)?.payload,a=o?.propertyId??o?.redirectUrl?.split("/").pop(),i=r?.[`/stingray/api/v1/rentals/homecards?propertyIds=${a}`]?.res?.text,s=i&&g(i),n=r?.["/stingray/api/home/details/belowTheFold"]?.res?.text,p=n&&g(n)?.payload,l=r?.[`/stingray/api/home/details/propertyParcelInfo?propertyId=${a}&pageType=6`]?.res?.text,d=l&&g(l)?.payload,_=s?.homes?.[0]?.rentalExtension?.rentalId,m=r?.[`/stingray/api/v1/rentals/${_}/floorPlans?`]?.res?.text,y=m&&g(m),u={hasPropertyHistory:!!p?.propertyHistoryInfo,propertyId:a,rentalId:_,initialInfo:o,homeCards:s,floorPlans:y,belowTheFold:p,propertyParcelInfo:d};return{timestamp:h(),...u}},R=async(e,t)=>{const{timeoutMs:r}=t??{},o=await K(),a={headers:{"Content-Type":"text/html; charset=utf-8","User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",...o&&{Cookie:o}},...r&&{signal:(s=>{const n=new AbortController;return setTimeout(()=>n.abort(),s||0),n.signal})(r)}},i=await fetch(e,a);if(i.status===202&&T(`Redfin bot filtering (202) for ${e} \u2014 restart the browser server to refresh the cookie.`),i.ok)return await i.text();{const s=await i.text().catch(()=>""),n=(s&&JSON.parse(s)?.error)??`${i.status} ${i.statusText}`;T(n)}},ee=async(e,t)=>await H(t??`https://www.redfin.com/us/home/${e}`),te=async({regionId:e,zipCode:t,timeoutMs:r,daysListed:o})=>{t&&(e=await N(t)??e);try{const a=await V({regionId:e,timeoutMs:r}),i=a?.numMatchedHomes,s=o?a?.homes?.filter(l=>h().diff(l?.rentalExtension?.freshnessTimestamp,"day")<o):a?.homes,n=s?.map(l=>l?.detailUrl),p=s?.length;return p?{timestamp:h(),...t&&{zipCode:t},regionId:e,numResults:p,totalResultCount:i,results:s,urls:n}:null}catch(a){throw console.log(a),a}},V=async({regionId:e,timeoutMs:t})=>{const r="https://www.redfin.com/stingray/api/v1/search/rentals",o={"Content-Type":"application/json; charset=utf-8","User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36"},a=`?al=1&isRentals=true&market=socal&num_homes=350&ord=days-on-redfin-asc&page_number=1&region_id=${e}&region_type=2&sf=1,2,3,5,6,7&start=0&status=9&uipt=1,2,3,4&use_max_pins=true&v=8`,i={headers:o,...t&&{signal:(n=>{const p=new AbortController;return setTimeout(()=>p.abort(),n||0),p.signal})(t)}},s=`${r}${a}`;return await(await fetch(s,i)).json()},B=async(e,t)=>{const{fromFile:r=!0}=t??{};return r?(await M("zillow","regionIds")??{})[String(e)]??null:await re(e)},re=async e=>{const t="https://www.zillow.com/graphql",r={operationName:"getAutocompleteResults",variables:{query:String(e),queryOptions:{maxResults:10,userIdentity:{abKey:"3176854b-962d-4266-8bb8-f9e8f2377158"},userSearchContext:"FOR_RENT",userLocation:{latitude:34.11,longitude:-118.23}},resultType:["REGIONS","FORSALE","RENTALS","SOLD","COMMUNITIES","BUILDER_COMMUNITIES"]},query:`query getAutocompleteResults($query: String!, $queryOptions: SearchAssistanceQueryOptions, $resultType: [SearchAssistanceResultType]) {
   searchAssistanceResult: zgsAutocompleteRequest(
     query: $query
     queryOptions: $queryOptions
@@ -129,7 +129,7 @@ fragment RentalCommunityResultFields on SearchAssistanceRentalCommunityResult {
 fragment BuilderCommunityResultFields on SearchAssistanceBuilderCommunityResult {
   plid
   __typename
-}`},o={accept:"*/*","accept-encoding":"gzip, deflate, br, zstd","accept-language":"en-US,en;q=0.7","client-id":"hops-homepage","content-type":"application/json",origin:"https://www.zillow.com",priority:"u=1, i",referer:"https://www.zillow.com/","sec-ch-ua":'"Brave";v="135", "Not-A.Brand";v="8", "Chromium";v="135"',"sec-ch-ua-mobile":"?0","sec-ch-ua-platform":'"macOS"',"user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36","sec-fetch-site":"same-origin","sec-fetch-dest":"empty","sec-fetch-mode":"cors",cookie:await b()};try{const{data:a}=await u.post(t,r,{headers:o})||{},{regionId:n}=a.data.searchAssistanceResult.results.filter(i=>i?.regionId)?.[0]||{};return n||console.log("no region id found for zip"),n}catch(a){const{status:n,message:i}=O(a)??{};if(E(n,i))T(i,n);else return console.log("zip code not found"),null}},U={Accept:"*/*","Accept-Language":"en-US,en;q=0.5","Content-Type":"application/json",Origin:"https://www.zillow.com",Referer:"https://www.zillow.com/",Priority:"u=1, i","Sec-Ch-Ua":'"Brave";v="147", "Not.A/Brand";v="8", "Chromium";v="147"',"Sec-Ch-Ua-Mobile":"?0","Sec-Ch-Ua-Platform":'"macOS"',"Sec-Fetch-Dest":"empty","Sec-Fetch-Mode":"cors","Sec-Fetch-Site":"same-origin","Sec-Gpc":"1","User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"},G=async({page:e,daysOnZillow:t,regionId:r,zipCode:o,mergePageResults:a,timeoutMs:n})=>{o&&(r=await B(o)??r),e=(a?1:e)??1;const i=await f({page:e,daysOnZillow:t,regionId:r,zipCode:o,timeoutMs:n}),s=i?.cat1?.searchList?.totalResultCount,p=i?.cat1?.searchList?.resultsPerPage,l=i?.cat1?.searchList?.totalPages;if(e&&e>l)return null;const d=a&&Array.from({length:l},(m,I)=>I+1),_=a&&l>1&&d?(await Promise.all(d?.map(async m=>await f({page:m,daysOnZillow:t,regionId:r,zipCode:o},!0)))).flat():i?.cat1?.searchResults?.listResults,y=_?.map(m=>m?.detailUrl),c=_?.length;return c?{timestamp:h(),...o&&{zipCode:o},regionId:r,numResults:c,totalResultCount:s,resultsPerPage:p,...a&&{mergePageResults:a},...d?{pagesCrawled:d?.length}:{page:e},totalPages:l,results:_,urls:y}:null},f=async({page:e,daysOnZillow:t,regionId:r,zipCode:o,timeoutMs:a},n)=>{const i="https://www.zillow.com/async-create-search-page-state",s={searchQueryState:{pagination:{currentPage:e??1},isMapVisible:!1,mapBounds:{east:-116.2057669921875,north:35.424914784130365,south:32.12915408246069,west:-120.3915580078125},regionSelection:[{regionId:r,regionType:7}],filterState:{sortSelection:{value:"days"},doz:{value:String(t??5)},isNewConstruction:{value:!1},isForSaleForeclosure:{value:!1},isForSaleByOwner:{value:!1},isForSaleByAgent:{value:!1},isForRent:{value:!0},isComingSoon:{value:!1},isAuction:{value:!1},isAllHomes:{value:!0}},isListVisible:!0,...o&&{usersSearchTerm:String(o)}},wants:{cat1:["listResults"]},requestId:6,isDebugRequest:!1},p=y=>{const c=new AbortController;return setTimeout(()=>c.abort(),y||0),c.signal},l=await b(),d={headers:{...U,cookie:l},...a&&{signal:p(a)}},{data:_}=await u.put(i,s,d)||{};return n?_?.cat1?.searchResults?.listResults:_},S=async(e,t)=>{const{timeoutMs:r}=t??{},o={headers:{Host:"www.zillow.com",accept:"*/*","content-type":"text/html; charset=utf-8","user-agent":"insomnia/10.3.0"},...r&&{signal:(n=>{const i=new AbortController;return setTimeout(()=>i.abort(),n||0),i.signal})(r)}},{data:a}=await u.options(e,o)||{};return a},j=async e=>({data:await S(e)}),J=async e=>{const t=await S(e);return x(t)},ee=(e,t)=>{if(t==="building")return e?.initialReduxState?.gdp?.building??null;{e=e&&JSON.parse(e?.gdpClientCache);const r=e&&Object.keys(e)?.[0];return e=e?.[r]?.property,e??null}},x=e=>{const t=M.load(e)("#__NEXT_DATA__").text(),r=t&&JSON.parse(t)?.props?.pageProps?.componentProps,o=r?.initialReduxState?.gdp?"building":"property",a=ee(r,o);return{timestamp:h(),...a}},w={accept:"*/*","accept-encoding":"gzip, deflate, br, zstd","accept-language":"en-US,en;q=0.9","client-id":"for-sale-sub-app-browser-client","content-type":"application/json","Cache-Control":"no-cache",Pragma:"no-cache",priority:"u=1, i","sec-ch-ua":'"Google Chrome";v="136", "Not(A:Brand";v="8", "Chromium";v="136"',"sec-ch-ua-mobile":"?0","sec-ch-ua-platform":'"macOS"',"sec-fetch-dest":"empty","sec-fetch-mode":"cors","sec-fetch-site":"same-origin","user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36","x-z-enable-oauth-conversion":"true"},z=async(e,t)=>{const{altFetch:r,onlyRemovedListings:o=!1}=t??{},a=(r?await re(e,t):await te(e,t))||{},{property:n,...i}=a;if(n){const{buyAbilityData:s,collections:p,nearbyCities:l,nearbyNeighborhoods:d,nearbyZipcodes:_,schools:y,nearbyHomes:c,listingMetadata:m,homeInsights:I,responsivePhotos:le,submitFlow:_e,contactFormRenderData:de,tourEligibility:ce,onsiteMessage:ye,ZoDsFsUpsellTop:me,streetViewMetadataUrlMapLightboxAddress:ue,responsivePhotosOriginalRatio:ge,thirdPartyVirtualTour:he,vrModel:Te,topNavJson:Ae,...A}=n||{};if(!A?.priceHistory)return null;if(o){const{homeStatus:D}=A??{};if(!D)return console.log("no home status",e),null;if(D==="FOR_RENT"){const[L]=A?.priceHistory??{};return L?.event?console.log("recent price history not removed",e,L?.event):console.log("no recent price history"),null}}return{timestamp:h(),...A,...i,...r&&{altFetch:r}}}},te=async(e,t)=>{const{timeoutMs:r}=t??{},o="https://www.zillow.com/graphql",a=`?extensions=%7B%22persistedQuery%22%3A%7B%22version%22%3A1%2C%22sha256Hash%22%3A%222a50ebbffc7b6fb6e70fad24b7738df5db7f2ed5b61c2b980542749accff1f17%22%7D%7D&variables=%7B%22zpid%22%3A${e}%2C%22platform%22%3A%22DESKTOP_WEB%22%2C%22formType%22%3A%22OPAQUE%22%2C%22contactFormRenderParameter%22%3A%7B%22zpid%22%3A${e}%2C%22platform%22%3A%22desktop%22%2C%22isDoubleScroll%22%3Atrue%7D%2C%22skipCFRD%22%3Afalse%2C%22ompPlatform%22%3A%22web%22%7D`,n=`${o}${a}`,i=d=>{const _=new AbortController;return setTimeout(()=>_.abort(),d||0),_.signal},s=await b(),p={headers:{...w,cookie:s},...r&&{signal:i(r)}},{data:l}=await u.get(n,p)||{};return l?.data},re=async(e,{timeoutMs:t})=>{const r="https://www.zillow.com/graphql",o={operationName:"ForRentShopperPlatformFullRenderQuery",variables:{zpid:e,altId:null},query:`query ForRentShopperPlatformFullRenderQuery($zpid: ID!, $altId: ID, $skipPropertyPrice: Boolean = false, $skipPropertyAddress: Boolean = false) {
+}`},o={accept:"*/*","accept-encoding":"gzip, deflate, br, zstd","accept-language":"en-US,en;q=0.7","client-id":"hops-homepage","content-type":"application/json",origin:"https://www.zillow.com",priority:"u=1, i",referer:"https://www.zillow.com/","sec-ch-ua":'"Brave";v="135", "Not-A.Brand";v="8", "Chromium";v="135"',"sec-ch-ua-mobile":"?0","sec-ch-ua-platform":'"macOS"',"user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36","sec-fetch-site":"same-origin","sec-fetch-dest":"empty","sec-fetch-mode":"cors",cookie:await b()};try{const{data:a}=await c.post(t,r,{headers:o})||{},{regionId:i}=a.data.searchAssistanceResult.results.filter(s=>s?.regionId)?.[0]||{};return i||console.log("no region id found for zip"),i}catch(a){const{status:i,message:s}=F(a)??{};if(E(i,s))T(s,i);else return console.log("zip code not found"),null}},U={Accept:"*/*","Accept-Language":"en-US,en;q=0.5","Content-Type":"application/json",Origin:"https://www.zillow.com",Referer:"https://www.zillow.com/",Priority:"u=1, i","Sec-Ch-Ua":'"Brave";v="147", "Not.A/Brand";v="8", "Chromium";v="147"',"Sec-Ch-Ua-Mobile":"?0","Sec-Ch-Ua-Platform":'"macOS"',"Sec-Fetch-Dest":"empty","Sec-Fetch-Mode":"cors","Sec-Fetch-Site":"same-origin","Sec-Gpc":"1","User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"},G=async({page:e,daysOnZillow:t,regionId:r,zipCode:o,mergePageResults:a,timeoutMs:i})=>{o&&(r=await B(o)??r),e=(a?1:e)??1;const s=await f({page:e,daysOnZillow:t,regionId:r,zipCode:o,timeoutMs:i}),n=s?.cat1?.searchList?.totalResultCount,p=s?.cat1?.searchList?.resultsPerPage,l=s?.cat1?.searchList?.totalPages;if(e&&e>l)return null;const d=a&&Array.from({length:l},(u,C)=>C+1),_=a&&l>1&&d?(await Promise.all(d?.map(async u=>await f({page:u,daysOnZillow:t,regionId:r,zipCode:o},!0)))).flat():s?.cat1?.searchResults?.listResults,m=_?.map(u=>u?.detailUrl),y=_?.length;return y?{timestamp:h(),...o&&{zipCode:o},regionId:r,numResults:y,totalResultCount:n,resultsPerPage:p,...a&&{mergePageResults:a},...d?{pagesCrawled:d?.length}:{page:e},totalPages:l,results:_,urls:m}:null},f=async({page:e,daysOnZillow:t,regionId:r,zipCode:o,timeoutMs:a},i)=>{const s="https://www.zillow.com/async-create-search-page-state",n={searchQueryState:{pagination:{currentPage:e??1},isMapVisible:!1,mapBounds:{east:-116.2057669921875,north:35.424914784130365,south:32.12915408246069,west:-120.3915580078125},regionSelection:[{regionId:r,regionType:7}],filterState:{sortSelection:{value:"days"},doz:{value:String(t??5)},isNewConstruction:{value:!1},isForSaleForeclosure:{value:!1},isForSaleByOwner:{value:!1},isForSaleByAgent:{value:!1},isForRent:{value:!0},isComingSoon:{value:!1},isAuction:{value:!1},isAllHomes:{value:!0}},isListVisible:!0,...o&&{usersSearchTerm:String(o)}},wants:{cat1:["listResults"]},requestId:6,isDebugRequest:!1},p=m=>{const y=new AbortController;return setTimeout(()=>y.abort(),m||0),y.signal},l=await b(),d={headers:{...U,cookie:l},...a&&{signal:p(a)}},{data:_}=await c.put(s,n,d)||{};return i?_?.cat1?.searchResults?.listResults:_},S=async(e,t)=>{const{timeoutMs:r}=t??{},o={headers:{Host:"www.zillow.com",accept:"*/*","content-type":"text/html; charset=utf-8","user-agent":"insomnia/10.3.0"},...r&&{signal:(i=>{const s=new AbortController;return setTimeout(()=>s.abort(),i||0),s.signal})(r)}},{data:a}=await c.options(e,o)||{};if(!a||a===""){const i=new Error("empty response");throw i.response={status:0,data:"empty response"},i}if(typeof a=="string"&&a.includes("px-captcha")){const i=new Error("captcha");throw i.response={status:403,data:"captcha"},i}return a},oe=async e=>({data:await S(e)}),ae=async e=>{const t=await S(e);return x(t)},ie=(e,t)=>{if(t==="building")return e?.initialReduxState?.gdp?.building??null;{e=e&&JSON.parse(e?.gdpClientCache);const r=e&&Object.keys(e)?.[0];return e=e?.[r]?.property,e??null}},x=e=>{const t=L.load(e)("#__NEXT_DATA__").text(),r=t&&JSON.parse(t)?.props?.pageProps?.componentProps,o=r?.initialReduxState?.gdp?"building":"property",a=ie(r,o);return{timestamp:h(),...a}},w={accept:"*/*","accept-encoding":"gzip, deflate, br, zstd","accept-language":"en-US,en;q=0.9","client-id":"for-sale-sub-app-browser-client","content-type":"application/json","Cache-Control":"no-cache",Pragma:"no-cache",priority:"u=1, i","sec-ch-ua":'"Google Chrome";v="136", "Not(A:Brand";v="8", "Chromium";v="136"',"sec-ch-ua-mobile":"?0","sec-ch-ua-platform":'"macOS"',"sec-fetch-dest":"empty","sec-fetch-mode":"cors","sec-fetch-site":"same-origin","user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36","x-z-enable-oauth-conversion":"true"},z=async(e,t)=>{const{altFetch:r,onlyRemovedListings:o=!1}=t??{},a=(r?await ne(e,t):await se(e,t))||{},{property:i,...s}=a;if(i){const{buyAbilityData:n,collections:p,nearbyCities:l,nearbyNeighborhoods:d,nearbyZipcodes:_,schools:m,nearbyHomes:y,listingMetadata:u,homeInsights:C,responsivePhotos:ue,submitFlow:ge,contactFormRenderData:he,tourEligibility:Te,onsiteMessage:Ae,ZoDsFsUpsellTop:be,streetViewMetadataUrlMapLightboxAddress:Se,responsivePhotosOriginalRatio:Pe,thirdPartyVirtualTour:Re,vrModel:fe,topNavJson:we,...A}=i||{};if(!A?.priceHistory)return null;if(o){const{homeStatus:I}=A??{};if(!I)return console.log("no home status",e),null;if(I==="FOR_RENT"){const[D]=A?.priceHistory??{};return D?.event?console.log("recent price history not removed",e,D?.event):console.log("no recent price history"),null}}return{timestamp:h(),...A,...s,...r&&{altFetch:r}}}},se=async(e,t)=>{const{timeoutMs:r}=t??{},o="https://www.zillow.com/graphql",a=`?extensions=%7B%22persistedQuery%22%3A%7B%22version%22%3A1%2C%22sha256Hash%22%3A%222a50ebbffc7b6fb6e70fad24b7738df5db7f2ed5b61c2b980542749accff1f17%22%7D%7D&variables=%7B%22zpid%22%3A${e}%2C%22platform%22%3A%22DESKTOP_WEB%22%2C%22formType%22%3A%22OPAQUE%22%2C%22contactFormRenderParameter%22%3A%7B%22zpid%22%3A${e}%2C%22platform%22%3A%22desktop%22%2C%22isDoubleScroll%22%3Atrue%7D%2C%22skipCFRD%22%3Afalse%2C%22ompPlatform%22%3A%22web%22%7D`,i=`${o}${a}`,s=d=>{const _=new AbortController;return setTimeout(()=>_.abort(),d||0),_.signal},n=await b(),p={headers:{...w,cookie:n},...r&&{signal:s(r)}},{data:l}=await c.get(i,p)||{};return l?.data},ne=async(e,{timeoutMs:t})=>{const r="https://www.zillow.com/graphql",o={operationName:"ForRentShopperPlatformFullRenderQuery",variables:{zpid:e,altId:null},query:`query ForRentShopperPlatformFullRenderQuery($zpid: ID!, $altId: ID, $skipPropertyPrice: Boolean = false, $skipPropertyAddress: Boolean = false) {
   property(zpid: $zpid, palsId: $altId) {
     ...FrActionBarContent_property
     ...FrMediaColumnContent_property
@@ -2398,7 +2398,7 @@ fragment ForRentSearchPageStateParamsFragment_property on Property {
 fragment RentalPrequalificationCard on ABTests {
   RC_QUALIFICATION_FLOW: abTest(trial: "RC_QUALIFICATION_FLOW")
 }
-`},a=p=>{const l=new AbortController;return setTimeout(()=>l.abort(),p||0),l.signal},n=await b(),i={headers:{...w,cookie:n},...t&&{signal:a(t)}},{data:s}=await u.post(r,o,i)||{};return s?.data},oe=async e=>{const t="https://www.zillow.com/graphql",r={operationName:"ListingContactDetailsQuery",variables:{zpid:e,altId:null},query:`query ListingContactDetailsQuery($zpid: ID!, $altId: ID) {
+`},a=p=>{const l=new AbortController;return setTimeout(()=>l.abort(),p||0),l.signal},i=await b(),s={headers:{...w,cookie:i},...t&&{signal:a(t)}},{data:n}=await c.post(r,o,s)||{};return n?.data},pe=async e=>{const t="https://www.zillow.com/graphql",r={operationName:"ListingContactDetailsQuery",variables:{zpid:e,altId:null},query:`query ListingContactDetailsQuery($zpid: ID!, $altId: ID) {
   viewer {
     roles {
       isLandlordLiaisonMember
@@ -2458,4 +2458,4 @@ fragment RentalPrequalificationCard on ABTests {
     }
   }
 }
-`},{data:o}=await u.post(t,r,{headers:w})||{};return o?.data?.property},E=(e,t)=>e===403&&t?.includes("captcha"),ae=e=>W(e),ie=async e=>{await u.post("http://localhost:8082/browser/open",{url:e})},C=async(e,t=0)=>{try{await X()||T("zip codes required, please run the createConfig script");const{fetchListings:r=!1}=e??{},o=3101,{results:a}=await G({regionId:o,daysOnZillow:1,mergePageResults:!1})??{},{zpid:n,detailUrl:i}=a?.[0]??{};return n&&r?await z(n):a?.length&&!r||(i&&!i.startsWith("/apartments")?await S(i):t<5?await C(e,t+1):T("Could not find a valid listing URL to check for bot filtering.")),!0}catch(r){const{status:o,message:a}=O(r);E(o,a)&&T("Zillow bot filtering! Please open https://zillow.com in your browser and complete the challenge to continue.")}},ne=async()=>await new Promise(e=>{const t=setInterval(async()=>{try{await ie("https://www.zillow.com/homes/for_rent/");const r=await C();r&&(e(r),clearInterval(t))}catch{}},500)}),k=()=>{const e=["Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36","Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.1 Safari/605.1.15","Mozilla/5.0 (Macintosh; Intel Mac OS X 13_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.1 Safari/605.1.15"],t=Math.floor(Math.random()*e.length);return e[t]},se=async(e,t)=>{const r=await Y.launch({headless:!0,timeout:1e5,defaultViewport:null}),o=await r.newPage();if(e.includes("file://"))await o.setJavaScriptEnabled(!1);else{const n=k();await o.setUserAgent(n)}await o.goto(e,{waitUntil:"networkidle2"});const a=await o.evaluate(()=>document.querySelector(".layout-content-container")?.clientHeight);a&&await o.setViewport({width:1024,height:a}),await o.screenshot({path:t,fullPage:!0,captureBeyondViewport:!0}),await o.close(),await r.close(),console.log(`screenshot complete, ${t}`)},pe=e=>{if(e==null||typeof e!="object")return!1;const t=e;return!(t.address===void 0||typeof t.bedrooms!="number"||typeof t.bedrooms!="number"||typeof t.daysOnZillow!="number"||typeof t.hdpUrl!="string"||typeof t.homeStatus!="string"||typeof t.homeType!="string"||typeof t.isRentalListingOffMarket!="boolean"||typeof t.price!="number"||typeof t.timeOnZillow!="string"||typeof t.zipcode!="string"||typeof t.zpid!="number")};export{C as checkForZillowBotFiltering,R as fetchHtmlFromRedfinListingUrl,S as fetchHtmlFromZillowListingUrl,V as fetchRedfinListingResults,P as fetchRedfinRegionIdByZipCode,f as fetchZillowListingResults,k as generateRandomUA,$ as getHtmlFromRedfinListingUrl,j as getHtmlFromZillowListingUrl,ae as getRandomZipCode,q as getRedfinListingDetailsById,K as getRedfinListingResults,N as getRedfinRegionIdByZipCode,oe as getZillowListingContactDetailsByZpid,z as getZillowListingDetailsByZpid,G as getZillowListingResults,B as getZillowRegionIdByZipCode,E as isZillowBotFiltering,pe as isZillowListingForRow,g as parseRedfinResponse,v as scrapeDataFromRedfinListingHtml,H as scrapeDataFromRedfinListingUrl,x as scrapeDataFromZillowListingHtml,J as scrapeDataFromZillowListingUrl,se as screenshotZillowListing,ne as waitForSolvedZillowCaptcha,U as zillowListingResultsHeaders};
+`},{data:o}=await c.post(t,r,{headers:w})||{};return o?.data?.property},E=(e,t)=>e===403&&t?.includes("captcha"),le=e=>q(e),k=async e=>{const{data:t}=await c.post("http://localhost:8082/browser/open",{url:e});return t?.browser},W=async()=>{const{data:e}=await c.post("http://localhost:8082/browser/status");return e?.browser},_e=async()=>{try{return(await W())?.status==="captcha"}catch{return!1}},Z=async()=>{await c.post("http://localhost:8082/cookie/refresh")},X=async(e,t=0)=>{try{await Q()||T("zip codes required, please run the createConfig script");const{fetchListings:r=!1}=e??{},o=3101,{results:a}=await G({regionId:o,daysOnZillow:1,mergePageResults:!1})??{},{zpid:i,detailUrl:s}=a?.[0]??{};return i&&r?await z(i):a?.length&&!r||(s&&!s.startsWith("/apartments")?await S(s):t<5?await X(e,t+1):T("Could not find a valid listing URL to check for bot filtering.")),!0}catch(r){const{status:o,message:a}=F(r);E(o,a)&&T("Zillow bot filtering! Please open https://zillow.com in your browser and complete the challenge to continue.")}},$=async()=>{const{data:e}=await c.post("http://localhost:8082/captcha/solve");return e?.solved===!0},de=async()=>{let e=!1;for(;;){try{const t=await k("https://www.zillow.com/homes/for_rent/");if(t?.status==="captcha"){if(!e&&await O()){e=!0,await $();continue}}else if(t?.status==="navigated")return await Z(),!0}catch{}await new Promise(t=>setTimeout(t,500))}},ce=async()=>{let e=!1,t=null,r=!1;for(;;){try{if(!e)(await k("https://www.zillow.com/homes/for_rent/"))?.status==="captcha"&&(e=!0,t=Date.now());else{if((await W())?.status==="navigated"){if((t?Date.now()-t:0)>=3e3)return await Z(),!0;e=!1,t=null,r=!1}if(!r&&await O()){r=!0,await $();continue}}}catch{}await new Promise(o=>setTimeout(o,e?500:100))}},Y=()=>{const e=["Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36","Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.1 Safari/605.1.15","Mozilla/5.0 (Macintosh; Intel Mac OS X 13_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.1 Safari/605.1.15"],t=Math.floor(Math.random()*e.length);return e[t]},ye=async(e,t)=>{const r=await j.launch({headless:!0,timeout:1e5,defaultViewport:null}),o=await r.newPage();if(e.includes("file://"))await o.setJavaScriptEnabled(!1);else{const i=Y();await o.setUserAgent(i)}await o.goto(e,{waitUntil:"networkidle2"});const a=await o.evaluate(()=>document.querySelector(".layout-content-container")?.clientHeight);a&&await o.setViewport({width:1024,height:a}),await o.screenshot({path:t,fullPage:!0,captureBeyondViewport:!0}),await o.close(),await r.close(),console.log(`screenshot complete, ${t}`)},me=e=>{if(e==null||typeof e!="object")return!1;const t=e;return!(t.address===void 0||typeof t.bedrooms!="number"||typeof t.bedrooms!="number"||typeof t.daysOnZillow!="number"||typeof t.hdpUrl!="string"||typeof t.homeStatus!="string"||typeof t.homeType!="string"||typeof t.isRentalListingOffMarket!="boolean"||typeof t.price!="number"||typeof t.timeOnZillow!="string"||typeof t.zipcode!="string"||typeof t.zpid!="number")};export{X as checkForZillowBotFiltering,R as fetchHtmlFromRedfinListingUrl,S as fetchHtmlFromZillowListingUrl,V as fetchRedfinListingResults,P as fetchRedfinRegionIdByZipCode,f as fetchZillowListingResults,Y as generateRandomUA,J as getHtmlFromRedfinListingUrl,oe as getHtmlFromZillowListingUrl,le as getRandomZipCode,ee as getRedfinListingDetailsById,te as getRedfinListingResults,N as getRedfinRegionIdByZipCode,pe as getZillowListingContactDetailsByZpid,z as getZillowListingDetailsByZpid,G as getZillowListingResults,B as getZillowRegionIdByZipCode,_e as isBrowserShowingCaptcha,E as isZillowBotFiltering,me as isZillowListingForRow,g as parseRedfinResponse,v as scrapeDataFromRedfinListingHtml,H as scrapeDataFromRedfinListingUrl,x as scrapeDataFromZillowListingHtml,ae as scrapeDataFromZillowListingUrl,ye as screenshotZillowListing,de as waitForSolvedZillowCaptcha,ce as waitForZillowCaptchaSolve,U as zillowListingResultsHeaders};

package/dist/shared/api.DfAvAXsZ.mjs

@@ -0,0 +1 @@
+import{getOutputPathFromConfig as i,getZipCodesFromConfig as t,getValueFromConfigFile as a}from"@rent-scraper/utils/config";const e=async()=>await i("redfin"),o=async()=>await t("redfin"),s=async()=>await a("redfin","daysListed"),l=async()=>await a("redfin","redfinCookie"),w=async()=>await a("zillow","zillowCookie"),n=async()=>await a("zillow","daysListed"),d=async()=>await a("zillow","autoCaptcha"),f=async()=>await a("zillow","limit"),g=async()=>await a("zillow","offset"),c=async()=>await i("zillow"),y=async()=>await t("zillow");export{n as a,d as b,f as c,g as d,c as e,y as f,w as g,e as h,o as i,s as j,l as k};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rent-scraper/api",
-  "version": "1.0.28",
+  "version": "1.0.30",
   "type": "module",
   "main": "./dist/index.mjs",
   "module": "./dist/index.mjs",
@@ -31,7 +31,7 @@
     "cheerio": "^1.1.0",
     "dayjs": "^1.11.13",
     "puppeteer": "^24.11.2",
-    "@rent-scraper/utils": "1.0.28"
+    "@rent-scraper/utils": "1.0.30"
   },
   "devDependencies": {
     "unbuild": "^3.5.0",

package/dist/shared/api.bGXs5gok.mjs

@@ -1 +0,0 @@
-import{getOutputPathFromConfig as a,getZipCodesFromConfig as t,getValueFromConfigFile as i}from"@rent-scraper/utils/config";const e=async()=>await a("redfin"),o=async()=>await t("redfin"),s=async()=>await i("redfin","daysListed"),n=async()=>await i("redfin","redfinCookie"),l=async()=>await i("zillow","zillowCookie"),d=async()=>await i("zillow","daysListed"),w=async()=>await a("zillow"),g=async()=>await t("zillow");export{d as a,w as b,g as c,e as d,o as e,s as f,l as g,n as h};