@rent-scraper/browser-server 1.0.30 → 1.0.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md
CHANGED
|
@@ -1,3 +1,40 @@
|
|
|
1
1
|
# @rent-scraper/browser-server
|
|
2
2
|
|
|
3
|
-
Local
|
|
3
|
+
Local HTTP server wrapping Puppeteer. Runs at `http://localhost:8082` and manages a persistent browser session for Zillow scraping — handling cookie capture, PerimeterX bot detection, and captcha solving.
|
|
4
|
+
|
|
5
|
+
## Why a separate server?
|
|
6
|
+
|
|
7
|
+
The browser session (and its PerimeterX trust cookies) must persist across the full scrape run. Running Puppeteer inside the scrape process would restart the session on every zip code fetch. The server keeps the browser alive and exposes it over HTTP so the scraping process can read cookies and trigger actions without owning the browser lifecycle.
|
|
8
|
+
|
|
9
|
+
## Endpoints
|
|
10
|
+
|
|
11
|
+
| Method | Path | Description |
|
|
12
|
+
| ------ | --------------------- | ------------------------------------------------------------------------------------ |
|
|
13
|
+
| `GET` | `/server` | Health check — returns `{ running: true }` |
|
|
14
|
+
| `POST` | `/browser/launch` | Launch the browser |
|
|
15
|
+
| `POST` | `/browser/open` | Navigate to a URL, returns `{ status: 'navigated' \| 'captcha' \| 'not connected' }` |
|
|
16
|
+
| `POST` | `/browser/status` | Check current page status without navigating |
|
|
17
|
+
| `POST` | `/browser/close` | Close the current page |
|
|
18
|
+
| `GET` | `/cookie` | Return the current Zillow cookie string |
|
|
19
|
+
| `POST` | `/cookie/save` | Extract and save the Zillow cookie from the browser session |
|
|
20
|
+
| `POST` | `/cookie/refresh` | Re-extract cookie after a navigation |
|
|
21
|
+
| `POST` | `/cookie/redfin/save` | Extract and save the Redfin cookie |
|
|
22
|
+
| `POST` | `/captcha/solve` | Auto-solve a PerimeterX captcha via keyboard hold |
|
|
23
|
+
| `POST` | `/server/shutdown` | Shut down the browser and stop the server |
|
|
24
|
+
|
|
25
|
+
## Captcha Solving
|
|
26
|
+
|
|
27
|
+
PerimeterX uses a press-and-hold challenge (`#px-captcha`). The solver (`solve-captcha.ts`) presses `Tab` to focus the element, holds `Space` for 10 seconds, then reloads to verify. Retries up to 5 times with a 3-second pause between attempts.
|
|
28
|
+
|
|
29
|
+
## Usage
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
# Start (production build)
|
|
33
|
+
pnpm run browser-server
|
|
34
|
+
|
|
35
|
+
# Start (dev, no build)
|
|
36
|
+
pnpm run browser-server:dev
|
|
37
|
+
|
|
38
|
+
# Debug mode (logs browser and server address)
|
|
39
|
+
pnpm run browser-server -- --debug
|
|
40
|
+
```
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import{h as r}from"../shared/browser-server.
|
|
2
|
+
import{h as r}from"../shared/browser-server.BZvS6Tr9.mjs";import"express";import"puppeteer";import"fs/promises";import"child_process";import"os";import"path";import"@rent-scraper/utils/config";import"@rent-scraper/utils";r();
|
package/dist/index.mjs
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export{c as closeBrowser,g as getBrowser,a as getBrowserStatus,d as getRedfinCookie,b as getZillowCookie,l as launchBrowser,o as openBrowser,r as refreshZillowCookie,h as runBrowserServer,e as saveRedfinCookie,f as saveZillowCookie,s as shutdownBrowser,w as waitForBrowser}from"./shared/browser-server.
|
|
1
|
+
export{c as closeBrowser,g as getBrowser,a as getBrowserStatus,d as getRedfinCookie,b as getZillowCookie,l as launchBrowser,o as openBrowser,r as refreshZillowCookie,h as runBrowserServer,e as saveRedfinCookie,f as saveZillowCookie,s as shutdownBrowser,w as waitForBrowser}from"./shared/browser-server.BZvS6Tr9.mjs";import{confirm as r,isCancel as a,cancel as s,outro as i}from"@clack/prompts";import{setTimeout as o}from"node:timers/promises";import"express";import"puppeteer";import"fs/promises";import"child_process";import"os";import"path";import"@rent-scraper/utils/config";import"@rent-scraper/utils";async function t(){const e=await r({message:"We need to launch your browser to continue",active:"OK",inactive:"Cancel"});if(a(e)||!e)return s("Create config canceled. Please try again."),process.exit(1);await o(1e3),i("Browser Launching..."),await o(1e3)}export{t as runConfirmBrowserLaunch};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import x from"express";import E from"puppeteer";import{mkdtemp as X,access as Y}from"fs/promises";import{exec as k}from"child_process";import{tmpdir as ee}from"os";import te from"path";import{checkForConfigFile as oe,waitForConfigFile as ne,getValueFromConfigFile as ae,updateConfigFile as M}from"@rent-scraper/utils/config";import{parseError as L}from"@rent-scraper/utils";const j={darwin:["/Applications/Brave Browser.app/Contents/MacOS/Brave Browser","/Applications/Google Chrome.app/Contents/MacOS/Google Chrome","/Applications/Chromium.app/Contents/MacOS/Chromium","/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge"],linux:["/usr/bin/brave-browser","/usr/bin/google-chrome","/usr/bin/google-chrome-stable","/usr/bin/chromium-browser","/usr/bin/chromium","/snap/bin/chromium"],win32:["C:\\Program Files\\BraveSoftware\\Brave-Browser\\Application\\brave.exe","C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe","C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe"]},H=async()=>{const e=j[process.platform]??[];for(const t of e)try{return await Y(t),t}catch{}return null},V="http://127.0.0.1:9222/json/version",se=async(e=15e3)=>{const t=Date.now();for(;Date.now()-t<e;)try{await E.connect({browserURL:V});return}catch{await new Promise(c=>setTimeout(c,500))}throw new Error("timed out waiting for browser")},$=async()=>{try{return await E.connect({browserURL:V})}catch{return null}},S=async()=>{const e=await(await $())?.pages();e&&await Promise.all(e.map(t=>t.close()))},N=async(e="zillow")=>{if(await $())return{status:"already launched"};await oe(e)||await ne(e);const t=await X(te.join(ee(),"chrome-remote-")),c=await ae(e,"browser"),a=c?(j[process.platform]??[]).find(s=>s.toLowerCase().includes(c.toLowerCase()))??await H():await H();if(console.log(`launching browser: ${a??"bundled chromium"}`),a)k(`"${a}" --remote-debugging-port=9222 --no-first-run --no-default-browser-check --user-data-dir="${t}"`,s=>{s&&console.error(`Error: ${s.message}`)});else{const s=["--remote-debugging-port=9222","--no-first-run","--no-default-browser-check",`--user-data-dir=${t}`,...process.platform==="linux"?["--no-sandbox","--disable-setuid-sandbox"]:[]];await E.launch({headless:!1,args:s,dumpio:!1})}return{status:"launched"}},W=async()=>{const e=await $();if(e){const t=(await e.pages())?.[0];return t?(await t.title()).includes("denied")?{status:"captcha"}:{status:"navigated"}:{status:"navigated"}}else return{status:"not connected"}},_=async e=>{const t=await $();if(t){const c=e,a=(await t.pages())?.[0]??await t.newPage();return(await a.title()).includes("denied")?{status:"captcha"}:(await a.goto(c,{waitUntil:"load"}),(await a.title()).includes("denied")?{status:"captcha"}:{status:"navigated"})}else return{status:"not connected"}},U=async()=>{const e=await $();if(e){try{await(await e.target().createCDPSession()).send("Browser.close")}catch{await e.disconnect()}return{status:"closed"}}else return{status:"not connected"}},G="http://127.0.0.1:9222/json/version",A=async(e=0)=>{const t=await E.connect({browserURL:G});e===0&&await _("https://www.zillow.com/homes/for_rent/");const c=await t.cookies(),[a]=c.filter(n=>n.name==="_pxvid"),[s]=c.filter(n=>n.name==="_px3");return a?(await S(),[a,s].filter(Boolean).map(n=>`${n.name}=${n.value}`).join("; ")):(console.log("refetching zillow cookie"),await new Promise(n=>setTimeout(n,2e3)),await A(e+1))},I=async(e=0)=>{const t=await E.connect({browserURL:G});e===0&&(await _("https://www.redfin.com"),await new Promise(n=>setTimeout(n,3e3)));const c=await t.pages(),a=c?.[0]?await c[0].title():"",s=(await t.cookies()).filter(n=>n.domain.includes("redfin.com"));return s.some(n=>n.name==="aws-waf-token")&&a.includes("Redfin")?(await S(),s.map(n=>`${n.name}=${n.value}`).join("; ")):(console.log("refetching redfin cookie"),await new Promise(n=>setTimeout(n,2e3)),await I(e+1))},Z=async()=>{try{const e=await I();e&&await M("redfin",{redfinCookie:e})}catch(e){const{status:t,message:c}=L(e);console.error(t,c)}},J=async()=>{try{const e=await(await E.connect({browserURL:G})).cookies(),[t]=e.filter(a=>a.name==="_pxvid"),[c]=e.filter(a=>a.name==="_px3");if(t){const a=[t,c].filter(Boolean).map(s=>`${s.name}=${s.value}`).join("; ");return await M("zillow",{zillowCookie:a}),a}}catch(e){const{status:t,message:c}=L(e);console.error(t,c)}},D=async()=>{try{const e=await A();e&&await M("zillow",{zillowCookie:e})}catch(e){const{status:t,message:c}=L(e);console.error(t,c)}};function re(e){return e&&e.__esModule&&Object.prototype.hasOwnProperty.call(e,"default")?e.default:e}var K,Q;function ie(){if(Q)return K;Q=1;function e(a,s){var n=a;s.slice(0,-1).forEach(function(w){n=n[w]||{}});var p=s[s.length-1];return p in n}function t(a){return typeof a=="number"||/^0x[0-9a-f]+$/i.test(a)?!0:/^[-+]?(?:\d+(?:\.\d*)?|\.\d+)(e[-+]?\d+)?$/.test(a)}function c(a,s){return s==="constructor"&&typeof a[s]=="function"||s==="__proto__"}return K=function(a,s){s||(s={});var n={bools:{},strings:{},unknownFn:null};typeof s.unknown=="function"&&(n.unknownFn=s.unknown),typeof s.boolean=="boolean"&&s.boolean?n.allBools=!0:[].concat(s.boolean).filter(Boolean).forEach(function(i){n.bools[i]=!0});var p={};function w(i){return p[i].some(function(d){return n.bools[d]})}Object.keys(s.alias||{}).forEach(function(i){p[i]=[].concat(s.alias[i]),p[i].forEach(function(d){p[d]=[i].concat(p[i].filter(function(C){return d!==C}))})}),[].concat(s.string).filter(Boolean).forEach(function(i){n.strings[i]=!0,p[i]&&[].concat(p[i]).forEach(function(d){n.strings[d]=!0})});var r=s.default||{},o={_:[]};function T(i,d){return n.allBools&&/^--[^=]+$/.test(d)||n.strings[i]||n.bools[i]||p[i]}function O(i,d,C){for(var u=i,F=0;F<d.length-1;F++){var y=d[F];if(c(u,y))return;u[y]===void 0&&(u[y]={}),(u[y]===Object.prototype||u[y]===Number.prototype||u[y]===String.prototype)&&(u[y]={}),u[y]===Array.prototype&&(u[y]=[]),u=u[y]}var B=d[d.length-1];c(u,B)||((u===Object.prototype||u===Number.prototype||u===String.prototype)&&(u={}),u===Array.prototype&&(u=[]),u[B]===void 0||n.bools[B]||typeof u[B]=="boolean"?u[B]=C:Array.isArray(u[B])?u[B].push(C):u[B]=[u[B],C])}function h(i,d,C){if(!(C&&n.unknownFn&&!T(i,C)&&n.unknownFn(C)===!1)){var u=!n.strings[i]&&t(d)?Number(d):d;O(o,i.split("."),u),(p[i]||[]).forEach(function(F){O(o,F.split("."),u)})}}Object.keys(n.bools).forEach(function(i){h(i,r[i]===void 0?!1:r[i])});var z=[];a.indexOf("--")!==-1&&(z=a.slice(a.indexOf("--")+1),a=a.slice(0,a.indexOf("--")));for(var m=0;m<a.length;m++){var l=a[m],f,v;if(/^--.+=/.test(l)){var q=l.match(/^--([^=]+)=([\s\S]*)$/);f=q[1];var R=q[2];n.bools[f]&&(R=R!=="false"),h(f,R,l)}else if(/^--no-.+/.test(l))f=l.match(/^--no-(.+)/)[1],h(f,!1,l);else if(/^--.+/.test(l))f=l.match(/^--(.+)/)[1],v=a[m+1],v!==void 0&&!/^(-|--)[^-]/.test(v)&&!n.bools[f]&&!n.allBools&&(!p[f]||!w(f))?(h(f,v,l),m+=1):/^(true|false)$/.test(v)?(h(f,v==="true",l),m+=1):h(f,n.strings[f]?"":!0,l);else if(/^-[^-]+/.test(l)){for(var g=l.slice(1,-1).split(""),P=!1,b=0;b<g.length;b++){if(v=l.slice(b+2),v==="-"){h(g[b],v,l);continue}if(/[A-Za-z]/.test(g[b])&&v[0]==="="){h(g[b],v.slice(1),l),P=!0;break}if(/[A-Za-z]/.test(g[b])&&/-?\d+(\.\d*)?(e-?\d+)?$/.test(v)){h(g[b],v,l),P=!0;break}if(g[b+1]&&g[b+1].match(/\W/)){h(g[b],l.slice(b+2),l),P=!0;break}else h(g[b],n.strings[g[b]]?"":!0,l)}f=l.slice(-1)[0],!P&&f!=="-"&&(a[m+1]&&!/^(-|--)[^-]/.test(a[m+1])&&!n.bools[f]&&(!p[f]||!w(f))?(h(f,a[m+1],l),m+=1):a[m+1]&&/^(true|false)$/.test(a[m+1])?(h(f,a[m+1]==="true",l),m+=1):h(f,n.strings[f]?"":!0,l))}else if((!n.unknownFn||n.unknownFn(l)!==!1)&&o._.push(n.strings._||!t(l)?l:Number(l)),s.stopEarly){o._.push.apply(o._,a.slice(m+1));break}}return Object.keys(r).forEach(function(i){e(o,i.split("."))||(O(o,i.split("."),r[i]),(p[i]||[]).forEach(function(d){O(o,d.split("."),r[i])}))}),s["--"]?o["--"]=z.slice():z.forEach(function(i){o._.push(i)}),o},K}var ce=ie();const le=re(ce),ue=1e4,fe=async e=>{await e.keyboard.press("Escape"),await new Promise(t=>setTimeout(t,500)),await e.keyboard.press("Tab"),await new Promise(t=>setTimeout(t,200)),await e.keyboard.down("Space"),await new Promise(t=>setTimeout(t,ue)),await e.keyboard.up("Space")},we=async()=>{try{const e=await $();if(!e)return!1;const t=(await e.pages())?.[0];if(!t)return!1;if(!(await t.title()).includes("denied"))return!0;for(let c=0;c<5;c++){try{await fe(t)}catch{}if(await t.reload({waitUntil:"load"}),!(await t.title()).includes("denied"))return!0;await new Promise(a=>setTimeout(a,3e3))}return!1}catch{return!1}};function de(e="zillow"){const t=x(),c=process.env.HOST??"127.0.0.1",a=process.env.PORT??8082;t.use(x.json());const s=le(process.argv.slice(2)).debug,n=t.listen(Number(a),async()=>{await N(e);const w=setInterval(async()=>{(await $())?.connected&&(clearInterval(w),s&&console.log("Browser listening at 127.0.0.1:9222"),e==="zillow"?await D():e==="redfin"&&(await Z(),await U()))},1e3);s&&console.log(`Server listening at ${c}:${a}`)}),p=async()=>(await U(),n.close(w=>{s&&console.log("server closed"),process.exit(w?1:0)}),{status:"shutdown"});return t.get("/server",(w,r)=>{try{r.send({running:!0})}catch(o){r.send(o)}}),t.post("/browser/launch",async(w,r)=>{try{const o=await N();r.send({browser:o})}catch(o){r.send(o)}}),t.post("/browser/status",async(w,r)=>{try{const o=await W();r.send({browser:o})}catch(o){r.send(o)}}),t.post("/browser/open",async(w,r)=>{try{const{url:o}=w?.body??{},T=await _(o);r.send({browser:T})}catch(o){r.send(o)}}),t.get("/cookie",async(w,r)=>{try{const o=await A();r.send({cookie:o})}catch(o){r.send(o)}}),t.post("/cookie/refresh",async(w,r)=>{try{const o=await J();r.send({cookie:o})}catch(o){r.send(o)}}),t.post("/cookie/save",async(w,r)=>{try{const o=await D();r.send({cookie:o})}catch(o){r.send(o)}}),t.post("/cookie/redfin/save",async(w,r)=>{try{const o=await Z();r.send({cookie:o})}catch(o){r.send(o)}}),t.post("/captcha/solve",async(w,r)=>{try{const o=await we();r.send({solved:o})}catch(o){r.send(o)}}),t.post("/browser/close",async(w,r)=>{try{const o=await S();r.send({browser:o})}catch(o){r.send(o)}}),t.post("/server/shutdown",(w,r)=>{try{const o=p();r.send({server:o})}catch(o){r.send(o)}}),process.on("SIGINT",p),process.on("SIGTERM",p),t}export{W as a,A as b,S as c,I as d,Z as e,D as f,$ as g,de as h,N as l,_ as o,J as r,U as s,se as w};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rent-scraper/browser-server",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.32",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": {
|
|
@@ -26,8 +26,8 @@
|
|
|
26
26
|
"express": "^4.21.2",
|
|
27
27
|
"picocolors": "^1.1.1",
|
|
28
28
|
"puppeteer": "^24.11.2",
|
|
29
|
-
"@rent-scraper/
|
|
30
|
-
"@rent-scraper/
|
|
29
|
+
"@rent-scraper/api": "1.0.32",
|
|
30
|
+
"@rent-scraper/utils": "1.0.32"
|
|
31
31
|
},
|
|
32
32
|
"devDependencies": {
|
|
33
33
|
"@types/express": "^5.0.3",
|
package/src/solve-captcha.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { Page } from 'puppeteer'
|
|
2
2
|
import { getBrowser } from './browser.js'
|
|
3
3
|
|
|
4
|
-
const HOLD_DURATION_MS = 10000
|
|
4
|
+
const HOLD_DURATION_MS = 10000
|
|
5
5
|
|
|
6
6
|
const pressAndHoldSpace = async (page: Page) => {
|
|
7
7
|
await page.keyboard.press('Escape')
|
|
@@ -23,44 +23,20 @@ export const solveZillowCaptcha = async (): Promise<boolean> => {
|
|
|
23
23
|
if (!page) return false
|
|
24
24
|
|
|
25
25
|
const title = await page.title()
|
|
26
|
-
if (!title.includes('denied')) return true
|
|
27
|
-
|
|
28
|
-
// Delete pxcts cookie — resets the challenge state so a fresh attempt starts clean
|
|
29
|
-
await page.deleteCookie({ name: 'pxcts', domain: '.zillow.com' })
|
|
26
|
+
if (!title.includes('denied')) return true
|
|
30
27
|
|
|
31
28
|
for (let attempt = 0; attempt < 5; attempt++) {
|
|
32
|
-
if (attempt > 0) {
|
|
33
|
-
// wait between attempts
|
|
34
|
-
await new Promise(resolve => setTimeout(resolve, 5000))
|
|
35
|
-
}
|
|
36
|
-
|
|
37
29
|
try {
|
|
38
|
-
// Try mouse hold on the captcha element first
|
|
39
|
-
const captchaEl = await page.$('#px-captcha')
|
|
40
|
-
if (captchaEl) {
|
|
41
|
-
const box = await captchaEl.boundingBox()
|
|
42
|
-
if (box) {
|
|
43
|
-
await page.mouse.move(box.x + box.width / 2, box.y + box.height / 2)
|
|
44
|
-
await page.mouse.down()
|
|
45
|
-
await new Promise(resolve => setTimeout(resolve, HOLD_DURATION_MS))
|
|
46
|
-
await page.mouse.up()
|
|
47
|
-
} else {
|
|
48
|
-
// element found but no bounding box — fall back to keyboard
|
|
49
|
-
await pressAndHoldSpace(page)
|
|
50
|
-
}
|
|
51
|
-
} else {
|
|
52
|
-
// no element found — fall back to keyboard Tab + Space
|
|
53
|
-
await pressAndHoldSpace(page)
|
|
54
|
-
}
|
|
55
|
-
} catch {
|
|
56
|
-
// any puppeteer error — try keyboard as last resort
|
|
57
30
|
await pressAndHoldSpace(page)
|
|
31
|
+
} catch {
|
|
32
|
+
// ignore puppeteer errors, try again
|
|
58
33
|
}
|
|
59
34
|
|
|
60
|
-
|
|
61
|
-
await new Promise(resolve => setTimeout(resolve, 1000))
|
|
35
|
+
await page.reload({ waitUntil: 'load' })
|
|
62
36
|
const currentTitle = await page.title()
|
|
63
37
|
if (!currentTitle.includes('denied')) return true
|
|
38
|
+
|
|
39
|
+
await new Promise(resolve => setTimeout(resolve, 3000))
|
|
64
40
|
}
|
|
65
41
|
|
|
66
42
|
return false
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import j from"express";import E from"puppeteer";import{mkdtemp as k,access as ee}from"fs/promises";import{exec as te}from"child_process";import{tmpdir as oe}from"os";import ae from"path";import{checkForConfigFile as ne,waitForConfigFile as se,getValueFromConfigFile as ie,updateConfigFile as M}from"@rent-scraper/utils/config";import{parseError as L}from"@rent-scraper/utils";const H={darwin:["/Applications/Brave Browser.app/Contents/MacOS/Brave Browser","/Applications/Google Chrome.app/Contents/MacOS/Google Chrome","/Applications/Chromium.app/Contents/MacOS/Chromium","/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge"],linux:["/usr/bin/brave-browser","/usr/bin/google-chrome","/usr/bin/google-chrome-stable","/usr/bin/chromium-browser","/usr/bin/chromium","/snap/bin/chromium"],win32:["C:\\Program Files\\BraveSoftware\\Brave-Browser\\Application\\brave.exe","C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe","C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe"]},V=async()=>{const t=H[process.platform]??[];for(const e of t)try{return await ee(e),e}catch{}return null},W="http://127.0.0.1:9222/json/version",re=async(t=15e3)=>{const e=Date.now();for(;Date.now()-e<t;)try{await E.connect({browserURL:W});return}catch{await new Promise(c=>setTimeout(c,500))}throw new Error("timed out waiting for browser")},$=async()=>{try{return await E.connect({browserURL:W})}catch{return null}},S=async()=>{const t=await(await $())?.pages();t&&await Promise.all(t.map(e=>e.close()))},N=async(t="zillow")=>{if(await $())return{status:"already launched"};await ne(t)||await se(t);const e=await k(ae.join(oe(),"chrome-remote-")),c=await ie(t,"browser"),o=c?(H[process.platform]??[]).find(s=>s.toLowerCase().includes(c.toLowerCase()))??await V():await V();if(console.log(`launching browser: ${o??"bundled chromium"}`),o)te(`"${o}" --remote-debugging-port=9222 --no-first-run --no-default-browser-check --user-data-dir="${e}"`,s=>{s&&console.error(`Error: ${s.message}`)});else{const s=["--remote-debugging-port=9222","--no-first-run","--no-default-browser-check",`--user-data-dir=${e}`,...process.platform==="linux"?["--no-sandbox","--disable-setuid-sandbox"]:[]];await E.launch({headless:!1,args:s,dumpio:!1})}return{status:"launched"}},J=async()=>{const t=await $();if(t){const e=(await t.pages())?.[0];return e?(await e.title()).includes("denied")?{status:"captcha"}:{status:"navigated"}:{status:"navigated"}}else return{status:"not connected"}},_=async t=>{const e=await $();if(e){const c=t,o=(await e.pages())?.[0]??await e.newPage();return(await o.title()).includes("denied")?{status:"captcha"}:(await o.goto(c,{waitUntil:"load"}),(await o.title()).includes("denied")?{status:"captcha"}:{status:"navigated"})}else return{status:"not connected"}},x=async()=>{const t=await $();if(t){try{await(await t.target().createCDPSession()).send("Browser.close")}catch{await t.disconnect()}return{status:"closed"}}else return{status:"not connected"}},G="http://127.0.0.1:9222/json/version",T=async(t=0)=>{const e=await E.connect({browserURL:G});t===0&&await _("https://www.zillow.com/homes/for_rent/");const c=await e.cookies(),[o]=c.filter(n=>n.name==="_pxvid"),[s]=c.filter(n=>n.name==="_px3");return o?(await S(),[o,s].filter(Boolean).map(n=>`${n.name}=${n.value}`).join("; ")):(console.log("refetching zillow cookie"),await new Promise(n=>setTimeout(n,2e3)),await T(t+1))},I=async(t=0)=>{const e=await E.connect({browserURL:G});t===0&&(await _("https://www.redfin.com"),await new Promise(n=>setTimeout(n,3e3)));const c=await e.pages(),o=c?.[0]?await c[0].title():"",s=(await e.cookies()).filter(n=>n.domain.includes("redfin.com"));return s.some(n=>n.name==="aws-waf-token")&&o.includes("Redfin")?(await S(),s.map(n=>`${n.name}=${n.value}`).join("; ")):(console.log("refetching redfin cookie"),await new Promise(n=>setTimeout(n,2e3)),await I(t+1))},U=async()=>{try{const t=await I();t&&await M("redfin",{redfinCookie:t})}catch(t){const{status:e,message:c}=L(t);console.error(e,c)}},Q=async()=>{try{const t=await(await E.connect({browserURL:G})).cookies(),[e]=t.filter(o=>o.name==="_pxvid"),[c]=t.filter(o=>o.name==="_px3");if(e){const o=[e,c].filter(Boolean).map(s=>`${s.name}=${s.value}`).join("; ");return await M("zillow",{zillowCookie:o}),o}}catch(t){const{status:e,message:c}=L(t);console.error(e,c)}},Z=async()=>{try{const t=await T();t&&await M("zillow",{zillowCookie:t})}catch(t){const{status:e,message:c}=L(t);console.error(e,c)}};function ce(t){return t&&t.__esModule&&Object.prototype.hasOwnProperty.call(t,"default")?t.default:t}var D,X;function le(){if(X)return D;X=1;function t(o,s){var n=o;s.slice(0,-1).forEach(function(f){n=n[f]||{}});var p=s[s.length-1];return p in n}function e(o){return typeof o=="number"||/^0x[0-9a-f]+$/i.test(o)?!0:/^[-+]?(?:\d+(?:\.\d*)?|\.\d+)(e[-+]?\d+)?$/.test(o)}function c(o,s){return s==="constructor"&&typeof o[s]=="function"||s==="__proto__"}return D=function(o,s){s||(s={});var n={bools:{},strings:{},unknownFn:null};typeof s.unknown=="function"&&(n.unknownFn=s.unknown),typeof s.boolean=="boolean"&&s.boolean?n.allBools=!0:[].concat(s.boolean).filter(Boolean).forEach(function(r){n.bools[r]=!0});var p={};function f(r){return p[r].some(function(d){return n.bools[d]})}Object.keys(s.alias||{}).forEach(function(r){p[r]=[].concat(s.alias[r]),p[r].forEach(function(d){p[d]=[r].concat(p[r].filter(function(C){return d!==C}))})}),[].concat(s.string).filter(Boolean).forEach(function(r){n.strings[r]=!0,p[r]&&[].concat(p[r]).forEach(function(d){n.strings[d]=!0})});var i=s.default||{},a={_:[]};function A(r,d){return n.allBools&&/^--[^=]+$/.test(d)||n.strings[r]||n.bools[r]||p[r]}function P(r,d,C){for(var u=r,F=0;F<d.length-1;F++){var y=d[F];if(c(u,y))return;u[y]===void 0&&(u[y]={}),(u[y]===Object.prototype||u[y]===Number.prototype||u[y]===String.prototype)&&(u[y]={}),u[y]===Array.prototype&&(u[y]=[]),u=u[y]}var B=d[d.length-1];c(u,B)||((u===Object.prototype||u===Number.prototype||u===String.prototype)&&(u={}),u===Array.prototype&&(u=[]),u[B]===void 0||n.bools[B]||typeof u[B]=="boolean"?u[B]=C:Array.isArray(u[B])?u[B].push(C):u[B]=[u[B],C])}function h(r,d,C){if(!(C&&n.unknownFn&&!A(r,C)&&n.unknownFn(C)===!1)){var u=!n.strings[r]&&e(d)?Number(d):d;P(a,r.split("."),u),(p[r]||[]).forEach(function(F){P(a,F.split("."),u)})}}Object.keys(n.bools).forEach(function(r){h(r,i[r]===void 0?!1:i[r])});var z=[];o.indexOf("--")!==-1&&(z=o.slice(o.indexOf("--")+1),o=o.slice(0,o.indexOf("--")));for(var m=0;m<o.length;m++){var l=o[m],w,v;if(/^--.+=/.test(l)){var q=l.match(/^--([^=]+)=([\s\S]*)$/);w=q[1];var R=q[2];n.bools[w]&&(R=R!=="false"),h(w,R,l)}else if(/^--no-.+/.test(l))w=l.match(/^--no-(.+)/)[1],h(w,!1,l);else if(/^--.+/.test(l))w=l.match(/^--(.+)/)[1],v=o[m+1],v!==void 0&&!/^(-|--)[^-]/.test(v)&&!n.bools[w]&&!n.allBools&&(!p[w]||!f(w))?(h(w,v,l),m+=1):/^(true|false)$/.test(v)?(h(w,v==="true",l),m+=1):h(w,n.strings[w]?"":!0,l);else if(/^-[^-]+/.test(l)){for(var g=l.slice(1,-1).split(""),O=!1,b=0;b<g.length;b++){if(v=l.slice(b+2),v==="-"){h(g[b],v,l);continue}if(/[A-Za-z]/.test(g[b])&&v[0]==="="){h(g[b],v.slice(1),l),O=!0;break}if(/[A-Za-z]/.test(g[b])&&/-?\d+(\.\d*)?(e-?\d+)?$/.test(v)){h(g[b],v,l),O=!0;break}if(g[b+1]&&g[b+1].match(/\W/)){h(g[b],l.slice(b+2),l),O=!0;break}else h(g[b],n.strings[g[b]]?"":!0,l)}w=l.slice(-1)[0],!O&&w!=="-"&&(o[m+1]&&!/^(-|--)[^-]/.test(o[m+1])&&!n.bools[w]&&(!p[w]||!f(w))?(h(w,o[m+1],l),m+=1):o[m+1]&&/^(true|false)$/.test(o[m+1])?(h(w,o[m+1]==="true",l),m+=1):h(w,n.strings[w]?"":!0,l))}else if((!n.unknownFn||n.unknownFn(l)!==!1)&&a._.push(n.strings._||!e(l)?l:Number(l)),s.stopEarly){a._.push.apply(a._,o.slice(m+1));break}}return Object.keys(i).forEach(function(r){t(a,r.split("."))||(P(a,r.split("."),i[r]),(p[r]||[]).forEach(function(d){P(a,d.split("."),i[r])}))}),s["--"]?a["--"]=z.slice():z.forEach(function(r){a._.push(r)}),a},D}var ue=le();const we=ce(ue),Y=1e4,K=async t=>{await t.keyboard.press("Escape"),await new Promise(e=>setTimeout(e,500)),await t.keyboard.press("Tab"),await new Promise(e=>setTimeout(e,200)),await t.keyboard.down("Space"),await new Promise(e=>setTimeout(e,Y)),await t.keyboard.up("Space")},fe=async()=>{try{const t=await $();if(!t)return!1;const e=(await t.pages())?.[0];if(!e)return!1;if(!(await e.title()).includes("denied"))return!0;await e.deleteCookie({name:"pxcts",domain:".zillow.com"});for(let c=0;c<5;c++){c>0&&await new Promise(o=>setTimeout(o,5e3));try{const o=await e.$("#px-captcha");if(o){const s=await o.boundingBox();s?(await e.mouse.move(s.x+s.width/2,s.y+s.height/2),await e.mouse.down(),await new Promise(n=>setTimeout(n,Y)),await e.mouse.up()):await K(e)}else await K(e)}catch{await K(e)}if(await new Promise(o=>setTimeout(o,1e3)),!(await e.title()).includes("denied"))return!0}return!1}catch{return!1}};function de(t="zillow"){const e=j(),c=process.env.HOST??"127.0.0.1",o=process.env.PORT??8082;e.use(j.json());const s=we(process.argv.slice(2)).debug,n=e.listen(Number(o),async()=>{await N(t);const f=setInterval(async()=>{(await $())?.connected&&(clearInterval(f),s&&console.log("Browser listening at 127.0.0.1:9222"),t==="zillow"?await Z():t==="redfin"&&(await U(),await x()))},1e3);s&&console.log(`Server listening at ${c}:${o}`)}),p=async()=>(await x(),n.close(f=>{s&&console.log("server closed"),process.exit(f?1:0)}),{status:"shutdown"});return e.get("/server",(f,i)=>{try{i.send({running:!0})}catch(a){i.send(a)}}),e.post("/browser/launch",async(f,i)=>{try{const a=await N();i.send({browser:a})}catch(a){i.send(a)}}),e.post("/browser/status",async(f,i)=>{try{const a=await J();i.send({browser:a})}catch(a){i.send(a)}}),e.post("/browser/open",async(f,i)=>{try{const{url:a}=f?.body??{},A=await _(a);i.send({browser:A})}catch(a){i.send(a)}}),e.get("/cookie",async(f,i)=>{try{const a=await T();i.send({cookie:a})}catch(a){i.send(a)}}),e.post("/cookie/refresh",async(f,i)=>{try{const a=await Q();i.send({cookie:a})}catch(a){i.send(a)}}),e.post("/cookie/save",async(f,i)=>{try{const a=await Z();i.send({cookie:a})}catch(a){i.send(a)}}),e.post("/cookie/redfin/save",async(f,i)=>{try{const a=await U();i.send({cookie:a})}catch(a){i.send(a)}}),e.post("/captcha/solve",async(f,i)=>{try{const a=await fe();i.send({solved:a})}catch(a){i.send(a)}}),e.post("/browser/close",async(f,i)=>{try{const a=await S();i.send({browser:a})}catch(a){i.send(a)}}),e.post("/server/shutdown",(f,i)=>{try{const a=p();i.send({server:a})}catch(a){i.send(a)}}),process.on("SIGINT",p),process.on("SIGTERM",p),e}export{J as a,T as b,S as c,I as d,U as e,Z as f,$ as g,de as h,N as l,_ as o,Q as r,x as s,re as w};
|