@rent-scraper/browser-server 1.0.0

package/dist/index.mjs

@@ -0,0 +1 @@
+export{c as closeBrowser,g as getBrowser,a as getZillowCookie,l as launchBrowser,o as openBrowser,r as runBrowserServer,b as saveZillowCookie,s as shutdownBrowser}from"./shared/browser-server.CTHRk1um.mjs";import{confirm as r,isCancel as a,cancel as s,outro as t}from"@clack/prompts";import{setTimeout as o}from"node:timers/promises";import"express";import"puppeteer";import"child_process";import"@rent-scraper/utils/config";import"@rent-scraper/utils";async function i(){const e=await r({message:"We need to launch your browser to continue",active:"OK",inactive:"Cancel"});if(a(e)||!e)return s("Create config canceled. Please try again."),process.exit(1);await o(1e3),t("Browser Launching..."),await o(1e3)}export{i as runConfirmBrowserLaunch};

package/dist/shared/browser-server.CTHRk1um.mjs

@@ -0,0 +1 @@
+import G from"express";import P from"puppeteer";import{exec as D}from"child_process";import{checkForConfigFile as Z,waitForConfigFile as L,getValueFromConfigFile as H,updateConfigFile as V}from"@rent-scraper/utils/config";import{parseError as W}from"@rent-scraper/utils";const J="http://127.0.0.1:9222/json/version",F=async()=>{try{return await P.connect({browserURL:J})}catch{return null}},y=async()=>{(await(await F())?.pages())?.forEach(async o=>o.close())},I=async(o="zillow")=>{if(await F())return{status:"already launched"};{await Z(o)||await L(o),await Z(o);const w=await H(o,"browser")==="brave"?"/Applications/Brave\\ Browser.app/Contents/MacOS/Brave\\ Browser":"/Applications/Google\\ Chrome.app/Contents/MacOS/Google\\ Chrome";return D(`${w} --remote-debugging-port=9222 --no-first-run --no-default-browser-check --user-data-dir=$(mktemp -d -t 'chrome-remote_data_dir')`,(d,n,c)=>{if(d){console.error(`Error: ${d.message}`);return}if(c){console.error(`Stderr: ${c}`);return}console.log(`Stdout: ${n}`)}),{status:"launched"}}},M=async(o="https://zillow.com")=>{const w=await F();if(w){const d=o,n=(await w.pages())?.[0]??await w.newPage();return(await n.title()).includes("denied")||await n.goto(d,{waitUntil:"load"}),{status:"opened"}}else return{status:"not connected"}},K=async()=>{const o=await F();return o?(await o.close(),await o.disconnect(),{status:"closed"}):{status:"not connected"}},S=async()=>{const o=await P.connect({browserURL:"http://127.0.0.1:9222/json/version"});await M("https://www.zillow.com/homes/for_rent/");const[w]=(await o.cookies()).filter(d=>d.name==="_pxvid");if(w)return await y(),w;console.log("refetching cookie"),setTimeout(async()=>{await S()},2e3)},N=async()=>{try{const{name:o,value:w}=await S()??{},d={zillowCookie:`${o}=${w}`};await V("zillow",d)}catch(o){const{status:w,message:d}=W(o);console.error(w,d)}};function Q(o){return o&&o.__esModule&&Object.prototype.hasOwnProperty.call(o,"default")?o.default:o}var R,U;function X(){if(U)return R;U=1;function o(n,c){var e=n;c.slice(0,-1).forEach(function(a){e=e[a]||{}});var l=c[c.length-1];return l in e}function w(n){return typeof n=="number"||/^0x[0-9a-f]+$/i.test(n)?!0:/^[-+]?(?:\d+(?:\.\d*)?|\.\d+)(e[-+]?\d+)?$/.test(n)}function d(n,c){return c==="constructor"&&typeof n[c]=="function"||c==="__proto__"}return R=function(n,c){c||(c={});var e={bools:{},strings:{},unknownFn:null};typeof c.unknown=="function"&&(e.unknownFn=c.unknown),typeof c.boolean=="boolean"&&c.boolean?e.allBools=!0:[].concat(c.boolean).filter(Boolean).forEach(function(t){e.bools[t]=!0});var l={};function a(t){return l[t].some(function(f){return e.bools[f]})}Object.keys(c.alias||{}).forEach(function(t){l[t]=[].concat(c.alias[t]),l[t].forEach(function(f){l[f]=[t].concat(l[t].filter(function($){return f!==$}))})}),[].concat(c.string).filter(Boolean).forEach(function(t){e.strings[t]=!0,l[t]&&[].concat(l[t]).forEach(function(f){e.strings[f]=!0})});var r=c.default||{},b={_:[]};function q(t,f){return e.allBools&&/^--[^=]+$/.test(f)||e.strings[t]||e.bools[t]||l[t]}function g(t,f,$){for(var i=t,O=0;O<f.length-1;O++){var B=f[O];if(d(i,B))return;i[B]===void 0&&(i[B]={}),(i[B]===Object.prototype||i[B]===Number.prototype||i[B]===String.prototype)&&(i[B]={}),i[B]===Array.prototype&&(i[B]=[]),i=i[B]}var _=f[f.length-1];d(i,_)||((i===Object.prototype||i===Number.prototype||i===String.prototype)&&(i={}),i===Array.prototype&&(i=[]),i[_]===void 0||e.bools[_]||typeof i[_]=="boolean"?i[_]=$:Array.isArray(i[_])?i[_].push($):i[_]=[i[_],$])}function h(t,f,$){if(!($&&e.unknownFn&&!q(t,$)&&e.unknownFn($)===!1)){var i=!e.strings[t]&&w(f)?Number(f):f;g(b,t.split("."),i),(l[t]||[]).forEach(function(O){g(b,O.split("."),i)})}}Object.keys(e.bools).forEach(function(t){h(t,r[t]===void 0?!1:r[t])});var A=[];n.indexOf("--")!==-1&&(A=n.slice(n.indexOf("--")+1),n=n.slice(0,n.indexOf("--")));for(var p=0;p<n.length;p++){var s=n[p],u,v;if(/^--.+=/.test(s)){var T=s.match(/^--([^=]+)=([\s\S]*)$/);u=T[1];var z=T[2];e.bools[u]&&(z=z!=="false"),h(u,z,s)}else if(/^--no-.+/.test(s))u=s.match(/^--no-(.+)/)[1],h(u,!1,s);else if(/^--.+/.test(s))u=s.match(/^--(.+)/)[1],v=n[p+1],v!==void 0&&!/^(-|--)[^-]/.test(v)&&!e.bools[u]&&!e.allBools&&(!l[u]||!a(u))?(h(u,v,s),p+=1):/^(true|false)$/.test(v)?(h(u,v==="true",s),p+=1):h(u,e.strings[u]?"":!0,s);else if(/^-[^-]+/.test(s)){for(var E=s.slice(1,-1).split(""),C=!1,m=0;m<E.length;m++){if(v=s.slice(m+2),v==="-"){h(E[m],v,s);continue}if(/[A-Za-z]/.test(E[m])&&v[0]==="="){h(E[m],v.slice(1),s),C=!0;break}if(/[A-Za-z]/.test(E[m])&&/-?\d+(\.\d*)?(e-?\d+)?$/.test(v)){h(E[m],v,s),C=!0;break}if(E[m+1]&&E[m+1].match(/\W/)){h(E[m],s.slice(m+2),s),C=!0;break}else h(E[m],e.strings[E[m]]?"":!0,s)}u=s.slice(-1)[0],!C&&u!=="-"&&(n[p+1]&&!/^(-|--)[^-]/.test(n[p+1])&&!e.bools[u]&&(!l[u]||!a(u))?(h(u,n[p+1],s),p+=1):n[p+1]&&/^(true|false)$/.test(n[p+1])?(h(u,n[p+1]==="true",s),p+=1):h(u,e.strings[u]?"":!0,s))}else if((!e.unknownFn||e.unknownFn(s)!==!1)&&b._.push(e.strings._||!w(s)?s:Number(s)),c.stopEarly){b._.push.apply(b._,n.slice(p+1));break}}return Object.keys(r).forEach(function(t){o(b,t.split("."))||(g(b,t.split("."),r[t]),(l[t]||[]).forEach(function(f){g(b,f.split("."),r[t])}))}),c["--"]?b["--"]=A.slice():A.forEach(function(t){b._.push(t)}),b},R}var Y=X();const j=Q(Y);function x(){const o=G(),w=process.env.HOST??"127.0.0.1",d=process.env.PORT??8082;o.use(G.json());const n=j(process.argv.slice(2)).debug,c=o.listen(8082,async()=>{await I();const l=setInterval(async()=>{(await F())?.connected&&(clearInterval(l),n&&console.log("Browser listening at 127.0.0.1:9222"),await N())},1e3);n&&console.log(`Server listening at ${w}:${d}`)}),e=()=>(c.close(async l=>{const a=await K();n&&(console.log(a),console.log("server closed")),process.exit(l?1:0)}),{status:"shutdown"});return o.get("/server",(l,a)=>{try{a.send({running:!0})}catch(r){a.send(r)}}),o.post("/browser/launch",async(l,a)=>{try{const r=await I();a.send({browser:r})}catch(r){a.send(r)}}),o.post("/browser/open",async(l,a)=>{try{const{url:r}=l?.body??{},b=await M(r);a.send({browser:b})}catch(r){a.send(r)}}),o.get("/cookie",async(l,a)=>{try{const r=await S();a.send({cookie:r})}catch(r){a.send(r)}}),o.get("/cookie/save",async(l,a)=>{try{const r=await N();a.send({cookie:r})}catch(r){a.send(r)}}),o.post("/browser/close",async(l,a)=>{try{const r=await y();a.send({browser:r})}catch(r){a.send(r)}}),o.post("/server/shutdown",(l,a)=>{try{const r=e();a.send({server:r})}catch(r){a.send(r)}}),process.on("SIGINT",e),process.on("SIGTERM",e),o}export{S as a,N as b,y as c,F as g,I as l,M as o,x as r,K as s};

package/eslint.config.ts

@@ -0,0 +1,2 @@
+import config from '../../eslint.config.js'
+export default config

package/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "@rent-scraper/browser-server",
+  "version": "1.0.0",
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "default": "./dist/index.mjs"
+    },
+    "./bin/run-browser-server": {
+      "types": "./dist/bin/run-browser-server.d.mts",
+      "default": "./dist/bin/run-browser-server.mjs"
+    }
+  },
+  "author": "Max Stein <maxwell.stein@gmail.com> (https://maxstein.net)",
+  "license": "MIT",
+  "description": "Local browser server to help scrape data for rental listings",
+  "scripts": {
+    "dev": "tsx ./src/bin/run-browser-server.ts",
+    "build": "unbuild",
+    "start": "node ./dist/bin/run-browser-server.mjs",
+    "cli:dev": "tsx src/cli.ts",
+    "cli": "node dist/cli.mjs",
+    "typecheck": "tsc --noEmit",
+    "lint": "eslint .",
+    "lint:fix": "eslint --fix .",
+    "prepublish": "npm run lint && npm run build"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/rent-brigade/rent-scraper.git",
+    "directory": "packages/browser-server"
+  },
+  "dependencies": {
+    "@clack/prompts": "alpha",
+    "@rent-scraper/api": "workspace:*",
+    "@rent-scraper/utils": "workspace:*",
+    "bumpp": "^10.2.3",
+    "express": "^4.21.2",
+    "picocolors": "^1.1.1",
+    "puppeteer": "^24.11.2"
+  },
+  "devDependencies": {
+    "@types/express": "^5.0.3",
+    "@types/qs": "^6.14.0",
+    "@types/range-parser": "^1.2.7",
+    "@types/send": "^0.17.5",
+    "tsx": "^4.20.3",
+    "unbuild": "^3.5.0"
+  },
+  "engines": {
+    "node": "22.x"
+  }
+}

package/src/bin/run-browser-server.ts

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+import { runBrowserServer } from '../server.js'
+
+runBrowserServer()

package/src/browser.ts

@@ -0,0 +1,79 @@
+import puppeteer from 'puppeteer'
+import { exec } from 'child_process'
+import { checkForConfigFile, getValueFromConfigFile, waitForConfigFile } from '@rent-scraper/utils/config'
+import type { ListingsSource } from '@rent-scraper/api'
+
+const wsChromeEndpointurl = 'http://127.0.0.1:9222/json/version'
+
+export const getBrowser = async () => {
+  try {
+    const browser = await puppeteer.connect({
+      browserURL: wsChromeEndpointurl,
+    })
+    return browser
+  } catch {
+    return null
+  }
+}
+
+export const closeBrowser = async () => {
+  const browser = await getBrowser()
+  const pages = await browser?.pages()
+  pages?.forEach(async page => page.close())
+}
+
+export const launchBrowser = async (source = 'zillow' as ListingsSource) => {
+  const browser = await getBrowser()
+  if (browser) {
+    return { status: 'already launched' }
+  } else {
+    if (!await checkForConfigFile(source)) {
+      await waitForConfigFile(source)
+    }
+    await checkForConfigFile(source)
+    const browserKey = await getValueFromConfigFile(source, 'browser')
+    const browserPath = browserKey === 'brave' ? '/Applications/Brave\\ Browser.app/Contents/MacOS/Brave\\ Browser' : '/Applications/Google\\ Chrome.app/Contents/MacOS/Google\\ Chrome'
+    exec(`${browserPath} --remote-debugging-port=9222 --no-first-run --no-default-browser-check --user-data-dir=$(mktemp -d -t 'chrome-remote_data_dir')`, (error, stdout, stderr) => {
+      if (error) {
+        console.error(`Error: ${error.message}`)
+        return
+      }
+      if (stderr) {
+        console.error(`Stderr: ${stderr}`)
+        return
+      }
+      console.log(`Stdout: ${stdout}`)
+    })
+    return { status: 'launched' }
+  }
+}
+
+export const openBrowser = async (url = 'https://zillow.com') => {
+  const browser = await getBrowser()
+  if (browser) {
+    const pageUrl = url
+    const pages = await browser.pages()
+    const page = pages?.[0] ?? await browser.newPage()
+    const pageTitle = await page.title()
+    // do not change page if captcha is showing
+    if (!pageTitle.includes('denied')) {
+      await page.goto(pageUrl, {
+        waitUntil: 'load',
+      })
+    }
+    return { status: 'opened' }
+  } else {
+    return { status: 'not connected' }
+  }
+}
+
+export const shutdownBrowser = async () => {
+  const browser = await getBrowser()
+  if (browser) {
+    await browser.close()
+    await browser.disconnect()
+    return { status: 'closed' }
+  } else {
+    return { status: 'not connected' }
+  }
+}

package/src/confirm-browser-launch.ts

@@ -0,0 +1,26 @@
+import {
+  outro,
+  confirm,
+  isCancel,
+  cancel,
+} from '@clack/prompts'
+import { setTimeout as sleep } from 'node:timers/promises'
+
+export async function runConfirmBrowserLaunch() {
+  const shouldContinue = await confirm({
+    message: 'We need to launch your browser to continue',
+    active: 'OK',
+    inactive: 'Cancel',
+  })
+
+  if (isCancel(shouldContinue) || !shouldContinue) {
+    cancel('Create config canceled. Please try again.')
+    return process.exit(1)
+  }
+
+  await sleep(1000)
+
+  outro('Browser Launching...')
+
+  await sleep(1000)
+}

package/src/cookie.ts

@@ -0,0 +1,39 @@
+import puppeteer from 'puppeteer'
+import { closeBrowser, openBrowser } from './browser.js'
+import { updateConfigFile } from '@rent-scraper/utils/config'
+import { parseError } from '@rent-scraper/utils'
+
+export const getZillowCookie = async () => {
+  const wsChromeEndpointurl = 'http://127.0.0.1:9222/json/version'
+  const browser = await puppeteer.connect({
+    browserURL: wsChromeEndpointurl,
+  })
+  await openBrowser('https://www.zillow.com/homes/for_rent/')
+  const [cookie] = (await browser.cookies()).filter(cookie => cookie.name === '_pxvid')
+  if (cookie) {
+    await closeBrowser()
+    return cookie
+  } else {
+    console.log('refetching cookie')
+    setTimeout(async () => {
+      await getZillowCookie()
+    }, 2000)
+  }
+}
+
+export const saveZillowCookie = async () => {
+  try {
+    const { name, value } = await getZillowCookie() ?? {}
+    const zillowCookie = `${name}=${value}`
+
+    const data = {
+      zillowCookie,
+    }
+
+    // update config file
+    await updateConfigFile('zillow', data)
+  } catch (error: any) {
+    const { status, message } = parseError(error)
+    console.error(status, message)
+  }
+}

package/src/index.ts

@@ -0,0 +1,4 @@
+export * from './browser.js'
+export * from './confirm-browser-launch.js'
+export * from './cookie.js'
+export * from './server.js'

package/src/server.ts

@@ -0,0 +1,111 @@
+import express from 'express'
+import minimist from 'minimist'
+import { getZillowCookie, saveZillowCookie } from './cookie.js'
+import { launchBrowser, closeBrowser, getBrowser, shutdownBrowser, openBrowser } from './browser.js'
+
+export function runBrowserServer() {
+  const app = express()
+  const host = process.env.HOST ?? '127.0.0.1'
+  const port = process.env.PORT ?? 8082
+  app.use(express.json())
+
+  const args = minimist(process.argv.slice(2))
+  const debug = args.debug
+
+  const server = app.listen(8082, async () => {
+    await launchBrowser()
+    const connecting = setInterval(async () => {
+      const browser = await getBrowser()
+      if (browser?.connected) {
+        clearInterval(connecting)
+        if (debug) {
+          console.log(`Browser listening at 127.0.0.1:9222`)
+        }
+        await saveZillowCookie()
+      }
+    }, 1000)
+    if (debug) {
+      console.log(`Server listening at ${host}:${port}`)
+    }
+  })
+
+  const shutdownServer = () => {
+    server.close(async (err) => {
+      const browser = await shutdownBrowser()
+      if (debug) {
+        console.log(browser)
+        console.log('server closed')
+      }
+      process.exit(err ? 1 : 0)
+    })
+    return { status: 'shutdown' }
+  }
+
+  app.get('/server', (_req, res) => {
+    try {
+      res.send({ running: true })
+    } catch (error) {
+      res.send(error)
+    }
+  })
+
+  app.post('/browser/launch', async (_req, res) => {
+    try {
+      const browser = await launchBrowser()
+      res.send({ browser })
+    } catch (error) {
+      res.send(error)
+    }
+  })
+
+  app.post('/browser/open', async (req, res) => {
+    try {
+      const { url } = req?.body ?? {}
+      const browser = await openBrowser(url)
+      res.send({ browser })
+    } catch (error) {
+      res.send(error)
+    }
+  })
+
+  app.get('/cookie', async (_req, res) => {
+    try {
+      const cookie = await getZillowCookie()
+      res.send({ cookie })
+    } catch (error) {
+      res.send(error)
+    }
+  })
+
+  app.get('/cookie/save', async (_req, res) => {
+    try {
+      const cookie = await saveZillowCookie()
+      res.send({ cookie })
+    } catch (error) {
+      res.send(error)
+    }
+  })
+
+  app.post('/browser/close', async (_req, res) => {
+    try {
+      const browser = await closeBrowser()
+      res.send({ browser })
+    } catch (error) {
+      res.send(error)
+    }
+  })
+
+  app.post('/server/shutdown', (_req, res) => {
+    try {
+      const server = shutdownServer()
+      res.send({ server })
+    } catch (error) {
+      res.send(error)
+    }
+  })
+
+  process.on('SIGINT', shutdownServer)
+  process.on('SIGTERM', shutdownServer)
+
+  return app
+}

package/tsconfig.json

@@ -0,0 +1,4 @@
+{
+  "extends": "../../tsconfig.json",
+  "includes": "*/**.ts"
+}