@rent-scraper/utils 1.0.0

package/README.md

@@ -0,0 +1,2 @@
+# @rent-scraper/utils
+Utility functions for rent-scraper

package/dist/config.d.mts

@@ -0,0 +1,31 @@
+import { ListingsSource, BrowserKey } from '@rent-scraper/api';
+
+interface ScrapeConfig {
+    outputPath: string;
+    source?: ListingsSource;
+    zipCodes: string;
+    daysListed?: number;
+    regionIds?: Record<number, number | null>;
+    browser?: BrowserKey;
+    zillowCookie?: string;
+}
+declare const getConfigFilePath: (source: ListingsSource) => Promise<string>;
+declare const checkForConfigFile: (source: ListingsSource) => Promise<boolean | undefined>;
+declare const checkForAndReadConfigFile: (source: ListingsSource) => Promise<ScrapeConfig>;
+declare const resetZillowCookie: () => Promise<void>;
+declare const waitForConfigFile: (source: ListingsSource) => Promise<unknown>;
+declare const waitForBrowserServer: () => Promise<unknown>;
+declare const checkBrowserServer: () => Promise<any>;
+declare const waitForZillowCookie: () => Promise<unknown>;
+declare const checkForZillowCookie: () => Promise<string | number | Record<number, number | null> | null | undefined>;
+declare const checkRequiredConfigValues: (source: ListingsSource, config: ScrapeConfig, task?: string) => string[];
+declare const getZipCodesFromConfig: (source: ListingsSource) => Promise<number[]>;
+declare const stringifyZipCodes: (zipCodes: number[]) => string;
+declare const readConfigFile: (source: ListingsSource) => Promise<ScrapeConfig>;
+declare const getOutputPathFromConfig: (source: ListingsSource) => Promise<string>;
+declare const getValueFromConfigFile: (source: ListingsSource, key: keyof ScrapeConfig) => Promise<ScrapeConfig[keyof ScrapeConfig] | null>;
+declare const updateConfigFile: (source: ListingsSource, payload: any) => Promise<void>;
+declare const writeConfigFile: (source: ListingsSource, config: ScrapeConfig) => Promise<void>;
+
+export { checkBrowserServer, checkForAndReadConfigFile, checkForConfigFile, checkForZillowCookie, checkRequiredConfigValues, getConfigFilePath, getOutputPathFromConfig, getValueFromConfigFile, getZipCodesFromConfig, readConfigFile, resetZillowCookie, stringifyZipCodes, updateConfigFile, waitForBrowserServer, waitForConfigFile, waitForZillowCookie, writeConfigFile };
+export type { ScrapeConfig };

package/dist/config.mjs

@@ -0,0 +1,4 @@
+import{checkForFile as A,parseYamlFile as O,throwError as F,writeYamlFile as k}from"@rent-scraper/utils";import v from"fs";import I from"path";import M from"util";import Y from"axios";import{spinner as H,log as Q}from"@clack/prompts";var p={},h={},g={},b;function G(){return b||(b=1,(function(e){Object.defineProperty(e,"__esModule",{value:!0}),e.USEFUL_NON_ROOT_PNPM_FIELDS=e.FULL_FILTERED_META_DIR=e.FULL_META_DIR=e.ABBREVIATED_META_DIR=e.WORKSPACE_MANIFEST_FILENAME=e.STORE_VERSION=e.LAYOUT_VERSION=e.ENGINE_NAME=e.MANIFEST_BASE_NAMES=e.LOCKFILE_VERSION=e.LOCKFILE_MAJOR_VERSION=e.WANTED_LOCKFILE=void 0,e.getNodeBinLocationForCurrentOS=t,e.getDenoBinLocationForCurrentOS=i,e.getBunBinLocationForCurrentOS=n,e.WANTED_LOCKFILE="pnpm-lock.yaml",e.LOCKFILE_MAJOR_VERSION="9",e.LOCKFILE_VERSION=`${e.LOCKFILE_MAJOR_VERSION}.0`,e.MANIFEST_BASE_NAMES=["package.json","package.json5","package.yaml"],e.ENGINE_NAME=`${process.platform};${process.arch};node${process.version.split(".")[0].substring(1)}`,e.LAYOUT_VERSION=5,e.STORE_VERSION="v10",e.WORKSPACE_MANIFEST_FILENAME="pnpm-workspace.yaml",e.ABBREVIATED_META_DIR="metadata-v1.3",e.FULL_META_DIR="metadata-full-v1.3",e.FULL_FILTERED_META_DIR="metadata-v1.3",e.USEFUL_NON_ROOT_PNPM_FIELDS=["executionEnv"];function t(c=process.platform){return c==="win32"?"node.exe":"bin/node"}function i(c=process.platform){return c==="win32"?"deno.exe":"deno"}function n(c=process.platform){return c==="win32"?"bun.exe":"bun"}})(g)),g}var T;function Z(){if(T)return h;T=1,Object.defineProperty(h,"__esModule",{value:!0}),h.LockfileMissingDependencyError=h.FetchError=h.PnpmError=void 0;const e=G();class t extends Error{code;hint;attempts;prefix;pkgsStack;constructor(s,r,f){super(r),this.code=s.startsWith("ERR_PNPM_")?s:`ERR_PNPM_${s}`,this.hint=f?.hint,this.attempts=f?.attempts}}h.PnpmError=t;class i extends t{response;request;constructor(s,r,f){const l={url:s.url};s.authHeaderValue&&(l.authHeaderValue=n(s.authHeaderValue));const a=`GET ${s.url}: ${r.statusText} - ${r.status}`;(r.status===401||r.status===403||r.status===404)&&(f=f?`${f}
+
+`:"",l.authHeaderValue?f+=`An authorization header was used: ${l.authHeaderValue}`:f+="No authorization header was set for the request."),super(`FETCH_${r.status}`,a,{hint:f}),this.request=l,this.response=r}}h.FetchError=i;function n(u){const[s,r]=u.split(" ");return r==null?"[hidden]":r.length<20?`${s} [hidden]`:`${s} ${r.substring(0,4)}[hidden]`}class c extends t{constructor(s){const r=`Broken lockfile: no entry for '${s}' in ${e.WANTED_LOCKFILE}`;super("LOCKFILE_MISSING_DEPENDENCY",r,{hint:`This issue is probably caused by a badly resolved merge conflict.
+To fix the lockfile, run 'pnpm install --no-frozen-lockfile'.`})}}return h.LockfileMissingDependencyError=c,h}var S={exports:{}},E={exports:{}},C,D;function J(){if(D)return C;D=1;class e{constructor(n){this.value=n,this.next=void 0}}class t{constructor(){this.clear()}enqueue(n){const c=new e(n);this._head?(this._tail.next=c,this._tail=c):(this._head=c,this._tail=c),this._size++}dequeue(){const n=this._head;if(n)return this._head=this._head.next,this._size--,n.value}clear(){this._head=void 0,this._tail=void 0,this._size=0}get size(){return this._size}*[Symbol.iterator](){let n=this._head;for(;n;)yield n.value,n=n.next}}return C=t,C}var N,$;function X(){if($)return N;$=1;const e=J();return N=i=>{if(!((Number.isInteger(i)||i===1/0)&&i>0))throw new TypeError("Expected `concurrency` to be a number from 1 and up");const n=new e;let c=0;const u=()=>{c--,n.size>0&&n.dequeue()()},s=async(l,a,...o)=>{c++;const d=(async()=>l(...o))();a(d);try{await d}catch{}u()},r=(l,a,...o)=>{n.enqueue(s.bind(null,l,a,...o)),(async()=>(await Promise.resolve(),c<i&&n.size>0&&n.dequeue()()))()},f=(l,...a)=>new Promise(o=>{r(l,o,...a)});return Object.defineProperties(f,{activeCount:{get:()=>c},pendingCount:{get:()=>n.size},clearQueue:{value:()=>{n.clear()}}}),f},N}var P,q;function ee(){if(q)return P;q=1;const e=X();class t extends Error{constructor(s){super(),this.value=s}}const i=async(u,s)=>s(await u),n=async u=>{const s=await Promise.all(u);if(s[1]===!0)throw new t(s[0]);return!1};return P=async(u,s,r)=>{r={concurrency:1/0,preserveOrder:!0,...r};const f=e(r.concurrency),l=[...u].map(o=>[o,f(i,o,s)]),a=e(r.preserveOrder?1:1/0);try{await Promise.all(l.map(o=>a(n,o)))}catch(o){if(o instanceof t)return o.value;throw o}},P}var z;function te(){if(z)return E.exports;z=1;const e=I,t=v,{promisify:i}=M,n=ee(),c=i(t.stat),u=i(t.lstat),s={directory:"isDirectory",file:"isFile"};function r({type:l}){if(!(l in s))throw new Error(`Invalid type specified: ${l}`)}const f=(l,a)=>l===void 0||a[s[l]]();return E.exports=async(l,a)=>{a={cwd:process.cwd(),type:"file",allowSymlinks:!0,...a},r(a);const o=a.allowSymlinks?c:u;return n(l,async d=>{try{const w=await o(e.resolve(a.cwd,d));return f(a.type,w)}catch{return!1}},a)},E.exports.sync=(l,a)=>{a={cwd:process.cwd(),allowSymlinks:!0,type:"file",...a},r(a);const o=a.allowSymlinks?t.statSync:t.lstatSync;for(const d of l)try{const w=o(e.resolve(a.cwd,d));if(f(a.type,w))return d}catch{}},E.exports}var _={exports:{}},V;function re(){if(V)return _.exports;V=1;const e=v,{promisify:t}=M,i=t(e.access);return _.exports=async n=>{try{return await i(n),!0}catch{return!1}},_.exports.sync=n=>{try{return e.accessSync(n),!0}catch{return!1}},_.exports}var B;function ne(){return B||(B=1,(function(e){const t=I,i=te(),n=re(),c=Symbol("findUp.stop");e.exports=async(u,s={})=>{let r=t.resolve(s.cwd||"");const{root:f}=t.parse(r),l=[].concat(u),a=async o=>{if(typeof u!="function")return i(l,o);const d=await u(o.cwd);return typeof d=="string"?i([d],o):d};for(;;){const o=await a({...s,cwd:r});if(o===c)return;if(o)return t.resolve(r,o);if(r===f)return;r=t.dirname(r)}},e.exports.sync=(u,s={})=>{let r=t.resolve(s.cwd||"");const{root:f}=t.parse(r),l=[].concat(u),a=o=>{if(typeof u!="function")return i.sync(l,o);const d=u(o.cwd);return typeof d=="string"?i.sync([d],o):d};for(;;){const o=a({...s,cwd:r});if(o===c)return;if(o)return t.resolve(r,o);if(r===f)return;r=t.dirname(r)}},e.exports.exists=n,e.exports.sync.exists=n.sync,e.exports.stop=c})(S)),S.exports}var K;function ie(){if(K)return p;K=1;var e=p&&p.__importDefault||function(a){return a&&a.__esModule?a:{default:a}};Object.defineProperty(p,"__esModule",{value:!0}),p.findWorkspaceDir=f;const t=e(v),i=e(I),n=Z(),c=e(ne()),u="NPM_CONFIG_WORKSPACE_DIR",s="pnpm-workspace.yaml",r=["pnpm-workspaces.yaml","pnpm-workspaces.yml","pnpm-workspace.yml"];async function f(a){const o=process.env[u]??process.env[u.toLowerCase()],d=o?i.default.join(o,s):await(0,c.default)([s,...r],{cwd:await l(a)});if(d&&i.default.basename(d)!==s)throw new n.PnpmError("BAD_WORKSPACE_MANIFEST_NAME",`The workspace manifest file should be named "pnpm-workspace.yaml". File found: ${d}`);return d&&i.default.dirname(d)}async function l(a){return new Promise(o=>{t.default.realpath.native(a,function(d,w){o(d!==null?a:w)})})}return p}var W=ie();const y=async e=>{const t=await W.findWorkspaceDir(process.cwd());return e==="redfin"?`${t}/config.redfin.yaml`:`${t}/config.zillow.yaml`},x=async e=>{const t=await y(e);return await A(t)},ae=async e=>{const t=await y(e);return await A(t)||F("Config file is required."),await O(t)},se=async()=>{const{zillowCookie:e,...t}=await m("zillow");await R("zillow",t)},oe=async e=>{try{return await new Promise(t=>{const i=setInterval(async()=>{const n=await x(e);n&&(t(n),clearInterval(i))},1e3)})}catch(t){F(t?.message)}},ce=async()=>{try{return await new Promise((e,t)=>{const i=H();i.start("Waiting for browser server");let n=0;const c=setInterval(async()=>{n++,n>10&&t(new Error("browser server is not running."));const u=await U();u&&(i.stop("Browser server is connected!"),e(u),clearInterval(c))},1e3)})}catch(e){F(e?.message)}},U=async()=>{try{const{data:e}=await Y.get("http://localhost:8082/server");return e.running}catch{return!1}},ue=async()=>await new Promise(e=>{const t=setInterval(async()=>{const i=await j();i&&(e(i),clearInterval(t))},1e3)}),j=async()=>await L("zillow","zillowCookie"),le=(e,t,i="init")=>{const{outputPath:n,zipCodes:c,browser:u,zillowCookie:s}=t??{},r=[];return n||r.push("outputPath"),c||r.push("zipCodes"),e==="zillow"&&!u&&r.push("browser"),i==="scrape"&&e==="zillow"&&!s&&r.push("zillowCookie"),r},fe=async e=>(await L(e,"zipCodes"))?.replace(/ /g,"").split(",").map(t=>Number(t)),de=e=>e.join(", "),m=async e=>{const t=await y(e);return await O(t)},he=async e=>await L(e,"outputPath"),L=async(e,t)=>(await m(e))?.[t]??null,pe=async(e,t)=>{const i=await m(e),n=Object.keys(t),c={...i,...t};await R(e,c),Q.success(`Updated ${e} config: ${n.join(", ")}`)},R=async(e,t)=>{const i=await W.findWorkspaceDir(process.cwd());return e==="redfin"?await k(`${i}/config.redfin.yaml`,t):await k(`${i}/config.zillow.yaml`,t)};export{U as checkBrowserServer,ae as checkForAndReadConfigFile,x as checkForConfigFile,j as checkForZillowCookie,le as checkRequiredConfigValues,y as getConfigFilePath,he as getOutputPathFromConfig,L as getValueFromConfigFile,fe as getZipCodesFromConfig,m as readConfigFile,se as resetZillowCookie,de as stringifyZipCodes,pe as updateConfigFile,ce as waitForBrowserServer,oe as waitForConfigFile,ue as waitForZillowCookie,R as writeConfigFile};

package/dist/index.d.mts

@@ -0,0 +1,59 @@
+import * as csv from 'csv';
+
+type Id = string | number;
+type Item = Record<string, unknown>;
+type List = Item[];
+declare const groupByKey: (list: any[], key: string | number) => any;
+declare const isObject: (data: Record<string, unknown>) => boolean;
+declare const isEmptyObject: (data: Record<string, unknown>) => boolean;
+declare function throwError(message: string, status?: number): void;
+declare const parseThrownError: (error: {
+    message: string;
+}) => any;
+declare const chunkArray: (array: unknown[], size: number) => unknown[];
+declare const parseNumber: (value: string | number, options?: {
+    round?: number;
+    zeros?: number;
+    type?: string;
+}) => any;
+declare const parseCurrency: (value: string | number) => string | number;
+declare const parsePercentage: (value: string | number, options?: {
+    round: number;
+}) => string | number;
+declare const parsePrice: (value: string | number) => number;
+declare const roundValue: (value: string | number, nearest?: number) => number;
+declare const priceToInteger: (value: string) => number;
+declare const percentageToDecimal: (value: string) => number;
+declare const validateBooleanString: (arg: string) => boolean;
+declare const parseError: (error: any) => {
+    status: any;
+    message: any;
+};
+declare const createDayStartInPST: (fourDigitYear: number, month: number, day: number) => Date;
+interface ReadDirectoryOptions {
+    extension?: string | string[];
+    prefix?: string;
+    prependDirectory?: boolean;
+    recursive?: boolean;
+}
+declare const readFilesInDirectory: (directory: string, options?: ReadDirectoryOptions) => Promise<string[]>;
+declare const readFilesInDirectorySync: (directory: string, options?: ReadDirectoryOptions) => string[];
+declare const checkForFile: (filePath: string) => Promise<boolean | undefined>;
+declare const parseAbsolutePath: (relPath: string) => string;
+declare const getRandomArrayValue: (array: unknown[]) => unknown;
+declare const parseCsvFile: (filePath: string, options: csv.parser.Options) => Promise<any[]>;
+declare const parseTxtFile: (filePath: string) => Promise<string[]>;
+declare const parseJsonFile: (filePath: string) => Promise<any>;
+declare const parseYamlFile: (filePath: string) => Promise<any>;
+declare const writeYamlFile: (filePath: string, data: any) => Promise<void>;
+declare const compareArrays: <T>(arr1: T[], arr2: T[]) => T[];
+declare class ErrorLog {
+    errors: string[];
+    constructor(errors?: string[]);
+    add(msg: string, error?: unknown): void;
+    get(): string[];
+    write(path: string, data: string): Promise<void>;
+}
+
+export { ErrorLog, checkForFile, chunkArray, compareArrays, createDayStartInPST, getRandomArrayValue, groupByKey, isEmptyObject, isObject, parseAbsolutePath, parseCsvFile, parseCurrency, parseError, parseJsonFile, parseNumber, parsePercentage, parsePrice, parseThrownError, parseTxtFile, parseYamlFile, percentageToDecimal, priceToInteger, readFilesInDirectory, readFilesInDirectorySync, roundValue, throwError, validateBooleanString, writeYamlFile };
+export type { Id, Item, List };

package/dist/index.mjs

@@ -0,0 +1 @@
+import{readdir as S,access as w,readFile as p,writeFile as y}from"fs/promises";import f,{readdirSync as N}from"fs";import c from"path";import*as b from"csv";import d from"yaml";const $=(e,r)=>e.reduce((t,a)=>({...t,[a[r]]:(t[a[r]]||[]).concat(a)}),{}),g=e=>typeof e=="object"&&!!e,E=e=>g(e)&&Object.keys(e).length===0;function F(e,r){throw new Error(JSON.stringify({status:r??400,message:e}))}const x=e=>JSON.parse(e?.message),h=(e,r)=>e.length<=r?[e]:[e.slice(0,r),...h(e.slice(r),r)],l=(e,r)=>{const{round:t,zeros:a,type:i}=r??{};let s=e;return s=Number(t?m(e,t):s),s=a?Number(s).toFixed(a):s,s=i==="%"?`${s}%`:s,s=i==="$"?`$${s}`:s,r?s:Number(s)},D=e=>l(e,{round:100,zeros:2,type:"$"}),O=(e,r)=>{const{round:t=100}=r??{};return l(e,{round:t,type:"%"})},v=e=>Number(l(e,{round:100,zeros:2})),m=(e,r=100)=>Math.round(Number(e)*r)/r,I=e=>{const r=e?.replace("$","")?.replace(",","");return Number(r)},T=e=>{const r=e?.replace("+","")?.replace("%","");return m(Number(r)/100,1e3)},A=e=>e==="true"||Number(e)===1,j=e=>{if(e?.response){const{status:r,data:t}=e?.response??{};return{status:r,message:t}}else{if(e?.errors)return{status:400,message:e?.errors?.map(r=>r).map(r=>r?.message).join("")};try{if(e?.body){const{status:r,message:t}=JSON.parse(e?.body)??{};return{status:r,message:t}}else{const{status:r,message:t}=JSON.parse(e?.message)??{};return{status:r,message:t}}}catch{console.log(e);const{status:r,message:t}=e??{};return{status:r||400,message:t||"Internal Server Error"}}}},J=(e,r,t)=>{const a=e.toString(),i=r.toString().padStart(2,"0"),s=t.toString().padStart(2,"0");return new Date(`${a}-${i}-${s}T00:00:00-08:00`)},P=async(e,r)=>{const{extension:t,prefix:a,prependDirectory:i,recursive:s=!1}=r??{},u=await S(e,{recursive:s});let o=t?u.filter(n=>Array.isArray(t)?t.includes(c.extname(n)):c.extname(n)===t):u;return o=a?o.filter(n=>n.startsWith(a)):o,o=o.filter(n=>n!==".DS_Store"),i?o.map(n=>`${e}/${n}`):o},k=(e,r)=>{const{extension:t,prefix:a,prependDirectory:i,recursive:s=!1}=r??{},u=N(e,{recursive:s});let o=t?u.filter(n=>Array.isArray(t)?t.includes(c.extname(n)):c.extname(n)===t):u;return o=a?o.filter(n=>n.startsWith(a)):o,o=o.filter(n=>n!==".DS_Store"),i?o.map(n=>`${e}/${n}`):o},z=async e=>{try{if(await w(e),(await p(e)).length)return!0}catch(r){if(r?.code==="EISDIR")return!0;if(r?.code==="ENOENT")return!1;throw r}},C=e=>c.resolve(c.join(process.env.INIT_CWD,e)),M=e=>e[Math.floor(Math.random()*e.length)],R=async(e,r)=>{const t=[],a=f.createReadStream(e).pipe(b.parse(r));for await(const i of a)t.push(i);return t},W=async e=>{const r=[],t=(await p(e,"utf8")).replace(/\n/g,",").replace(/ /g,"").split(",");for(const a of t)r.push(a);return r},_=async e=>JSON.parse(await p(e,"utf8")),B=async e=>{try{return d.parse(await p(e,"utf8"))}catch(r){r?.code==="ENOENT"&&await f.promises.writeFile(e,"")}},V=async(e,r)=>{await y(e,d.stringify(r))},Y=(e,r)=>e?.filter(t=>!r?.includes(t));class K{errors;constructor(r=[]){this.errors=r}add(r,t){t instanceof Error&&(r+=`: ${t.message}`),this.errors.push(r)}get(){return this.errors}async write(r,t){await y(r,t)}}export{K as ErrorLog,z as checkForFile,h as chunkArray,Y as compareArrays,J as createDayStartInPST,M as getRandomArrayValue,$ as groupByKey,E as isEmptyObject,g as isObject,C as parseAbsolutePath,R as parseCsvFile,D as parseCurrency,j as parseError,_ as parseJsonFile,l as parseNumber,O as parsePercentage,v as parsePrice,x as parseThrownError,W as parseTxtFile,B as parseYamlFile,T as percentageToDecimal,I as priceToInteger,P as readFilesInDirectory,k as readFilesInDirectorySync,m as roundValue,F as throwError,A as validateBooleanString,V as writeYamlFile};

package/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "@rent-scraper/utils",
+  "version": "1.0.0",
+  "type": "module",
+  "main": "./dist/index.mjs",
+  "module": "./dist/index.mjs",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "default": "./dist/index.mjs"
+    },
+    "./config": {
+      "types": "./dist/config.d.mts",
+      "default": "./dist/config.mjs"
+    }
+  },
+  "author": "Max Stein <maxwell.stein@gmail.com> (https://maxstein.net)",
+  "license": "MIT",
+  "description": "Utility functions for rent-scraper",
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "unbuild",
+    "typecheck": "tsc --noEmit",
+    "lint": "eslint .",
+    "lint:fix": "eslint --fix .",
+    "prepublish": "npm run lint && npm run build"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/rent-brigade/rent-scraper.git",
+    "directory": "packages/utils"
+  },
+  "dependencies": {
+    "@clack/prompts": "alpha",
+    "@rent-scraper/api": "workspace:*",
+    "axios": "^1.10.0",
+    "bumpp": "^10.2.3",
+    "csv": "^6.3.11",
+    "yaml": "^2.8.0"
+  },
+  "devDependencies": {
+    "unbuild": "^3.5.0"
+  },
+  "engines": {
+    "node": "22.x"
+  }
+}