@rent-scraper/utils 1.0.15 → 1.0.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/config.d.mts CHANGED
@@ -8,12 +8,13 @@ interface ScrapeConfig {
8
8
  regionIds?: Record<number, number | null>;
9
9
  browser?: BrowserKey;
10
10
  zillowCookie?: string;
11
+ redfinCookie?: string;
11
12
  }
12
13
  declare const pointerFilePath: string;
13
14
  declare const globalDir: string;
14
15
  declare const getConfigFilePath: (source: ListingsSource) => Promise<any>;
15
16
  declare const checkForConfigFile: (source: ListingsSource, configFilePath?: string) => Promise<boolean | undefined>;
16
- declare const checkForPointerFile: () => Promise<boolean | undefined>;
17
+ declare const checkForPointerFile: () => Promise<boolean>;
17
18
  declare const checkForAndReadConfigFile: (source: ListingsSource) => Promise<ScrapeConfig | undefined>;
18
19
  declare const resetZillowCookie: () => Promise<void>;
19
20
  declare const waitForConfigFile: (source: ListingsSource) => Promise<unknown>;
@@ -21,6 +22,8 @@ declare const waitForBrowserServer: () => Promise<unknown>;
21
22
  declare const checkBrowserServer: () => Promise<any>;
22
23
  declare const waitForZillowCookie: () => Promise<unknown>;
23
24
  declare const checkForZillowCookie: () => Promise<string | number | Record<number, number | null> | null | undefined>;
25
+ declare const waitForRedfinCookie: () => Promise<unknown>;
26
+ declare const checkForRedfinCookie: () => Promise<string | number | Record<number, number | null> | null | undefined>;
24
27
  declare const checkRequiredConfigValues: (source: ListingsSource, config?: ScrapeConfig, task?: string) => string[];
25
28
  declare const getZipCodesFromConfig: (source: ListingsSource) => Promise<number[]>;
26
29
  declare const stringifyZipCodes: (zipCodes: number[]) => string;
@@ -32,5 +35,5 @@ declare const getValueFromConfigFile: (source: ListingsSource, key: keyof Scrape
32
35
  declare const updateConfigFile: (source: ListingsSource, payload: any) => Promise<void>;
33
36
  declare const writeConfigFile: (source: ListingsSource, config: ScrapeConfig) => Promise<any>;
34
37
 
35
- export { checkBrowserServer, checkForAndReadConfigFile, checkForConfigFile, checkForPointerFile, checkForZillowCookie, checkRequiredConfigValues, getConfigFilePath, getOutputPathFromConfig, getValueFromConfigFile, getZipCodesFromConfig, globalDir, pointerFilePath, readConfigFile, readPointerFile, resetZillowCookie, stringifyZipCodes, updateConfigFile, waitForBrowserServer, waitForConfigFile, waitForZillowCookie, writeConfigFile, writePointerFile };
38
+ export { checkBrowserServer, checkForAndReadConfigFile, checkForConfigFile, checkForPointerFile, checkForRedfinCookie, checkForZillowCookie, checkRequiredConfigValues, getConfigFilePath, getOutputPathFromConfig, getValueFromConfigFile, getZipCodesFromConfig, globalDir, pointerFilePath, readConfigFile, readPointerFile, resetZillowCookie, stringifyZipCodes, updateConfigFile, waitForBrowserServer, waitForConfigFile, waitForRedfinCookie, waitForZillowCookie, writeConfigFile, writePointerFile };
36
39
  export type { ScrapeConfig };
package/dist/config.mjs CHANGED
@@ -1,4 +1,4 @@
1
- import{checkForFile as P,parseYamlFile as _,throwError as m,writeYamlFile as b}from"@rent-scraper/utils";import G from"axios";import S from"fs";import h from"path";import D from"util";import{spinner as Z,log as J}from"@clack/prompts";import{stat as X,mkdir as C}from"fs/promises";import ee from"os";import te from"env-paths";var y={},p={},N={},T;function re(){return T||(T=1,(function(e){Object.defineProperty(e,"__esModule",{value:!0}),e.USEFUL_NON_ROOT_PNPM_FIELDS=e.FULL_FILTERED_META_DIR=e.FULL_META_DIR=e.ABBREVIATED_META_DIR=e.WORKSPACE_MANIFEST_FILENAME=e.STORE_VERSION=e.LAYOUT_VERSION=e.ENGINE_NAME=e.MANIFEST_BASE_NAMES=e.LOCKFILE_VERSION=e.LOCKFILE_MAJOR_VERSION=e.WANTED_LOCKFILE=void 0,e.getNodeBinLocationForCurrentOS=t,e.getDenoBinLocationForCurrentOS=r,e.getBunBinLocationForCurrentOS=n,e.WANTED_LOCKFILE="pnpm-lock.yaml",e.LOCKFILE_MAJOR_VERSION="9",e.LOCKFILE_VERSION=`${e.LOCKFILE_MAJOR_VERSION}.0`,e.MANIFEST_BASE_NAMES=["package.json","package.json5","package.yaml"],e.ENGINE_NAME=`${process.platform};${process.arch};node${process.version.split(".")[0].substring(1)}`,e.LAYOUT_VERSION=5,e.STORE_VERSION="v10",e.WORKSPACE_MANIFEST_FILENAME="pnpm-workspace.yaml",e.ABBREVIATED_META_DIR="metadata-v1.3",e.FULL_META_DIR="metadata-full-v1.3",e.FULL_FILTERED_META_DIR="metadata-v1.3",e.USEFUL_NON_ROOT_PNPM_FIELDS=["executionEnv"];function t(c=process.platform){return c==="win32"?"node.exe":"bin/node"}function r(c=process.platform){return c==="win32"?"deno.exe":"deno"}function n(c=process.platform){return c==="win32"?"bun.exe":"bun"}})(N)),N}var $;function ne(){if($)return p;$=1,Object.defineProperty(p,"__esModule",{value:!0}),p.LockfileMissingDependencyError=p.FetchError=p.PnpmError=void 0;const e=re();class t extends Error{code;hint;attempts;prefix;pkgsStack;constructor(a,i,f){super(i),this.code=a.startsWith("ERR_PNPM_")?a:`ERR_PNPM_${a}`,this.hint=f?.hint,this.attempts=f?.attempts}}p.PnpmError=t;class r extends t{response;request;constructor(a,i,f){const l={url:a.url};a.authHeaderValue&&(l.authHeaderValue=n(a.authHeaderValue));const s=`GET ${a.url}: ${i.statusText} - ${i.status}`;(i.status===401||i.status===403||i.status===404)&&(f=f?`${f}
1
+ import{checkForFile as P,parseYamlFile as _,throwError as m,writeYamlFile as b}from"@rent-scraper/utils";import Z from"axios";import C from"fs";import h from"path";import D from"util";import{spinner as J,log as X}from"@clack/prompts";import{stat as ee,mkdir as S}from"fs/promises";import te from"os";import re from"env-paths";var y={},w={},R={},T;function ne(){return T||(T=1,(function(e){Object.defineProperty(e,"__esModule",{value:!0}),e.USEFUL_NON_ROOT_PNPM_FIELDS=e.FULL_FILTERED_META_DIR=e.FULL_META_DIR=e.ABBREVIATED_META_DIR=e.WORKSPACE_MANIFEST_FILENAME=e.STORE_VERSION=e.LAYOUT_VERSION=e.ENGINE_NAME=e.MANIFEST_BASE_NAMES=e.LOCKFILE_VERSION=e.LOCKFILE_MAJOR_VERSION=e.WANTED_LOCKFILE=void 0,e.getNodeBinLocationForCurrentOS=t,e.getDenoBinLocationForCurrentOS=r,e.getBunBinLocationForCurrentOS=n,e.WANTED_LOCKFILE="pnpm-lock.yaml",e.LOCKFILE_MAJOR_VERSION="9",e.LOCKFILE_VERSION=`${e.LOCKFILE_MAJOR_VERSION}.0`,e.MANIFEST_BASE_NAMES=["package.json","package.json5","package.yaml"],e.ENGINE_NAME=`${process.platform};${process.arch};node${process.version.split(".")[0].substring(1)}`,e.LAYOUT_VERSION=5,e.STORE_VERSION="v10",e.WORKSPACE_MANIFEST_FILENAME="pnpm-workspace.yaml",e.ABBREVIATED_META_DIR="metadata-v1.3",e.FULL_META_DIR="metadata-full-v1.3",e.FULL_FILTERED_META_DIR="metadata-v1.3",e.USEFUL_NON_ROOT_PNPM_FIELDS=["executionEnv"];function t(c=process.platform){return c==="win32"?"node.exe":"bin/node"}function r(c=process.platform){return c==="win32"?"deno.exe":"deno"}function n(c=process.platform){return c==="win32"?"bun.exe":"bun"}})(R)),R}var $;function ie(){if($)return w;$=1,Object.defineProperty(w,"__esModule",{value:!0}),w.LockfileMissingDependencyError=w.FetchError=w.PnpmError=void 0;const e=ne();class t extends Error{code;hint;attempts;prefix;pkgsStack;constructor(a,i,f){super(i),this.code=a.startsWith("ERR_PNPM_")?a:`ERR_PNPM_${a}`,this.hint=f?.hint,this.attempts=f?.attempts}}w.PnpmError=t;class r extends t{response;request;constructor(a,i,f){const l={url:a.url};a.authHeaderValue&&(l.authHeaderValue=n(a.authHeaderValue));const s=`GET ${a.url}: ${i.statusText} - ${i.status}`;(i.status===401||i.status===403||i.status===404)&&(f=f?`${f}
2
2
 
3
- `:"",l.authHeaderValue?f+=`An authorization header was used: ${l.authHeaderValue}`:f+="No authorization header was set for the request."),super(`FETCH_${i.status}`,s,{hint:f}),this.request=l,this.response=i}}p.FetchError=r;function n(u){const[a,i]=u.split(" ");return i==null?"[hidden]":i.length<20?`${a} [hidden]`:`${a} ${i.substring(0,4)}[hidden]`}class c extends t{constructor(a){const i=`Broken lockfile: no entry for '${a}' in ${e.WANTED_LOCKFILE}`;super("LOCKFILE_MISSING_DEPENDENCY",i,{hint:`This issue is probably caused by a badly resolved merge conflict.
4
- To fix the lockfile, run 'pnpm install --no-frozen-lockfile'.`})}}return p.LockfileMissingDependencyError=c,p}var R={exports:{}},L={exports:{}},A,q;function ie(){if(q)return A;q=1;class e{constructor(n){this.value=n,this.next=void 0}}class t{constructor(){this.clear()}enqueue(n){const c=new e(n);this._head?(this._tail.next=c,this._tail=c):(this._head=c,this._tail=c),this._size++}dequeue(){const n=this._head;if(n)return this._head=this._head.next,this._size--,n.value}clear(){this._head=void 0,this._tail=void 0,this._size=0}get size(){return this._size}*[Symbol.iterator](){let n=this._head;for(;n;)yield n.value,n=n.next}}return A=t,A}var O,z;function ae(){if(z)return O;z=1;const e=ie();return O=r=>{if(!((Number.isInteger(r)||r===1/0)&&r>0))throw new TypeError("Expected `concurrency` to be a number from 1 and up");const n=new e;let c=0;const u=()=>{c--,n.size>0&&n.dequeue()()},a=async(l,s,...o)=>{c++;const d=(async()=>l(...o))();s(d);try{await d}catch{}u()},i=(l,s,...o)=>{n.enqueue(a.bind(null,l,s,...o)),(async()=>(await Promise.resolve(),c<r&&n.size>0&&n.dequeue()()))()},f=(l,...s)=>new Promise(o=>{i(l,o,...s)});return Object.defineProperties(f,{activeCount:{get:()=>c},pendingCount:{get:()=>n.size},clearQueue:{value:()=>{n.clear()}}}),f},O}var k,V;function se(){if(V)return k;V=1;const e=ae();class t extends Error{constructor(a){super(),this.value=a}}const r=async(u,a)=>a(await u),n=async u=>{const a=await Promise.all(u);if(a[1]===!0)throw new t(a[0]);return!1};return k=async(u,a,i)=>{i={concurrency:1/0,preserveOrder:!0,...i};const f=e(i.concurrency),l=[...u].map(o=>[o,f(r,o,a)]),s=e(i.preserveOrder?1:1/0);try{await Promise.all(l.map(o=>s(n,o)))}catch(o){if(o instanceof t)return o.value;throw o}},k}var B;function oe(){if(B)return L.exports;B=1;const e=h,t=S,{promisify:r}=D,n=se(),c=r(t.stat),u=r(t.lstat),a={directory:"isDirectory",file:"isFile"};function i({type:l}){if(!(l in a))throw new Error(`Invalid type specified: ${l}`)}const f=(l,s)=>l===void 0||s[a[l]]();return L.exports=async(l,s)=>{s={cwd:process.cwd(),type:"file",allowSymlinks:!0,...s},i(s);const o=s.allowSymlinks?c:u;return n(l,async d=>{try{const E=await o(e.resolve(s.cwd,d));return f(s.type,E)}catch{return!1}},s)},L.exports.sync=(l,s)=>{s={cwd:process.cwd(),allowSymlinks:!0,type:"file",...s},i(s);const o=s.allowSymlinks?t.statSync:t.lstatSync;for(const d of l)try{const E=o(e.resolve(s.cwd,d));if(f(s.type,E))return d}catch{}},L.exports}var v={exports:{}},x;function ce(){if(x)return v.exports;x=1;const e=S,{promisify:t}=D,r=t(e.access);return v.exports=async n=>{try{return await r(n),!0}catch{return!1}},v.exports.sync=n=>{try{return e.accessSync(n),!0}catch{return!1}},v.exports}var K;function ue(){return K||(K=1,(function(e){const t=h,r=oe(),n=ce(),c=Symbol("findUp.stop");e.exports=async(u,a={})=>{let i=t.resolve(a.cwd||"");const{root:f}=t.parse(i),l=[].concat(u),s=async o=>{if(typeof u!="function")return r(l,o);const d=await u(o.cwd);return typeof d=="string"?r([d],o):d};for(;;){const o=await s({...a,cwd:i});if(o===c)return;if(o)return t.resolve(i,o);if(i===f)return;i=t.dirname(i)}},e.exports.sync=(u,a={})=>{let i=t.resolve(a.cwd||"");const{root:f}=t.parse(i),l=[].concat(u),s=o=>{if(typeof u!="function")return r.sync(l,o);const d=u(o.cwd);return typeof d=="string"?r.sync([d],o):d};for(;;){const o=s({...a,cwd:i});if(o===c)return;if(o)return t.resolve(i,o);if(i===f)return;i=t.dirname(i)}},e.exports.exists=n,e.exports.sync.exists=n.sync,e.exports.stop=c})(R)),R.exports}var j;function le(){if(j)return y;j=1;var e=y&&y.__importDefault||function(s){return s&&s.__esModule?s:{default:s}};Object.defineProperty(y,"__esModule",{value:!0}),y.findWorkspaceDir=f;const t=e(S),r=e(h),n=ne(),c=e(ue()),u="NPM_CONFIG_WORKSPACE_DIR",a="pnpm-workspace.yaml",i=["pnpm-workspaces.yaml","pnpm-workspaces.yml","pnpm-workspace.yml"];async function f(s){const o=process.env[u]??process.env[u.toLowerCase()],d=o?r.default.join(o,a):await(0,c.default)([a,...i],{cwd:await l(s)});if(d&&r.default.basename(d)!==a)throw new n.PnpmError("BAD_WORKSPACE_MANIFEST_NAME",`The workspace manifest file should be named "pnpm-workspace.yaml". File found: ${d}`);return d&&r.default.dirname(d)}async function l(s){return new Promise(o=>{t.default.realpath.native(s,function(d,E){o(d!==null?s:E)})})}return y}var fe=le();const de=te("rent-scraper").config,w=h.join(de,"config.yaml"),he=h.join(ee.homedir(),"rent-scraper"),F=async e=>{const t=await fe.findWorkspaceDir(process.cwd());if(t)return e==="redfin"?h.join(t,"config.redfin.yaml"):h.join(t,"config.zillow.yaml");if(await W())return await Q(e)},U=async(e,t)=>{const r=t??await F(e);if(r)return await P(r)},W=async()=>await P(w),pe=async e=>{const t=await F(e);if(t)return await P(t)||m("Config file is required."),await _(t)},we=async()=>{const e=await g("zillow");if(e){const{zillowCookie:t,...r}=e??{};await M("zillow",r)}},ye=async e=>{try{return await new Promise(t=>{const r=setInterval(async()=>{const n=await U(e);n&&(t(n),clearInterval(r))},1e3)})}catch(t){m(t?.message)}},Ee=async()=>{try{return await new Promise((e,t)=>{const r=Z();r.start("Waiting for browser server");let n=0;const c=setInterval(async()=>{n++,n>10&&t(new Error("browser server is not running."));const u=await Y();u&&(r.stop("Browser server is connected!"),e(u),clearInterval(c))},1e3)})}catch(e){m(e?.message)}},Y=async()=>{try{const{data:e}=await G.get("http://localhost:8082/server");return e.running}catch{return!1}},_e=async()=>await new Promise(e=>{const t=setInterval(async()=>{const r=await H();r&&(e(r),clearInterval(t))},1e3)}),H=async()=>await I("zillow","zillowCookie"),me=(e,t,r="init")=>{const{outputPath:n,zipCodes:c,browser:u,zillowCookie:a}=t??{},i=[];return n||i.push("outputPath"),c||i.push("zipCodes"),e==="zillow"&&!u&&i.push("browser"),r==="scrape"&&e==="zillow"&&!a&&i.push("zillowCookie"),i},Fe=async e=>(await I(e,"zipCodes"))?.replace(/ /g,"").split(",").map(t=>Number(t)),Le=e=>e.join(", "),g=async e=>{const t=await F(e);if(t)return await _(t)},ve=async(e,t,r)=>{h.isAbsolute(t)||m("configDirectory must be an absolute path");const n=await X(t).catch(()=>null);n&&!n.isDirectory()&&m(`${t} exists but is not a directory`),await C(t,{recursive:!0});const c=h.dirname(w);await C(c,{recursive:!0}),r=r??`config.${e}.yaml`;const u=h.join(t,r),a={...await _(w),[e]:u};return await b(w,a),(await _(w)??{})?.[e]},Q=async e=>{try{return(await _(w))?.[e]}catch(t){console.error(`Failed to read pointer file at ${w}: ${t.message}`)}},ge=async e=>await I(e,"outputPath"),I=async(e,t)=>(await g(e))?.[t]??null,Ie=async(e,t)=>{const r=await g(e),n=Object.keys(t),c={...r,...t};await M(e,c),J.success(`Updated ${e} config: ${n.join(", ")}`)},M=async(e,t)=>{const r=await F(e);return await C(h.dirname(r),{recursive:!0}),await b(r,t),r};export{Y as checkBrowserServer,pe as checkForAndReadConfigFile,U as checkForConfigFile,W as checkForPointerFile,H as checkForZillowCookie,me as checkRequiredConfigValues,F as getConfigFilePath,ge as getOutputPathFromConfig,I as getValueFromConfigFile,Fe as getZipCodesFromConfig,he as globalDir,w as pointerFilePath,g as readConfigFile,Q as readPointerFile,we as resetZillowCookie,Le as stringifyZipCodes,Ie as updateConfigFile,Ee as waitForBrowserServer,ye as waitForConfigFile,_e as waitForZillowCookie,M as writeConfigFile,ve as writePointerFile};
3
+ `:"",l.authHeaderValue?f+=`An authorization header was used: ${l.authHeaderValue}`:f+="No authorization header was set for the request."),super(`FETCH_${i.status}`,s,{hint:f}),this.request=l,this.response=i}}w.FetchError=r;function n(u){const[a,i]=u.split(" ");return i==null?"[hidden]":i.length<20?`${a} [hidden]`:`${a} ${i.substring(0,4)}[hidden]`}class c extends t{constructor(a){const i=`Broken lockfile: no entry for '${a}' in ${e.WANTED_LOCKFILE}`;super("LOCKFILE_MISSING_DEPENDENCY",i,{hint:`This issue is probably caused by a badly resolved merge conflict.
4
+ To fix the lockfile, run 'pnpm install --no-frozen-lockfile'.`})}}return w.LockfileMissingDependencyError=c,w}var k={exports:{}},L={exports:{}},N,q;function ae(){if(q)return N;q=1;class e{constructor(n){this.value=n,this.next=void 0}}class t{constructor(){this.clear()}enqueue(n){const c=new e(n);this._head?(this._tail.next=c,this._tail=c):(this._head=c,this._tail=c),this._size++}dequeue(){const n=this._head;if(n)return this._head=this._head.next,this._size--,n.value}clear(){this._head=void 0,this._tail=void 0,this._size=0}get size(){return this._size}*[Symbol.iterator](){let n=this._head;for(;n;)yield n.value,n=n.next}}return N=t,N}var A,z;function se(){if(z)return A;z=1;const e=ae();return A=r=>{if(!((Number.isInteger(r)||r===1/0)&&r>0))throw new TypeError("Expected `concurrency` to be a number from 1 and up");const n=new e;let c=0;const u=()=>{c--,n.size>0&&n.dequeue()()},a=async(l,s,...o)=>{c++;const d=(async()=>l(...o))();s(d);try{await d}catch{}u()},i=(l,s,...o)=>{n.enqueue(a.bind(null,l,s,...o)),(async()=>(await Promise.resolve(),c<r&&n.size>0&&n.dequeue()()))()},f=(l,...s)=>new Promise(o=>{i(l,o,...s)});return Object.defineProperties(f,{activeCount:{get:()=>c},pendingCount:{get:()=>n.size},clearQueue:{value:()=>{n.clear()}}}),f},A}var O,V;function oe(){if(V)return O;V=1;const e=se();class t extends Error{constructor(a){super(),this.value=a}}const r=async(u,a)=>a(await u),n=async u=>{const a=await Promise.all(u);if(a[1]===!0)throw new t(a[0]);return!1};return O=async(u,a,i)=>{i={concurrency:1/0,preserveOrder:!0,...i};const f=e(i.concurrency),l=[...u].map(o=>[o,f(r,o,a)]),s=e(i.preserveOrder?1:1/0);try{await Promise.all(l.map(o=>s(n,o)))}catch(o){if(o instanceof t)return o.value;throw o}},O}var B;function ce(){if(B)return L.exports;B=1;const e=h,t=C,{promisify:r}=D,n=oe(),c=r(t.stat),u=r(t.lstat),a={directory:"isDirectory",file:"isFile"};function i({type:l}){if(!(l in a))throw new Error(`Invalid type specified: ${l}`)}const f=(l,s)=>l===void 0||s[a[l]]();return L.exports=async(l,s)=>{s={cwd:process.cwd(),type:"file",allowSymlinks:!0,...s},i(s);const o=s.allowSymlinks?c:u;return n(l,async d=>{try{const E=await o(e.resolve(s.cwd,d));return f(s.type,E)}catch{return!1}},s)},L.exports.sync=(l,s)=>{s={cwd:process.cwd(),allowSymlinks:!0,type:"file",...s},i(s);const o=s.allowSymlinks?t.statSync:t.lstatSync;for(const d of l)try{const E=o(e.resolve(s.cwd,d));if(f(s.type,E))return d}catch{}},L.exports}var g={exports:{}},x;function ue(){if(x)return g.exports;x=1;const e=C,{promisify:t}=D,r=t(e.access);return g.exports=async n=>{try{return await r(n),!0}catch{return!1}},g.exports.sync=n=>{try{return e.accessSync(n),!0}catch{return!1}},g.exports}var K;function le(){return K||(K=1,(function(e){const t=h,r=ce(),n=ue(),c=Symbol("findUp.stop");e.exports=async(u,a={})=>{let i=t.resolve(a.cwd||"");const{root:f}=t.parse(i),l=[].concat(u),s=async o=>{if(typeof u!="function")return r(l,o);const d=await u(o.cwd);return typeof d=="string"?r([d],o):d};for(;;){const o=await s({...a,cwd:i});if(o===c)return;if(o)return t.resolve(i,o);if(i===f)return;i=t.dirname(i)}},e.exports.sync=(u,a={})=>{let i=t.resolve(a.cwd||"");const{root:f}=t.parse(i),l=[].concat(u),s=o=>{if(typeof u!="function")return r.sync(l,o);const d=u(o.cwd);return typeof d=="string"?r.sync([d],o):d};for(;;){const o=s({...a,cwd:i});if(o===c)return;if(o)return t.resolve(i,o);if(i===f)return;i=t.dirname(i)}},e.exports.exists=n,e.exports.sync.exists=n.sync,e.exports.stop=c})(k)),k.exports}var j;function fe(){if(j)return y;j=1;var e=y&&y.__importDefault||function(s){return s&&s.__esModule?s:{default:s}};Object.defineProperty(y,"__esModule",{value:!0}),y.findWorkspaceDir=f;const t=e(C),r=e(h),n=ie(),c=e(le()),u="NPM_CONFIG_WORKSPACE_DIR",a="pnpm-workspace.yaml",i=["pnpm-workspaces.yaml","pnpm-workspaces.yml","pnpm-workspace.yml"];async function f(s){const o=process.env[u]??process.env[u.toLowerCase()],d=o?r.default.join(o,a):await(0,c.default)([a,...i],{cwd:await l(s)});if(d&&r.default.basename(d)!==a)throw new n.PnpmError("BAD_WORKSPACE_MANIFEST_NAME",`The workspace manifest file should be named "pnpm-workspace.yaml". File found: ${d}`);return d&&r.default.dirname(d)}async function l(s){return new Promise(o=>{t.default.realpath.native(s,function(d,E){o(d!==null?s:E)})})}return y}var de=fe();const he=re("rent-scraper").config,p=h.join(he,"config.yaml"),we=h.join(te.homedir(),"rent-scraper"),F=async e=>{const t=await de.findWorkspaceDir(process.cwd());if(t)return e==="redfin"?h.join(t,"config.redfin.yaml"):h.join(t,"config.zillow.yaml");if(await W())return await G(e)},U=async(e,t)=>{const r=t??await F(e);if(r)return await P(r)},W=async()=>await P(p),pe=async e=>{const t=await F(e);if(t)return await P(t)||m("Config file is required."),await _(t)},ye=async()=>{const e=await I("zillow");if(e){const{zillowCookie:t,...r}=e??{};await M("zillow",r)}},Ee=async e=>{try{return await new Promise(t=>{const r=setInterval(async()=>{const n=await U(e);n&&(t(n),clearInterval(r))},1e3)})}catch(t){m(t?.message)}},_e=async()=>{try{return await new Promise((e,t)=>{const r=J();r.start("Waiting for browser server");let n=0;const c=setInterval(async()=>{n++,n>10&&t(new Error("browser server is not running."));const u=await Y();u&&(r.stop("Browser server is connected!"),e(u),clearInterval(c))},1e3)})}catch(e){m(e?.message)}},Y=async()=>{try{const{data:e}=await Z.get("http://localhost:8082/server");return e.running}catch{return!1}},me=async()=>await new Promise(e=>{const t=setInterval(async()=>{const r=await H();r&&(e(r),clearInterval(t))},1e3)}),H=async()=>await v("zillow","zillowCookie"),Fe=async()=>await new Promise(e=>{const t=setInterval(async()=>{const r=await Q();r&&(e(r),clearInterval(t))},1e3)}),Q=async()=>await v("redfin","redfinCookie"),ve=(e,t,r="init")=>{const{outputPath:n,zipCodes:c,browser:u,zillowCookie:a}=t??{},i=[];return n||i.push("outputPath"),c||i.push("zipCodes"),e==="zillow"&&!u&&i.push("browser"),r==="scrape"&&e==="zillow"&&!a&&i.push("zillowCookie"),i},Le=async e=>(await v(e,"zipCodes"))?.replace(/ /g,"").split(",").map(t=>Number(t)),ge=e=>e.join(", "),I=async e=>{const t=await F(e);if(t)return await _(t)},Ie=async(e,t,r)=>{h.isAbsolute(t)||m("configDirectory must be an absolute path");const n=await ee(t).catch(()=>null);n&&!n.isDirectory()&&m(`${t} exists but is not a directory`),await S(t,{recursive:!0});const c=h.dirname(p);await S(c,{recursive:!0}),r=r??`config.${e}.yaml`;const u=h.join(t,r),a={...await _(p),[e]:u};return await b(p,a),(await _(p)??{})?.[e]},G=async e=>{try{return(await _(p))?.[e]}catch(t){console.error(`Failed to read pointer file at ${p}: ${t.message}`)}},Pe=async e=>await v(e,"outputPath"),v=async(e,t)=>(await I(e))?.[t]??null,Ce=async(e,t)=>{const r=await I(e),n=Object.keys(t),c={...r,...t};await M(e,c),X.success(`Updated ${e} config: ${n.join(", ")}`)},M=async(e,t)=>{const r=await F(e);return await S(h.dirname(r),{recursive:!0}),await b(r,t),r};export{Y as checkBrowserServer,pe as checkForAndReadConfigFile,U as checkForConfigFile,W as checkForPointerFile,Q as checkForRedfinCookie,H as checkForZillowCookie,ve as checkRequiredConfigValues,F as getConfigFilePath,Pe as getOutputPathFromConfig,v as getValueFromConfigFile,Le as getZipCodesFromConfig,we as globalDir,p as pointerFilePath,I as readConfigFile,G as readPointerFile,ye as resetZillowCookie,ge as stringifyZipCodes,Ce as updateConfigFile,_e as waitForBrowserServer,Ee as waitForConfigFile,Fe as waitForRedfinCookie,me as waitForZillowCookie,M as writeConfigFile,Ie as writePointerFile};
package/dist/index.d.mts CHANGED
@@ -38,7 +38,7 @@ interface ReadDirectoryOptions {
38
38
  }
39
39
  declare const readFilesInDirectory: (directory: string, options?: ReadDirectoryOptions) => Promise<string[]>;
40
40
  declare const readFilesInDirectorySync: (directory: string, options?: ReadDirectoryOptions) => string[];
41
- declare const checkForFile: (filePath: string) => Promise<boolean | undefined>;
41
+ declare const checkForFile: (filePath: string) => Promise<boolean>;
42
42
  declare const parseAbsolutePath: (relPath: string) => string;
43
43
  declare const getRandomArrayValue: (array: unknown[]) => unknown;
44
44
  declare const parseCsvFile: (filePath: string, options: csv.parser.Options) => Promise<any[]>;
package/dist/index.mjs CHANGED
@@ -1 +1 @@
1
- import{readdir as ue,access as oe,readFile as L,writeFile as Q}from"fs/promises";import G,{readdirSync as ce}from"fs";import E from"path";import*as fe from"csv";import X from"yaml";function le(e){return e&&e.__esModule&&Object.prototype.hasOwnProperty.call(e,"default")?e.default:e}var z={exports:{}},he=z.exports,ee;function de(){return ee||(ee=1,(function(e,s){(function(i,h){e.exports=h()})(he,(function(){var i=1e3,h=6e4,D=36e5,m="millisecond",M="second",$="minute",d="hour",O="day",C="week",b="month",R="quarter",_="year",k="date",Z="Invalid Date",ne=/^(\d{4})[-/]?(\d{1,2})?[-/]?(\d{0,2})[Tt\s]*(\d{1,2})?:?(\d{1,2})?:?(\d{1,2})?[.:]?(\d+)?$/,se=/\[([^\]]+)]|Y{1,4}|M{1,4}|D{1,2}|d{1,4}|H{1,2}|h{1,2}|a|A|m{1,2}|s{1,2}|Z{1,2}|SSS/g,ae={name:"en",weekdays:"Sunday_Monday_Tuesday_Wednesday_Thursday_Friday_Saturday".split("_"),months:"January_February_March_April_May_June_July_August_September_October_November_December".split("_"),ordinal:function(u){var n=["th","st","nd","rd"],t=u%100;return"["+u+(n[(t-20)%10]||n[t]||n[0])+"]"}},U=function(u,n,t){var a=String(u);return!a||a.length>=n?u:""+Array(n+1-a.length).join(t)+u},ie={s:U,z:function(u){var n=-u.utcOffset(),t=Math.abs(n),a=Math.floor(t/60),r=t%60;return(n<=0?"+":"-")+U(a,2,"0")+":"+U(r,2,"0")},m:function u(n,t){if(n.date()<t.date())return-u(t,n);var a=12*(t.year()-n.year())+(t.month()-n.month()),r=n.clone().add(a,b),o=t-r<0,c=n.clone().add(a+(o?-1:1),b);return+(-(a+(t-r)/(o?r-c:c-r))||0)},a:function(u){return u<0?Math.ceil(u)||0:Math.floor(u)},p:function(u){return{M:b,y:_,w:C,d:O,D:k,h:d,m:$,s:M,ms:m,Q:R}[u]||String(u||"").toLowerCase().replace(/s$/,"")},u:function(u){return u===void 0}},A="en",N={};N[A]=ae;var B="$isDayjsObject",V=function(u){return u instanceof W||!(!u||!u[B])},H=function u(n,t,a){var r;if(!n)return A;if(typeof n=="string"){var o=n.toLowerCase();N[o]&&(r=o),t&&(N[o]=t,r=o);var c=n.split("-");if(!r&&c.length>1)return u(c[0])}else{var l=n.name;N[l]=n,r=l}return!a&&r&&(A=r),r||!a&&A},y=function(u,n){if(V(u))return u.clone();var t=typeof n=="object"?n:{};return t.date=u,t.args=arguments,new W(t)},f=ie;f.l=H,f.i=V,f.w=function(u,n){return y(u,{locale:n.$L,utc:n.$u,x:n.$x,$offset:n.$offset})};var W=(function(){function u(t){this.$L=H(t.locale,null,!0),this.parse(t),this.$x=this.$x||t.x||{},this[B]=!0}var n=u.prototype;return n.parse=function(t){this.$d=(function(a){var r=a.date,o=a.utc;if(r===null)return new Date(NaN);if(f.u(r))return new Date;if(r instanceof Date)return new Date(r);if(typeof r=="string"&&!/Z$/i.test(r)){var c=r.match(ne);if(c){var l=c[2]-1||0,p=(c[7]||"0").substring(0,3);return o?new Date(Date.UTC(c[1],l,c[3]||1,c[4]||0,c[5]||0,c[6]||0,p)):new Date(c[1],l,c[3]||1,c[4]||0,c[5]||0,c[6]||0,p)}}return new Date(r)})(t),this.init()},n.init=function(){var t=this.$d;this.$y=t.getFullYear(),this.$M=t.getMonth(),this.$D=t.getDate(),this.$W=t.getDay(),this.$H=t.getHours(),this.$m=t.getMinutes(),this.$s=t.getSeconds(),this.$ms=t.getMilliseconds()},n.$utils=function(){return f},n.isValid=function(){return this.$d.toString()!==Z},n.isSame=function(t,a){var r=y(t);return this.startOf(a)<=r&&r<=this.endOf(a)},n.isAfter=function(t,a){return y(t)<this.startOf(a)},n.isBefore=function(t,a){return this.endOf(a)<y(t)},n.$g=function(t,a,r){return f.u(t)?this[a]:this.set(r,t)},n.unix=function(){return Math.floor(this.valueOf()/1e3)},n.valueOf=function(){return this.$d.getTime()},n.startOf=function(t,a){var r=this,o=!!f.u(a)||a,c=f.p(t),l=function(j,v){var x=f.w(r.$u?Date.UTC(r.$y,v,j):new Date(r.$y,v,j),r);return o?x:x.endOf(O)},p=function(j,v){return f.w(r.toDate()[j].apply(r.toDate("s"),(o?[0,0,0,0]:[23,59,59,999]).slice(v)),r)},g=this.$W,S=this.$M,w=this.$D,F="set"+(this.$u?"UTC":"");switch(c){case _:return o?l(1,0):l(31,11);case b:return o?l(1,S):l(0,S+1);case C:var T=this.$locale().weekStart||0,I=(g<T?g+7:g)-T;return l(o?w-I:w+(6-I),S);case O:case k:return p(F+"Hours",0);case d:return p(F+"Minutes",1);case $:return p(F+"Seconds",2);case M:return p(F+"Milliseconds",3);default:return this.clone()}},n.endOf=function(t){return this.startOf(t,!1)},n.$set=function(t,a){var r,o=f.p(t),c="set"+(this.$u?"UTC":""),l=(r={},r[O]=c+"Date",r[k]=c+"Date",r[b]=c+"Month",r[_]=c+"FullYear",r[d]=c+"Hours",r[$]=c+"Minutes",r[M]=c+"Seconds",r[m]=c+"Milliseconds",r)[o],p=o===O?this.$D+(a-this.$W):a;if(o===b||o===_){var g=this.clone().set(k,1);g.$d[l](p),g.init(),this.$d=g.set(k,Math.min(this.$D,g.daysInMonth())).$d}else l&&this.$d[l](p);return this.init(),this},n.set=function(t,a){return this.clone().$set(t,a)},n.get=function(t){return this[f.p(t)]()},n.add=function(t,a){var r,o=this;t=Number(t);var c=f.p(a),l=function(S){var w=y(o);return f.w(w.date(w.date()+Math.round(S*t)),o)};if(c===b)return this.set(b,this.$M+t);if(c===_)return this.set(_,this.$y+t);if(c===O)return l(1);if(c===C)return l(7);var p=(r={},r[$]=h,r[d]=D,r[M]=i,r)[c]||1,g=this.$d.getTime()+t*p;return f.w(g,this)},n.subtract=function(t,a){return this.add(-1*t,a)},n.format=function(t){var a=this,r=this.$locale();if(!this.isValid())return r.invalidDate||Z;var o=t||"YYYY-MM-DDTHH:mm:ssZ",c=f.z(this),l=this.$H,p=this.$m,g=this.$M,S=r.weekdays,w=r.months,F=r.meridiem,T=function(v,x,Y,J){return v&&(v[x]||v(a,o))||Y[x].slice(0,J)},I=function(v){return f.s(l%12||12,v,"0")},j=F||function(v,x,Y){var J=v<12?"AM":"PM";return Y?J.toLowerCase():J};return o.replace(se,(function(v,x){return x||(function(Y){switch(Y){case"YY":return String(a.$y).slice(-2);case"YYYY":return f.s(a.$y,4,"0");case"M":return g+1;case"MM":return f.s(g+1,2,"0");case"MMM":return T(r.monthsShort,g,w,3);case"MMMM":return T(w,g);case"D":return a.$D;case"DD":return f.s(a.$D,2,"0");case"d":return String(a.$W);case"dd":return T(r.weekdaysMin,a.$W,S,2);case"ddd":return T(r.weekdaysShort,a.$W,S,3);case"dddd":return S[a.$W];case"H":return String(l);case"HH":return f.s(l,2,"0");case"h":return I(1);case"hh":return I(2);case"a":return j(l,p,!0);case"A":return j(l,p,!1);case"m":return String(p);case"mm":return f.s(p,2,"0");case"s":return String(a.$s);case"ss":return f.s(a.$s,2,"0");case"SSS":return f.s(a.$ms,3,"0");case"Z":return c}return null})(v)||c.replace(":","")}))},n.utcOffset=function(){return 15*-Math.round(this.$d.getTimezoneOffset()/15)},n.diff=function(t,a,r){var o,c=this,l=f.p(a),p=y(t),g=(p.utcOffset()-this.utcOffset())*h,S=this-p,w=function(){return f.m(c,p)};switch(l){case _:o=w()/12;break;case b:o=w();break;case R:o=w()/3;break;case C:o=(S-g)/6048e5;break;case O:o=(S-g)/864e5;break;case d:o=S/D;break;case $:o=S/h;break;case M:o=S/i;break;default:o=S}return r?o:f.a(o)},n.daysInMonth=function(){return this.endOf(b).$D},n.$locale=function(){return N[this.$L]},n.locale=function(t,a){if(!t)return this.$L;var r=this.clone(),o=H(t,a,!0);return o&&(r.$L=o),r},n.clone=function(){return f.w(this.$d,this)},n.toDate=function(){return new Date(this.valueOf())},n.toJSON=function(){return this.isValid()?this.toISOString():null},n.toISOString=function(){return this.$d.toISOString()},n.toString=function(){return this.$d.toUTCString()},u})(),K=W.prototype;return y.prototype=K,[["$ms",m],["$s",M],["$m",$],["$H",d],["$W",O],["$M",b],["$y",_],["$D",k]].forEach((function(u){K[u[1]]=function(n){return this.$g(n,u[0],u[1])}})),y.extend=function(u,n){return u.$i||(u(n,W,y),u.$i=!0),y},y.locale=H,y.isDayjs=V,y.unix=function(u){return y(1e3*u)},y.en=N[A],y.Ls=N,y.p={},y}))})(z)),z.exports}var me=de();const $e=le(me),pe=(e,s)=>e.reduce((i,h)=>({...i,[h[s]]:(i[h[s]]||[]).concat(h)}),{}),te=e=>typeof e=="object"&&!!e,ye=e=>te(e)&&Object.keys(e).length===0;function ge(e,s){throw new Error(JSON.stringify({status:s??400,message:e}))}const Se=e=>JSON.parse(e?.message),re=(e,s)=>e.length<=s?[e]:[e.slice(0,s),...re(e.slice(s),s)],P=(e,s)=>{const{round:i,zeros:h,type:D}=s??{};let m=e;return m=Number(i?q(e,i):m),m=h?Number(m).toFixed(h):m,m=D==="%"?`${m}%`:m,m=D==="$"?`$${m}`:m,s?m:Number(m)},ve=e=>P(e,{round:100,zeros:2,type:"$"}),De=(e,s)=>{const{round:i=100}=s??{};return P(e,{round:i,type:"%"})},Me=e=>Number(P(e,{round:100,zeros:2})),q=(e,s=100)=>Math.round(Number(e)*s)/s,we=e=>{const s=e?.replace("$","")?.replace(",","");return Number(s)},be=e=>{const s=e?.replace("+","")?.replace("%","");return q(Number(s)/100,1e3)},Oe=e=>e==="true"||Number(e)===1,_e=e=>{if(e?.response){const{status:s,data:i}=e?.response??{};return{status:s,message:i}}else{if(e?.errors)return{status:400,message:e?.errors?.map(s=>s).map(s=>s?.message).join("")};if(e?.message==="canceled")return{status:400,message:"request timed out at "+$e().format()};try{if(e?.body){const{status:s,message:i}=JSON.parse(e?.body)??{};return{status:s,message:i}}else{const{status:s,message:i}=JSON.parse(e?.message)??{};return{status:s,message:i}}}catch{const{status:s,message:i}=e??{};return console.log({status:s,message:i}),{status:s||400,message:i||"Internal Server Error"}}}},xe=(e,s,i)=>{const h=e.toString(),D=s.toString().padStart(2,"0"),m=i.toString().padStart(2,"0");return new Date(`${h}-${D}-${m}T00:00:00-08:00`)},Ne=async(e,s)=>{const{extension:i,prefix:h,prependDirectory:D,recursive:m=!1}=s??{},M=await ue(e,{recursive:m});let $=i?M.filter(d=>Array.isArray(i)?i.includes(E.extname(d)):E.extname(d)===i):M;return $=h?$.filter(d=>d.startsWith(h)):$,$=$.filter(d=>d!==".DS_Store"),D?$.map(d=>`${e}/${d}`):$},Te=(e,s)=>{const{extension:i,prefix:h,prependDirectory:D,recursive:m=!1}=s??{},M=ce(e,{recursive:m});let $=i?M.filter(d=>Array.isArray(i)?i.includes(E.extname(d)):E.extname(d)===i):M;return $=h?$.filter(d=>d.startsWith(h)):$,$=$.filter(d=>d!==".DS_Store"),D?$.map(d=>`${e}/${d}`):$},je=async e=>{try{if(await oe(e),(await L(e)).length)return!0}catch(s){if(s?.code==="EISDIR")return!0;if(s?.code==="ENOENT")return!1;throw s}},ke=e=>E.resolve(E.join(process.env.INIT_CWD,e)),Fe=e=>e[Math.floor(Math.random()*e.length)],Ee=async(e,s)=>{const i=[],h=G.createReadStream(e).pipe(fe.parse(s));for await(const D of h)i.push(D);return i},Ae=async e=>{const s=[],i=(await L(e,"utf8")).replace(/\n/g,",").replace(/ /g,"").split(",");for(const h of i)s.push(h);return s},Ie=async e=>JSON.parse(await L(e,"utf8")),Ye=async e=>{try{return X.parse(await L(e,"utf8"))}catch(s){s?.code==="ENOENT"&&await G.promises.writeFile(e,"")}},Ce=async(e,s)=>{await Q(e,X.stringify(s))},He=(e,s)=>e?.filter(i=>!s?.includes(i));class We{errors;constructor(s=[]){this.errors=s}add(s,i){i instanceof Error&&(s+=`: ${i.message}`),this.errors.push(s)}get(){return this.errors}async write(s,i){await Q(s,i)}}export{We as ErrorLog,je as checkForFile,re as chunkArray,He as compareArrays,xe as createDayStartInPST,Fe as getRandomArrayValue,pe as groupByKey,ye as isEmptyObject,te as isObject,ke as parseAbsolutePath,Ee as parseCsvFile,ve as parseCurrency,_e as parseError,Ie as parseJsonFile,P as parseNumber,De as parsePercentage,Me as parsePrice,Se as parseThrownError,Ae as parseTxtFile,Ye as parseYamlFile,be as percentageToDecimal,we as priceToInteger,Ne as readFilesInDirectory,Te as readFilesInDirectorySync,q as roundValue,ge as throwError,Oe as validateBooleanString,Ce as writeYamlFile};
1
+ import{readdir as ue,stat as oe,readFile as V,writeFile as Q}from"fs/promises";import G,{readdirSync as ce}from"fs";import A from"path";import*as fe from"csv";import X from"yaml";function le(e){return e&&e.__esModule&&Object.prototype.hasOwnProperty.call(e,"default")?e.default:e}var L={exports:{}},he=L.exports,ee;function de(){return ee||(ee=1,(function(e,n){(function(i,h){e.exports=h()})(he,(function(){var i=1e3,h=6e4,D=36e5,m="millisecond",M="second",$="minute",d="hour",O="day",C="week",b="month",Z="quarter",_="year",k="date",B="Invalid Date",ne=/^(\d{4})[-/]?(\d{1,2})?[-/]?(\d{0,2})[Tt\s]*(\d{1,2})?:?(\d{1,2})?:?(\d{1,2})?[.:]?(\d+)?$/,se=/\[([^\]]+)]|Y{1,4}|M{1,4}|D{1,2}|d{1,4}|H{1,2}|h{1,2}|a|A|m{1,2}|s{1,2}|Z{1,2}|SSS/g,ae={name:"en",weekdays:"Sunday_Monday_Tuesday_Wednesday_Thursday_Friday_Saturday".split("_"),months:"January_February_March_April_May_June_July_August_September_October_November_December".split("_"),ordinal:function(u){var s=["th","st","nd","rd"],t=u%100;return"["+u+(s[(t-20)%10]||s[t]||s[0])+"]"}},P=function(u,s,t){var a=String(u);return!a||a.length>=s?u:""+Array(s+1-a.length).join(t)+u},ie={s:P,z:function(u){var s=-u.utcOffset(),t=Math.abs(s),a=Math.floor(t/60),r=t%60;return(s<=0?"+":"-")+P(a,2,"0")+":"+P(r,2,"0")},m:function u(s,t){if(s.date()<t.date())return-u(t,s);var a=12*(t.year()-s.year())+(t.month()-s.month()),r=s.clone().add(a,b),o=t-r<0,c=s.clone().add(a+(o?-1:1),b);return+(-(a+(t-r)/(o?r-c:c-r))||0)},a:function(u){return u<0?Math.ceil(u)||0:Math.floor(u)},p:function(u){return{M:b,y:_,w:C,d:O,D:k,h:d,m:$,s:M,ms:m,Q:Z}[u]||String(u||"").toLowerCase().replace(/s$/,"")},u:function(u){return u===void 0}},E="en",N={};N[E]=ae;var R="$isDayjsObject",U=function(u){return u instanceof W||!(!u||!u[R])},H=function u(s,t,a){var r;if(!s)return E;if(typeof s=="string"){var o=s.toLowerCase();N[o]&&(r=o),t&&(N[o]=t,r=o);var c=s.split("-");if(!r&&c.length>1)return u(c[0])}else{var l=s.name;N[l]=s,r=l}return!a&&r&&(E=r),r||!a&&E},y=function(u,s){if(U(u))return u.clone();var t=typeof s=="object"?s:{};return t.date=u,t.args=arguments,new W(t)},f=ie;f.l=H,f.i=U,f.w=function(u,s){return y(u,{locale:s.$L,utc:s.$u,x:s.$x,$offset:s.$offset})};var W=(function(){function u(t){this.$L=H(t.locale,null,!0),this.parse(t),this.$x=this.$x||t.x||{},this[R]=!0}var s=u.prototype;return s.parse=function(t){this.$d=(function(a){var r=a.date,o=a.utc;if(r===null)return new Date(NaN);if(f.u(r))return new Date;if(r instanceof Date)return new Date(r);if(typeof r=="string"&&!/Z$/i.test(r)){var c=r.match(ne);if(c){var l=c[2]-1||0,p=(c[7]||"0").substring(0,3);return o?new Date(Date.UTC(c[1],l,c[3]||1,c[4]||0,c[5]||0,c[6]||0,p)):new Date(c[1],l,c[3]||1,c[4]||0,c[5]||0,c[6]||0,p)}}return new Date(r)})(t),this.init()},s.init=function(){var t=this.$d;this.$y=t.getFullYear(),this.$M=t.getMonth(),this.$D=t.getDate(),this.$W=t.getDay(),this.$H=t.getHours(),this.$m=t.getMinutes(),this.$s=t.getSeconds(),this.$ms=t.getMilliseconds()},s.$utils=function(){return f},s.isValid=function(){return this.$d.toString()!==B},s.isSame=function(t,a){var r=y(t);return this.startOf(a)<=r&&r<=this.endOf(a)},s.isAfter=function(t,a){return y(t)<this.startOf(a)},s.isBefore=function(t,a){return this.endOf(a)<y(t)},s.$g=function(t,a,r){return f.u(t)?this[a]:this.set(r,t)},s.unix=function(){return Math.floor(this.valueOf()/1e3)},s.valueOf=function(){return this.$d.getTime()},s.startOf=function(t,a){var r=this,o=!!f.u(a)||a,c=f.p(t),l=function(j,S){var x=f.w(r.$u?Date.UTC(r.$y,S,j):new Date(r.$y,S,j),r);return o?x:x.endOf(O)},p=function(j,S){return f.w(r.toDate()[j].apply(r.toDate("s"),(o?[0,0,0,0]:[23,59,59,999]).slice(S)),r)},g=this.$W,v=this.$M,w=this.$D,F="set"+(this.$u?"UTC":"");switch(c){case _:return o?l(1,0):l(31,11);case b:return o?l(1,v):l(0,v+1);case C:var T=this.$locale().weekStart||0,Y=(g<T?g+7:g)-T;return l(o?w-Y:w+(6-Y),v);case O:case k:return p(F+"Hours",0);case d:return p(F+"Minutes",1);case $:return p(F+"Seconds",2);case M:return p(F+"Milliseconds",3);default:return this.clone()}},s.endOf=function(t){return this.startOf(t,!1)},s.$set=function(t,a){var r,o=f.p(t),c="set"+(this.$u?"UTC":""),l=(r={},r[O]=c+"Date",r[k]=c+"Date",r[b]=c+"Month",r[_]=c+"FullYear",r[d]=c+"Hours",r[$]=c+"Minutes",r[M]=c+"Seconds",r[m]=c+"Milliseconds",r)[o],p=o===O?this.$D+(a-this.$W):a;if(o===b||o===_){var g=this.clone().set(k,1);g.$d[l](p),g.init(),this.$d=g.set(k,Math.min(this.$D,g.daysInMonth())).$d}else l&&this.$d[l](p);return this.init(),this},s.set=function(t,a){return this.clone().$set(t,a)},s.get=function(t){return this[f.p(t)]()},s.add=function(t,a){var r,o=this;t=Number(t);var c=f.p(a),l=function(v){var w=y(o);return f.w(w.date(w.date()+Math.round(v*t)),o)};if(c===b)return this.set(b,this.$M+t);if(c===_)return this.set(_,this.$y+t);if(c===O)return l(1);if(c===C)return l(7);var p=(r={},r[$]=h,r[d]=D,r[M]=i,r)[c]||1,g=this.$d.getTime()+t*p;return f.w(g,this)},s.subtract=function(t,a){return this.add(-1*t,a)},s.format=function(t){var a=this,r=this.$locale();if(!this.isValid())return r.invalidDate||B;var o=t||"YYYY-MM-DDTHH:mm:ssZ",c=f.z(this),l=this.$H,p=this.$m,g=this.$M,v=r.weekdays,w=r.months,F=r.meridiem,T=function(S,x,I,J){return S&&(S[x]||S(a,o))||I[x].slice(0,J)},Y=function(S){return f.s(l%12||12,S,"0")},j=F||function(S,x,I){var J=S<12?"AM":"PM";return I?J.toLowerCase():J};return o.replace(se,(function(S,x){return x||(function(I){switch(I){case"YY":return String(a.$y).slice(-2);case"YYYY":return f.s(a.$y,4,"0");case"M":return g+1;case"MM":return f.s(g+1,2,"0");case"MMM":return T(r.monthsShort,g,w,3);case"MMMM":return T(w,g);case"D":return a.$D;case"DD":return f.s(a.$D,2,"0");case"d":return String(a.$W);case"dd":return T(r.weekdaysMin,a.$W,v,2);case"ddd":return T(r.weekdaysShort,a.$W,v,3);case"dddd":return v[a.$W];case"H":return String(l);case"HH":return f.s(l,2,"0");case"h":return Y(1);case"hh":return Y(2);case"a":return j(l,p,!0);case"A":return j(l,p,!1);case"m":return String(p);case"mm":return f.s(p,2,"0");case"s":return String(a.$s);case"ss":return f.s(a.$s,2,"0");case"SSS":return f.s(a.$ms,3,"0");case"Z":return c}return null})(S)||c.replace(":","")}))},s.utcOffset=function(){return 15*-Math.round(this.$d.getTimezoneOffset()/15)},s.diff=function(t,a,r){var o,c=this,l=f.p(a),p=y(t),g=(p.utcOffset()-this.utcOffset())*h,v=this-p,w=function(){return f.m(c,p)};switch(l){case _:o=w()/12;break;case b:o=w();break;case Z:o=w()/3;break;case C:o=(v-g)/6048e5;break;case O:o=(v-g)/864e5;break;case d:o=v/D;break;case $:o=v/h;break;case M:o=v/i;break;default:o=v}return r?o:f.a(o)},s.daysInMonth=function(){return this.endOf(b).$D},s.$locale=function(){return N[this.$L]},s.locale=function(t,a){if(!t)return this.$L;var r=this.clone(),o=H(t,a,!0);return o&&(r.$L=o),r},s.clone=function(){return f.w(this.$d,this)},s.toDate=function(){return new Date(this.valueOf())},s.toJSON=function(){return this.isValid()?this.toISOString():null},s.toISOString=function(){return this.$d.toISOString()},s.toString=function(){return this.$d.toUTCString()},u})(),K=W.prototype;return y.prototype=K,[["$ms",m],["$s",M],["$m",$],["$H",d],["$W",O],["$M",b],["$y",_],["$D",k]].forEach((function(u){K[u[1]]=function(s){return this.$g(s,u[0],u[1])}})),y.extend=function(u,s){return u.$i||(u(s,W,y),u.$i=!0),y},y.locale=H,y.isDayjs=U,y.unix=function(u){return y(1e3*u)},y.en=N[E],y.Ls=N,y.p={},y}))})(L)),L.exports}var me=de();const $e=le(me),pe=(e,n)=>e.reduce((i,h)=>({...i,[h[n]]:(i[h[n]]||[]).concat(h)}),{}),te=e=>typeof e=="object"&&!!e,ye=e=>te(e)&&Object.keys(e).length===0;function ge(e,n){throw new Error(JSON.stringify({status:n??400,message:e}))}const ve=e=>JSON.parse(e?.message),re=(e,n)=>e.length<=n?[e]:[e.slice(0,n),...re(e.slice(n),n)],z=(e,n)=>{const{round:i,zeros:h,type:D}=n??{};let m=e;return m=Number(i?q(e,i):m),m=h?Number(m).toFixed(h):m,m=D==="%"?`${m}%`:m,m=D==="$"?`$${m}`:m,n?m:Number(m)},Se=e=>z(e,{round:100,zeros:2,type:"$"}),De=(e,n)=>{const{round:i=100}=n??{};return z(e,{round:i,type:"%"})},Me=e=>Number(z(e,{round:100,zeros:2})),q=(e,n=100)=>Math.round(Number(e)*n)/n,we=e=>{const n=e?.replace("$","")?.replace(",","");return Number(n)},be=e=>{const n=e?.replace("+","")?.replace("%","");return q(Number(n)/100,1e3)},Oe=e=>e==="true"||Number(e)===1,_e=e=>{if(e?.response){const{status:n,data:i}=e?.response??{};return{status:n,message:i}}else{if(e?.errors)return{status:400,message:e?.errors?.map(n=>n).map(n=>n?.message).join("")};if(e?.message==="canceled")return{status:400,message:"request timed out at "+$e().format()};try{if(e?.body){const{status:n,message:i}=JSON.parse(e?.body)??{};return{status:n,message:i}}else{const{status:n,message:i}=JSON.parse(e?.message)??{};return{status:n,message:i}}}catch{const{status:n,message:i}=e??{};return console.log({status:n,message:i}),{status:n||400,message:i||"Internal Server Error"}}}},xe=(e,n,i)=>{const h=e.toString(),D=n.toString().padStart(2,"0"),m=i.toString().padStart(2,"0");return new Date(`${h}-${D}-${m}T00:00:00-08:00`)},Ne=async(e,n)=>{const{extension:i,prefix:h,prependDirectory:D,recursive:m=!1}=n??{},M=await ue(e,{recursive:m});let $=i?M.filter(d=>Array.isArray(i)?i.includes(A.extname(d)):A.extname(d)===i):M;return $=h?$.filter(d=>d.startsWith(h)):$,$=$.filter(d=>d!==".DS_Store"),D?$.map(d=>`${e}/${d}`):$},Te=(e,n)=>{const{extension:i,prefix:h,prependDirectory:D,recursive:m=!1}=n??{},M=ce(e,{recursive:m});let $=i?M.filter(d=>Array.isArray(i)?i.includes(A.extname(d)):A.extname(d)===i):M;return $=h?$.filter(d=>d.startsWith(h)):$,$=$.filter(d=>d!==".DS_Store"),D?$.map(d=>`${e}/${d}`):$},je=async e=>{try{const n=await oe(e);return n.isDirectory()||n.size>0}catch(n){if(n?.code==="ENOENT")return!1;throw n}},ke=e=>A.resolve(A.join(process.env.INIT_CWD,e)),Fe=e=>e[Math.floor(Math.random()*e.length)],Ae=async(e,n)=>{const i=[],h=G.createReadStream(e).pipe(fe.parse(n));for await(const D of h)i.push(D);return i},Ee=async e=>{const n=[],i=(await V(e,"utf8")).replace(/\n/g,",").replace(/ /g,"").split(",");for(const h of i)n.push(h);return n},Ye=async e=>JSON.parse(await V(e,"utf8")),Ie=async e=>{try{return X.parse(await V(e,"utf8"))}catch(n){n?.code==="ENOENT"&&await G.promises.writeFile(e,"")}},Ce=async(e,n)=>{await Q(e,X.stringify(n))},He=(e,n)=>e?.filter(i=>!n?.includes(i));class We{errors;constructor(n=[]){this.errors=n}add(n,i){i instanceof Error&&(n+=`: ${i.message}`),this.errors.push(n)}get(){return this.errors}async write(n,i){await Q(n,i)}}export{We as ErrorLog,je as checkForFile,re as chunkArray,He as compareArrays,xe as createDayStartInPST,Fe as getRandomArrayValue,pe as groupByKey,ye as isEmptyObject,te as isObject,ke as parseAbsolutePath,Ae as parseCsvFile,Se as parseCurrency,_e as parseError,Ye as parseJsonFile,z as parseNumber,De as parsePercentage,Me as parsePrice,ve as parseThrownError,Ee as parseTxtFile,Ie as parseYamlFile,be as percentageToDecimal,we as priceToInteger,Ne as readFilesInDirectory,Te as readFilesInDirectorySync,q as roundValue,ge as throwError,Oe as validateBooleanString,Ce as writeYamlFile};
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@rent-scraper/utils",
3
- "version": "1.0.15",
3
+ "version": "1.0.16",
4
4
  "type": "module",
5
5
  "main": "./dist/index.mjs",
6
6
  "module": "./dist/index.mjs",
@@ -32,7 +32,7 @@
32
32
  "csv": "^6.3.11",
33
33
  "env-paths": "^3.0.0",
34
34
  "yaml": "^2.8.0",
35
- "@rent-scraper/api": "1.0.15"
35
+ "@rent-scraper/api": "1.0.16"
36
36
  },
37
37
  "devDependencies": {
38
38
  "unbuild": "^3.5.0"