@rent-scraper/utils 1.0.19 → 1.0.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config.d.mts +2 -2
- package/dist/config.mjs +3 -3
- package/package.json +2 -2
package/dist/config.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { ListingsSource
|
|
1
|
+
import { ListingsSource } from '@rent-scraper/api';
|
|
2
2
|
|
|
3
3
|
interface ScrapeConfig {
|
|
4
4
|
outputPath: string;
|
|
@@ -6,7 +6,7 @@ interface ScrapeConfig {
|
|
|
6
6
|
zipCodes: string;
|
|
7
7
|
daysListed?: number;
|
|
8
8
|
regionIds?: Record<number, number | null>;
|
|
9
|
-
browser?:
|
|
9
|
+
browser?: string;
|
|
10
10
|
zillowCookie?: string;
|
|
11
11
|
redfinCookie?: string;
|
|
12
12
|
}
|
package/dist/config.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import{checkForFile as P,parseYamlFile as _,throwError as m,writeYamlFile as b}from"@rent-scraper/utils";import Z from"axios";import
|
|
1
|
+
import{checkForFile as P,parseYamlFile as _,throwError as m,writeYamlFile as b}from"@rent-scraper/utils";import Z from"axios";import R from"fs";import h from"path";import D from"util";import{spinner as J,log as X}from"@clack/prompts";import{stat as ee,mkdir as S}from"fs/promises";import te from"os";import re from"env-paths";var y={},w={},k={},T;function ne(){return T||(T=1,(function(e){Object.defineProperty(e,"__esModule",{value:!0}),e.USEFUL_NON_ROOT_PNPM_FIELDS=e.FULL_FILTERED_META_DIR=e.FULL_META_DIR=e.ABBREVIATED_META_DIR=e.WORKSPACE_MANIFEST_FILENAME=e.STORE_VERSION=e.LAYOUT_VERSION=e.ENGINE_NAME=e.MANIFEST_BASE_NAMES=e.LOCKFILE_VERSION=e.LOCKFILE_MAJOR_VERSION=e.WANTED_LOCKFILE=void 0,e.getNodeBinLocationForCurrentOS=t,e.getDenoBinLocationForCurrentOS=r,e.getBunBinLocationForCurrentOS=n,e.WANTED_LOCKFILE="pnpm-lock.yaml",e.LOCKFILE_MAJOR_VERSION="9",e.LOCKFILE_VERSION=`${e.LOCKFILE_MAJOR_VERSION}.0`,e.MANIFEST_BASE_NAMES=["package.json","package.json5","package.yaml"],e.ENGINE_NAME=`${process.platform};${process.arch};node${process.version.split(".")[0].substring(1)}`,e.LAYOUT_VERSION=5,e.STORE_VERSION="v10",e.WORKSPACE_MANIFEST_FILENAME="pnpm-workspace.yaml",e.ABBREVIATED_META_DIR="metadata-v1.3",e.FULL_META_DIR="metadata-full-v1.3",e.FULL_FILTERED_META_DIR="metadata-v1.3",e.USEFUL_NON_ROOT_PNPM_FIELDS=["executionEnv"];function t(c=process.platform){return c==="win32"?"node.exe":"bin/node"}function r(c=process.platform){return c==="win32"?"deno.exe":"deno"}function n(c=process.platform){return c==="win32"?"bun.exe":"bun"}})(k)),k}var $;function ie(){if($)return w;$=1,Object.defineProperty(w,"__esModule",{value:!0}),w.LockfileMissingDependencyError=w.FetchError=w.PnpmError=void 0;const e=ne();class t extends Error{code;hint;attempts;prefix;pkgsStack;constructor(i,a,f){super(a),this.code=i.startsWith("ERR_PNPM_")?i:`ERR_PNPM_${i}`,this.hint=f?.hint,this.attempts=f?.attempts}}w.PnpmError=t;class r extends t{response;request;constructor(i,a,f){const l={url:i.url};i.authHeaderValue&&(l.authHeaderValue=n(i.authHeaderValue));const s=`GET ${i.url}: ${a.statusText} - ${a.status}`;(a.status===401||a.status===403||a.status===404)&&(f=f?`${f}
|
|
2
2
|
|
|
3
|
-
`:"",l.authHeaderValue?f+=`An authorization header was used: ${l.authHeaderValue}`:f+="No authorization header was set for the request."),super(`FETCH_${
|
|
4
|
-
To fix the lockfile, run 'pnpm install --no-frozen-lockfile'.`})}}return
|
|
3
|
+
`:"",l.authHeaderValue?f+=`An authorization header was used: ${l.authHeaderValue}`:f+="No authorization header was set for the request."),super(`FETCH_${a.status}`,s,{hint:f}),this.request=l,this.response=a}}w.FetchError=r;function n(u){const[i,a]=u.split(" ");return a==null?"[hidden]":a.length<20?`${i} [hidden]`:`${i} ${a.substring(0,4)}[hidden]`}class c extends t{constructor(i){const a=`Broken lockfile: no entry for '${i}' in ${e.WANTED_LOCKFILE}`;super("LOCKFILE_MISSING_DEPENDENCY",a,{hint:`This issue is probably caused by a badly resolved merge conflict.
|
|
4
|
+
To fix the lockfile, run 'pnpm install --no-frozen-lockfile'.`})}}return w.LockfileMissingDependencyError=c,w}var N={exports:{}},g={exports:{}},A,q;function ae(){if(q)return A;q=1;class e{constructor(n){this.value=n,this.next=void 0}}class t{constructor(){this.clear()}enqueue(n){const c=new e(n);this._head?(this._tail.next=c,this._tail=c):(this._head=c,this._tail=c),this._size++}dequeue(){const n=this._head;if(n)return this._head=this._head.next,this._size--,n.value}clear(){this._head=void 0,this._tail=void 0,this._size=0}get size(){return this._size}*[Symbol.iterator](){let n=this._head;for(;n;)yield n.value,n=n.next}}return A=t,A}var O,z;function se(){if(z)return O;z=1;const e=ae();return O=r=>{if(!((Number.isInteger(r)||r===1/0)&&r>0))throw new TypeError("Expected `concurrency` to be a number from 1 and up");const n=new e;let c=0;const u=()=>{c--,n.size>0&&n.dequeue()()},i=async(l,s,...o)=>{c++;const d=(async()=>l(...o))();s(d);try{await d}catch{}u()},a=(l,s,...o)=>{n.enqueue(i.bind(null,l,s,...o)),(async()=>(await Promise.resolve(),c<r&&n.size>0&&n.dequeue()()))()},f=(l,...s)=>new Promise(o=>{a(l,o,...s)});return Object.defineProperties(f,{activeCount:{get:()=>c},pendingCount:{get:()=>n.size},clearQueue:{value:()=>{n.clear()}}}),f},O}var M,V;function oe(){if(V)return M;V=1;const e=se();class t extends Error{constructor(i){super(),this.value=i}}const r=async(u,i)=>i(await u),n=async u=>{const i=await Promise.all(u);if(i[1]===!0)throw new t(i[0]);return!1};return M=async(u,i,a)=>{a={concurrency:1/0,preserveOrder:!0,...a};const f=e(a.concurrency),l=[...u].map(o=>[o,f(r,o,i)]),s=e(a.preserveOrder?1:1/0);try{await Promise.all(l.map(o=>s(n,o)))}catch(o){if(o instanceof t)return o.value;throw o}},M}var B;function ce(){if(B)return g.exports;B=1;const e=h,t=R,{promisify:r}=D,n=oe(),c=r(t.stat),u=r(t.lstat),i={directory:"isDirectory",file:"isFile"};function a({type:l}){if(!(l in i))throw new Error(`Invalid type specified: ${l}`)}const f=(l,s)=>l===void 0||s[i[l]]();return g.exports=async(l,s)=>{s={cwd:process.cwd(),type:"file",allowSymlinks:!0,...s},a(s);const o=s.allowSymlinks?c:u;return n(l,async d=>{try{const E=await o(e.resolve(s.cwd,d));return f(s.type,E)}catch{return!1}},s)},g.exports.sync=(l,s)=>{s={cwd:process.cwd(),allowSymlinks:!0,type:"file",...s},a(s);const o=s.allowSymlinks?t.statSync:t.lstatSync;for(const d of l)try{const E=o(e.resolve(s.cwd,d));if(f(s.type,E))return d}catch{}},g.exports}var I={exports:{}},x;function ue(){if(x)return I.exports;x=1;const e=R,{promisify:t}=D,r=t(e.access);return I.exports=async n=>{try{return await r(n),!0}catch{return!1}},I.exports.sync=n=>{try{return e.accessSync(n),!0}catch{return!1}},I.exports}var K;function le(){return K||(K=1,(function(e){const t=h,r=ce(),n=ue(),c=Symbol("findUp.stop");e.exports=async(u,i={})=>{let a=t.resolve(i.cwd||"");const{root:f}=t.parse(a),l=[].concat(u),s=async o=>{if(typeof u!="function")return r(l,o);const d=await u(o.cwd);return typeof d=="string"?r([d],o):d};for(;;){const o=await s({...i,cwd:a});if(o===c)return;if(o)return t.resolve(a,o);if(a===f)return;a=t.dirname(a)}},e.exports.sync=(u,i={})=>{let a=t.resolve(i.cwd||"");const{root:f}=t.parse(a),l=[].concat(u),s=o=>{if(typeof u!="function")return r.sync(l,o);const d=u(o.cwd);return typeof d=="string"?r.sync([d],o):d};for(;;){const o=s({...i,cwd:a});if(o===c)return;if(o)return t.resolve(a,o);if(a===f)return;a=t.dirname(a)}},e.exports.exists=n,e.exports.sync.exists=n.sync,e.exports.stop=c})(N)),N.exports}var j;function fe(){if(j)return y;j=1;var e=y&&y.__importDefault||function(s){return s&&s.__esModule?s:{default:s}};Object.defineProperty(y,"__esModule",{value:!0}),y.findWorkspaceDir=f;const t=e(R),r=e(h),n=ie(),c=e(le()),u="NPM_CONFIG_WORKSPACE_DIR",i="pnpm-workspace.yaml",a=["pnpm-workspaces.yaml","pnpm-workspaces.yml","pnpm-workspace.yml"];async function f(s){const o=process.env[u]??process.env[u.toLowerCase()],d=o?r.default.join(o,i):await(0,c.default)([i,...a],{cwd:await l(s)});if(d&&r.default.basename(d)!==i)throw new n.PnpmError("BAD_WORKSPACE_MANIFEST_NAME",`The workspace manifest file should be named "pnpm-workspace.yaml". File found: ${d}`);return d&&r.default.dirname(d)}async function l(s){return new Promise(o=>{t.default.realpath.native(s,function(d,E){o(d!==null?s:E)})})}return y}var de=fe();const he=re("rent-scraper").config,p=h.join(he,"config.yaml"),we=h.join(te.homedir(),"rent-scraper"),F=async e=>{const t=await de.findWorkspaceDir(process.cwd());if(t)return e==="redfin"?h.join(t,"config.redfin.yaml"):h.join(t,"config.zillow.yaml");if(await W())return await G(e)},U=async(e,t)=>{const r=t??await F(e);if(r)return await P(r)},W=async()=>await P(p),pe=async e=>{const t=await F(e);if(t)return await P(t)||m("Config file is required."),await _(t)},ye=async()=>{const e=await v("zillow");if(e){const{zillowCookie:t,...r}=e??{};await C("zillow",r)}},Ee=async()=>{const e=await v("redfin");if(e){const{redfinCookie:t,...r}=e??{};await C("redfin",r)}},_e=async e=>{try{return await new Promise(t=>{const r=setInterval(async()=>{const n=await U(e);n&&(t(n),clearInterval(r))},1e3)})}catch(t){m(t?.message)}},me=async()=>{try{return await new Promise((e,t)=>{const r=J();r.start("Waiting for browser server");let n=0;const c=setInterval(async()=>{n++,n>10&&t(new Error("browser server is not running."));const u=await Y();u&&(r.stop("Browser server is connected!"),e(u),clearInterval(c))},1e3)})}catch(e){m(e?.message)}},Y=async()=>{try{const{data:e}=await Z.get("http://localhost:8082/server");return e.running}catch{return!1}},Fe=async()=>await new Promise(e=>{const t=setInterval(async()=>{const r=await H();r&&(e(r),clearInterval(t))},1e3)}),H=async()=>await L("zillow","zillowCookie"),ve=async()=>await new Promise(e=>{const t=setInterval(async()=>{const r=await Q();r&&(e(r),clearInterval(t))},1e3)}),Q=async()=>await L("redfin","redfinCookie"),Le=(e,t,r="init")=>{const{outputPath:n,zipCodes:c,zillowCookie:u}=t??{},i=[];return n||i.push("outputPath"),c||i.push("zipCodes"),r==="scrape"&&e==="zillow"&&!u&&i.push("zillowCookie"),i},ge=async e=>(await L(e,"zipCodes"))?.replace(/ /g,"").split(",").map(t=>Number(t)),Ie=e=>e.join(", "),v=async e=>{const t=await F(e);if(t)return await _(t)},Ce=async(e,t,r)=>{h.isAbsolute(t)||m("configDirectory must be an absolute path");const n=await ee(t).catch(()=>null);n&&!n.isDirectory()&&m(`${t} exists but is not a directory`),await S(t,{recursive:!0});const c=h.dirname(p);await S(c,{recursive:!0}),r=r??`config.${e}.yaml`;const u=h.join(t,r),i={...await _(p),[e]:u};return await b(p,i),(await _(p)??{})?.[e]},G=async e=>{try{return(await _(p))?.[e]}catch(t){console.error(`Failed to read pointer file at ${p}: ${t.message}`)}},Pe=async e=>await L(e,"outputPath"),L=async(e,t)=>(await v(e))?.[t]??null,Re=async(e,t)=>{const r=await v(e),n=Object.keys(t),c={...r,...t};await C(e,c),X.success(`Updated ${e} config: ${n.join(", ")}`)},C=async(e,t)=>{const r=await F(e);return await S(h.dirname(r),{recursive:!0}),await b(r,t),r};export{Y as checkBrowserServer,pe as checkForAndReadConfigFile,U as checkForConfigFile,W as checkForPointerFile,Q as checkForRedfinCookie,H as checkForZillowCookie,Le as checkRequiredConfigValues,F as getConfigFilePath,Pe as getOutputPathFromConfig,L as getValueFromConfigFile,ge as getZipCodesFromConfig,we as globalDir,p as pointerFilePath,v as readConfigFile,G as readPointerFile,Ee as resetRedfinCookie,ye as resetZillowCookie,Ie as stringifyZipCodes,Re as updateConfigFile,me as waitForBrowserServer,_e as waitForConfigFile,ve as waitForRedfinCookie,Fe as waitForZillowCookie,C as writeConfigFile,Ce as writePointerFile};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rent-scraper/utils",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.20",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "./dist/index.mjs",
|
|
6
6
|
"module": "./dist/index.mjs",
|
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
"csv": "^6.3.11",
|
|
33
33
|
"env-paths": "^3.0.0",
|
|
34
34
|
"yaml": "^2.8.0",
|
|
35
|
-
"@rent-scraper/api": "1.0.
|
|
35
|
+
"@rent-scraper/api": "1.0.20"
|
|
36
36
|
},
|
|
37
37
|
"devDependencies": {
|
|
38
38
|
"unbuild": "^3.5.0"
|