@rent-scraper/utils 1.0.13 → 1.0.14-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config.d.mts +2 -1
- package/dist/config.mjs +2 -2
- package/package.json +2 -2
package/dist/config.d.mts
CHANGED
|
@@ -9,6 +9,7 @@ interface ScrapeConfig {
|
|
|
9
9
|
browser?: BrowserKey;
|
|
10
10
|
zillowCookie?: string;
|
|
11
11
|
}
|
|
12
|
+
declare const globalDir: string;
|
|
12
13
|
declare const getConfigFilePath: (source: ListingsSource) => Promise<string>;
|
|
13
14
|
declare const checkForConfigFile: (source: ListingsSource) => Promise<boolean | undefined>;
|
|
14
15
|
declare const checkForAndReadConfigFile: (source: ListingsSource) => Promise<ScrapeConfig>;
|
|
@@ -27,5 +28,5 @@ declare const getValueFromConfigFile: (source: ListingsSource, key: keyof Scrape
|
|
|
27
28
|
declare const updateConfigFile: (source: ListingsSource, payload: any) => Promise<void>;
|
|
28
29
|
declare const writeConfigFile: (source: ListingsSource, config: ScrapeConfig) => Promise<void>;
|
|
29
30
|
|
|
30
|
-
export { checkBrowserServer, checkForAndReadConfigFile, checkForConfigFile, checkForZillowCookie, checkRequiredConfigValues, getConfigFilePath, getOutputPathFromConfig, getValueFromConfigFile, getZipCodesFromConfig, readConfigFile, resetZillowCookie, stringifyZipCodes, updateConfigFile, waitForBrowserServer, waitForConfigFile, waitForZillowCookie, writeConfigFile };
|
|
31
|
+
export { checkBrowserServer, checkForAndReadConfigFile, checkForConfigFile, checkForZillowCookie, checkRequiredConfigValues, getConfigFilePath, getOutputPathFromConfig, getValueFromConfigFile, getZipCodesFromConfig, globalDir, readConfigFile, resetZillowCookie, stringifyZipCodes, updateConfigFile, waitForBrowserServer, waitForConfigFile, waitForZillowCookie, writeConfigFile };
|
|
31
32
|
export type { ScrapeConfig };
|
package/dist/config.mjs
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import{checkForFile as
|
|
1
|
+
import{checkForFile as I,parseYamlFile as k,throwError as g,writeYamlFile as M}from"@rent-scraper/utils";import S from"fs";import p from"path";import b from"util";import H from"axios";import{spinner as Q,log as G}from"@clack/prompts";import{mkdir as Z}from"fs/promises";import J from"os";var w={},h={},C={},T;function X(){return T||(T=1,(function(e){Object.defineProperty(e,"__esModule",{value:!0}),e.USEFUL_NON_ROOT_PNPM_FIELDS=e.FULL_FILTERED_META_DIR=e.FULL_META_DIR=e.ABBREVIATED_META_DIR=e.WORKSPACE_MANIFEST_FILENAME=e.STORE_VERSION=e.LAYOUT_VERSION=e.ENGINE_NAME=e.MANIFEST_BASE_NAMES=e.LOCKFILE_VERSION=e.LOCKFILE_MAJOR_VERSION=e.WANTED_LOCKFILE=void 0,e.getNodeBinLocationForCurrentOS=t,e.getDenoBinLocationForCurrentOS=i,e.getBunBinLocationForCurrentOS=n,e.WANTED_LOCKFILE="pnpm-lock.yaml",e.LOCKFILE_MAJOR_VERSION="9",e.LOCKFILE_VERSION=`${e.LOCKFILE_MAJOR_VERSION}.0`,e.MANIFEST_BASE_NAMES=["package.json","package.json5","package.yaml"],e.ENGINE_NAME=`${process.platform};${process.arch};node${process.version.split(".")[0].substring(1)}`,e.LAYOUT_VERSION=5,e.STORE_VERSION="v10",e.WORKSPACE_MANIFEST_FILENAME="pnpm-workspace.yaml",e.ABBREVIATED_META_DIR="metadata-v1.3",e.FULL_META_DIR="metadata-full-v1.3",e.FULL_FILTERED_META_DIR="metadata-v1.3",e.USEFUL_NON_ROOT_PNPM_FIELDS=["executionEnv"];function t(c=process.platform){return c==="win32"?"node.exe":"bin/node"}function i(c=process.platform){return c==="win32"?"deno.exe":"deno"}function n(c=process.platform){return c==="win32"?"bun.exe":"bun"}})(C)),C}var D;function ee(){if(D)return h;D=1,Object.defineProperty(h,"__esModule",{value:!0}),h.LockfileMissingDependencyError=h.FetchError=h.PnpmError=void 0;const e=X();class t extends Error{code;hint;attempts;prefix;pkgsStack;constructor(s,r,f){super(r),this.code=s.startsWith("ERR_PNPM_")?s:`ERR_PNPM_${s}`,this.hint=f?.hint,this.attempts=f?.attempts}}h.PnpmError=t;class i extends t{response;request;constructor(s,r,f){const l={url:s.url};s.authHeaderValue&&(l.authHeaderValue=n(s.authHeaderValue));const a=`GET ${s.url}: ${r.statusText} - ${r.status}`;(r.status===401||r.status===403||r.status===404)&&(f=f?`${f}
|
|
2
2
|
|
|
3
3
|
`:"",l.authHeaderValue?f+=`An authorization header was used: ${l.authHeaderValue}`:f+="No authorization header was set for the request."),super(`FETCH_${r.status}`,a,{hint:f}),this.request=l,this.response=r}}h.FetchError=i;function n(u){const[s,r]=u.split(" ");return r==null?"[hidden]":r.length<20?`${s} [hidden]`:`${s} ${r.substring(0,4)}[hidden]`}class c extends t{constructor(s){const r=`Broken lockfile: no entry for '${s}' in ${e.WANTED_LOCKFILE}`;super("LOCKFILE_MISSING_DEPENDENCY",r,{hint:`This issue is probably caused by a badly resolved merge conflict.
|
|
4
|
-
To fix the lockfile, run 'pnpm install --no-frozen-lockfile'.`})}}return h.LockfileMissingDependencyError=c,h}var
|
|
4
|
+
To fix the lockfile, run 'pnpm install --no-frozen-lockfile'.`})}}return h.LockfileMissingDependencyError=c,h}var N={exports:{}},y={exports:{}},P,q;function te(){if(q)return P;q=1;class e{constructor(n){this.value=n,this.next=void 0}}class t{constructor(){this.clear()}enqueue(n){const c=new e(n);this._head?(this._tail.next=c,this._tail=c):(this._head=c,this._tail=c),this._size++}dequeue(){const n=this._head;if(n)return this._head=this._head.next,this._size--,n.value}clear(){this._head=void 0,this._tail=void 0,this._size=0}get size(){return this._size}*[Symbol.iterator](){let n=this._head;for(;n;)yield n.value,n=n.next}}return P=t,P}var R,$;function re(){if($)return R;$=1;const e=te();return R=i=>{if(!((Number.isInteger(i)||i===1/0)&&i>0))throw new TypeError("Expected `concurrency` to be a number from 1 and up");const n=new e;let c=0;const u=()=>{c--,n.size>0&&n.dequeue()()},s=async(l,a,...o)=>{c++;const d=(async()=>l(...o))();a(d);try{await d}catch{}u()},r=(l,a,...o)=>{n.enqueue(s.bind(null,l,a,...o)),(async()=>(await Promise.resolve(),c<i&&n.size>0&&n.dequeue()()))()},f=(l,...a)=>new Promise(o=>{r(l,o,...a)});return Object.defineProperties(f,{activeCount:{get:()=>c},pendingCount:{get:()=>n.size},clearQueue:{value:()=>{n.clear()}}}),f},R}var A,z;function ne(){if(z)return A;z=1;const e=re();class t extends Error{constructor(s){super(),this.value=s}}const i=async(u,s)=>s(await u),n=async u=>{const s=await Promise.all(u);if(s[1]===!0)throw new t(s[0]);return!1};return A=async(u,s,r)=>{r={concurrency:1/0,preserveOrder:!0,...r};const f=e(r.concurrency),l=[...u].map(o=>[o,f(i,o,s)]),a=e(r.preserveOrder?1:1/0);try{await Promise.all(l.map(o=>a(n,o)))}catch(o){if(o instanceof t)return o.value;throw o}},A}var V;function ie(){if(V)return y.exports;V=1;const e=p,t=S,{promisify:i}=b,n=ne(),c=i(t.stat),u=i(t.lstat),s={directory:"isDirectory",file:"isFile"};function r({type:l}){if(!(l in s))throw new Error(`Invalid type specified: ${l}`)}const f=(l,a)=>l===void 0||a[s[l]]();return y.exports=async(l,a)=>{a={cwd:process.cwd(),type:"file",allowSymlinks:!0,...a},r(a);const o=a.allowSymlinks?c:u;return n(l,async d=>{try{const E=await o(e.resolve(a.cwd,d));return f(a.type,E)}catch{return!1}},a)},y.exports.sync=(l,a)=>{a={cwd:process.cwd(),allowSymlinks:!0,type:"file",...a},r(a);const o=a.allowSymlinks?t.statSync:t.lstatSync;for(const d of l)try{const E=o(e.resolve(a.cwd,d));if(f(a.type,E))return d}catch{}},y.exports}var m={exports:{}},B;function ae(){if(B)return m.exports;B=1;const e=S,{promisify:t}=b,i=t(e.access);return m.exports=async n=>{try{return await i(n),!0}catch{return!1}},m.exports.sync=n=>{try{return e.accessSync(n),!0}catch{return!1}},m.exports}var K;function se(){return K||(K=1,(function(e){const t=p,i=ie(),n=ae(),c=Symbol("findUp.stop");e.exports=async(u,s={})=>{let r=t.resolve(s.cwd||"");const{root:f}=t.parse(r),l=[].concat(u),a=async o=>{if(typeof u!="function")return i(l,o);const d=await u(o.cwd);return typeof d=="string"?i([d],o):d};for(;;){const o=await a({...s,cwd:r});if(o===c)return;if(o)return t.resolve(r,o);if(r===f)return;r=t.dirname(r)}},e.exports.sync=(u,s={})=>{let r=t.resolve(s.cwd||"");const{root:f}=t.parse(r),l=[].concat(u),a=o=>{if(typeof u!="function")return i.sync(l,o);const d=u(o.cwd);return typeof d=="string"?i.sync([d],o):d};for(;;){const o=a({...s,cwd:r});if(o===c)return;if(o)return t.resolve(r,o);if(r===f)return;r=t.dirname(r)}},e.exports.exists=n,e.exports.sync.exists=n.sync,e.exports.stop=c})(N)),N.exports}var W;function oe(){if(W)return w;W=1;var e=w&&w.__importDefault||function(a){return a&&a.__esModule?a:{default:a}};Object.defineProperty(w,"__esModule",{value:!0}),w.findWorkspaceDir=f;const t=e(S),i=e(p),n=ee(),c=e(se()),u="NPM_CONFIG_WORKSPACE_DIR",s="pnpm-workspace.yaml",r=["pnpm-workspaces.yaml","pnpm-workspaces.yml","pnpm-workspace.yml"];async function f(a){const o=process.env[u]??process.env[u.toLowerCase()],d=o?i.default.join(o,s):await(0,c.default)([s,...r],{cwd:await l(a)});if(d&&i.default.basename(d)!==s)throw new n.PnpmError("BAD_WORKSPACE_MANIFEST_NAME",`The workspace manifest file should be named "pnpm-workspace.yaml". File found: ${d}`);return d&&i.default.dirname(d)}async function l(a){return new Promise(o=>{t.default.realpath.native(a,function(d,E){o(d!==null?a:E)})})}return w}var j=oe();const _=p.join(J.homedir(),"rent-scraper"),L=async e=>{const t=await j.findWorkspaceDir(process.cwd());t||await I(_)||await Z(_,{recursive:!0});const i=t??_;return e==="redfin"?p.join(i,"config.redfin.yaml"):p.join(i,"config.zillow.yaml")},x=async e=>{const t=await L(e);return await I(t)},ce=async e=>{const t=await L(e);return await I(t)||g("Config file is required."),await k(t)},ue=async()=>{const{zillowCookie:e,...t}=await v("zillow");await O("zillow",t)},le=async e=>{try{return await new Promise(t=>{const i=setInterval(async()=>{const n=await x(e);n&&(t(n),clearInterval(i))},1e3)})}catch(t){g(t?.message)}},fe=async()=>{try{return await new Promise((e,t)=>{const i=Q();i.start("Waiting for browser server");let n=0;const c=setInterval(async()=>{n++,n>10&&t(new Error("browser server is not running."));const u=await U();u&&(i.stop("Browser server is connected!"),e(u),clearInterval(c))},1e3)})}catch(e){g(e?.message)}},U=async()=>{try{const{data:e}=await H.get("http://localhost:8082/server");return e.running}catch{return!1}},de=async()=>await new Promise(e=>{const t=setInterval(async()=>{const i=await Y();i&&(e(i),clearInterval(t))},1e3)}),Y=async()=>await F("zillow","zillowCookie"),he=(e,t,i="init")=>{const{outputPath:n,zipCodes:c,browser:u,zillowCookie:s}=t??{},r=[];return n||r.push("outputPath"),c||r.push("zipCodes"),e==="zillow"&&!u&&r.push("browser"),i==="scrape"&&e==="zillow"&&!s&&r.push("zillowCookie"),r},pe=async e=>(await F(e,"zipCodes"))?.replace(/ /g,"").split(",").map(t=>Number(t)),we=e=>e.join(", "),v=async e=>{const t=await L(e);return await k(t)},Ee=async e=>await F(e,"outputPath"),F=async(e,t)=>(await v(e))?.[t]??null,_e=async(e,t)=>{const i=await v(e),n=Object.keys(t),c={...i,...t};await O(e,c),G.success(`Updated ${e} config: ${n.join(", ")}`)},O=async(e,t)=>{const i=await j.findWorkspaceDir(process.cwd())??_;return e==="redfin"?await M(p.join(i,"config.redfin.yaml"),t):await M(p.join(i,"config.zillow.yaml"),t)};export{U as checkBrowserServer,ce as checkForAndReadConfigFile,x as checkForConfigFile,Y as checkForZillowCookie,he as checkRequiredConfigValues,L as getConfigFilePath,Ee as getOutputPathFromConfig,F as getValueFromConfigFile,pe as getZipCodesFromConfig,_ as globalDir,v as readConfigFile,ue as resetZillowCookie,we as stringifyZipCodes,_e as updateConfigFile,fe as waitForBrowserServer,le as waitForConfigFile,de as waitForZillowCookie,O as writeConfigFile};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rent-scraper/utils",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.14-beta.1",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "./dist/index.mjs",
|
|
6
6
|
"module": "./dist/index.mjs",
|
|
@@ -31,7 +31,7 @@
|
|
|
31
31
|
"bumpp": "^10.2.3",
|
|
32
32
|
"csv": "^6.3.11",
|
|
33
33
|
"yaml": "^2.8.0",
|
|
34
|
-
"@rent-scraper/api": "1.0.
|
|
34
|
+
"@rent-scraper/api": "1.0.14-beta.1"
|
|
35
35
|
},
|
|
36
36
|
"devDependencies": {
|
|
37
37
|
"unbuild": "^3.5.0"
|