@rent-scraper/create-config 1.0.14-beta.1 → 1.0.15-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/run-check-config.mjs +1 -1
- package/dist/bin/run-create-config.mjs +1 -1
- package/dist/bin/run-generate-region-ids.mjs +1 -1
- package/dist/bin/run-init-config.mjs +1 -1
- package/dist/index.mjs +1 -1
- package/dist/shared/{create-config.hQGNlBni.mjs → create-config.DS6TtXI8.mjs} +1 -1
- package/dist/shared/create-config.DeJWWWT0.mjs +1 -0
- package/dist/shared/create-config.R8v2gKOZ.mjs +1 -0
- package/dist/shared/create-config.hUS3xkVg.mjs +1 -0
- package/package.json +4 -4
- package/dist/shared/create-config.B5xWKlnW.mjs +0 -1
- package/dist/shared/create-config.C7pSTZYd.mjs +0 -1
- package/dist/shared/create-config.CtEJpaDX.mjs +0 -1
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import s from"minimist";import{parseError as e}from"@rent-scraper/utils";import{log as t}from"@clack/prompts";import{r as i}from"../shared/create-config.
|
|
2
|
+
import s from"minimist";import{parseError as e}from"@rent-scraper/utils";import{log as t}from"@clack/prompts";import{r as i}from"../shared/create-config.DeJWWWT0.mjs";import"@rent-scraper/utils/config";import"@rent-scraper/browser-server";const m=s(process.argv.slice(2)),c=m.source??"zillow";i(c).then(()=>{process.exit(0)}).catch(r=>{const{message:o}=e(r);t.error(o),process.exit(1)});
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import t from"minimist";import{log as i}from"@clack/prompts";import{parseError as m}from"@rent-scraper/utils";import{r as e}from"../shared/create-config.
|
|
2
|
+
import t from"minimist";import{log as i}from"@clack/prompts";import{parseError as m}from"@rent-scraper/utils";import{r as e}from"../shared/create-config.R8v2gKOZ.mjs";import"../shared/create-config.hUS3xkVg.mjs";import"@pnpm/find-workspace-dir";import"@rent-scraper/utils/config";import"node:path";import"node:timers/promises";import"picocolors";import"node:fs/promises";const s=t(process.argv.slice(2)),p=s.source??"zillow";e(p).then(()=>process.exit(0)).catch(r=>{const{message:o}=m(r);i.error(o),process.exit(1)});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import r from"minimist";import{r as o}from"../shared/create-config.
|
|
1
|
+
import r from"minimist";import{r as o}from"../shared/create-config.DS6TtXI8.mjs";import"axios";import"@rent-scraper/api";import"@rent-scraper/utils";import"@rent-scraper/utils/config";import"@clack/prompts";import"node:timers/promises";const t=r(process.argv.slice(2)),i=t.source??"zillow";o(i).then(()=>{process.exit(0)}).catch(()=>{process.exit(1)});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{r}from"../shared/create-config.
|
|
1
|
+
import{r}from"../shared/create-config.hUS3xkVg.mjs";import"@clack/prompts";import"@pnpm/find-workspace-dir";import"@rent-scraper/utils";import"@rent-scraper/utils/config";import"minimist";import"node:path";import"node:timers/promises";import"picocolors";r().catch(console.error);
|
package/dist/index.mjs
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export{r as runCreateConfig}from"./shared/create-config.
|
|
1
|
+
export{r as runCreateConfig}from"./shared/create-config.R8v2gKOZ.mjs";export{r as runInitConfig}from"./shared/create-config.hUS3xkVg.mjs";export{a as generateRedfinRegionIds,g as generateZillowRegionIds,r as runGenerateRegionIds}from"./shared/create-config.DS6TtXI8.mjs";export{r as runCheckConfig}from"./shared/create-config.DeJWWWT0.mjs";import"@rent-scraper/utils/config";import"@clack/prompts";import"node:timers/promises";import"node:fs/promises";import"@rent-scraper/utils";import"@pnpm/find-workspace-dir";import"minimist";import"node:path";import"picocolors";import"axios";import"@rent-scraper/api";import"@rent-scraper/browser-server";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import p,{isCancel as u}from"axios";import{checkForZillowBotFiltering as c,waitForSolvedZillowCaptcha as y,getRedfinRegionIdByZipCode as C,getZillowRegionIdByZipCode as h}from"@rent-scraper/api";import{parseError as n,throwError as d}from"@rent-scraper/utils";import{checkForAndReadConfigFile as R,getZipCodesFromConfig as
|
|
1
|
+
import p,{isCancel as u}from"axios";import{checkForZillowBotFiltering as c,waitForSolvedZillowCaptcha as y,getRedfinRegionIdByZipCode as C,getZillowRegionIdByZipCode as h}from"@rent-scraper/api";import{parseError as n,throwError as d}from"@rent-scraper/utils";import{checkForAndReadConfigFile as R,getZipCodesFromConfig as l,stringifyZipCodes as f,updateConfigFile as g}from"@rent-scraper/utils/config";import{log as r,spinner as I,confirm as F,cancel as Z,outro as b}from"@clack/prompts";import{setTimeout as s}from"node:timers/promises";const v=async()=>{await p.post("http://localhost:8082/browser/close")},z=async e=>{try{if((await R(e))?.regionIds){r.success("Region Ids have been found in config...");return}r.step("Region Ids were not found in your Config file");const o=I();if(e==="redfin")await m();else{try{await c()}catch(t){const{message:i}=n(t);r.error(i);const a=await F({message:"You need to complete a captcha in your browser. Press Return to launch your browser and continue.",active:"OK",inactive:"Cancel"});if(u(a)||!a)return Z("Create config canceled. Please try again."),process.exit(1);await s(1e3),b("Browser Launching..."),await s(1e3),await y()}await v(),o.start("Generating Region Ids"),await w(),await s(1e3),o.stop("Region Ids Generated and added to Config file"),await s(1e3)}}catch(o){const{message:t}=n(o);r.error(t)}};async function w(){try{const e=await l("zillow");e||d("Zip codes must be added to config file."),await c();const o=Object.fromEntries((await Promise.all(e.map(async i=>{const a=await h(i,{fromFile:!1});return[i,a]}))).filter(i=>i)),t={zipCodes:f(e),regionIds:o};await g("zillow",t)}catch(e){const{status:o,message:t}=n(e);r.error(`${o}, ${t}`)}}async function m(){try{const e=await l("redfin");e||d("Zip codes must be added to config file.");const o=Object.fromEntries((await Promise.all(e.map(async i=>{const a=await C(i,{fromFile:!1});return[i,a]}))).filter(i=>i)),t={zipCodes:f(e),regionIds:o};await g("redfin",t)}catch(e){const{status:o,message:t}=n(e);console.error(o,t)}}export{m as a,w as g,z as r};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{resetZillowCookie as t,waitForBrowserServer as f,checkForZillowCookie as c,waitForZillowCookie as l,checkForAndReadConfigFile as w,checkRequiredConfigValues as s}from"@rent-scraper/utils/config";import{log as e}from"@clack/prompts";import{runConfirmBrowserLaunch as m,runBrowserServer as g}from"@rent-scraper/browser-server";import{throwError as n}from"@rent-scraper/utils";async function u(o){o==="zillow"&&(await m(),g(),await t(),e.info("Waiting for Server"),await f(),await c()||(e.info("Waiting for Zillow cookie"),await l()));const r=await w(o);r||n("config required");const i=s(o,r,"scrape");if(i.length){const a=`Required fields missing in config.${o}.yaml file: ${i.join(", ")}`;n(a)}}export{u as r};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{r}from"./create-config.hUS3xkVg.mjs";import{readConfigFile as s,writeConfigFile as d,checkForConfigFile as m,checkRequiredConfigValues as h}from"@rent-scraper/utils/config";import{log as u,select as c,isCancel as C,cancel as b,multiselect as v,outro as k}from"@clack/prompts";import{setTimeout as f}from"node:timers/promises";import{rename as p}from"node:fs/promises";import{throwError as y,parseAbsolutePath as o}from"@rent-scraper/utils";async function $(e){const i=await s(e);i||y("config required");const l=i.zipCodes.split(", ").length,w=[`Output path: ${o(i.outputPath)}`,...i.browser?[`Browser: ${i.browser}`]:[],`Zip codes: ${l>10?i.zipCodes.split(", ").splice(0,10).join(", ")+"...":i.zipCodes}`].filter(t=>t);u.warn(`Config file found at ${o("./config."+e+".yaml")}`);let a=await c({message:"Would you like to:",initialValue:"keep",options:[{value:"keep",label:"Keep this config"},{value:"edit",label:"Edit this config"},{value:"review",label:"Review this config"},{value:"new",label:"Create a new config"}]});if(C(a))return b("Operation cancelled"),process.exit(1);if(a==="review"||a==="edit")for(const t of w)await f(300),u.info(t);const n=a==="edit"&&await v({message:"Which fields would you like to edit?",options:[{value:"outputPath",label:"Output path"},{value:"browser",label:"Browser"},{value:"zipCodes",label:"Zip codes"}],required:!1});if(n&&Array.isArray(n)&&n.length){const t=await s(e);if(t){await f(1e3),await p(o(`./config.${e}.yaml`),o(`./config.${e}.yaml.bak`));for(const g of n)delete t[g];await f(1e3),await d(e,t),await r(e)}}if(a==="new"&&(await p(o(`./config.${e}.yaml`),o(`./config.${e}.yaml.bak`)),await r()),a=a==="review"?await c({message:"Would you like to:",initialValue:"keep",options:[{value:"keep",label:"Keep this config"},{value:"edit",label:"Edit this config"},{value:"new",label:"Create a new config"}]}):a,a==="keep")return k("Config file saved!"),process.exit(0)}async function F(e){if(e&&await m(e)){const i=await s(e),l=h(e,i);l.length?(console.log({missingFields:l}),await r(e)):await $(e)}else await r(e)}export{F as r};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import{intro as $,log as r,select as d,isCancel as t,cancel as l,text as c,confirm as O,tasks as V,outro as W}from"@clack/prompts";import{findWorkspaceDir as F}from"@pnpm/find-workspace-dir";import{parseAbsolutePath as j}from"@rent-scraper/utils";import{checkForPointerFile as L,readPointerFile as T,globalDir as z,writePointerFile as Z,readConfigFile as A,getConfigFilePath as B,writeConfigFile as D}from"@rent-scraper/utils/config";import G from"minimist";import P from"node:path";import{setTimeout as f}from"node:timers/promises";import R from"picocolors";const S=G(process.argv.slice(2)),g=S.source,x=i=>/(^\d{5}$)|(^\d{5}-\d{4}$)/.test(i);async function I(i){if($(R.inverse(" create config ")),g&&(i=g),!await F(process.cwd())&&(!await L()||!i||!await T(i))){if(g)r.info(`Using source ${i==="redfin"?"Redfin":"Zillow"}`);else if(i=await d({message:"Which listings would you like to scrape?",initialValue:"zillow",options:[{value:"zillow",label:"Zillow"},{value:"redfin",label:"Redfin"}]}),t(i))return l("Operation cancelled"),process.exit(1);const e=(await c({message:"Where would like you to save the config file?",placeholder:z,defaultValue:z}))?.trim();if(await Z(i,e),t(e))return l("Operation cancelled"),process.exit(1)}const u=i&&await A(i);if(i=i??await d({message:"Which listings would you like to scrape?",initialValue:"zillow",options:[{value:"zillow",label:"Zillow"},{value:"redfin",label:"Redfin"}]}),t(i))return l("Operation cancelled"),process.exit(1);const h=P.dirname(await B(i)),s=u?.outputPath??(await c({message:"Where would you like the data to be stored?",placeholder:h,defaultValue:h}))?.trim();if(t(s))return l("Operation cancelled"),process.exit(1);const n=u?.browser??(i==="zillow"?await d({message:"Which browser would you use for scraping?",initialValue:"chrome",options:[{value:"chrome",label:"Google Chrome"},{value:"brave",label:"Brave Browser"}]}):null);if(t(n))return l("Operation cancelled"),process.exit(1);let a=(u?.zipCodes??await c({message:"What zip codes would you like to scrape?",placeholder:"Example: 90026, 90039, 90027",validate(e){if(!e||e?.length===0)return"Please enter a list of zip codes"}})).replace(/ /g,"").split(",").filter(e=>e);const o=a.filter(e=>!x(e));if(a=a.filter(e=>!o.includes(e)),o.length){r.error("Invalid zips!");const e=await d({message:`The following zip codes are invalid: ${o.join(", ")}.`,initialValue:"remove",options:[{value:"remove",label:"Please remove them."},{value:"edit",label:"Let me edit them."}]});if(e==="remove"&&(a=a.filter(w=>!o.includes(w))),e==="edit"){const w=(await c({message:"Please fix the following zip codes:",placeholder:o.join(", "),initialValue:o.join(", "),validate(b){if(b?.length===0)return"Please enter a list of zip codes";const y=b?.replace(/ /g,"").split(",").filter(m=>m)?.filter(m=>!x(m));if(y?.length)return`The following zip codes are invalid: ${y.join(", ")}.`}})).replace(/ /g,"").split(",");a=[...a,...w]}}if(a=[...new Set(a)],o.length||r.success("Great! All your zips are valid!"),t(a))return l("Operation cancelled"),process.exit(1);const p=u?.daysListed??(await c({message:"How many days would you like to search listings? (Max. 90)",placeholder:"1",defaultValue:"1",validate(e){if(e&&e==="0")return"Please enter a number.";if(Number(e)>90)return"Please enter a number less than 90."}}))?.trim();if(t(p))return l("Operation cancelled"),process.exit(1);await f(1e3);const k=[`Listings source: ${i}`,`Output path: ${s}`,...n?[`Browser: ${n}`]:[],`Zip codes: ${a.join(", ")}`,`Days listed: ${p}`].filter(e=>e);r.info(""),r.info("Please review your answers:"),k.map(e=>{r.warn(e)}),r.info("");const v=await O({message:"Would you like to save this config?"});if(t(v)||!v)return l("Create config canceled. Please try again."),process.exit(1);const C={outputPath:P.isAbsolute(s)?s:j(s),...n&&{browser:n},zipCodes:a.join(", "),daysListed:Number(p)};return await V([{title:"Saving config to file",task:async()=>(await f(3e3),`Saved config to ${await D(i,C)}`)}]),await f(1e3),W("Config file saved!"),await f(1e3),i}export{I as r};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@rent-scraper/create-config",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.15-beta.1",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": {
|
|
@@ -46,9 +46,9 @@
|
|
|
46
46
|
"minimist": "^1.2.8",
|
|
47
47
|
"picocolors": "^1.1.1",
|
|
48
48
|
"puppeteer": "^24.11.2",
|
|
49
|
-
"@rent-scraper/api": "1.0.
|
|
50
|
-
"@rent-scraper/browser-server": "1.0.
|
|
51
|
-
"@rent-scraper/utils": "1.0.
|
|
49
|
+
"@rent-scraper/api": "1.0.15-beta.1",
|
|
50
|
+
"@rent-scraper/browser-server": "1.0.15-beta.1",
|
|
51
|
+
"@rent-scraper/utils": "1.0.15-beta.1"
|
|
52
52
|
},
|
|
53
53
|
"devDependencies": {
|
|
54
54
|
"@types/minimist": "^1.2.5",
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{intro as P,select as w,isCancel as o,cancel as l,text as u,log as r,confirm as C,tasks as $,outro as k}from"@clack/prompts";import{findWorkspaceDir as W}from"@pnpm/find-workspace-dir";import{parseAbsolutePath as j}from"@rent-scraper/utils";import{readConfigFile as O,globalDir as V,writeConfigFile as L}from"@rent-scraper/utils/config";import A from"node:path";import{setTimeout as d}from"node:timers/promises";import B from"picocolors";const y=a=>/(^\d{5}$)|(^\d{5}-\d{4}$)/.test(a);async function D(a){P(B.inverse(" create config "));const c=a&&await O(a);if(a=a??await w({message:"Which listings would you like to scrape?",initialValue:"zillow",options:[{value:"zillow",label:"Zillow"},{value:"redfin",label:"Redfin"}]}),o(a))return l("Operation cancelled"),process.exit(1);const g=await W(process.cwd())?"./rent-data":V,s=c?.outputPath??(await u({message:"Where would like you the data to be stored?",placeholder:g,defaultValue:g}))?.trim();if(o(s))return l("Operation cancelled"),process.exit(1);const n=c?.browser??(a==="zillow"?await w({message:"Which browser would you use for scraping?",initialValue:"chrome",options:[{value:"chrome",label:"Google Chrome"},{value:"brave",label:"Brave Browser"}]}):null);if(o(n))return l("Operation cancelled"),process.exit(1);let i=(c?.zipCodes??await u({message:"What zip codes would you like to scrape?",placeholder:"Example: 90026, 90039, 90027",validate(e){if(!e||e?.length===0)return"Please enter a list of zip codes"}})).replace(/ /g,"").split(",").filter(e=>e);const t=i.filter(e=>!y(e));if(i=i.filter(e=>!t.includes(e)),t.length){r.error("Invalid zips!");const e=await w({message:`The following zip codes are invalid: ${t.join(", ")}.`,initialValue:"remove",options:[{value:"remove",label:"Please remove them."},{value:"edit",label:"Let me edit them."}]});if(e==="remove"&&(i=i.filter(p=>!t.includes(p))),e==="edit"){const p=(await u({message:"Please fix the following zip codes:",placeholder:t.join(", "),initialValue:t.join(", "),validate(v){if(v?.length===0)return"Please enter a list of zip codes";const b=v?.replace(/ /g,"").split(",").filter(m=>m)?.filter(m=>!y(m));if(b?.length)return`The following zip codes are invalid: ${b.join(", ")}.`}})).replace(/ /g,"").split(",");i=[...i,...p]}}if(i=[...new Set(i)],t.length||r.success("Great! All your zips are valid!"),o(i))return l("Operation cancelled"),process.exit(1);const f=c?.daysListed??(await u({message:"How many days would you like to search listings? (Max. 90)",placeholder:"1",defaultValue:"1",validate(e){if(e&&e==="0")return"Please enter a number.";if(Number(e)>90)return"Please enter a number less than 90."}}))?.trim();if(o(f))return l("Operation cancelled"),process.exit(1);await d(1e3);const z=[`Listings source: ${a}`,`Output path: ${s}`,...n?[`Browser: ${n}`]:[],`Zip codes: ${i.join(", ")}`,`Days listed: ${f}`].filter(e=>e);r.info(""),r.info("Please review your answers:"),z.map(e=>{r.warn(e)}),r.info("");const h=await C({message:"Would you like to save this config?"});if(o(h)||!h)return l("Create config canceled. Please try again."),process.exit(1);const x={outputPath:A.isAbsolute(s)?s:j(s),...n&&{browser:n},zipCodes:i.join(", "),daysListed:Number(f)};return await $([{title:"Saving config to file",task:async()=>(await d(3e3),await L(a,x),"Saved config to file")}]),await d(1e3),k("Config file saved!"),await d(1e3),a}export{D as r};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{resetZillowCookie as a,waitForBrowserServer as t,checkForZillowCookie as w,waitForZillowCookie as l,checkForAndReadConfigFile as f,checkRequiredConfigValues as c}from"@rent-scraper/utils/config";import{log as r}from"@clack/prompts";import{runConfirmBrowserLaunch as s,runBrowserServer as g}from"@rent-scraper/browser-server";async function m(o){o==="zillow"&&(await s(),g(),await a(),r.info("Waiting for Server"),await t(),await w()||(r.info("Waiting for Zillow cookie"),await l()));const e=await f(o),i=c(o,e,"scrape");if(i.length){const n=`Required fields missing in config.${o}.yaml file: ${i.join(", ")}`;throw new Error(n)}}export{m as r};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{r as s}from"./create-config.B5xWKlnW.mjs";import{readConfigFile as r,writeConfigFile as d,checkForConfigFile as m,checkRequiredConfigValues as h}from"@rent-scraper/utils/config";import{log as u,select as c,isCancel as C,cancel as b,multiselect as v,outro as k}from"@clack/prompts";import{setTimeout as f}from"node:timers/promises";import{rename as p}from"node:fs/promises";import{parseAbsolutePath as o}from"@rent-scraper/utils";async function y(e){const a=await r(e),l=a.zipCodes.split(", ").length,w=[`Output path: ${o(a.outputPath)}`,...a.browser?[`Browser: ${a.browser}`]:[],`Zip codes: ${l>10?a.zipCodes.split(", ").splice(0,10).join(", ")+"...":a.zipCodes}`].filter(t=>t);u.warn(`Config file found at ${o("./config."+e+".yaml")}`);let i=await c({message:"Would you like to:",initialValue:"keep",options:[{value:"keep",label:"Keep this config"},{value:"edit",label:"Edit this config"},{value:"review",label:"Review this config"},{value:"new",label:"Create a new config"}]});if(C(i))return b("Operation cancelled"),process.exit(1);if(i==="review"||i==="edit")for(const t of w)await f(300),u.info(t);const n=i==="edit"&&await v({message:"Which fields would you like to edit?",options:[{value:"outputPath",label:"Output path"},{value:"browser",label:"Browser"},{value:"zipCodes",label:"Zip codes"}],required:!1});if(n&&Array.isArray(n)&&n.length){const t=await r(e);await f(1e3),await p(o(`./config.${e}.yaml`),o(`./config.${e}.yaml.bak`));for(const g of n)delete t[g];await f(1e3),await d(e,t),await s(e)}if(i==="new"&&(await p(o(`./config.${e}.yaml`),o(`./config.${e}.yaml.bak`)),await s()),i=i==="review"?await c({message:"Would you like to:",initialValue:"keep",options:[{value:"keep",label:"Keep this config"},{value:"edit",label:"Edit this config"},{value:"new",label:"Create a new config"}]}):i,i==="keep")return k("Config file saved!"),process.exit(0)}async function $(e){if(e&&await m(e)){const a=await r(e),l=h(e,a);l.length?(console.log({missingFields:l}),await s(e)):await y(e)}else await s()}export{$ as r};
|