@apounited/mac 1.2.0 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -3
- package/dist/main.js +1 -1
- package/package.json +2 -1
package/README.md
CHANGED
|
@@ -34,10 +34,10 @@ Available crawl endpoints: `categories`, `customers`, `documents`, `products`.
|
|
|
34
34
|
|
|
35
35
|
| Options | Short | Required | Values | Description |
|
|
36
36
|
| ---------------- | ----- | ---------- | ------------ | -------------------------------- |
|
|
37
|
-
| `--username` | `-u` | `true` |
|
|
38
|
-
| `--password` | `-p` | `true` |
|
|
37
|
+
| `--username` | `-u` | `true` | `<string>` | Mauve API username |
|
|
38
|
+
| `--password` | `-p` | `true` | `<string>` | Mauve API password |
|
|
39
39
|
| `--format` | `-f` | `optional` | `json` `csv` | Export file format |
|
|
40
|
-
| `--max-requests` | `-r` | `optional` |
|
|
40
|
+
| `--max-requests` | `-r` | `optional` | `<number>` | Maximum number of crawl requests |
|
|
41
41
|
|
|
42
42
|
### CI
|
|
43
43
|
|
|
@@ -47,6 +47,14 @@ Runs `crawl` command using Environment Variables instead of CLI arguments.
|
|
|
47
47
|
bunx @apounited/mac ci
|
|
48
48
|
```
|
|
49
49
|
|
|
50
|
+
| Name | Required | Values | Description |
|
|
51
|
+
| -------------------------------- | ---------- | ----------------------------------------------- | -------------------------------- |
|
|
52
|
+
| `MAUVE_API_CRAWLER_ENDPOINT` | `true` | `categories` `customers` `documents` `products` | Mauve API crawl endpoint |
|
|
53
|
+
| `MAUVE_API_CRAWLER_USERNAME` | `true` | `<string>` | Mauve API username |
|
|
54
|
+
| `MAUVE_API_CRAWLER_PASSWORD` | `true` | `<string>` | Mauve API password |
|
|
55
|
+
| `MAUVE_API_CRAWLER_FORMAT` | `optional` | `json` `csv` | Export file format |
|
|
56
|
+
| `MAUVE_API_CRAWLER_MAX_REQUESTS` | `optional` | `<number>` | Maximum number of crawl requests |
|
|
57
|
+
|
|
50
58
|
### Clean
|
|
51
59
|
|
|
52
60
|
Deletes `./storage` directory relative to the execution path.
|
package/dist/main.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
var S=Object.defineProperty;var U=(n)=>n;function C(n,e){this[n]=U.bind(null,e)}var l=(n,e)=>{for(var t in e)S(n,t,{get:e[t],enumerable:!0,configurable:!0,set:C.bind(e,t)})};var w=(n,e)=>()=>(n&&(e=n(n=0)),e);function A({username:n,password:e}){if(!n||!e)console.error("Error: Username and password are required."),process.exit(1);return{auth:Buffer.from(`${n}:${e}`).toString("base64")}}function g({baseUrl:n="https://api.mauve.de",endpoint:e="/",limit:t=1000,offset:r=0}){let s={limit:t,offset:r,include:""},o={categories:[],customers:[],documents:[],products:[]};return s.include=o[e].join(","),{params:s,req:function(p){let a=new URLSearchParams(p).toString();return`${n.replace(/\/$/,"")}/${e.replace(/^\//,"")}?${a}`}}}var x={};l(x,{crawl:()=>P});import{BasicCrawler as M}from"@crawlee/basic";async function P(n){let{auth:e}=A({username:n.username,password:n.password}),{params:t,req:r}=g({endpoint:n.endpoint}),s=new M({maxConcurrency:1,maxRequestsPerCrawl:n.maxRequests,autoscaledPoolOptions:{loggingIntervalSecs:null},statisticsOptions:{logIntervalSecs:9999},async requestHandler({sendRequest:o,addRequests:v,pushData:p,log:a}){let{ok:b,body:i}=await o({responseType:"json",http2:!1,headers:{Authorization:`Basic ${e}`}});if(b){let m="categories"in i&&i.categories||"customers"in i&&i.customers||"documents"in i&&i.documents||"products"in i&&i.products;if(m){if(t.offset+=m.length,t.offset&&m.length===t.limit)await v([r(t)]);await p(m),a.info(`Fetched ${m.length} ${n.endpoint} (total: ${t.offset})`)}else a.warning("Response might not contain valid keys"),await s.teardown(),process.exit()}else if("responseStatus"in i)a.error(`${i.responseStatus.errorCode}: ${i.responseStatus.message}`),await s.teardown(),process.exit(1)}});await s.run([r(t)]),await s.exportData(`storage/data/${n.username.toLowerCase()}/${n.endpoint}.${n.format?.toLowerCase()??"json"}`)}var d=()=>{};var E={};l(E,{ci:()=>
|
|
2
|
+
var S=Object.defineProperty;var U=(n)=>n;function C(n,e){this[n]=U.bind(null,e)}var l=(n,e)=>{for(var t in e)S(n,t,{get:e[t],enumerable:!0,configurable:!0,set:C.bind(e,t)})};var w=(n,e)=>()=>(n&&(e=n(n=0)),e);function A({username:n,password:e}){if(!n||!e)console.error("Error: Username and password are required."),process.exit(1);return{auth:Buffer.from(`${n}:${e}`).toString("base64")}}function g({baseUrl:n="https://api.mauve.de",endpoint:e="/",limit:t=1000,offset:r=0}){let s={limit:t,offset:r,include:""},o={categories:[],customers:[],documents:[],products:[]};return s.include=o[e].join(","),{params:s,req:function(p){let a=new URLSearchParams(p).toString();return`${n.replace(/\/$/,"")}/${e.replace(/^\//,"")}?${a}`}}}var x={};l(x,{crawl:()=>P});import{BasicCrawler as M}from"@crawlee/basic";async function P(n){let{auth:e}=A({username:n.username,password:n.password}),{params:t,req:r}=g({endpoint:n.endpoint}),s=new M({maxConcurrency:1,maxRequestsPerCrawl:n.maxRequests,autoscaledPoolOptions:{loggingIntervalSecs:null},statisticsOptions:{logIntervalSecs:9999},async requestHandler({sendRequest:o,addRequests:v,pushData:p,log:a}){let{ok:b,body:i}=await o({responseType:"json",http2:!1,headers:{Authorization:`Basic ${e}`}});if(b){let m="categories"in i&&i.categories||"customers"in i&&i.customers||"documents"in i&&i.documents||"products"in i&&i.products;if(m){if(t.offset+=m.length,t.offset&&m.length===t.limit)await v([r(t)]);await p(m),a.info(`Fetched ${m.length} ${n.endpoint} (total: ${t.offset})`)}else a.warning("Response might not contain valid keys"),await s.teardown(),process.exit()}else if("responseStatus"in i&&i.responseStatus)a.error(`${i.responseStatus.errorCode}: ${i.responseStatus.message}`),await s.teardown(),process.exit(1)}});await s.run([r(t)]),await s.exportData(`storage/data/${n.username.toLowerCase()}/${n.endpoint}.${n.format?.toLowerCase()??"json"}`)}var d=()=>{};var E={};l(E,{ci:()=>L});async function L(){let n=process.env.MAUVE_API_CRAWLER_ENDPOINT,e=process.env.MAUVE_API_CRAWLER_USERNAME,t=process.env.MAUVE_API_CRAWLER_PASSWORD,r=process.env.MAUVE_API_CRAWLER_FORMAT,s=process.env.MAUVE_API_CRAWLER_MAX_REQUESTS;if(!n||!e||!t)console.error("No endpoint, username or password provided."),process.exit(1);let{crawl:o}=await Promise.resolve().then(() => (d(),x));await o({endpoint:n,username:e,password:t,format:r&&["json","csv"].includes(r)?r:void 0,maxRequests:s?Number(s):void 0})}var R={};l(R,{clean:()=>y});import{rimraf as j}from"rimraf";async function y(){await j("storage")}var _=()=>{};import h from"cac";var f={name:"@apounited/mac",version:"1.2.1",license:"UNLICENSED",author:"Sergej Samsonenko",description:"Crawls and extracts data from the Mauve API.",type:"module",keywords:["mauve","api","crawler"],files:["dist"],publishConfig:{access:"public"},main:"./dist/main.js",bin:{mac:"./dist/main.js"},scripts:{build:"bun build.ts",dev:"bun src/main.ts",lint:"bunx oxlint",fmt:"bunx oxfmt",clean:"bunx rimraf storage",deps:"(bunx taze -r -w) && (bun i)",prepublishOnly:"bunx pinst -d",postpublish:"bunx pinst -e",postinstall:"bunx simple-git-hooks"},dependencies:{"@crawlee/basic":"^3.16.0",cac:"^7.0.0",rimraf:"^6.1.3"},devDependencies:{"@apify/tsconfig":"^0.1.2","@apounited/schemas":"^0.1.0","@types/bun":"latest","oxlint-tsgolint":"^0.22.1",typescript:"^6.0.3"},"simple-git-hooks":{"pre-commit":"bunx lint-staged","commit-msg":"bunx @apounited/verify-commit-msg@latest $1"},"lint-staged":{"*.{js,ts}":["bunx oxlint --fix"],"*":["bunx oxfmt --no-error-on-unmatched-pattern"]}};var c=h(f.name);c.command("crawl <endpoint>","Specify the endpoint to crawl (categories, customers, documents, products)").option("-u, --username <username>","Username for authentication").option("-p, --password <password>","Password for authentication").option("-f, --format [format]","Export file format (json, csv)").option("-r, --max-requests [number]","Maximum number of requests to make").example("crawl customers --username demo --password 'demo' --format csv --max-requests 5").action(async(n,e)=>{let{crawl:t}=await Promise.resolve().then(() => (d(),x));await t({endpoint:n,...e})});c.command("ci","Run crawl command using Environment Variables instead of CLI arguments").action(async()=>{let{ci:n}=await Promise.resolve().then(() => E);await n()});c.command("clean","Delete ./storage directory and all its contents").action(async()=>{let{clean:n}=await Promise.resolve().then(() => (_(),R));await n()});c.help();c.version(f.version);c.parse();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@apounited/mac",
|
|
3
|
-
"version": "1.2.
|
|
3
|
+
"version": "1.2.1",
|
|
4
4
|
"license": "UNLICENSED",
|
|
5
5
|
"author": "Sergej Samsonenko",
|
|
6
6
|
"description": "Crawls and extracts data from the Mauve API.",
|
|
@@ -38,6 +38,7 @@
|
|
|
38
38
|
},
|
|
39
39
|
"devDependencies": {
|
|
40
40
|
"@apify/tsconfig": "^0.1.2",
|
|
41
|
+
"@apounited/schemas": "^0.1.0",
|
|
41
42
|
"@types/bun": "latest",
|
|
42
43
|
"oxlint-tsgolint": "^0.22.1",
|
|
43
44
|
"typescript": "^6.0.3"
|