@apounited/mac 1.2.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -13
- package/dist/main.js +1 -1
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -32,12 +32,15 @@ npx -y @apounited/mac@latest crawl customers --username demo --password 'demo'
|
|
|
32
32
|
|
|
33
33
|
Available crawl endpoints: `categories`, `customers`, `documents`, `products`.
|
|
34
34
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
|
38
|
-
|
|
|
39
|
-
| `--
|
|
40
|
-
| `--
|
|
35
|
+
> You can use [Query String Generator](https://simpliconvert.com/query_string_generator/) to generate query strings.
|
|
36
|
+
|
|
37
|
+
| Options | Short | Required | Values | Description |
|
|
38
|
+
| ---------------- | ----- | ---------- | ------------ | --------------------------------------------- |
|
|
39
|
+
| `--username` | `-u` | `true` | `<string>` | Mauve API username |
|
|
40
|
+
| `--password` | `-p` | `true` | `<string>` | Mauve API password |
|
|
41
|
+
| `--query` | `-q` | `optional` | `<string>` | A query to be appanded to the API request URL |
|
|
42
|
+
| `--format` | `-f` | `optional` | `json` `csv` | Export file format |
|
|
43
|
+
| `--max-requests` | `-r` | `optional` | `<number>` | Maximum number of crawl requests |
|
|
41
44
|
|
|
42
45
|
### CI
|
|
43
46
|
|
|
@@ -47,13 +50,14 @@ Runs `crawl` command using Environment Variables instead of CLI arguments.
|
|
|
47
50
|
bunx @apounited/mac ci
|
|
48
51
|
```
|
|
49
52
|
|
|
50
|
-
| Name | Required | Values | Description
|
|
51
|
-
| -------------------------------- | ---------- | ----------------------------------------------- |
|
|
52
|
-
| `MAUVE_API_CRAWLER_ENDPOINT` | `true` | `categories` `customers` `documents` `products` | Mauve API crawl endpoint
|
|
53
|
-
| `MAUVE_API_CRAWLER_USERNAME` | `true` | `<string>` | Mauve API username
|
|
54
|
-
| `MAUVE_API_CRAWLER_PASSWORD` | `true` | `<string>` | Mauve API password
|
|
55
|
-
| `
|
|
56
|
-
| `
|
|
53
|
+
| Name | Required | Values | Description |
|
|
54
|
+
| -------------------------------- | ---------- | ----------------------------------------------- | --------------------------------------------- |
|
|
55
|
+
| `MAUVE_API_CRAWLER_ENDPOINT` | `true` | `categories` `customers` `documents` `products` | Mauve API crawl endpoint |
|
|
56
|
+
| `MAUVE_API_CRAWLER_USERNAME` | `true` | `<string>` | Mauve API username |
|
|
57
|
+
| `MAUVE_API_CRAWLER_PASSWORD` | `true` | `<string>` | Mauve API password |
|
|
58
|
+
| `MAUVE_API_CRAWLER_QUERY` | `optional` | `<string>` | A query to be appanded to the API request URL |
|
|
59
|
+
| `MAUVE_API_CRAWLER_FORMAT` | `optional` | `json` `csv` | Export file format |
|
|
60
|
+
| `MAUVE_API_CRAWLER_MAX_REQUESTS` | `optional` | `<number>` | Maximum number of crawl requests |
|
|
57
61
|
|
|
58
62
|
### Clean
|
|
59
63
|
|
package/dist/main.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
var
|
|
2
|
+
var L=Object.defineProperty;var P=(e)=>e;function C(e,n){this[e]=P.bind(null,n)}var f=(e,n)=>{for(var t in n)L(e,t,{get:n[t],enumerable:!0,configurable:!0,set:C.bind(n,t)})};var A=(e,n)=>()=>(e&&(n=e(e=0)),n);function E({username:e,password:n}){if(!e||!n)console.error("Error: Username and password are required."),process.exit(1);return{auth:Buffer.from(`${e}:${n}`).toString("base64")}}function R({baseUrl:e="https://api.mauve.de",endpoint:n="/",query:t=""}){return{params:{limit:1000,offset:0,...Object.fromEntries([...new URLSearchParams(t)].map(([r,s])=>{if(["limit","offset"].includes(r.toLowerCase()))return[r,Number(s)];else return[r,s]}))},req:function(r){let s=new URLSearchParams(r).toString();return`${e.replace(/\/$/,"")}/${n.replace(/^\//,"")}?${s}`}}}var x={};f(x,{crawl:()=>M});import{BasicCrawler as S}from"@crawlee/basic";async function M(e){let{auth:n}=E({username:e.username,password:e.password}),{params:t,req:a}=R({endpoint:e.endpoint,query:e.query}),c=t.offset,r=new S({maxConcurrency:1,maxRequestsPerCrawl:e.maxRequests,autoscaledPoolOptions:{loggingIntervalSecs:null},statisticsOptions:{logIntervalSecs:9999},async requestHandler({sendRequest:s,addRequests:_,pushData:b,log:p}){let{ok:U,body:i}=await s({responseType:"json",http2:!1,headers:{Authorization:`Basic ${n}`}});if(U){let m="categories"in i&&i.categories||"customers"in i&&i.customers||"documents"in i&&i.documents||"products"in i&&i.products;if(m){if(t.offset+=m.length,t.offset&&m.length===t.limit)await _([a(t)]);await b(m),p.info(`Fetched ${m.length} ${e.endpoint} (total: ${t.offset-c})`)}else p.warning("Response might not contain valid keys"),await r.teardown(),process.exit()}else if("responseStatus"in i&&i.responseStatus)p.error(`${i.responseStatus.errorCode}: ${i.responseStatus.message}`),await r.teardown(),process.exit(1)}});await r.run([a(t)]),await r.exportData(`storage/data/${e.username.toLowerCase()}/${e.endpoint}.${e.format?.toLowerCase()??"json"}`)}var d=()=>{};var w={};f(w,{ci:()=>h});async function h(){let e=process.env.MAUVE_API_CRAWLER_ENDPOINT,n=process.env.MAUVE_API_CRAWLER_USERNAME,t=process.env.MAUVE_API_CRAWLER_PASSWORD,a=process.env.MAUVE_API_CRAWLER_QUERY,c=process.env.MAUVE_API_CRAWLER_FORMAT,r=process.env.MAUVE_API_CRAWLER_MAX_REQUESTS;if(!e||!n||!t)console.error("No endpoint, username or password provided."),process.exit(1);let{crawl:s}=await Promise.resolve().then(() => (d(),x));await s({endpoint:e,username:n,password:t,query:a,format:c&&["json","csv"].includes(c)?c:void 0,maxRequests:r?Number(r):void 0})}var g={};f(g,{clean:()=>y});import{rimraf as j}from"rimraf";async function y(){await j("storage")}var v=()=>{};import N from"cac";var l={name:"@apounited/mac",version:"1.3.0",license:"UNLICENSED",author:"Sergej Samsonenko",description:"Crawls and extracts data from the Mauve API.",type:"module",keywords:["mauve","api","crawler"],files:["dist"],publishConfig:{access:"public"},main:"./dist/main.js",bin:{mac:"./dist/main.js"},scripts:{build:"bun build.ts",dev:"bun src/main.ts",lint:"bunx oxlint",fmt:"bunx oxfmt",clean:"bunx rimraf storage",deps:"(bunx taze -r -w) && (bun i)",prepublishOnly:"bunx pinst -d",postpublish:"bunx pinst -e",postinstall:"bunx simple-git-hooks"},dependencies:{"@crawlee/basic":"^3.16.0",cac:"^7.0.0",rimraf:"^6.1.3"},devDependencies:{"@apify/tsconfig":"^0.1.2","@apounited/schemas":"^0.1.0","@types/bun":"latest","oxlint-tsgolint":"^0.23.0",typescript:"^6.0.3"},"simple-git-hooks":{"pre-commit":"bunx lint-staged","commit-msg":"bunx @apounited/verify-commit-msg@latest $1"},"lint-staged":{"*.{js,ts}":["bunx oxlint --fix"],"*":["bunx oxfmt --no-error-on-unmatched-pattern"]}};var o=N(l.name);o.command("crawl <endpoint>","Specify the endpoint to crawl (categories, customers, documents, products)").option("-u, --username <username>","Username for authentication").option("-p, --password <password>","Password for authentication").option("-q, --query [string]","A query to be appanded to the API request URL").option("-f, --format [format]","Export file format (json, csv)").option("-r, --max-requests [number]","Maximum number of requests to make").example("crawl customers --username demo --password 'demo' --format csv --max-requests 5").action(async(e,n)=>{let{crawl:t}=await Promise.resolve().then(() => (d(),x));await t({endpoint:e,...n})});o.command("ci","Run crawl command using Environment Variables instead of CLI arguments").action(async()=>{let{ci:e}=await Promise.resolve().then(() => w);await e()});o.command("clean","Delete ./storage directory and all its contents").action(async()=>{let{clean:e}=await Promise.resolve().then(() => (v(),g));await e()});o.help();o.version(l.version);o.parse();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@apounited/mac",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.3.0",
|
|
4
4
|
"license": "UNLICENSED",
|
|
5
5
|
"author": "Sergej Samsonenko",
|
|
6
6
|
"description": "Crawls and extracts data from the Mauve API.",
|
|
@@ -40,7 +40,7 @@
|
|
|
40
40
|
"@apify/tsconfig": "^0.1.2",
|
|
41
41
|
"@apounited/schemas": "^0.1.0",
|
|
42
42
|
"@types/bun": "latest",
|
|
43
|
-
"oxlint-tsgolint": "^0.
|
|
43
|
+
"oxlint-tsgolint": "^0.23.0",
|
|
44
44
|
"typescript": "^6.0.3"
|
|
45
45
|
},
|
|
46
46
|
"simple-git-hooks": {
|