@apounited/mac 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -10
- package/dist/main.js +1 -1
- package/package.json +4 -3
package/README.md
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
# @apounited/mac
|
|
6
6
|
|
|
7
|
-
>
|
|
7
|
+
> Mauve API-Crawler (MAC)
|
|
8
8
|
|
|
9
9
|
Crawls and extracts data from the [Mauve API](https://api.mauve.de)
|
|
10
10
|
|
|
@@ -16,25 +16,36 @@ The crawlers can be invoked via `bunx`, `pnpx` or `npx` with `@apounited/mac`.
|
|
|
16
16
|
|
|
17
17
|
> Data is saved to `./storage` relative to the execution path.
|
|
18
18
|
|
|
19
|
+
> Pin to specific version with `@apounited/mac@1.0.0`.
|
|
20
|
+
|
|
19
21
|
#### Bun
|
|
20
22
|
|
|
21
23
|
```bash
|
|
22
|
-
bunx @apounited/mac@latest crawl customers --username demo --password demo
|
|
24
|
+
bunx @apounited/mac@latest crawl customers --username demo --password 'demo'
|
|
23
25
|
```
|
|
24
26
|
|
|
25
27
|
#### Node
|
|
26
28
|
|
|
27
29
|
```bash
|
|
28
|
-
npx -y @apounited/mac@latest crawl customers --username demo --password demo
|
|
30
|
+
npx -y @apounited/mac@latest crawl customers --username demo --password 'demo'
|
|
29
31
|
```
|
|
30
32
|
|
|
31
|
-
Available
|
|
33
|
+
Available crawl endpoints: `categories`, `customers`, `documents`, `products`.
|
|
34
|
+
|
|
35
|
+
| Options | Short | Required | Values | Description |
|
|
36
|
+
| ---------------- | ----- | ---------- | ------------ | -------------------------------- |
|
|
37
|
+
| `--username` | `-u` | `true` | `string` | Mauve Customer ID |
|
|
38
|
+
| `--password` | `-p` | `true` | `string` | Mauve API Password |
|
|
39
|
+
| `--format` | `-f` | `optional` | `json` `csv` | Export file format |
|
|
40
|
+
| `--max-requests` | `-r` | `optional` | `number` | Maximum number of crawl requests |
|
|
41
|
+
|
|
42
|
+
### CI
|
|
32
43
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
44
|
+
Runs `crawl` command using Environment Variables instead of CLI arguments.
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
bunx @apounited/mac ci
|
|
48
|
+
```
|
|
38
49
|
|
|
39
50
|
### Clean
|
|
40
51
|
|
|
@@ -49,7 +60,7 @@ bunx @apounited/mac clean
|
|
|
49
60
|
Run the main CLI entry point without building:
|
|
50
61
|
|
|
51
62
|
```bash
|
|
52
|
-
bun dev
|
|
63
|
+
bun dev crawl customers --username demo --password 'demo'
|
|
53
64
|
```
|
|
54
65
|
|
|
55
66
|
## Production
|
package/dist/main.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
var
|
|
2
|
+
var S=Object.defineProperty;var U=(n)=>n;function C(n,e){this[n]=U.bind(null,e)}var l=(n,e)=>{for(var t in e)S(n,t,{get:e[t],enumerable:!0,configurable:!0,set:C.bind(e,t)})};var w=(n,e)=>()=>(n&&(e=n(n=0)),e);function A({username:n,password:e}){if(!n||!e)console.error("Error: Username and password are required."),process.exit(1);return{auth:Buffer.from(`${n}:${e}`).toString("base64")}}function g({baseUrl:n="https://api.mauve.de",endpoint:e="/",limit:t=1000,offset:r=0}){let s={limit:t,offset:r,include:""},o={categories:[],customers:[],documents:[],products:[]};return s.include=o[e].join(","),{params:s,req:function(p){let a=new URLSearchParams(p).toString();return`${n.replace(/\/$/,"")}/${e.replace(/^\//,"")}?${a}`}}}var x={};l(x,{crawl:()=>P});import{BasicCrawler as M}from"@crawlee/basic";async function P(n){let{auth:e}=A({username:n.username,password:n.password}),{params:t,req:r}=g({endpoint:n.endpoint}),s=new M({maxConcurrency:1,maxRequestsPerCrawl:n.maxRequests,autoscaledPoolOptions:{loggingIntervalSecs:null},statisticsOptions:{logIntervalSecs:9999},async requestHandler({sendRequest:o,addRequests:v,pushData:p,log:a}){let{ok:b,body:i}=await o({responseType:"json",http2:!1,headers:{Authorization:`Basic ${e}`}});if(b){let m="categories"in i&&i.categories||"customers"in i&&i.customers||"documents"in i&&i.documents||"products"in i&&i.products;if(m){if(t.offset+=m.length,t.offset&&m.length===t.limit)await v([r(t)]);await p(m),a.info(`Fetched ${m.length} ${n.endpoint} (total: ${t.offset})`)}else a.warning("Response might not contain valid keys"),await s.teardown(),process.exit()}else if("responseStatus"in i)a.error(`${i.responseStatus.errorCode}: ${i.responseStatus.message}`),await s.teardown(),process.exit(1)}});await s.run([r(t)]),await s.exportData(`storage/data/${n.username.toLowerCase()}/${n.endpoint}.${n.format?.toLowerCase()??"json"}`)}var d=()=>{};var E={};l(E,{ci:()=>y});async function y(){let n=process.env.MAUVE_API_CRAWLER_ENDPOINT,e=process.env.MAUVE_API_CRAWLER_USERNAME,t=process.env.MAUVE_API_CRAWLER_PASSWORD,r=process.env.MAUVE_API_CRAWLER_FORMAT,s=process.env.MAUVE_API_CRAWLER_MAX_REQUESTS;if(!n||!e||!t)console.error("No endpoint, username or password provided."),process.exit(1);let{crawl:o}=await Promise.resolve().then(() => (d(),x));await o({endpoint:n,username:e,password:t,format:r&&["json","csv"].includes(r)?r:void 0,maxRequests:s?Number(s):void 0})}var R={};l(R,{clean:()=>j});import{rimraf as L}from"rimraf";async function j(){await L("storage")}var _=()=>{};import N from"cac";var f={name:"@apounited/mac",version:"1.2.0",license:"UNLICENSED",author:"Sergej Samsonenko",description:"Crawls and extracts data from the Mauve API.",type:"module",keywords:["mauve","api","crawler"],files:["dist"],publishConfig:{access:"public"},main:"./dist/main.js",bin:{mac:"./dist/main.js"},scripts:{build:"bun build.ts",dev:"bun src/main.ts",lint:"bunx oxlint",fmt:"bunx oxfmt",clean:"bunx rimraf storage",deps:"(bunx taze -r -w) && (bun i)",prepublishOnly:"bunx pinst -d",postpublish:"bunx pinst -e",postinstall:"bunx simple-git-hooks"},dependencies:{"@crawlee/basic":"^3.16.0",cac:"^7.0.0",rimraf:"^6.1.3"},devDependencies:{"@apify/tsconfig":"^0.1.2","@types/bun":"latest","oxlint-tsgolint":"^0.22.1",typescript:"^6.0.3"},"simple-git-hooks":{"pre-commit":"bunx lint-staged","commit-msg":"bunx @apounited/verify-commit-msg@latest $1"},"lint-staged":{"*.{js,ts}":["bunx oxlint --fix"],"*":["bunx oxfmt --no-error-on-unmatched-pattern"]}};var c=N(f.name);c.command("crawl <endpoint>","Specify the endpoint to crawl (categories, customers, documents, products)").option("-u, --username <username>","Username for authentication").option("-p, --password <password>","Password for authentication").option("-f, --format [format]","Export file format (json, csv)").option("-r, --max-requests [number]","Maximum number of requests to make").example("crawl customers --username demo --password 'demo' --format csv --max-requests 5").action(async(n,e)=>{let{crawl:t}=await Promise.resolve().then(() => (d(),x));await t({endpoint:n,...e})});c.command("ci","Run crawl command using Environment Variables instead of CLI arguments").action(async()=>{let{ci:n}=await Promise.resolve().then(() => E);await n()});c.command("clean","Delete ./storage directory and all its contents").action(async()=>{let{clean:n}=await Promise.resolve().then(() => (_(),R));await n()});c.help();c.version(f.version);c.parse();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@apounited/mac",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.2.0",
|
|
4
4
|
"license": "UNLICENSED",
|
|
5
5
|
"author": "Sergej Samsonenko",
|
|
6
6
|
"description": "Crawls and extracts data from the Mauve API.",
|
|
@@ -27,8 +27,9 @@
|
|
|
27
27
|
"fmt": "bunx oxfmt",
|
|
28
28
|
"clean": "bunx rimraf storage",
|
|
29
29
|
"deps": "(bunx taze -r -w) && (bun i)",
|
|
30
|
-
"prepublishOnly": "
|
|
31
|
-
"postpublish": "
|
|
30
|
+
"prepublishOnly": "bunx pinst -d",
|
|
31
|
+
"postpublish": "bunx pinst -e",
|
|
32
|
+
"_postinstall": "bunx simple-git-hooks"
|
|
32
33
|
},
|
|
33
34
|
"dependencies": {
|
|
34
35
|
"@crawlee/basic": "^3.16.0",
|