@xcrawl/cli 0.2.2 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/README.md +42 -53
- package/dist/commands/status.js +2 -2
- package/dist/commands/status.js.map +1 -1
- package/dist/core/constants.d.ts +1 -0
- package/dist/core/constants.js +2 -1
- package/dist/core/constants.js.map +1 -1
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this project will be documented in this file.
|
|
4
4
|
|
|
5
|
+
## [0.2.3] - 2026-03-17
|
|
6
|
+
|
|
7
|
+
### Changed
|
|
8
|
+
- Updated `status` command to always call `https://api.xcrawl.com/web_v1/user/credit-user-info`.
|
|
9
|
+
- Removed `--api-base-url` override from `status` to avoid ambiguous routing behavior.
|
|
10
|
+
- Refined `README.md` structure to a more user-facing layout and removed the local development section.
|
|
11
|
+
|
|
5
12
|
## [0.2.0] - 2026-03-12
|
|
6
13
|
|
|
7
14
|
### Added
|
package/README.md
CHANGED
|
@@ -1,68 +1,71 @@
|
|
|
1
1
|
# XCrawl CLI
|
|
2
2
|
|
|
3
|
-
XCrawl CLI is
|
|
3
|
+
XCrawl CLI is the official command-line interface for XCrawl.
|
|
4
|
+
Use it to scrape pages, run search queries, map sites, and manage crawl jobs from your terminal.
|
|
4
5
|
|
|
5
|
-
##
|
|
6
|
+
## Installation
|
|
6
7
|
|
|
7
|
-
|
|
8
|
+
Run directly with `npx`:
|
|
8
9
|
|
|
9
10
|
```bash
|
|
10
11
|
npx -y @xcrawl/cli@latest doctor
|
|
11
12
|
```
|
|
12
13
|
|
|
13
|
-
|
|
14
|
+
Install globally with npm:
|
|
14
15
|
|
|
15
16
|
```bash
|
|
16
17
|
npm install -g @xcrawl/cli
|
|
17
18
|
xcrawl --help
|
|
18
19
|
```
|
|
19
20
|
|
|
20
|
-
##
|
|
21
|
+
## Quick Start
|
|
21
22
|
|
|
22
|
-
|
|
23
|
+
Authenticate:
|
|
23
24
|
|
|
24
25
|
```bash
|
|
25
|
-
# Save API key locally
|
|
26
26
|
xcrawl login --api-key <your_api_key>
|
|
27
|
-
|
|
28
|
-
# Clear local API key
|
|
29
|
-
xcrawl logout
|
|
30
|
-
|
|
31
|
-
# Or use env var
|
|
32
|
-
export XCRAWL_API_KEY=<your_api_key>
|
|
33
27
|
```
|
|
34
28
|
|
|
35
|
-
|
|
29
|
+
Run core commands:
|
|
36
30
|
|
|
37
31
|
```bash
|
|
38
|
-
# Scrape a page
|
|
39
32
|
xcrawl scrape https://example.com --format markdown
|
|
40
|
-
|
|
41
|
-
# Search
|
|
42
33
|
xcrawl search "xcrawl cli" --limit 10
|
|
43
|
-
|
|
44
|
-
# Show account and credit package status
|
|
45
34
|
xcrawl status
|
|
46
|
-
|
|
47
|
-
# Map links in a site
|
|
48
35
|
xcrawl map https://example.com --limit 10
|
|
49
|
-
|
|
50
|
-
# Start a crawl
|
|
51
36
|
xcrawl crawl https://example.com
|
|
52
|
-
|
|
53
|
-
# Check crawl status
|
|
54
37
|
xcrawl crawl status <job-id>
|
|
55
38
|
```
|
|
56
39
|
|
|
57
|
-
Default shortcut:
|
|
40
|
+
Default shortcut for scrape:
|
|
58
41
|
|
|
59
42
|
```bash
|
|
60
43
|
xcrawl https://example.com
|
|
61
|
-
# same as:
|
|
62
|
-
xcrawl scrape https://example.com
|
|
63
44
|
```
|
|
64
45
|
|
|
65
|
-
##
|
|
46
|
+
## Authentication
|
|
47
|
+
|
|
48
|
+
Use either local login or environment variable:
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
xcrawl login --api-key <your_api_key>
|
|
52
|
+
xcrawl logout
|
|
53
|
+
export XCRAWL_API_KEY=<your_api_key>
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
## Common Commands
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
xcrawl scrape <url...> [--format markdown|json|html|screenshot|text] [--output <path>] [--json]
|
|
60
|
+
xcrawl search <query> [--limit <n>] [--json]
|
|
61
|
+
xcrawl map <url> [--limit <n>] [--json]
|
|
62
|
+
xcrawl crawl <url> [--wait]
|
|
63
|
+
xcrawl crawl status <job-id>
|
|
64
|
+
xcrawl status [--json]
|
|
65
|
+
xcrawl doctor [--json]
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Batch scrape example:
|
|
66
69
|
|
|
67
70
|
```bash
|
|
68
71
|
xcrawl scrape --input ./urls.txt --concurrency 3 --json
|
|
@@ -70,7 +73,9 @@ xcrawl scrape --input ./urls.txt --concurrency 3 --json
|
|
|
70
73
|
|
|
71
74
|
`urls.txt` should contain one URL per line. Lines starting with `#` are ignored.
|
|
72
75
|
|
|
73
|
-
##
|
|
76
|
+
## Configuration
|
|
77
|
+
|
|
78
|
+
Manage config values:
|
|
74
79
|
|
|
75
80
|
```bash
|
|
76
81
|
xcrawl config keys
|
|
@@ -78,7 +83,7 @@ xcrawl config get api-base-url
|
|
|
78
83
|
xcrawl config set api-base-url https://run.xcrawl.com
|
|
79
84
|
```
|
|
80
85
|
|
|
81
|
-
|
|
86
|
+
Configuration priority:
|
|
82
87
|
1. CLI flags
|
|
83
88
|
2. Environment variables
|
|
84
89
|
3. Local config file `~/.xcrawl/config.json`
|
|
@@ -93,28 +98,12 @@ Environment variables:
|
|
|
93
98
|
|
|
94
99
|
## Output
|
|
95
100
|
|
|
96
|
-
- Default
|
|
97
|
-
- `--json
|
|
98
|
-
- `--output
|
|
99
|
-
- Multi-URL scrape defaults to `.xcrawl/` when no output path is provided
|
|
101
|
+
- Default output is human-readable text.
|
|
102
|
+
- Use `--json` for machine-readable output.
|
|
103
|
+
- Use `--output <path>` to save output to a file.
|
|
104
|
+
- Multi-URL scrape defaults to `.xcrawl/` when no output path is provided.
|
|
100
105
|
|
|
101
|
-
##
|
|
106
|
+
## API Routing Notes
|
|
102
107
|
|
|
103
108
|
- Default API base URL is `https://run.xcrawl.com`.
|
|
104
|
-
-
|
|
105
|
-
|
|
106
|
-
## Local Development
|
|
107
|
-
|
|
108
|
-
```bash
|
|
109
|
-
npm install
|
|
110
|
-
npm run build
|
|
111
|
-
npm run test
|
|
112
|
-
npm run lint
|
|
113
|
-
```
|
|
114
|
-
|
|
115
|
-
Real API smoke test:
|
|
116
|
-
|
|
117
|
-
```bash
|
|
118
|
-
export XCRAWL_API_KEY=<your_api_key>
|
|
119
|
-
npm run smoke
|
|
120
|
-
```
|
|
109
|
+
- `status` always calls `https://api.xcrawl.com/web_v1/user/credit-user-info`.
|
package/dist/commands/status.js
CHANGED
|
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.registerStatusCommand = registerStatusCommand;
|
|
4
4
|
const status_1 = require("../api/status");
|
|
5
5
|
const auth_1 = require("../core/auth");
|
|
6
|
+
const constants_1 = require("../core/constants");
|
|
6
7
|
const output_1 = require("../core/output");
|
|
7
8
|
const text_1 = require("../formatters/text");
|
|
8
9
|
const validate_1 = require("../utils/validate");
|
|
@@ -12,7 +13,6 @@ function registerStatusCommand(program, context) {
|
|
|
12
13
|
.command('status')
|
|
13
14
|
.description('Show account profile and credit package status')
|
|
14
15
|
.option('--api-key <key>', 'Override API key')
|
|
15
|
-
.option('--api-base-url <url>', 'Override API base URL')
|
|
16
16
|
.option('--timeout <ms>', 'Request timeout in milliseconds')
|
|
17
17
|
.option('--debug', 'Enable debug output')
|
|
18
18
|
.option('--json', 'Output result as JSON')
|
|
@@ -20,11 +20,11 @@ function registerStatusCommand(program, context) {
|
|
|
20
20
|
.action(async (options) => {
|
|
21
21
|
const runtime = await (0, shared_1.resolveCommandRuntimeConfig)(context, {
|
|
22
22
|
apiKey: options.apiKey,
|
|
23
|
-
apiBaseUrl: options.apiBaseUrl,
|
|
24
23
|
timeoutMs: (0, validate_1.parsePositiveInt)(options.timeout, 'timeout'),
|
|
25
24
|
debug: options.debug
|
|
26
25
|
});
|
|
27
26
|
runtime.apiKey = (0, auth_1.requireApiKey)(runtime.apiKey);
|
|
27
|
+
runtime.apiBaseUrl = constants_1.ACCOUNT_API_BASE_URL;
|
|
28
28
|
const client = context.createApiClient(runtime);
|
|
29
29
|
const outputPath = (0, shared_1.resolveOutputPath)(context, options.output);
|
|
30
30
|
const result = await (0, status_1.fetchStatus)(client);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"status.js","sourceRoot":"","sources":["../../src/commands/status.ts"],"names":[],"mappings":";;AAmBA,
|
|
1
|
+
{"version":3,"file":"status.js","sourceRoot":"","sources":["../../src/commands/status.ts"],"names":[],"mappings":";;AAmBA,sDA+BC;AAhDD,0CAA4C;AAC5C,uCAA6C;AAC7C,iDAAyD;AACzD,2CAA8C;AAC9C,6CAAkD;AAElD,gDAAqD;AACrD,qCAA0E;AAU1E,SAAgB,qBAAqB,CAAC,OAAgB,EAAE,OAAmB;IACzE,OAAO;SACJ,OAAO,CAAC,QAAQ,CAAC;SACjB,WAAW,CAAC,gDAAgD,CAAC;SAC7D,MAAM,CAAC,iBAAiB,EAAE,kBAAkB,CAAC;SAC7C,MAAM,CAAC,gBAAgB,EAAE,iCAAiC,CAAC;SAC3D,MAAM,CAAC,SAAS,EAAE,qBAAqB,CAAC;SACxC,MAAM,CAAC,QAAQ,EAAE,uBAAuB,CAAC;SACzC,MAAM,CAAC,iBAAiB,EAAE,uBAAuB,CAAC;SAClD,MAAM,CAAC,KAAK,EAAE,OAAsB,EAAE,EAAE;QACvC,MAAM,OAAO,GAAG,MAAM,IAAA,oCAA2B,EAAC,OAAO,EAAE;YACzD,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,SAAS,EAAE,IAAA,2BAAgB,EAAC,OAAO,CAAC,OAAO,EAAE,SAAS,CAAC;YACvD,KAAK,EAAE,OAAO,CAAC,KAAK;SACrB,CAAC,CAAC;QAEH,OAAO,CAAC,MAAM,GAAG,IAAA,oBAAa,EAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC/C,OAAO,CAAC,UAAU,GAAG,gCAAoB,CAAC;QAE1C,MAAM,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAChD,MAAM,UAAU,GAAG,IAAA,0BAAiB,EAAC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QAC9D,MAAM,MAAM,GAAG,MAAM,IAAA,oBAAW,EAAC,MAAM,CAAC,CAAC;QAEzC,MAAM,IAAA,qBAAY,EAAC;YACjB,GAAG,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE;YAC/B,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,UAAU;YACV,UAAU,EAAE,mBAAY;SACzB,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACP,CAAC"}
|
package/dist/core/constants.d.ts
CHANGED
package/dist/core/constants.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.DEFAULT_RUNTIME_CONFIG = exports.DEFAULT_API_BASE_URL = void 0;
|
|
3
|
+
exports.DEFAULT_RUNTIME_CONFIG = exports.ACCOUNT_API_BASE_URL = exports.DEFAULT_API_BASE_URL = void 0;
|
|
4
4
|
exports.DEFAULT_API_BASE_URL = 'https://run.xcrawl.com';
|
|
5
|
+
exports.ACCOUNT_API_BASE_URL = 'https://api.xcrawl.com';
|
|
5
6
|
exports.DEFAULT_RUNTIME_CONFIG = {
|
|
6
7
|
apiBaseUrl: exports.DEFAULT_API_BASE_URL,
|
|
7
8
|
defaultFormat: 'markdown',
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../src/core/constants.ts"],"names":[],"mappings":";;;AAEa,QAAA,oBAAoB,GAAG,wBAAwB,CAAC;AAEhD,QAAA,sBAAsB,GAAkB;IACnD,UAAU,EAAE,4BAAoB;IAChC,aAAa,EAAE,UAAU;IACzB,SAAS,EAAE,SAAS;IACpB,SAAS,EAAE,KAAK;IAChB,KAAK,EAAE,KAAK;CACb,CAAC"}
|
|
1
|
+
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../src/core/constants.ts"],"names":[],"mappings":";;;AAEa,QAAA,oBAAoB,GAAG,wBAAwB,CAAC;AAChD,QAAA,oBAAoB,GAAG,wBAAwB,CAAC;AAEhD,QAAA,sBAAsB,GAAkB;IACnD,UAAU,EAAE,4BAAoB;IAChC,aAAa,EAAE,UAAU;IACzB,SAAS,EAAE,SAAS;IACpB,SAAS,EAAE,KAAK;IAChB,KAAK,EAAE,KAAK;CACb,CAAC"}
|