@hunt-labs/bytekit-cli 0.1.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +72 -0
- package/LICENSE +21 -0
- package/README.md +143 -0
- package/dist/client.d.ts +49 -3
- package/dist/client.js +118 -11
- package/dist/commands/account.d.ts +0 -1
- package/dist/commands/account.js +7 -3
- package/dist/commands/bulk.d.ts +0 -1
- package/dist/commands/bulk.js +0 -1
- package/dist/commands/fetch.d.ts +0 -1
- package/dist/commands/fetch.js +0 -1
- package/dist/commands/monitors.d.ts +0 -1
- package/dist/commands/monitors.js +0 -1
- package/dist/commands/scrape.d.ts +0 -1
- package/dist/commands/scrape.js +11 -3
- package/dist/commands/screenshots.d.ts +0 -1
- package/dist/commands/screenshots.js +1 -2
- package/dist/commands/search.d.ts +10 -1
- package/dist/commands/search.js +23 -4
- package/dist/commands/sitemap.d.ts +0 -1
- package/dist/commands/sitemap.js +0 -1
- package/dist/index.d.ts +15 -1
- package/dist/index.js +34 -6
- package/dist/lib/url-lines.d.ts +0 -1
- package/dist/lib/url-lines.js +0 -1
- package/dist/output.d.ts +80 -1
- package/dist/output.js +117 -1
- package/package.json +25 -9
- package/dist/client.d.ts.map +0 -1
- package/dist/client.js.map +0 -1
- package/dist/commands/account.d.ts.map +0 -1
- package/dist/commands/account.js.map +0 -1
- package/dist/commands/bulk.d.ts.map +0 -1
- package/dist/commands/bulk.js.map +0 -1
- package/dist/commands/fetch.d.ts.map +0 -1
- package/dist/commands/fetch.js.map +0 -1
- package/dist/commands/monitors.d.ts.map +0 -1
- package/dist/commands/monitors.js.map +0 -1
- package/dist/commands/recordings.d.ts +0 -6
- package/dist/commands/recordings.d.ts.map +0 -1
- package/dist/commands/recordings.js +0 -37
- package/dist/commands/recordings.js.map +0 -1
- package/dist/commands/scrape.d.ts.map +0 -1
- package/dist/commands/scrape.js.map +0 -1
- package/dist/commands/screenshots.d.ts.map +0 -1
- package/dist/commands/screenshots.js.map +0 -1
- package/dist/commands/search.d.ts.map +0 -1
- package/dist/commands/search.js.map +0 -1
- package/dist/commands/sitemap.d.ts.map +0 -1
- package/dist/commands/sitemap.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/lib/url-lines.d.ts.map +0 -1
- package/dist/lib/url-lines.js.map +0 -1
- package/dist/output.d.ts.map +0 -1
- package/dist/output.js.map +0 -1
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to `@hunt-labs/bytekit-cli` are documented here.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
This file records only the delta between versions. For how to use the package
|
|
9
|
+
today, see [README.md](./README.md), which remains the authoritative usage guide.
|
|
10
|
+
|
|
11
|
+
## [0.2.2] - 2026-07-16
|
|
12
|
+
|
|
13
|
+
### Changed
|
|
14
|
+
|
|
15
|
+
- Re-release of 0.2.1. The 0.2.1 artifacts never reached npm — the prod publish's
|
|
16
|
+
`publish-sdk` step failed on a CI test-isolation flake (`cjs-require.test.ts`
|
|
17
|
+
rebuilding the shared `dist`), which is now fixed. No consumer-facing changes
|
|
18
|
+
beyond 0.2.1.
|
|
19
|
+
|
|
20
|
+
## [0.2.1] - 2026-07-16
|
|
21
|
+
|
|
22
|
+
### Changed
|
|
23
|
+
|
|
24
|
+
- `BYTEKIT_API_KEY` / `BYTEKIT_BASE_URL` are now the primary environment
|
|
25
|
+
variables (precedence: flag > `BYTEKIT_*` > legacy `RAPIDCRAWL_*`).
|
|
26
|
+
`RAPIDCRAWL_API_KEY` / `RAPIDCRAWL_BASE_URL` continue to work as silent
|
|
27
|
+
fallbacks. `--help` text, the missing-key error, and the README now reference
|
|
28
|
+
`BYTEKIT_*`. (#2421)
|
|
29
|
+
|
|
30
|
+
### Removed
|
|
31
|
+
|
|
32
|
+
- The `recordings` command (the endpoint is permanently `410` deactivated). (#2421)
|
|
33
|
+
|
|
34
|
+
## [0.2.0] - 2026-07-15
|
|
35
|
+
|
|
36
|
+
Additive, lower-risk release — no breaking changes to existing commands.
|
|
37
|
+
|
|
38
|
+
### Added
|
|
39
|
+
|
|
40
|
+
- `scrape create --raw` / `-o, --output <file>`: write the raw response body to
|
|
41
|
+
stdout or a file (#2425).
|
|
42
|
+
- Local `search --limit` validation via a commander `argParser`, so an invalid limit
|
|
43
|
+
is rejected before a request is sent (#2425).
|
|
44
|
+
- One automatic retry on HTTP 429, honoring a clamped `Retry-After` header, in the
|
|
45
|
+
CLI transport (#2425).
|
|
46
|
+
|
|
47
|
+
### Fixed
|
|
48
|
+
|
|
49
|
+
- Entrypoint guard canonicalized so the `bytekit` bin runs correctly when invoked
|
|
50
|
+
through an npm/npx symlink (#2418).
|
|
51
|
+
|
|
52
|
+
### Changed
|
|
53
|
+
|
|
54
|
+
- Package renamed to `@hunt-labs/bytekit-cli` (the `@bytekit` npm scope was
|
|
55
|
+
unobtainable); the `bin` name stays `bytekit`. The prior `rapidcrawl-cli` name
|
|
56
|
+
remains installable at its last version but is deprecated.
|
|
57
|
+
- Consumer call sites propagate the SDK's narrowed, spec-derived request types (#2426).
|
|
58
|
+
- Publish-ready packaging metadata: dropped `main`, unpinned the SDK dependency to
|
|
59
|
+
`workspace:^`, added `engines`, `keywords`, `homepage`, `bugs`, `README.md`,
|
|
60
|
+
`LICENSE`, and an MIT `license` field.
|
|
61
|
+
|
|
62
|
+
## [0.1.0] - 2026
|
|
63
|
+
|
|
64
|
+
### Added
|
|
65
|
+
|
|
66
|
+
- Initial release of the ByteKit CLI, wrapping the ByteKit SDK as shell
|
|
67
|
+
subcommands, published under the original `rapidcrawl-cli` name.
|
|
68
|
+
|
|
69
|
+
[0.2.2]: https://github.com/Hunt-Labs-Inc/ByteKit/tree/main/packages/cli
|
|
70
|
+
[0.2.1]: https://github.com/Hunt-Labs-Inc/ByteKit/tree/main/packages/cli
|
|
71
|
+
[0.2.0]: https://github.com/Hunt-Labs-Inc/ByteKit/tree/main/packages/cli
|
|
72
|
+
[0.1.0]: https://github.com/Hunt-Labs-Inc/ByteKit/tree/main/packages/cli
|
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Hunt Labs, Inc.
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
# @hunt-labs/bytekit-cli
|
|
2
|
+
|
|
3
|
+
Command-line interface for the [ByteKit](https://bytekit.com) API — scrape,
|
|
4
|
+
fetch, screenshot, search, crawl sitemaps, and manage page monitors from your
|
|
5
|
+
shell. Wraps [`@hunt-labs/bytekit-sdk`](https://www.npmjs.com/package/@hunt-labs/bytekit-sdk).
|
|
6
|
+
|
|
7
|
+
## Install
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
npm install -g @hunt-labs/bytekit-cli
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
This installs the `bytekit` binary on your `PATH`.
|
|
14
|
+
|
|
15
|
+
To run it without installing:
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
npx @hunt-labs/bytekit-cli scrape create --url https://example.com
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
> **Always use the fully-qualified `@hunt-labs/bytekit-cli` name.** The bare
|
|
22
|
+
> package name `bytekit` on the public npm registry belongs to an unrelated
|
|
23
|
+
> third-party project — running it by that short name fetches someone else's
|
|
24
|
+
> code, not this CLI. The short `bytekit` name is only ever the _binary_ you get
|
|
25
|
+
> after installing `@hunt-labs/bytekit-cli`.
|
|
26
|
+
|
|
27
|
+
Requires Node.js 18+.
|
|
28
|
+
|
|
29
|
+
## Authentication
|
|
30
|
+
|
|
31
|
+
Create an API key at [app.bytekit.com](https://app.bytekit.com) (keys are
|
|
32
|
+
prefixed `sk_live_`), then either export it:
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
export BYTEKIT_API_KEY=sk_live_xxx
|
|
36
|
+
bytekit scrape create --url https://example.com
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
or pass it per-invocation with the global `--key` flag (which takes precedence
|
|
40
|
+
over the environment variable):
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
bytekit --key sk_live_xxx scrape create --url https://example.com
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Precedence is `--key` flag → `BYTEKIT_API_KEY` → `RAPIDCRAWL_API_KEY` (legacy
|
|
47
|
+
fallback, deprecated — kept for existing scripts, will eventually be removed).
|
|
48
|
+
|
|
49
|
+
Without a key, the CLI exits with `Error: API key required. Pass --key or set BYTEKIT_API_KEY.`
|
|
50
|
+
|
|
51
|
+
## Base-URL override
|
|
52
|
+
|
|
53
|
+
Requests go to `https://api.bytekit.com` by default. Point the CLI at staging, a
|
|
54
|
+
local gateway, or a proxy with the global `--base-url` flag or the
|
|
55
|
+
`BYTEKIT_BASE_URL` environment variable (the flag wins; `RAPIDCRAWL_BASE_URL` is
|
|
56
|
+
a deprecated legacy fallback):
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
bytekit --base-url https://api-stg.bytekit.com scrape create --url https://example.com
|
|
60
|
+
export BYTEKIT_BASE_URL=http://localhost:3001
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
## Global options
|
|
64
|
+
|
|
65
|
+
| Flag | Env var | Purpose |
|
|
66
|
+
| ------------------ | ----------------------------------------------------------- | -------------------------------------------------------------------- |
|
|
67
|
+
| `--key <key>` | `BYTEKIT_API_KEY` (legacy fallback: `RAPIDCRAWL_API_KEY`) | API key used for every request. |
|
|
68
|
+
| `--base-url <url>` | `BYTEKIT_BASE_URL` (legacy fallback: `RAPIDCRAWL_BASE_URL`) | API base URL (default `https://api.bytekit.com`). |
|
|
69
|
+
| `--json` | — | Print the raw JSON response instead of the human-readable rendering. |
|
|
70
|
+
|
|
71
|
+
## Commands
|
|
72
|
+
|
|
73
|
+
### `scrape` — scrape URL content
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
bytekit scrape create --url https://example.com --format markdown
|
|
77
|
+
bytekit scrape get sc_123abc
|
|
78
|
+
bytekit scrape bulk create --file urls.txt --webhook-url https://your.app/hooks/bytekit
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
### `fetch` — raw, unprocessed response bodies
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
bytekit fetch create --url https://example.com
|
|
85
|
+
bytekit fetch bulk create --file urls.txt --webhook-url https://your.app/hooks/bytekit
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
### `search` — ranked web search
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
bytekit search "ai web scraping" --limit 10 --country us
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
### `screenshots` — capture a page
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
bytekit screenshots create --url https://example.com --full-page
|
|
98
|
+
bytekit screenshots get ss_123abc
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
### `bulk` — bulk screenshot jobs
|
|
102
|
+
|
|
103
|
+
```bash
|
|
104
|
+
bytekit bulk create --file urls.txt --webhook-url https://your.app/hooks/bytekit
|
|
105
|
+
bulk_id=bulk_123abc
|
|
106
|
+
bytekit bulk get "$bulk_id"
|
|
107
|
+
bytekit bulk screenshots list "$bulk_id"
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
`urls.txt` is newline-delimited: one plain URL (or `{"url":"https://..."}`) per line.
|
|
111
|
+
|
|
112
|
+
### `monitors` — watch a page for changes
|
|
113
|
+
|
|
114
|
+
```bash
|
|
115
|
+
bytekit monitors create --url https://example.com/pricing --interval-type daily \
|
|
116
|
+
--webhook-url https://your.app/hooks/bytekit
|
|
117
|
+
bytekit monitors list
|
|
118
|
+
bytekit monitors captures list mon_123abc
|
|
119
|
+
bytekit monitors delete mon_123abc
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
### `sitemap` — crawl a site's URL graph
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
bytekit sitemap create --url https://example.com
|
|
126
|
+
bytekit sitemap get sm_123abc
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
### `account` — account details and API keys
|
|
130
|
+
|
|
131
|
+
```bash
|
|
132
|
+
bytekit account get
|
|
133
|
+
bytekit account api-keys list
|
|
134
|
+
bytekit account api-keys create --name "ci-key"
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
## Documentation
|
|
138
|
+
|
|
139
|
+
Full API reference: [bytekit.com/docs](https://bytekit.com/docs).
|
|
140
|
+
|
|
141
|
+
## License
|
|
142
|
+
|
|
143
|
+
MIT — see [LICENSE](./LICENSE).
|
package/dist/client.d.ts
CHANGED
|
@@ -3,11 +3,57 @@ export interface GlobalOptions {
|
|
|
3
3
|
key?: string | undefined;
|
|
4
4
|
baseUrl?: string | undefined;
|
|
5
5
|
}
|
|
6
|
+
/**
|
|
7
|
+
* The API host the SDK itself defaults to. The SDK's copy is module-private, so the CLI declares
|
|
8
|
+
* its own and always passes an explicit `baseUrl` to the constructor — that way the SDK client and
|
|
9
|
+
* the CLI-owned transport below can never disagree about where a request is going. A test pins
|
|
10
|
+
* this constant so the two cannot drift silently.
|
|
11
|
+
*/
|
|
12
|
+
export declare const DEFAULT_BASE_URL = "https://api.bytekit.com";
|
|
13
|
+
/** Backoff used when a 429 arrives with no usable `Retry-After` header. */
|
|
14
|
+
export declare const DEFAULT_RETRY_AFTER_MS = 1000;
|
|
15
|
+
/**
|
|
16
|
+
* Floor for the honored backoff. The gateway sends `Retry-After: resetEpoch - now`, which on a
|
|
17
|
+
* 1-rps free-plan key is routinely `0` — and can be negative under clock skew. Sleeping 0 ms would
|
|
18
|
+
* burn the single retry against a limiter that has not reset yet.
|
|
19
|
+
*/
|
|
20
|
+
export declare const MIN_RETRY_AFTER_MS = 1000;
|
|
21
|
+
/** Ceiling for the honored backoff, so a hostile or absurd header cannot hang the CLI. */
|
|
22
|
+
export declare const MAX_RETRY_AFTER_MS = 60000;
|
|
23
|
+
/** Exactly one retry. A second 429 is terminal — no loop. */
|
|
24
|
+
export declare const MAX_RETRIES = 1;
|
|
25
|
+
/**
|
|
26
|
+
* Parse a `Retry-After` header into a clamped millisecond delay.
|
|
27
|
+
*
|
|
28
|
+
* Only the delta-seconds form is honored (the gateway never sends the HTTP-date form). Anything
|
|
29
|
+
* absent, empty, non-integer or non-numeric falls back to {@link DEFAULT_RETRY_AFTER_MS}; a valid
|
|
30
|
+
* value is clamped into `[MIN_RETRY_AFTER_MS, MAX_RETRY_AFTER_MS]`.
|
|
31
|
+
*/
|
|
32
|
+
export declare function parseRetryAfterMs(raw: string | null): number;
|
|
33
|
+
/**
|
|
34
|
+
* Build the CLI's HTTP transport: the SDK's `request` contract, plus a single bounded retry on 429.
|
|
35
|
+
*
|
|
36
|
+
* Why the CLI owns the `fetch` instead of wrapping the SDK's `request()`: the SDK throws a
|
|
37
|
+
* header-less `RapidCrawlError` and discards the `Response`, so a delegating wrapper can see
|
|
38
|
+
* `status === 429` but can never read `Retry-After`. Owning the fetch is the only way to honor the
|
|
39
|
+
* server's advertised backoff without editing the SDK.
|
|
40
|
+
*
|
|
41
|
+
* Every SDK resource dereferences `this.client.request` at call time, so installing this as an own
|
|
42
|
+
* property of the client (see {@link buildClient}) routes *all* commands through it from one seam.
|
|
43
|
+
*
|
|
44
|
+
* Retries apply to 429 only. Non-429 statuses and network rejections propagate untouched.
|
|
45
|
+
*/
|
|
46
|
+
export declare function createRetryingRequest(apiKey: string, baseUrl: string): RapidCrawl['request'];
|
|
6
47
|
/**
|
|
7
48
|
* Build an authenticated SDK client from global CLI options.
|
|
8
49
|
*
|
|
9
|
-
* Resolution order
|
|
10
|
-
*
|
|
50
|
+
* Resolution order (explicit input beats environment; new brand beats legacy brand):
|
|
51
|
+
* apiKey: --key flag → BYTEKIT_API_KEY → RAPIDCRAWL_API_KEY (legacy fallback).
|
|
52
|
+
* baseUrl: --base-url flag → BYTEKIT_BASE_URL → RAPIDCRAWL_BASE_URL (legacy) → DEFAULT_BASE_URL.
|
|
53
|
+
* The legacy RAPIDCRAWL_* vars stay silently honored so existing setups keep working; exits with a
|
|
54
|
+
* descriptive error when no key is available.
|
|
55
|
+
*
|
|
56
|
+
* The returned client's `request` is replaced by the retrying transport, so every command inherits
|
|
57
|
+
* the 429 retry from this one seam.
|
|
11
58
|
*/
|
|
12
59
|
export declare function buildClient(opts: GlobalOptions): RapidCrawl;
|
|
13
|
-
//# sourceMappingURL=client.d.ts.map
|
package/dist/client.js
CHANGED
|
@@ -1,20 +1,127 @@
|
|
|
1
|
-
import { RapidCrawl } from '@hunt-labs/bytekit-sdk';
|
|
1
|
+
import { RapidCrawl, RapidCrawlError } from '@hunt-labs/bytekit-sdk';
|
|
2
|
+
/**
|
|
3
|
+
* The API host the SDK itself defaults to. The SDK's copy is module-private, so the CLI declares
|
|
4
|
+
* its own and always passes an explicit `baseUrl` to the constructor — that way the SDK client and
|
|
5
|
+
* the CLI-owned transport below can never disagree about where a request is going. A test pins
|
|
6
|
+
* this constant so the two cannot drift silently.
|
|
7
|
+
*/
|
|
8
|
+
export const DEFAULT_BASE_URL = 'https://api.bytekit.com';
|
|
9
|
+
/** Backoff used when a 429 arrives with no usable `Retry-After` header. */
|
|
10
|
+
export const DEFAULT_RETRY_AFTER_MS = 1_000;
|
|
11
|
+
/**
|
|
12
|
+
* Floor for the honored backoff. The gateway sends `Retry-After: resetEpoch - now`, which on a
|
|
13
|
+
* 1-rps free-plan key is routinely `0` — and can be negative under clock skew. Sleeping 0 ms would
|
|
14
|
+
* burn the single retry against a limiter that has not reset yet.
|
|
15
|
+
*/
|
|
16
|
+
export const MIN_RETRY_AFTER_MS = 1_000;
|
|
17
|
+
/** Ceiling for the honored backoff, so a hostile or absurd header cannot hang the CLI. */
|
|
18
|
+
export const MAX_RETRY_AFTER_MS = 60_000;
|
|
19
|
+
/** Exactly one retry. A second 429 is terminal — no loop. */
|
|
20
|
+
export const MAX_RETRIES = 1;
|
|
21
|
+
/**
|
|
22
|
+
* Parse a `Retry-After` header into a clamped millisecond delay.
|
|
23
|
+
*
|
|
24
|
+
* Only the delta-seconds form is honored (the gateway never sends the HTTP-date form). Anything
|
|
25
|
+
* absent, empty, non-integer or non-numeric falls back to {@link DEFAULT_RETRY_AFTER_MS}; a valid
|
|
26
|
+
* value is clamped into `[MIN_RETRY_AFTER_MS, MAX_RETRY_AFTER_MS]`.
|
|
27
|
+
*/
|
|
28
|
+
export function parseRetryAfterMs(raw) {
|
|
29
|
+
if (raw === null)
|
|
30
|
+
return DEFAULT_RETRY_AFTER_MS;
|
|
31
|
+
const trimmed = raw.trim();
|
|
32
|
+
if (!/^-?\d+$/.test(trimmed))
|
|
33
|
+
return DEFAULT_RETRY_AFTER_MS;
|
|
34
|
+
const seconds = Number(trimmed);
|
|
35
|
+
if (!Number.isFinite(seconds))
|
|
36
|
+
return DEFAULT_RETRY_AFTER_MS;
|
|
37
|
+
return Math.min(Math.max(seconds * 1000, MIN_RETRY_AFTER_MS), MAX_RETRY_AFTER_MS);
|
|
38
|
+
}
|
|
39
|
+
function sleep(ms) {
|
|
40
|
+
return new Promise((resolve) => {
|
|
41
|
+
setTimeout(resolve, ms);
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Shape a failed response into the SDK's OWN error class, using the SDK's own body parsing.
|
|
46
|
+
*
|
|
47
|
+
* Reusing `RapidCrawlError` keeps `{status, code, message}` identical to what a direct SDK call
|
|
48
|
+
* would throw, so `handleError()`'s `'status' in err` branch still renders
|
|
49
|
+
* `Error: <message> (status: <n>)` and exits 1.
|
|
50
|
+
*/
|
|
51
|
+
async function toApiError(response) {
|
|
52
|
+
let code = 'internal_error';
|
|
53
|
+
let message = response.statusText;
|
|
54
|
+
try {
|
|
55
|
+
const json = (await response.json());
|
|
56
|
+
code = json.error?.code ?? code;
|
|
57
|
+
message = json.error?.message ?? message;
|
|
58
|
+
}
|
|
59
|
+
catch {
|
|
60
|
+
// leave defaults
|
|
61
|
+
}
|
|
62
|
+
return new RapidCrawlError(response.status, code, message);
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Build the CLI's HTTP transport: the SDK's `request` contract, plus a single bounded retry on 429.
|
|
66
|
+
*
|
|
67
|
+
* Why the CLI owns the `fetch` instead of wrapping the SDK's `request()`: the SDK throws a
|
|
68
|
+
* header-less `RapidCrawlError` and discards the `Response`, so a delegating wrapper can see
|
|
69
|
+
* `status === 429` but can never read `Retry-After`. Owning the fetch is the only way to honor the
|
|
70
|
+
* server's advertised backoff without editing the SDK.
|
|
71
|
+
*
|
|
72
|
+
* Every SDK resource dereferences `this.client.request` at call time, so installing this as an own
|
|
73
|
+
* property of the client (see {@link buildClient}) routes *all* commands through it from one seam.
|
|
74
|
+
*
|
|
75
|
+
* Retries apply to 429 only. Non-429 statuses and network rejections propagate untouched.
|
|
76
|
+
*/
|
|
77
|
+
export function createRetryingRequest(apiKey, baseUrl) {
|
|
78
|
+
return async function request(method, path, body) {
|
|
79
|
+
for (let attempt = 0;; attempt++) {
|
|
80
|
+
const init = {
|
|
81
|
+
method,
|
|
82
|
+
headers: {
|
|
83
|
+
Authorization: `Bearer ${apiKey}`,
|
|
84
|
+
'Content-Type': 'application/json',
|
|
85
|
+
},
|
|
86
|
+
};
|
|
87
|
+
if (body !== undefined) {
|
|
88
|
+
init.body = JSON.stringify(body);
|
|
89
|
+
}
|
|
90
|
+
const response = await fetch(`${baseUrl}${path}`, init);
|
|
91
|
+
if (response.ok) {
|
|
92
|
+
return (await response.json());
|
|
93
|
+
}
|
|
94
|
+
if (response.status === 429 && attempt < MAX_RETRIES) {
|
|
95
|
+
await sleep(parseRetryAfterMs(response.headers.get('Retry-After')));
|
|
96
|
+
continue;
|
|
97
|
+
}
|
|
98
|
+
throw await toApiError(response);
|
|
99
|
+
}
|
|
100
|
+
};
|
|
101
|
+
}
|
|
2
102
|
/**
|
|
3
103
|
* Build an authenticated SDK client from global CLI options.
|
|
4
104
|
*
|
|
5
|
-
* Resolution order
|
|
6
|
-
*
|
|
105
|
+
* Resolution order (explicit input beats environment; new brand beats legacy brand):
|
|
106
|
+
* apiKey: --key flag → BYTEKIT_API_KEY → RAPIDCRAWL_API_KEY (legacy fallback).
|
|
107
|
+
* baseUrl: --base-url flag → BYTEKIT_BASE_URL → RAPIDCRAWL_BASE_URL (legacy) → DEFAULT_BASE_URL.
|
|
108
|
+
* The legacy RAPIDCRAWL_* vars stay silently honored so existing setups keep working; exits with a
|
|
109
|
+
* descriptive error when no key is available.
|
|
110
|
+
*
|
|
111
|
+
* The returned client's `request` is replaced by the retrying transport, so every command inherits
|
|
112
|
+
* the 429 retry from this one seam.
|
|
7
113
|
*/
|
|
8
114
|
export function buildClient(opts) {
|
|
9
|
-
const apiKey = opts.key ?? process.env['RAPIDCRAWL_API_KEY'];
|
|
115
|
+
const apiKey = opts.key ?? process.env['BYTEKIT_API_KEY'] ?? process.env['RAPIDCRAWL_API_KEY'];
|
|
10
116
|
if (!apiKey) {
|
|
11
|
-
console.error('Error: API key required. Pass --key or set
|
|
117
|
+
console.error('Error: API key required. Pass --key or set BYTEKIT_API_KEY.');
|
|
12
118
|
process.exit(1);
|
|
13
119
|
}
|
|
14
|
-
const baseUrl = opts.baseUrl ??
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
});
|
|
120
|
+
const baseUrl = opts.baseUrl ??
|
|
121
|
+
process.env['BYTEKIT_BASE_URL'] ??
|
|
122
|
+
process.env['RAPIDCRAWL_BASE_URL'] ??
|
|
123
|
+
DEFAULT_BASE_URL;
|
|
124
|
+
const client = new RapidCrawl({ apiKey, baseUrl });
|
|
125
|
+
client.request = createRetryingRequest(apiKey, baseUrl);
|
|
126
|
+
return client;
|
|
19
127
|
}
|
|
20
|
-
//# sourceMappingURL=client.js.map
|
package/dist/commands/account.js
CHANGED
|
@@ -12,7 +12,10 @@ export function makeAccountCommand(getGlobal) {
|
|
|
12
12
|
const g = getGlobal();
|
|
13
13
|
const client = buildClient(g);
|
|
14
14
|
try {
|
|
15
|
-
|
|
15
|
+
// The API's field is `label`, not `name` — sending `{ name }` created an
|
|
16
|
+
// UNLABELLED key, because the gateway ignored the unknown field (#2426:
|
|
17
|
+
// the SDK's newly derived request type is what surfaced this).
|
|
18
|
+
const result = await client.account.apiKeys.create({ label: opts.name });
|
|
16
19
|
printResult(result, g.json);
|
|
17
20
|
}
|
|
18
21
|
catch (err) {
|
|
@@ -55,7 +58,9 @@ export function makeAccountCommand(getGlobal) {
|
|
|
55
58
|
const g = getGlobal();
|
|
56
59
|
const client = buildClient(g);
|
|
57
60
|
try {
|
|
58
|
-
|
|
61
|
+
// As above: the API's field is `label`, so `--name` never actually
|
|
62
|
+
// renamed the key.
|
|
63
|
+
const result = await client.account.apiKeys.update(id, { label: opts.name });
|
|
59
64
|
printResult(result, g.json);
|
|
60
65
|
}
|
|
61
66
|
catch (err) {
|
|
@@ -93,4 +98,3 @@ export function makeAccountCommand(getGlobal) {
|
|
|
93
98
|
});
|
|
94
99
|
return cmd;
|
|
95
100
|
}
|
|
96
|
-
//# sourceMappingURL=account.js.map
|
package/dist/commands/bulk.d.ts
CHANGED
package/dist/commands/bulk.js
CHANGED
package/dist/commands/fetch.d.ts
CHANGED
package/dist/commands/fetch.js
CHANGED
package/dist/commands/scrape.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Command } from 'commander';
|
|
2
2
|
import { buildClient } from '../client.js';
|
|
3
|
-
import { printResult, handleError } from '../output.js';
|
|
3
|
+
import { printResult, handleError, resolveOutputMode, emitResult } from '../output.js';
|
|
4
4
|
import { parseUrlLines } from '../lib/url-lines.js';
|
|
5
5
|
export function makeScrapeCommand(getGlobal) {
|
|
6
6
|
const cmd = new Command('scrape').description('Scrape URL content');
|
|
@@ -43,6 +43,8 @@ export function makeScrapeCommand(getGlobal) {
|
|
|
43
43
|
.description('Create a scrape job')
|
|
44
44
|
.requiredOption('--url <url>', 'URL to scrape')
|
|
45
45
|
.option('--format <format>', 'Output format (e.g. markdown, html); sent as the formats array')
|
|
46
|
+
.option('--raw', 'Print the requested format body raw, with no JSON wrapper')
|
|
47
|
+
.option('-o, --output <file>', 'Write the output to the given file path (implies --raw unless --json is set)')
|
|
46
48
|
.action(async (opts) => {
|
|
47
49
|
const g = getGlobal();
|
|
48
50
|
const client = buildClient(g);
|
|
@@ -53,8 +55,15 @@ export function makeScrapeCommand(getGlobal) {
|
|
|
53
55
|
if (opts.format !== undefined)
|
|
54
56
|
body['formats'] = [opts.format];
|
|
55
57
|
try {
|
|
58
|
+
// FIRST statement inside the try — deliberately NOT hoisted next to buildClient() where
|
|
59
|
+
// the other handlers put pre-request logic. index.ts drives commander with .parse(), not
|
|
60
|
+
// .parseAsync(), so nothing awaits this action's promise: a throw raised above the try
|
|
61
|
+
// would escape as an unhandled rejection and print a RAW STACK TRACE instead of
|
|
62
|
+
// handleError()'s clean one-liner. buildClient() does no network I/O, so rejecting the
|
|
63
|
+
// --json/--raw conflict here still happens before any request is issued.
|
|
64
|
+
const mode = resolveOutputMode({ json: g.json, raw: opts.raw, output: opts.output });
|
|
56
65
|
const result = await client.scrape.create(body);
|
|
57
|
-
|
|
66
|
+
emitResult(result, mode, { file: opts.output }, opts.format);
|
|
58
67
|
}
|
|
59
68
|
catch (err) {
|
|
60
69
|
handleError(err);
|
|
@@ -76,4 +85,3 @@ export function makeScrapeCommand(getGlobal) {
|
|
|
76
85
|
});
|
|
77
86
|
return cmd;
|
|
78
87
|
}
|
|
79
|
-
//# sourceMappingURL=scrape.js.map
|
|
@@ -13,7 +13,7 @@ export function makeScreenshotsCommand(getGlobal) {
|
|
|
13
13
|
const client = buildClient(g);
|
|
14
14
|
const body = { url: opts.url };
|
|
15
15
|
if (opts.fullPage === true)
|
|
16
|
-
body
|
|
16
|
+
body.full_page = true;
|
|
17
17
|
try {
|
|
18
18
|
const result = await client.screenshots.create(body);
|
|
19
19
|
printResult(result, g.json);
|
|
@@ -38,4 +38,3 @@ export function makeScreenshotsCommand(getGlobal) {
|
|
|
38
38
|
});
|
|
39
39
|
return cmd;
|
|
40
40
|
}
|
|
41
|
-
//# sourceMappingURL=screenshots.js.map
|
|
@@ -1,6 +1,15 @@
|
|
|
1
1
|
import { Command } from 'commander';
|
|
2
2
|
import { type GlobalOptions } from '../client.js';
|
|
3
|
+
/**
|
|
4
|
+
* Validate `--limit` locally, before any HTTP request is issued.
|
|
5
|
+
*
|
|
6
|
+
* Commander runs an option's argParser during parsing — ahead of the action callback — so a
|
|
7
|
+
* rejection here means the client is never built and no request body is constructed. The strict
|
|
8
|
+
* `/^\d+$/` test (rather than Number.isInteger alone) deterministically rejects `abc`, ``, `1.5`,
|
|
9
|
+
* `-5`, `1e2`, `0x10`, `+1` and `Infinity`, all of which `Number()` would otherwise coerce into a
|
|
10
|
+
* NaN (→ `limit: null` on the wire) or a silently-wrong value.
|
|
11
|
+
*/
|
|
12
|
+
export declare function parseLimit(value: string): number;
|
|
3
13
|
export declare function makeSearchCommand(getGlobal: () => GlobalOptions & {
|
|
4
14
|
json: boolean;
|
|
5
15
|
}): Command;
|
|
6
|
-
//# sourceMappingURL=search.d.ts.map
|
package/dist/commands/search.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { Command } from 'commander';
|
|
1
|
+
import { Command, InvalidArgumentError } from 'commander';
|
|
2
2
|
import { buildClient } from '../client.js';
|
|
3
3
|
import { printResult, handleError } from '../output.js';
|
|
4
4
|
/**
|
|
@@ -14,12 +14,32 @@ function renderTable(response) {
|
|
|
14
14
|
console.log(` ${result.snippet}`);
|
|
15
15
|
}
|
|
16
16
|
}
|
|
17
|
+
const LIMIT_MIN = 1;
|
|
18
|
+
const LIMIT_MAX = 100;
|
|
19
|
+
/**
|
|
20
|
+
* Validate `--limit` locally, before any HTTP request is issued.
|
|
21
|
+
*
|
|
22
|
+
* Commander runs an option's argParser during parsing — ahead of the action callback — so a
|
|
23
|
+
* rejection here means the client is never built and no request body is constructed. The strict
|
|
24
|
+
* `/^\d+$/` test (rather than Number.isInteger alone) deterministically rejects `abc`, ``, `1.5`,
|
|
25
|
+
* `-5`, `1e2`, `0x10`, `+1` and `Infinity`, all of which `Number()` would otherwise coerce into a
|
|
26
|
+
* NaN (→ `limit: null` on the wire) or a silently-wrong value.
|
|
27
|
+
*/
|
|
28
|
+
export function parseLimit(value) {
|
|
29
|
+
const invalid = new InvalidArgumentError(`--limit must be a whole number between ${LIMIT_MIN} and ${LIMIT_MAX}.`);
|
|
30
|
+
if (!/^\d+$/.test(value))
|
|
31
|
+
throw invalid;
|
|
32
|
+
const n = Number(value);
|
|
33
|
+
if (n < LIMIT_MIN || n > LIMIT_MAX)
|
|
34
|
+
throw invalid;
|
|
35
|
+
return n;
|
|
36
|
+
}
|
|
17
37
|
export function makeSearchCommand(getGlobal) {
|
|
18
38
|
return new Command('search')
|
|
19
39
|
.description('Run a web search and return ranked results')
|
|
20
40
|
.argument('<query>', 'Search query')
|
|
21
41
|
.option('--type <type>', 'Search index to query: web (default), news, or images')
|
|
22
|
-
.option('--limit <n>', 'Maximum number of results to return (1–100)')
|
|
42
|
+
.option('--limit <n>', 'Maximum number of results to return (1–100)', parseLimit)
|
|
23
43
|
.option('--country <code>', 'Two-letter country code to localize results (e.g. us, de)')
|
|
24
44
|
.option('--language <code>', 'Two-letter language code for results (e.g. en, fr)')
|
|
25
45
|
.option('--date-range <window>', 'Restrict results to a recency window: any (default), hour, day, week, month, or year')
|
|
@@ -34,7 +54,7 @@ export function makeSearchCommand(getGlobal) {
|
|
|
34
54
|
if (opts.type !== undefined)
|
|
35
55
|
body.type = opts.type;
|
|
36
56
|
if (opts.limit !== undefined)
|
|
37
|
-
body.limit =
|
|
57
|
+
body.limit = opts.limit;
|
|
38
58
|
if (opts.country !== undefined)
|
|
39
59
|
body.country = opts.country;
|
|
40
60
|
if (opts.language !== undefined)
|
|
@@ -55,4 +75,3 @@ export function makeSearchCommand(getGlobal) {
|
|
|
55
75
|
}
|
|
56
76
|
});
|
|
57
77
|
}
|
|
58
|
-
//# sourceMappingURL=search.js.map
|
package/dist/commands/sitemap.js
CHANGED