@hunt-labs/bytekit-cli 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +51 -0
- package/README.md +53 -11
- package/dist/client.d.ts +19 -9
- package/dist/client.js +140 -18
- package/dist/commands/account.js +49 -0
- package/dist/commands/fetch.d.ts +14 -0
- package/dist/commands/fetch.js +23 -6
- package/dist/commands/monitors.js +26 -3
- package/dist/commands/scrape.js +88 -0
- package/dist/commands/screenshots.d.ts +12 -0
- package/dist/commands/screenshots.js +105 -6
- package/dist/commands/search.js +5 -1
- package/dist/commands/sitemap.js +8 -1
- package/dist/lib/download-artifact.d.ts +13 -0
- package/dist/lib/download-artifact.js +25 -0
- package/dist/lib/fetch-body.d.ts +18 -0
- package/dist/lib/fetch-body.js +6 -0
- package/dist/lib/parse-flags.d.ts +16 -0
- package/dist/lib/parse-flags.js +32 -0
- package/dist/output.d.ts +50 -0
- package/dist/output.js +99 -5
- package/package.json +2 -2
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,56 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
8
8
|
This file records only the delta between versions. For how to use the package
|
|
9
9
|
today, see [README.md](./README.md), which remains the authoritative usage guide.
|
|
10
10
|
|
|
11
|
+
## [0.3.0] - 2026-07-21
|
|
12
|
+
|
|
13
|
+
### Removed (**BREAKING**)
|
|
14
|
+
|
|
15
|
+
- The legacy `RAPIDCRAWL_API_KEY` and `RAPIDCRAWL_BASE_URL` environment-variable
|
|
16
|
+
fallbacks (kept as a one-minor-version bridge in #2421) are removed. Use
|
|
17
|
+
`BYTEKIT_API_KEY` and `BYTEKIT_BASE_URL` (or the `--key` / `--base-url` flags).
|
|
18
|
+
(#2473)
|
|
19
|
+
|
|
20
|
+
### Fixed
|
|
21
|
+
|
|
22
|
+
- `bytekit fetch get` / `fetch create` no longer crash on non-JSON bodies. The
|
|
23
|
+
transport JSON-parsed every 2xx response, but `/v1/fetch` returns the fetched
|
|
24
|
+
content AS the body. The fetch command now prints the raw body by default, or
|
|
25
|
+
the `X-Fetch-*` metadata as JSON with `--json`; `fetch bulk` and every other
|
|
26
|
+
command still parse JSON. (#2467)
|
|
27
|
+
- `204 No Content` / empty-body responses (e.g. `monitors delete`,
|
|
28
|
+
`account api-keys revoke`) no longer throw a JSON-parse error, including after
|
|
29
|
+
a 429 retry. (#2467, #2468)
|
|
30
|
+
- `search --type images` no longer prints an `undefined` snippet line for
|
|
31
|
+
results that omit the field. (#2479)
|
|
32
|
+
|
|
33
|
+
### Added
|
|
34
|
+
|
|
35
|
+
- `scrape create` accepts the full `ScrapeRequest` surface: `--country`,
|
|
36
|
+
`--timeout-ms`, `--markdown-mode`, `--async`, `--webhook-url`,
|
|
37
|
+
`--token-budget`, `--clean-markdown`, `--include-tags`, `--exclude-tags`,
|
|
38
|
+
`--mobile`. Each flag is omitted from the request when absent. (#2480)
|
|
39
|
+
- `screenshots create` accepts `--device`, `--viewport WxH`, `--format`,
|
|
40
|
+
`--quality`, `--wait-until`, `--clip x,y,w,h`, and `--async`; `-o <file>`
|
|
41
|
+
on `screenshots create`/`get` downloads the captured image bytes (`-o -` for
|
|
42
|
+
stdout). `-o` cannot be combined with `--json` on screenshots. (#2480)
|
|
43
|
+
- `monitors list` accepts `--limit`, `--cursor`, and `--status` query
|
|
44
|
+
filters. (#2480)
|
|
45
|
+
- `account api-keys reveal <ak_id>`: prints the key masked by default
|
|
46
|
+
(`sk_live_…2345`); pass `--show` for the full plaintext. (#2480)
|
|
47
|
+
- Per-attempt transport timeout (120s, mirroring the SDK default): a hung
|
|
48
|
+
request now fails with `code: 'timeout'` instead of hanging forever. Network
|
|
49
|
+
failures are mapped onto the SDK's connection-error taxonomy
|
|
50
|
+
(`timeout` / `aborted` / `connection`), and error output renders the API or
|
|
51
|
+
connection error code. (#2479)
|
|
52
|
+
- A queued async job (HTTP 202) under `--raw` prints the job envelope with its
|
|
53
|
+
`sc_` id instead of an empty body. (#2479)
|
|
54
|
+
|
|
55
|
+
### Changed (**BREAKING**)
|
|
56
|
+
|
|
57
|
+
- `fetch --format` is validated locally against the spec union
|
|
58
|
+
`'markdown' | 'html'`; off-spec values are rejected with a clear error before
|
|
59
|
+
a request is sent (previously forwarded as-is). (#2475)
|
|
60
|
+
|
|
11
61
|
## [0.2.2] - 2026-07-16
|
|
12
62
|
|
|
13
63
|
### Changed
|
|
@@ -66,6 +116,7 @@ Additive, lower-risk release — no breaking changes to existing commands.
|
|
|
66
116
|
- Initial release of the ByteKit CLI, wrapping the ByteKit SDK as shell
|
|
67
117
|
subcommands, published under the original `rapidcrawl-cli` name.
|
|
68
118
|
|
|
119
|
+
[0.3.0]: https://github.com/Hunt-Labs-Inc/ByteKit/tree/main/packages/cli
|
|
69
120
|
[0.2.2]: https://github.com/Hunt-Labs-Inc/ByteKit/tree/main/packages/cli
|
|
70
121
|
[0.2.1]: https://github.com/Hunt-Labs-Inc/ByteKit/tree/main/packages/cli
|
|
71
122
|
[0.2.0]: https://github.com/Hunt-Labs-Inc/ByteKit/tree/main/packages/cli
|
package/README.md
CHANGED
|
@@ -43,8 +43,7 @@ over the environment variable):
|
|
|
43
43
|
bytekit --key sk_live_xxx scrape create --url https://example.com
|
|
44
44
|
```
|
|
45
45
|
|
|
46
|
-
Precedence is `--key` flag → `BYTEKIT_API_KEY
|
|
47
|
-
fallback, deprecated — kept for existing scripts, will eventually be removed).
|
|
46
|
+
Precedence is `--key` flag → `BYTEKIT_API_KEY`.
|
|
48
47
|
|
|
49
48
|
Without a key, the CLI exits with `Error: API key required. Pass --key or set BYTEKIT_API_KEY.`
|
|
50
49
|
|
|
@@ -52,8 +51,7 @@ Without a key, the CLI exits with `Error: API key required. Pass --key or set BY
|
|
|
52
51
|
|
|
53
52
|
Requests go to `https://api.bytekit.com` by default. Point the CLI at staging, a
|
|
54
53
|
local gateway, or a proxy with the global `--base-url` flag or the
|
|
55
|
-
`BYTEKIT_BASE_URL` environment variable (the flag wins
|
|
56
|
-
a deprecated legacy fallback):
|
|
54
|
+
`BYTEKIT_BASE_URL` environment variable (the flag wins):
|
|
57
55
|
|
|
58
56
|
```bash
|
|
59
57
|
bytekit --base-url https://api-stg.bytekit.com scrape create --url https://example.com
|
|
@@ -62,11 +60,11 @@ export BYTEKIT_BASE_URL=http://localhost:3001
|
|
|
62
60
|
|
|
63
61
|
## Global options
|
|
64
62
|
|
|
65
|
-
| Flag | Env var
|
|
66
|
-
| ------------------ |
|
|
67
|
-
| `--key <key>` | `BYTEKIT_API_KEY`
|
|
68
|
-
| `--base-url <url>` | `BYTEKIT_BASE_URL`
|
|
69
|
-
| `--json` | —
|
|
63
|
+
| Flag | Env var | Purpose |
|
|
64
|
+
| ------------------ | ------------------ | --------------------------------------------------------------------------------------------------------------------------------------- |
|
|
65
|
+
| `--key <key>` | `BYTEKIT_API_KEY` | API key used for every request. |
|
|
66
|
+
| `--base-url <url>` | `BYTEKIT_BASE_URL` | API base URL (default `https://api.bytekit.com`). |
|
|
67
|
+
| `--json` | — | Print the raw JSON response instead of the human-readable rendering (for `fetch`, prints the `X-Fetch-*` metadata instead of the body). |
|
|
70
68
|
|
|
71
69
|
## Commands
|
|
72
70
|
|
|
@@ -78,6 +76,27 @@ bytekit scrape get sc_123abc
|
|
|
78
76
|
bytekit scrape bulk create --file urls.txt --webhook-url https://your.app/hooks/bytekit
|
|
79
77
|
```
|
|
80
78
|
|
|
79
|
+
`scrape create` also accepts the full `ScrapeRequest` surface, each flag omitted
|
|
80
|
+
from the request when absent (a boolean flag never sends `false`):
|
|
81
|
+
|
|
82
|
+
| Flag | Effect |
|
|
83
|
+
| ------------------------ | ------------------------------------------------------- |
|
|
84
|
+
| `--country <cc>` | Route through a country (e.g. `us`, `de`). |
|
|
85
|
+
| `--timeout-ms <n>` | Per-request timeout in milliseconds. |
|
|
86
|
+
| `--markdown-mode <mode>` | Markdown extraction mode: `article`, `raw`, `llm`. |
|
|
87
|
+
| `--async` | Queue the scrape and return a job envelope immediately. |
|
|
88
|
+
| `--webhook-url <url>` | Webhook notified when an async scrape completes. |
|
|
89
|
+
| `--token-budget <n>` | Cap the markdown returned to `n` tokens. |
|
|
90
|
+
| `--clean-markdown` | LLM-clean the markdown (billed 3x). |
|
|
91
|
+
| `--include-tags <csv>` | Comma-separated HTML tags to keep. |
|
|
92
|
+
| `--exclude-tags <csv>` | Comma-separated HTML tags to drop. |
|
|
93
|
+
| `--mobile` | Render with a mobile viewport/user-agent. |
|
|
94
|
+
|
|
95
|
+
`--async` combined with `--raw` (or a bare `-o`) prints the queued job envelope
|
|
96
|
+
(the `sc_` id + status) rather than a format body, since a queued scrape has no
|
|
97
|
+
body yet. On `scrape`, `-o` writes the same text rendering the terminal would
|
|
98
|
+
show, so it is legal alongside `--json`.
|
|
99
|
+
|
|
81
100
|
### `fetch` — raw, unprocessed response bodies
|
|
82
101
|
|
|
83
102
|
```bash
|
|
@@ -95,9 +114,22 @@ bytekit search "ai web scraping" --limit 10 --country us
|
|
|
95
114
|
|
|
96
115
|
```bash
|
|
97
116
|
bytekit screenshots create --url https://example.com --full-page
|
|
98
|
-
bytekit screenshots
|
|
117
|
+
bytekit screenshots create --url https://example.com --device mobile --format png -o shot.png
|
|
118
|
+
bytekit screenshots get ss_123abc -o shot.png
|
|
99
119
|
```
|
|
100
120
|
|
|
121
|
+
`screenshots create` accepts `--device` (`desktop`, `mobile`), `--viewport WxH`
|
|
122
|
+
(e.g. `1280x720`), `--format` (`jpeg`, `png`), `--quality <n>`, `--wait-until`
|
|
123
|
+
(`load`, `domcontentloaded`, `networkidle`), `--clip x,y,w,h`, and `--async`
|
|
124
|
+
(queue and return a job envelope). Each flag is omitted from the request when
|
|
125
|
+
absent.
|
|
126
|
+
|
|
127
|
+
`-o <file>` downloads the captured image bytes to that file (use `-o -` for
|
|
128
|
+
stdout). Because `-o` writes BINARY on `screenshots`, it cannot be combined with
|
|
129
|
+
`--json` (that would emit binary into a JSON pipeline) — pass one or the other.
|
|
130
|
+
`--async` + `-o` prints the queued job envelope and skips the download (the
|
|
131
|
+
image is not ready yet).
|
|
132
|
+
|
|
101
133
|
### `bulk` — bulk screenshot jobs
|
|
102
134
|
|
|
103
135
|
```bash
|
|
@@ -114,11 +146,15 @@ bytekit bulk screenshots list "$bulk_id"
|
|
|
114
146
|
```bash
|
|
115
147
|
bytekit monitors create --url https://example.com/pricing --interval-type daily \
|
|
116
148
|
--webhook-url https://your.app/hooks/bytekit
|
|
117
|
-
bytekit monitors list
|
|
149
|
+
bytekit monitors list --limit 25 --status active
|
|
118
150
|
bytekit monitors captures list mon_123abc
|
|
119
151
|
bytekit monitors delete mon_123abc
|
|
120
152
|
```
|
|
121
153
|
|
|
154
|
+
`monitors list` accepts `--limit <n>`, `--cursor <cursor>`, and `--status`
|
|
155
|
+
(`active`, `processing`, `paused`, `cancelled`, `suspended`) as query filters,
|
|
156
|
+
each omitted when absent.
|
|
157
|
+
|
|
122
158
|
### `sitemap` — crawl a site's URL graph
|
|
123
159
|
|
|
124
160
|
```bash
|
|
@@ -132,8 +168,14 @@ bytekit sitemap get sm_123abc
|
|
|
132
168
|
bytekit account get
|
|
133
169
|
bytekit account api-keys list
|
|
134
170
|
bytekit account api-keys create --name "ci-key"
|
|
171
|
+
bytekit account api-keys reveal ak_123abc # masked by default
|
|
172
|
+
bytekit account api-keys reveal ak_123abc --show # full plaintext
|
|
135
173
|
```
|
|
136
174
|
|
|
175
|
+
`account api-keys reveal` prints a live secret, so it MASKS the key by default
|
|
176
|
+
(`sk_live_…2345`). Pass `--show` for the full plaintext, or `--json` for the
|
|
177
|
+
full envelope (for scripts).
|
|
178
|
+
|
|
137
179
|
## Documentation
|
|
138
180
|
|
|
139
181
|
Full API reference: [bytekit.com/docs](https://bytekit.com/docs).
|
package/dist/client.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { ByteKit } from '@hunt-labs/bytekit-sdk';
|
|
2
2
|
export interface GlobalOptions {
|
|
3
3
|
key?: string | undefined;
|
|
4
4
|
baseUrl?: string | undefined;
|
|
@@ -22,6 +22,17 @@ export declare const MIN_RETRY_AFTER_MS = 1000;
|
|
|
22
22
|
export declare const MAX_RETRY_AFTER_MS = 60000;
|
|
23
23
|
/** Exactly one retry. A second 429 is terminal — no loop. */
|
|
24
24
|
export declare const MAX_RETRIES = 1;
|
|
25
|
+
/**
|
|
26
|
+
* Default CLIENT-SIDE abort budget for a single fetch attempt, in ms.
|
|
27
|
+
*
|
|
28
|
+
* Mirrors the SDK's own `DEFAULT_TIMEOUT_MS` (120s). Overriding `client.request` with this
|
|
29
|
+
* transport had discarded the SDK's default, so a hung server hung the CLI forever; this restores
|
|
30
|
+
* the same bound. The budget is **per attempt**, not per command: the single 429 retry (and its
|
|
31
|
+
* backoff sleep) gets a fresh budget, so the retry can never be cut short by time already spent on
|
|
32
|
+
* the first attempt. A future `--timeout` flag would override this default; today only the default
|
|
33
|
+
* exists.
|
|
34
|
+
*/
|
|
35
|
+
export declare const DEFAULT_TIMEOUT_MS = 120000;
|
|
25
36
|
/**
|
|
26
37
|
* Parse a `Retry-After` header into a clamped millisecond delay.
|
|
27
38
|
*
|
|
@@ -34,7 +45,7 @@ export declare function parseRetryAfterMs(raw: string | null): number;
|
|
|
34
45
|
* Build the CLI's HTTP transport: the SDK's `request` contract, plus a single bounded retry on 429.
|
|
35
46
|
*
|
|
36
47
|
* Why the CLI owns the `fetch` instead of wrapping the SDK's `request()`: the SDK throws a
|
|
37
|
-
* header-less `
|
|
48
|
+
* header-less `ByteKitError` and discards the `Response`, so a delegating wrapper can see
|
|
38
49
|
* `status === 429` but can never read `Retry-After`. Owning the fetch is the only way to honor the
|
|
39
50
|
* server's advertised backoff without editing the SDK.
|
|
40
51
|
*
|
|
@@ -43,17 +54,16 @@ export declare function parseRetryAfterMs(raw: string | null): number;
|
|
|
43
54
|
*
|
|
44
55
|
* Retries apply to 429 only. Non-429 statuses and network rejections propagate untouched.
|
|
45
56
|
*/
|
|
46
|
-
export declare function createRetryingRequest(apiKey: string, baseUrl: string):
|
|
57
|
+
export declare function createRetryingRequest(apiKey: string, baseUrl: string, timeoutMs?: number): ByteKit['request'];
|
|
47
58
|
/**
|
|
48
59
|
* Build an authenticated SDK client from global CLI options.
|
|
49
60
|
*
|
|
50
|
-
* Resolution order (explicit input beats environment
|
|
51
|
-
* apiKey: --key flag → BYTEKIT_API_KEY
|
|
52
|
-
* baseUrl: --base-url flag → BYTEKIT_BASE_URL →
|
|
53
|
-
*
|
|
54
|
-
* descriptive error when no key is available.
|
|
61
|
+
* Resolution order (explicit input beats environment):
|
|
62
|
+
* apiKey: --key flag → BYTEKIT_API_KEY.
|
|
63
|
+
* baseUrl: --base-url flag → BYTEKIT_BASE_URL → DEFAULT_BASE_URL.
|
|
64
|
+
* Exits with a descriptive error when no key is available.
|
|
55
65
|
*
|
|
56
66
|
* The returned client's `request` is replaced by the retrying transport, so every command inherits
|
|
57
67
|
* the 429 retry from this one seam.
|
|
58
68
|
*/
|
|
59
|
-
export declare function buildClient(opts: GlobalOptions):
|
|
69
|
+
export declare function buildClient(opts: GlobalOptions): ByteKit;
|
package/dist/client.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { ByteKit, ByteKitError, ByteKitConnectionError } from '@hunt-labs/bytekit-sdk';
|
|
2
2
|
/**
|
|
3
3
|
* The API host the SDK itself defaults to. The SDK's copy is module-private, so the CLI declares
|
|
4
4
|
* its own and always passes an explicit `baseUrl` to the constructor — that way the SDK client and
|
|
@@ -18,6 +18,69 @@ export const MIN_RETRY_AFTER_MS = 1_000;
|
|
|
18
18
|
export const MAX_RETRY_AFTER_MS = 60_000;
|
|
19
19
|
/** Exactly one retry. A second 429 is terminal — no loop. */
|
|
20
20
|
export const MAX_RETRIES = 1;
|
|
21
|
+
/**
|
|
22
|
+
* Default CLIENT-SIDE abort budget for a single fetch attempt, in ms.
|
|
23
|
+
*
|
|
24
|
+
* Mirrors the SDK's own `DEFAULT_TIMEOUT_MS` (120s). Overriding `client.request` with this
|
|
25
|
+
* transport had discarded the SDK's default, so a hung server hung the CLI forever; this restores
|
|
26
|
+
* the same bound. The budget is **per attempt**, not per command: the single 429 retry (and its
|
|
27
|
+
* backoff sleep) gets a fresh budget, so the retry can never be cut short by time already spent on
|
|
28
|
+
* the first attempt. A future `--timeout` flag would override this default; today only the default
|
|
29
|
+
* exists.
|
|
30
|
+
*/
|
|
31
|
+
export const DEFAULT_TIMEOUT_MS = 120_000;
|
|
32
|
+
/** The abort reason used when a per-attempt budget elapses; `name` maps it to `code: 'timeout'`. */
|
|
33
|
+
function timeoutReason(ms) {
|
|
34
|
+
const reason = new Error(`ByteKit: request timed out after ${ms}ms`);
|
|
35
|
+
reason.name = 'TimeoutError';
|
|
36
|
+
return reason;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* A per-attempt abort budget: an `AbortController` fired by a `setTimeout`.
|
|
40
|
+
*
|
|
41
|
+
* Deliberately NOT `AbortSignal.timeout` (which the SDK prefers): that schedules on a timer the
|
|
42
|
+
* test runner's fake timers cannot advance, so the transport's timeout would be untestable under
|
|
43
|
+
* `vi.useFakeTimers()`. CLI requests are short-lived and single-shot, so the GC advantage
|
|
44
|
+
* `AbortSignal.timeout` buys the long-running SDK does not apply here. The timer is `unref`'d so a
|
|
45
|
+
* pending budget never keeps the process alive, and `cleanup()` clears it on every exit path.
|
|
46
|
+
*/
|
|
47
|
+
function createAttemptBudget(ms) {
|
|
48
|
+
const controller = new AbortController();
|
|
49
|
+
const timer = setTimeout(() => controller.abort(timeoutReason(ms)), ms);
|
|
50
|
+
timer.unref?.();
|
|
51
|
+
return {
|
|
52
|
+
signal: controller.signal,
|
|
53
|
+
cleanup: () => clearTimeout(timer),
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Map a `fetch` rejection (no HTTP response was produced) onto the SDK's `ByteKitConnectionError`
|
|
58
|
+
* taxonomy, so a CLI failure carries the same `{name, code, cause}` a direct SDK call would.
|
|
59
|
+
*
|
|
60
|
+
* `timedOut` is true when this attempt's own budget fired the abort, which is the reliable
|
|
61
|
+
* discriminator (undici rejects with the signal's reason, whose `name` we also inspect as a
|
|
62
|
+
* fallback for a caller-style AbortError).
|
|
63
|
+
*/
|
|
64
|
+
function toConnectionError(err, method, path, timedOut) {
|
|
65
|
+
const rawName = typeof err === 'object' && err !== null && 'name' in err ? err.name : undefined;
|
|
66
|
+
const name = typeof rawName === 'string' ? rawName : '';
|
|
67
|
+
let code;
|
|
68
|
+
if (timedOut || name === 'TimeoutError') {
|
|
69
|
+
code = 'timeout';
|
|
70
|
+
}
|
|
71
|
+
else if (name === 'AbortError') {
|
|
72
|
+
code = 'aborted';
|
|
73
|
+
}
|
|
74
|
+
else {
|
|
75
|
+
code = 'connection';
|
|
76
|
+
}
|
|
77
|
+
const detail = code === 'timeout'
|
|
78
|
+
? 'request timed out'
|
|
79
|
+
: code === 'aborted'
|
|
80
|
+
? 'request aborted'
|
|
81
|
+
: 'connection failed before a response was received';
|
|
82
|
+
return new ByteKitConnectionError(code, `ByteKit: ${method} ${path} — ${detail}`, { cause: err });
|
|
83
|
+
}
|
|
21
84
|
/**
|
|
22
85
|
* Parse a `Retry-After` header into a clamped millisecond delay.
|
|
23
86
|
*
|
|
@@ -44,7 +107,7 @@ function sleep(ms) {
|
|
|
44
107
|
/**
|
|
45
108
|
* Shape a failed response into the SDK's OWN error class, using the SDK's own body parsing.
|
|
46
109
|
*
|
|
47
|
-
* Reusing `
|
|
110
|
+
* Reusing `ByteKitError` keeps `{status, code, message}` identical to what a direct SDK call
|
|
48
111
|
* would throw, so `handleError()`'s `'status' in err` branch still renders
|
|
49
112
|
* `Error: <message> (status: <n>)` and exits 1.
|
|
50
113
|
*/
|
|
@@ -59,13 +122,41 @@ async function toApiError(response) {
|
|
|
59
122
|
catch {
|
|
60
123
|
// leave defaults
|
|
61
124
|
}
|
|
62
|
-
return new
|
|
125
|
+
return new ByteKitError(response.status, code, message);
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* Is this request targeting a `/v1/fetch` CONTENT endpoint (POST or GET), whose body IS the fetched
|
|
129
|
+
* content rather than a JSON envelope?
|
|
130
|
+
*
|
|
131
|
+
* Keyed off the request TARGET, never the response content-type: a fetched page can itself serve
|
|
132
|
+
* `application/json`, and that body must still be returned raw, not parsed. The `/v1/fetch/bulk`
|
|
133
|
+
* job endpoints are NOT content endpoints — they return a JSON envelope and keep parsing as JSON.
|
|
134
|
+
*/
|
|
135
|
+
function isFetchContentPath(path) {
|
|
136
|
+
return (path.split('?')[0] ?? path) === '/v1/fetch';
|
|
137
|
+
}
|
|
138
|
+
/** Title-Case an HTTP header name segment-by-segment: `x-fetch-status` → `X-Fetch-Status`. */
|
|
139
|
+
function titleCaseHeader(name) {
|
|
140
|
+
return name
|
|
141
|
+
.split('-')
|
|
142
|
+
.map((seg) => seg.charAt(0).toUpperCase() + seg.slice(1).toLowerCase())
|
|
143
|
+
.join('-');
|
|
144
|
+
}
|
|
145
|
+
/** Collect the `X-Fetch-*` response headers into a Title-Cased metadata object. */
|
|
146
|
+
function collectFetchMetadata(headers) {
|
|
147
|
+
const meta = {};
|
|
148
|
+
headers.forEach((value, key) => {
|
|
149
|
+
if (key.toLowerCase().startsWith('x-fetch-')) {
|
|
150
|
+
meta[titleCaseHeader(key)] = value;
|
|
151
|
+
}
|
|
152
|
+
});
|
|
153
|
+
return meta;
|
|
63
154
|
}
|
|
64
155
|
/**
|
|
65
156
|
* Build the CLI's HTTP transport: the SDK's `request` contract, plus a single bounded retry on 429.
|
|
66
157
|
*
|
|
67
158
|
* Why the CLI owns the `fetch` instead of wrapping the SDK's `request()`: the SDK throws a
|
|
68
|
-
* header-less `
|
|
159
|
+
* header-less `ByteKitError` and discards the `Response`, so a delegating wrapper can see
|
|
69
160
|
* `status === 429` but can never read `Retry-After`. Owning the fetch is the only way to honor the
|
|
70
161
|
* server's advertised backoff without editing the SDK.
|
|
71
162
|
*
|
|
@@ -74,7 +165,7 @@ async function toApiError(response) {
|
|
|
74
165
|
*
|
|
75
166
|
* Retries apply to 429 only. Non-429 statuses and network rejections propagate untouched.
|
|
76
167
|
*/
|
|
77
|
-
export function createRetryingRequest(apiKey, baseUrl) {
|
|
168
|
+
export function createRetryingRequest(apiKey, baseUrl, timeoutMs = DEFAULT_TIMEOUT_MS) {
|
|
78
169
|
return async function request(method, path, body) {
|
|
79
170
|
for (let attempt = 0;; attempt++) {
|
|
80
171
|
const init = {
|
|
@@ -87,9 +178,44 @@ export function createRetryingRequest(apiKey, baseUrl) {
|
|
|
87
178
|
if (body !== undefined) {
|
|
88
179
|
init.body = JSON.stringify(body);
|
|
89
180
|
}
|
|
90
|
-
|
|
181
|
+
// Per-attempt abort budget: a fresh timer for THIS fetch. A budget of 0 disables it.
|
|
182
|
+
const budget = timeoutMs > 0 ? createAttemptBudget(timeoutMs) : undefined;
|
|
183
|
+
if (budget !== undefined) {
|
|
184
|
+
init.signal = budget.signal;
|
|
185
|
+
}
|
|
186
|
+
let response;
|
|
187
|
+
try {
|
|
188
|
+
response = await fetch(`${baseUrl}${path}`, init);
|
|
189
|
+
}
|
|
190
|
+
catch (err) {
|
|
191
|
+
// No HTTP response was produced (timeout, abort, socket/DNS/TLS). Re-throw as the SDK's
|
|
192
|
+
// own ByteKitConnectionError so a hung server can never hang the CLI and a network failure
|
|
193
|
+
// no longer surfaces as a bare `Error: fetch failed`.
|
|
194
|
+
throw toConnectionError(err, method, path, budget?.signal.aborted === true);
|
|
195
|
+
}
|
|
196
|
+
finally {
|
|
197
|
+
// Clear the timer on every exit path — success, HTTP error, or abort — so a healthy
|
|
198
|
+
// request leaves no dangling 120s timer.
|
|
199
|
+
budget?.cleanup();
|
|
200
|
+
}
|
|
91
201
|
if (response.ok) {
|
|
92
|
-
|
|
202
|
+
// A /v1/fetch content response body IS the fetched content (HTML/markdown/binary), with its
|
|
203
|
+
// metadata in X-Fetch-* headers. Return the raw bytes untouched — JSON-parsing it is the
|
|
204
|
+
// #2467 DOA crash. Every other endpoint returns a JSON envelope, parsed below.
|
|
205
|
+
if (isFetchContentPath(path)) {
|
|
206
|
+
const bodyBytes = Buffer.from(await response.arrayBuffer());
|
|
207
|
+
const fetchResult = {
|
|
208
|
+
__bytekitFetchBody: true,
|
|
209
|
+
body: bodyBytes,
|
|
210
|
+
metadata: collectFetchMetadata(response.headers),
|
|
211
|
+
};
|
|
212
|
+
return fetchResult;
|
|
213
|
+
}
|
|
214
|
+
const text = await response.text();
|
|
215
|
+
if (response.status === 204 || text.trim().length === 0) {
|
|
216
|
+
return undefined;
|
|
217
|
+
}
|
|
218
|
+
return JSON.parse(text);
|
|
93
219
|
}
|
|
94
220
|
if (response.status === 429 && attempt < MAX_RETRIES) {
|
|
95
221
|
await sleep(parseRetryAfterMs(response.headers.get('Retry-After')));
|
|
@@ -102,26 +228,22 @@ export function createRetryingRequest(apiKey, baseUrl) {
|
|
|
102
228
|
/**
|
|
103
229
|
* Build an authenticated SDK client from global CLI options.
|
|
104
230
|
*
|
|
105
|
-
* Resolution order (explicit input beats environment
|
|
106
|
-
* apiKey: --key flag → BYTEKIT_API_KEY
|
|
107
|
-
* baseUrl: --base-url flag → BYTEKIT_BASE_URL →
|
|
108
|
-
*
|
|
109
|
-
* descriptive error when no key is available.
|
|
231
|
+
* Resolution order (explicit input beats environment):
|
|
232
|
+
* apiKey: --key flag → BYTEKIT_API_KEY.
|
|
233
|
+
* baseUrl: --base-url flag → BYTEKIT_BASE_URL → DEFAULT_BASE_URL.
|
|
234
|
+
* Exits with a descriptive error when no key is available.
|
|
110
235
|
*
|
|
111
236
|
* The returned client's `request` is replaced by the retrying transport, so every command inherits
|
|
112
237
|
* the 429 retry from this one seam.
|
|
113
238
|
*/
|
|
114
239
|
export function buildClient(opts) {
|
|
115
|
-
const apiKey = opts.key ?? process.env['BYTEKIT_API_KEY']
|
|
240
|
+
const apiKey = opts.key ?? process.env['BYTEKIT_API_KEY'];
|
|
116
241
|
if (!apiKey) {
|
|
117
242
|
console.error('Error: API key required. Pass --key or set BYTEKIT_API_KEY.');
|
|
118
243
|
process.exit(1);
|
|
119
244
|
}
|
|
120
|
-
const baseUrl = opts.baseUrl ??
|
|
121
|
-
|
|
122
|
-
process.env['RAPIDCRAWL_BASE_URL'] ??
|
|
123
|
-
DEFAULT_BASE_URL;
|
|
124
|
-
const client = new RapidCrawl({ apiKey, baseUrl });
|
|
245
|
+
const baseUrl = opts.baseUrl ?? process.env['BYTEKIT_BASE_URL'] ?? DEFAULT_BASE_URL;
|
|
246
|
+
const client = new ByteKit({ apiKey, baseUrl });
|
|
125
247
|
client.request = createRetryingRequest(apiKey, baseUrl);
|
|
126
248
|
return client;
|
|
127
249
|
}
|
package/dist/commands/account.js
CHANGED
|
@@ -1,6 +1,28 @@
|
|
|
1
1
|
import { Command } from 'commander';
|
|
2
2
|
import { buildClient } from '../client.js';
|
|
3
3
|
import { printResult, handleError } from '../output.js';
|
|
4
|
+
/** Visible lead-in of the masked form: the `sk_live_`/`sk_test_` prefix (or first 8 chars). */
|
|
5
|
+
const MASK_PREFIX_CHARS = 8;
|
|
6
|
+
/** Visible tail of the masked form, shown only when enough of the middle stays hidden. */
|
|
7
|
+
const MASK_SUFFIX_CHARS = 4;
|
|
8
|
+
/** Minimum number of characters the `…` must actually hide before a suffix may be shown. */
|
|
9
|
+
const MIN_MASKED_CHARS = 8;
|
|
10
|
+
/**
|
|
11
|
+
* Mask a plaintext API key for human display: show the `sk_live_`/`sk_test_` prefix (or the first
|
|
12
|
+
* 8 chars) and the last 4, with the secret middle replaced by `…`. The last-4 suffix is shown ONLY
|
|
13
|
+
* when at least MIN_MASKED_CHARS of the middle stay hidden (key length >= 8 + 4 + 8 = 20): on a
|
|
14
|
+
* 13–19-char key, `slice(0, 8) + slice(-4)` would hide as little as one character, and at 12 or
|
|
15
|
+
* fewer the prefix and suffix overlap outright — either way the "mask" would leak most of a live
|
|
16
|
+
* secret, so everything after the recognized prefix is masked instead (#2480).
|
|
17
|
+
*/
|
|
18
|
+
function maskKey(key) {
|
|
19
|
+
const prefixMatch = /^(sk_(?:live|test)_)/.exec(key);
|
|
20
|
+
if (key.length < MASK_PREFIX_CHARS + MASK_SUFFIX_CHARS + MIN_MASKED_CHARS) {
|
|
21
|
+
const prefix = prefixMatch ? prefixMatch[1] : key.slice(0, MASK_PREFIX_CHARS);
|
|
22
|
+
return `${prefix}…`;
|
|
23
|
+
}
|
|
24
|
+
return `${key.slice(0, MASK_PREFIX_CHARS)}…${key.slice(-MASK_SUFFIX_CHARS)}`;
|
|
25
|
+
}
|
|
4
26
|
export function makeAccountCommand(getGlobal) {
|
|
5
27
|
const cmd = new Command('account').description('Account and API key operations');
|
|
6
28
|
const apiKeysCmd = new Command('api-keys').description('API key management');
|
|
@@ -67,6 +89,33 @@ export function makeAccountCommand(getGlobal) {
|
|
|
67
89
|
handleError(err);
|
|
68
90
|
}
|
|
69
91
|
});
|
|
92
|
+
apiKeysCmd
|
|
93
|
+
.command('reveal <id>')
|
|
94
|
+
.description('Reveal an API key value (masked by default; use --show for plaintext)')
|
|
95
|
+
.option('--show', 'Print the full plaintext key instead of the masked form')
|
|
96
|
+
.action(async (id, opts) => {
|
|
97
|
+
const g = getGlobal();
|
|
98
|
+
const client = buildClient(g);
|
|
99
|
+
try {
|
|
100
|
+
const result = await client.account.apiKeys.reveal(id);
|
|
101
|
+
// Masking decision (#2480 E10): `reveal` prints a LIVE secret to a terminal that may be
|
|
102
|
+
// logged or screen-shared, so plaintext is opt-in. Default (human) mode masks the key
|
|
103
|
+
// (prefix + last-4, e.g. `sk_live_…2345`); `--show` prints the full plaintext; `--json`
|
|
104
|
+
// emits the full envelope verbatim for machine consumption.
|
|
105
|
+
if (g.json) {
|
|
106
|
+
printResult(result, true);
|
|
107
|
+
}
|
|
108
|
+
else if (opts.show === true) {
|
|
109
|
+
console.log(result.key);
|
|
110
|
+
}
|
|
111
|
+
else {
|
|
112
|
+
console.log(maskKey(result.key));
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
catch (err) {
|
|
116
|
+
handleError(err);
|
|
117
|
+
}
|
|
118
|
+
});
|
|
70
119
|
apiKeysCmd
|
|
71
120
|
.command('revoke <id>')
|
|
72
121
|
.description('Revoke an API key (idempotent)')
|
package/dist/commands/fetch.d.ts
CHANGED
|
@@ -1,5 +1,19 @@
|
|
|
1
1
|
import { Command } from 'commander';
|
|
2
2
|
import { type GlobalOptions } from '../client.js';
|
|
3
|
+
/** The `/v1/fetch` spec union for `format` — mirrors `FetchOpts['format']` in the SDK. */
|
|
4
|
+
type FetchFormat = 'markdown' | 'html';
|
|
5
|
+
/**
|
|
6
|
+
* Validate `--format` locally, before any HTTP request is issued.
|
|
7
|
+
*
|
|
8
|
+
* The SDK's `FetchOpts.format` is narrowed to the spec union `'markdown' | 'html'`
|
|
9
|
+
* (#2475), so an off-spec value like `pdf` is a compile-time error at the call site.
|
|
10
|
+
* Commander runs an option's argParser during parsing — ahead of the action callback —
|
|
11
|
+
* so rejecting here means the client is never built and no request is issued, and the
|
|
12
|
+
* caller gets an actionable message instead of a server 422 (same convention as
|
|
13
|
+
* `parseLimit` in search.ts).
|
|
14
|
+
*/
|
|
15
|
+
export declare function parseFormat(value: string): FetchFormat;
|
|
3
16
|
export declare function makeFetchCommand(getGlobal: () => GlobalOptions & {
|
|
4
17
|
json: boolean;
|
|
5
18
|
}): Command;
|
|
19
|
+
export {};
|
package/dist/commands/fetch.js
CHANGED
|
@@ -1,7 +1,24 @@
|
|
|
1
|
-
import { Command } from 'commander';
|
|
1
|
+
import { Command, InvalidArgumentError } from 'commander';
|
|
2
2
|
import { buildClient } from '../client.js';
|
|
3
|
-
import { printResult, handleError } from '../output.js';
|
|
3
|
+
import { printResult, emitFetchResult, handleError } from '../output.js';
|
|
4
4
|
import { parseUrlLines } from '../lib/url-lines.js';
|
|
5
|
+
const FETCH_FORMATS = ['markdown', 'html'];
|
|
6
|
+
/**
|
|
7
|
+
* Validate `--format` locally, before any HTTP request is issued.
|
|
8
|
+
*
|
|
9
|
+
* The SDK's `FetchOpts.format` is narrowed to the spec union `'markdown' | 'html'`
|
|
10
|
+
* (#2475), so an off-spec value like `pdf` is a compile-time error at the call site.
|
|
11
|
+
* Commander runs an option's argParser during parsing — ahead of the action callback —
|
|
12
|
+
* so rejecting here means the client is never built and no request is issued, and the
|
|
13
|
+
* caller gets an actionable message instead of a server 422 (same convention as
|
|
14
|
+
* `parseLimit` in search.ts).
|
|
15
|
+
*/
|
|
16
|
+
export function parseFormat(value) {
|
|
17
|
+
if (!FETCH_FORMATS.includes(value)) {
|
|
18
|
+
throw new InvalidArgumentError(`--format must be one of: ${FETCH_FORMATS.join(', ')}.`);
|
|
19
|
+
}
|
|
20
|
+
return value;
|
|
21
|
+
}
|
|
5
22
|
export function makeFetchCommand(getGlobal) {
|
|
6
23
|
const cmd = new Command('fetch').description('Fetch URL content operations');
|
|
7
24
|
const bulkCmd = new Command('bulk').description('Bulk fetch operations');
|
|
@@ -41,7 +58,7 @@ export function makeFetchCommand(getGlobal) {
|
|
|
41
58
|
.command('create')
|
|
42
59
|
.description('Fetch URL content (POST)')
|
|
43
60
|
.requiredOption('--url <url>', 'URL to fetch')
|
|
44
|
-
.option('--format <format>', 'Response format')
|
|
61
|
+
.option('--format <format>', 'Response format (markdown or html)', parseFormat)
|
|
45
62
|
.action(async (opts) => {
|
|
46
63
|
const g = getGlobal();
|
|
47
64
|
const client = buildClient(g);
|
|
@@ -50,7 +67,7 @@ export function makeFetchCommand(getGlobal) {
|
|
|
50
67
|
url: opts.url,
|
|
51
68
|
...(opts.format !== undefined ? { format: opts.format } : {}),
|
|
52
69
|
});
|
|
53
|
-
|
|
70
|
+
emitFetchResult(result, g.json);
|
|
54
71
|
}
|
|
55
72
|
catch (err) {
|
|
56
73
|
handleError(err);
|
|
@@ -60,7 +77,7 @@ export function makeFetchCommand(getGlobal) {
|
|
|
60
77
|
.command('get')
|
|
61
78
|
.description('Fetch URL content (GET via query params)')
|
|
62
79
|
.requiredOption('--url <url>', 'URL to fetch')
|
|
63
|
-
.option('--format <format>', 'Response format')
|
|
80
|
+
.option('--format <format>', 'Response format (markdown or html)', parseFormat)
|
|
64
81
|
.action(async (opts) => {
|
|
65
82
|
const g = getGlobal();
|
|
66
83
|
const client = buildClient(g);
|
|
@@ -69,7 +86,7 @@ export function makeFetchCommand(getGlobal) {
|
|
|
69
86
|
url: opts.url,
|
|
70
87
|
...(opts.format !== undefined ? { format: opts.format } : {}),
|
|
71
88
|
});
|
|
72
|
-
|
|
89
|
+
emitFetchResult(result, g.json);
|
|
73
90
|
}
|
|
74
91
|
catch (err) {
|
|
75
92
|
handleError(err);
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import { Command } from 'commander';
|
|
2
2
|
import { buildClient } from '../client.js';
|
|
3
3
|
import { printResult, handleError } from '../output.js';
|
|
4
|
+
import { parseEnum, parseIntFlag } from '../lib/parse-flags.js';
|
|
5
|
+
/** The `listMonitors` `status` query enum. */
|
|
6
|
+
const MONITOR_STATUSES = ['active', 'processing', 'paused', 'cancelled', 'suspended'];
|
|
4
7
|
export function makeMonitorsCommand(getGlobal) {
|
|
5
8
|
const cmd = new Command('monitors').description('Page-change monitor operations');
|
|
6
9
|
const capturesCmd = new Command('captures').description('Monitor capture sub-resources');
|
|
@@ -29,13 +32,18 @@ export function makeMonitorsCommand(getGlobal) {
|
|
|
29
32
|
.action(async (opts) => {
|
|
30
33
|
const g = getGlobal();
|
|
31
34
|
const client = buildClient(g);
|
|
35
|
+
// Key-checked subset of the SDK's MonitorCreateRequest. The CLI intentionally omits the
|
|
36
|
+
// spec-required-with-@default fields (type, change_threshold, notify_on) and relies on
|
|
37
|
+
// the server-side defaults, so the final call needs an explicit assertion — type-level
|
|
38
|
+
// only; the wire body is unchanged (pinned by __tests__/request-shapes.test.ts).
|
|
32
39
|
const body = {
|
|
33
40
|
url: opts.url,
|
|
41
|
+
// Commander delivers a plain string; the server validates the enum (422 on a bad value).
|
|
34
42
|
interval_type: opts.intervalType,
|
|
35
43
|
webhook_url: opts.webhookUrl,
|
|
36
44
|
};
|
|
37
45
|
if (opts.cron !== undefined)
|
|
38
|
-
body
|
|
46
|
+
body.cron_expression = opts.cron;
|
|
39
47
|
try {
|
|
40
48
|
const result = await client.monitors.create(body);
|
|
41
49
|
printResult(result, g.json);
|
|
@@ -47,11 +55,26 @@ export function makeMonitorsCommand(getGlobal) {
|
|
|
47
55
|
cmd
|
|
48
56
|
.command('list')
|
|
49
57
|
.description('List all monitors')
|
|
50
|
-
.
|
|
58
|
+
.option('--limit <n>', 'Maximum number of monitors to return', (v) => parseIntFlag('--limit', v))
|
|
59
|
+
.option('--cursor <cursor>', 'Pagination cursor from a previous page')
|
|
60
|
+
.option('--status <status>', `Filter by status: ${MONITOR_STATUSES.join(', ')} (default active)`, (v) => parseEnum('--status', MONITOR_STATUSES, v))
|
|
61
|
+
.action(async (opts) => {
|
|
51
62
|
const g = getGlobal();
|
|
52
63
|
const client = buildClient(g);
|
|
64
|
+
// Build the params object with the same omit-when-absent rule as the body builders. When NO
|
|
65
|
+
// query flag is passed, call list() with ZERO arguments (never list({}) / list(undefined)) so
|
|
66
|
+
// the inverse-case request-shape assertion `monitors.list` args length 0 stays green (#2480).
|
|
67
|
+
const params = {};
|
|
68
|
+
if (opts.limit !== undefined)
|
|
69
|
+
params.limit = opts.limit;
|
|
70
|
+
if (opts.cursor !== undefined)
|
|
71
|
+
params.cursor = opts.cursor;
|
|
72
|
+
if (opts.status !== undefined)
|
|
73
|
+
params.status = opts.status;
|
|
53
74
|
try {
|
|
54
|
-
const result =
|
|
75
|
+
const result = Object.keys(params).length > 0
|
|
76
|
+
? await client.monitors.list(params)
|
|
77
|
+
: await client.monitors.list();
|
|
55
78
|
printResult(result, g.json);
|
|
56
79
|
}
|
|
57
80
|
catch (err) {
|