@hunt-labs/bytekit-cli 0.3.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +230 -0
- package/README.md +187 -29
- package/dist/client.d.ts +128 -11
- package/dist/client.js +369 -95
- package/dist/commands/account.d.ts +11 -0
- package/dist/commands/account.js +10 -127
- package/dist/commands/bulk.js +36 -4
- package/dist/commands/fetch.js +91 -10
- package/dist/commands/monitors.js +92 -11
- package/dist/commands/scrape.js +131 -20
- package/dist/commands/screenshots.js +86 -1
- package/dist/commands/sitemap.js +38 -6
- package/dist/commands/usage.d.ts +13 -0
- package/dist/commands/usage.js +81 -0
- package/dist/commands/webhook-deliveries.d.ts +11 -0
- package/dist/commands/webhook-deliveries.js +59 -0
- package/dist/index.js +4 -0
- package/dist/lib/download-artifact.d.ts +30 -1
- package/dist/lib/download-artifact.js +89 -5
- package/dist/lib/parse-flags.d.ts +30 -0
- package/dist/lib/parse-flags.js +58 -0
- package/dist/lib/url-lines.d.ts +46 -8
- package/dist/lib/url-lines.js +58 -19
- package/dist/output.d.ts +47 -9
- package/dist/output.js +103 -16
- package/package.json +3 -3
- package/dist/lib/fetch-body.d.ts +0 -18
- package/dist/lib/fetch-body.js +0 -6
|
@@ -9,5 +9,34 @@
|
|
|
9
9
|
* test that stubs `globalThis.fetch` in a `beforeEach` — after this module was imported — still
|
|
10
10
|
* sees its stub (G-049). A non-2xx response throws with the HTTP status so `handleError` renders a
|
|
11
11
|
* clean one-liner and exits 1.
|
|
12
|
+
*
|
|
13
|
+
* TIMEOUT (#2587): this path sat entirely outside the transport seam — no abort signal, no
|
|
14
|
+
* connection-error taxonomy — so a stalled artifact host (headers, then a body that never
|
|
15
|
+
* completes) hung the CLI forever, even though the 0.5.0 CHANGELOG's "a stalled body hung the CLI
|
|
16
|
+
* forever" note (#2546) reads as if every response body were covered. #2546 only covered API-call
|
|
17
|
+
* bodies read through `sendWithRetry`; this download never rode that seam. The fix REUSES
|
|
18
|
+
* `sendWithRetry`'s own per-attempt abort budget and connection-error taxonomy
|
|
19
|
+
* (`createAttemptBudget` / `toConnectionError`, exported from `../client.js`) rather than a second
|
|
20
|
+
* timeout implementation, and mirrors the same default (`DEFAULT_TIMEOUT_MS`, 120 s per attempt) —
|
|
21
|
+
* there is no retry here, since artifact GETs are not the API's 429 domain.
|
|
22
|
+
*
|
|
23
|
+
* PARTIAL-FILE POLICY (#2587 dod-1 / E1, decided + recorded, scoped to the DOWNLOAD phase only —
|
|
24
|
+
* cr-4): the response is buffered COMPLETELY in memory (`res.arrayBuffer()`) before either write
|
|
25
|
+
* path (`writeFileSync` / `process.stdout.write`) runs, so an ABORT — whether from the timeout or
|
|
26
|
+
* any other connection failure — always fails BEFORE a single byte is written; no partial file is
|
|
27
|
+
* ever left at `dest` from a download-side failure. This does NOT cover a failure DURING the write
|
|
28
|
+
* itself: `writeFileSync` opens `dest` with `O_TRUNC` and writes in a loop, so e.g. an `ENOSPC`
|
|
29
|
+
* partway through a large write leaves a created, truncated file at `dest` (any pre-existing file
|
|
30
|
+
* there already clobbered by the `O_TRUNC` open). That failure mode is accepted as-is for now (no
|
|
31
|
+
* write-to-temp-then-`renameSync` here) — the CLI still exits 1, and the write phase's tiny
|
|
32
|
+
* (post-buffering) failure window is a separate, unaddressed edge from the abort guarantee above.
|
|
33
|
+
*
|
|
34
|
+
* @param timeoutMs Per-attempt abort budget in ms; defaults to {@link DEFAULT_TIMEOUT_MS} (120 s).
|
|
35
|
+
* `<= 0` DISABLES the budget entirely (no `AbortSignal` is attached, restoring the exact
|
|
36
|
+
* unbounded-hang behavior this ticket fixes) rather than meaning "no override" — there is no
|
|
37
|
+
* sentinel for that. #2587 cr-8: no CLI command reaches this parameter today
|
|
38
|
+
* (`commands/screenshots.ts` always calls the 2-arg form), so a future caller — e.g. an
|
|
39
|
+
* anticipated `--timeout` flag threaded through here — must NOT pass `0` to mean "use the
|
|
40
|
+
* default"; it must omit the argument or pass `DEFAULT_TIMEOUT_MS` explicitly.
|
|
12
41
|
*/
|
|
13
|
-
export declare function downloadArtifact(url: string, dest: string): Promise<void>;
|
|
42
|
+
export declare function downloadArtifact(url: string, dest: string, timeoutMs?: number): Promise<void>;
|
|
@@ -1,4 +1,25 @@
|
|
|
1
1
|
import { writeFileSync } from 'node:fs';
|
|
2
|
+
import { createAttemptBudget, toConnectionError, DEFAULT_TIMEOUT_MS } from '../client.js';
|
|
3
|
+
/**
|
|
4
|
+
* Redact a URL for error-message rendering: origin + pathname only, query string dropped.
|
|
5
|
+
*
|
|
6
|
+
* #2587 cr-2: `image_url` for a completed screenshot is an AWS SigV4 PRESIGNED URL
|
|
7
|
+
* (`X-Amz-Credential=…&X-Amz-Signature=…`, `gateway/src/lib/signed-url.ts`), unlike every other
|
|
8
|
+
* `toConnectionError` call site in `client.ts` which passes a bare relative API path with no
|
|
9
|
+
* credential material. Rendering the full URL in `handleError`'s unmodified `err.message` would
|
|
10
|
+
* print a live 600s-valid signature to stderr on every new failure path this diff adds. On a
|
|
11
|
+
* malformed URL (should not happen — `image_url` is always an absolute URL from the API) fall
|
|
12
|
+
* back to the original string rather than throwing out of an error-handling path.
|
|
13
|
+
*/
|
|
14
|
+
function redactedPath(url) {
|
|
15
|
+
try {
|
|
16
|
+
const u = new URL(url);
|
|
17
|
+
return `${u.origin}${u.pathname}`;
|
|
18
|
+
}
|
|
19
|
+
catch {
|
|
20
|
+
return url;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
2
23
|
/**
|
|
3
24
|
* Download a screenshot artifact from its `image_url` and write the raw bytes to `dest`.
|
|
4
25
|
*
|
|
@@ -10,13 +31,76 @@ import { writeFileSync } from 'node:fs';
|
|
|
10
31
|
* test that stubs `globalThis.fetch` in a `beforeEach` — after this module was imported — still
|
|
11
32
|
* sees its stub (G-049). A non-2xx response throws with the HTTP status so `handleError` renders a
|
|
12
33
|
* clean one-liner and exits 1.
|
|
34
|
+
*
|
|
35
|
+
* TIMEOUT (#2587): this path sat entirely outside the transport seam — no abort signal, no
|
|
36
|
+
* connection-error taxonomy — so a stalled artifact host (headers, then a body that never
|
|
37
|
+
* completes) hung the CLI forever, even though the 0.5.0 CHANGELOG's "a stalled body hung the CLI
|
|
38
|
+
* forever" note (#2546) reads as if every response body were covered. #2546 only covered API-call
|
|
39
|
+
* bodies read through `sendWithRetry`; this download never rode that seam. The fix REUSES
|
|
40
|
+
* `sendWithRetry`'s own per-attempt abort budget and connection-error taxonomy
|
|
41
|
+
* (`createAttemptBudget` / `toConnectionError`, exported from `../client.js`) rather than a second
|
|
42
|
+
* timeout implementation, and mirrors the same default (`DEFAULT_TIMEOUT_MS`, 120 s per attempt) —
|
|
43
|
+
* there is no retry here, since artifact GETs are not the API's 429 domain.
|
|
44
|
+
*
|
|
45
|
+
* PARTIAL-FILE POLICY (#2587 dod-1 / E1, decided + recorded, scoped to the DOWNLOAD phase only —
|
|
46
|
+
* cr-4): the response is buffered COMPLETELY in memory (`res.arrayBuffer()`) before either write
|
|
47
|
+
* path (`writeFileSync` / `process.stdout.write`) runs, so an ABORT — whether from the timeout or
|
|
48
|
+
* any other connection failure — always fails BEFORE a single byte is written; no partial file is
|
|
49
|
+
* ever left at `dest` from a download-side failure. This does NOT cover a failure DURING the write
|
|
50
|
+
* itself: `writeFileSync` opens `dest` with `O_TRUNC` and writes in a loop, so e.g. an `ENOSPC`
|
|
51
|
+
* partway through a large write leaves a created, truncated file at `dest` (any pre-existing file
|
|
52
|
+
* there already clobbered by the `O_TRUNC` open). That failure mode is accepted as-is for now (no
|
|
53
|
+
* write-to-temp-then-`renameSync` here) — the CLI still exits 1, and the write phase's tiny
|
|
54
|
+
* (post-buffering) failure window is a separate, unaddressed edge from the abort guarantee above.
|
|
55
|
+
*
|
|
56
|
+
* @param timeoutMs Per-attempt abort budget in ms; defaults to {@link DEFAULT_TIMEOUT_MS} (120 s).
|
|
57
|
+
* `<= 0` DISABLES the budget entirely (no `AbortSignal` is attached, restoring the exact
|
|
58
|
+
* unbounded-hang behavior this ticket fixes) rather than meaning "no override" — there is no
|
|
59
|
+
* sentinel for that. #2587 cr-8: no CLI command reaches this parameter today
|
|
60
|
+
* (`commands/screenshots.ts` always calls the 2-arg form), so a future caller — e.g. an
|
|
61
|
+
* anticipated `--timeout` flag threaded through here — must NOT pass `0` to mean "use the
|
|
62
|
+
* default"; it must omit the argument or pass `DEFAULT_TIMEOUT_MS` explicitly.
|
|
13
63
|
*/
|
|
14
|
-
export async function downloadArtifact(url, dest) {
|
|
15
|
-
const
|
|
16
|
-
|
|
17
|
-
|
|
64
|
+
export async function downloadArtifact(url, dest, timeoutMs = DEFAULT_TIMEOUT_MS) {
|
|
65
|
+
const budget = timeoutMs > 0 ? createAttemptBudget(timeoutMs) : undefined;
|
|
66
|
+
const safePath = redactedPath(url);
|
|
67
|
+
let bytes;
|
|
68
|
+
try {
|
|
69
|
+
let res;
|
|
70
|
+
try {
|
|
71
|
+
res = await globalThis.fetch(url, budget ? { signal: budget.signal } : {});
|
|
72
|
+
}
|
|
73
|
+
catch (err) {
|
|
74
|
+
// No HTTP response was produced (timeout, abort, socket/DNS/TLS) — re-throw as the SDK's own
|
|
75
|
+
// ByteKitConnectionError, exactly like the API transport, so a hung/broken artifact host can
|
|
76
|
+
// never hang the CLI and `handleError` renders the same taxonomy either way. `safePath` (not
|
|
77
|
+
// `url`) is what gets rendered — see {@link redactedPath}.
|
|
78
|
+
throw toConnectionError(err, 'GET', safePath, budget?.signal.aborted === true);
|
|
79
|
+
}
|
|
80
|
+
if (!res.ok) {
|
|
81
|
+
// #2587 cr2-1: no drain here, unlike client.ts's 429 path. cr-11 originally added one
|
|
82
|
+
// (mirroring client.ts's corrected 429-drain mechanism, cr-3), reasoning that a full
|
|
83
|
+
// `text()` read lets undici return the connection to the pool. That reasoning does not
|
|
84
|
+
// apply on THIS path: `commands/screenshots.ts` throws this error straight into
|
|
85
|
+
// `handleError`, which `process.exit(1)`s immediately after — there is no next request in
|
|
86
|
+
// this process for a reclaimed socket to serve. A drain here was bounded only by the FULL
|
|
87
|
+
// per-attempt abort budget (or, under `timeoutMs <= 0`, by nothing at all) for zero benefit,
|
|
88
|
+
// so it is removed outright rather than merely capped.
|
|
89
|
+
throw new Error(`artifact download failed: HTTP ${res.status}`);
|
|
90
|
+
}
|
|
91
|
+
try {
|
|
92
|
+
bytes = Buffer.from(await res.arrayBuffer());
|
|
93
|
+
}
|
|
94
|
+
catch (err) {
|
|
95
|
+
// A body read that never completes is aborted by the same budget; undici then rejects the
|
|
96
|
+
// stream (mirrors #2546's body-read guard). Map it into the same taxonomy as above.
|
|
97
|
+
throw toConnectionError(err, 'GET', safePath, budget?.signal.aborted === true);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
finally {
|
|
101
|
+
// Clear the timer on every exit path so a healthy download leaves no dangling budget timer.
|
|
102
|
+
budget?.cleanup();
|
|
18
103
|
}
|
|
19
|
-
const bytes = Buffer.from(await res.arrayBuffer());
|
|
20
104
|
if (dest === '-') {
|
|
21
105
|
process.stdout.write(bytes);
|
|
22
106
|
return;
|
|
@@ -14,3 +14,33 @@ export declare function parseIntFlag(flag: string, value: string): number;
|
|
|
14
14
|
export declare function parseEnum<T extends string>(flag: string, allowed: readonly T[], value: string): T;
|
|
15
15
|
/** Split a comma-separated flag value into a trimmed, non-empty string array. */
|
|
16
16
|
export declare function parseCsv(value: string): string[];
|
|
17
|
+
/**
|
|
18
|
+
* Parse a finite decimal flag value (#2545). Unlike {@link parseIntFlag}, the `/^\d+(\.\d+)?$/`
|
|
19
|
+
* test accepts a fractional value such as `2.5` — required for spec `type: number` fields like
|
|
20
|
+
* `monitors change_threshold` (default 5, range 0–100) that `parseIntFlag`'s `/^\d+$/` would reject
|
|
21
|
+
* at parse time. Rejects `abc`, ``, `-5`, `1.` and `1e2` with the flag named in the message.
|
|
22
|
+
*/
|
|
23
|
+
export declare function parseNumberFlag(flag: string, value: string): number;
|
|
24
|
+
/**
|
|
25
|
+
* Parse a `cache_ttl` flag value (#2545). Mirrors the gateway validator's own format grammar
|
|
26
|
+
* (`^(0|\d+[hd])$`, `packages/gateway/src/validators/shared.ts`) locally so an off-grammar value
|
|
27
|
+
* (`30m`, `soon`) is rejected at parse time with an actionable message instead of a server 422.
|
|
28
|
+
* The string is returned verbatim — the validator accepts the string `'0'`, and the range cap
|
|
29
|
+
* (7d/168h for scrape+fetch) stays a server-side concern.
|
|
30
|
+
*/
|
|
31
|
+
export declare function parseCacheTtl(flag: string, value: string): string;
|
|
32
|
+
/**
|
|
33
|
+
* Parse a JSON-object- or JSON-array-valued flag (#2545). The ONLY `JSON.parse` call site for CLI
|
|
34
|
+
* flag values — every JSON-shaped flag (`--cookies`, `--headers`, `--custom`,
|
|
35
|
+
* `--webhook-headers`, `--options`, `--scrape-options`) routes through here, so command modules
|
|
36
|
+
* carry no raw `JSON.parse` (grep-gated). Rejects malformed JSON AND a value whose top-level shape
|
|
37
|
+
* does not match `expect`, with a copy-pasteable example in the message (E5).
|
|
38
|
+
*/
|
|
39
|
+
export declare function parseJsonFlag(flag: string, value: string, expect: 'object' | 'array'): unknown;
|
|
40
|
+
/**
|
|
41
|
+
* Commander collector for a repeatable, CSV-aware `--format` flag (#2545). Each occurrence is
|
|
42
|
+
* CSV-split and appended to the accumulator, so `--format raw_html --format markdown` and
|
|
43
|
+
* `--format raw_html,markdown` both yield `['raw_html','markdown']`. Commander seeds `prev` with
|
|
44
|
+
* the option default (or the previous return) on each occurrence.
|
|
45
|
+
*/
|
|
46
|
+
export declare function collectFormats(value: string, prev: string[] | undefined): string[];
|
package/dist/lib/parse-flags.js
CHANGED
|
@@ -30,3 +30,61 @@ export function parseCsv(value) {
|
|
|
30
30
|
.map((s) => s.trim())
|
|
31
31
|
.filter((s) => s.length > 0);
|
|
32
32
|
}
|
|
33
|
+
/**
|
|
34
|
+
* Parse a finite decimal flag value (#2545). Unlike {@link parseIntFlag}, the `/^\d+(\.\d+)?$/`
|
|
35
|
+
* test accepts a fractional value such as `2.5` — required for spec `type: number` fields like
|
|
36
|
+
* `monitors change_threshold` (default 5, range 0–100) that `parseIntFlag`'s `/^\d+$/` would reject
|
|
37
|
+
* at parse time. Rejects `abc`, ``, `-5`, `1.` and `1e2` with the flag named in the message.
|
|
38
|
+
*/
|
|
39
|
+
export function parseNumberFlag(flag, value) {
|
|
40
|
+
if (!/^\d+(\.\d+)?$/.test(value)) {
|
|
41
|
+
throw new InvalidArgumentError(`${flag} must be a number.`);
|
|
42
|
+
}
|
|
43
|
+
return Number(value);
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Parse a `cache_ttl` flag value (#2545). Mirrors the gateway validator's own format grammar
|
|
47
|
+
* (`^(0|\d+[hd])$`, `packages/gateway/src/validators/shared.ts`) locally so an off-grammar value
|
|
48
|
+
* (`30m`, `soon`) is rejected at parse time with an actionable message instead of a server 422.
|
|
49
|
+
* The string is returned verbatim — the validator accepts the string `'0'`, and the range cap
|
|
50
|
+
* (7d/168h for scrape+fetch) stays a server-side concern.
|
|
51
|
+
*/
|
|
52
|
+
export function parseCacheTtl(flag, value) {
|
|
53
|
+
if (!/^(0|\d+[hd])$/.test(value)) {
|
|
54
|
+
throw new InvalidArgumentError(`${flag} must be 0, Nh (hours), or Nd (days) — e.g. 48h, 2d, 0.`);
|
|
55
|
+
}
|
|
56
|
+
return value;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Parse a JSON-object- or JSON-array-valued flag (#2545). The ONLY `JSON.parse` call site for CLI
|
|
60
|
+
* flag values — every JSON-shaped flag (`--cookies`, `--headers`, `--custom`,
|
|
61
|
+
* `--webhook-headers`, `--options`, `--scrape-options`) routes through here, so command modules
|
|
62
|
+
* carry no raw `JSON.parse` (grep-gated). Rejects malformed JSON AND a value whose top-level shape
|
|
63
|
+
* does not match `expect`, with a copy-pasteable example in the message (E5).
|
|
64
|
+
*/
|
|
65
|
+
export function parseJsonFlag(flag, value, expect) {
|
|
66
|
+
const example = expect === 'array' ? `[{"name":"a","value":"b"}]` : `{"X-Header":"value"}`;
|
|
67
|
+
let parsed;
|
|
68
|
+
try {
|
|
69
|
+
parsed = JSON.parse(value);
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
throw new InvalidArgumentError(`${flag} must be valid JSON (a ${expect}), e.g. '${example}'.`);
|
|
73
|
+
}
|
|
74
|
+
const matches = expect === 'array'
|
|
75
|
+
? Array.isArray(parsed)
|
|
76
|
+
: typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed);
|
|
77
|
+
if (!matches) {
|
|
78
|
+
throw new InvalidArgumentError(`${flag} must be a JSON ${expect}, e.g. '${example}'.`);
|
|
79
|
+
}
|
|
80
|
+
return parsed;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Commander collector for a repeatable, CSV-aware `--format` flag (#2545). Each occurrence is
|
|
84
|
+
* CSV-split and appended to the accumulator, so `--format raw_html --format markdown` and
|
|
85
|
+
* `--format raw_html,markdown` both yield `['raw_html','markdown']`. Commander seeds `prev` with
|
|
86
|
+
* the option default (or the previous return) on each occurrence.
|
|
87
|
+
*/
|
|
88
|
+
export function collectFormats(value, prev) {
|
|
89
|
+
return [...(prev ?? []), ...parseCsv(value)];
|
|
90
|
+
}
|
package/dist/lib/url-lines.d.ts
CHANGED
|
@@ -1,12 +1,50 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* One parsed `--file` line: a bare URL string, or a per-item config object carrying `url`.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* `
|
|
6
|
-
* the
|
|
7
|
-
|
|
4
|
+
* The two arms are not interchangeable on the wire. `/v1/bulk` and `/v1/scrape/bulk` accept
|
|
5
|
+
* `urls` (strings) XOR `items` (objects); `/v1/fetch/bulk` has no `items` property at all and
|
|
6
|
+
* takes a heterogeneous `urls` union. Callers pick the right shape — see {@link toBulkItems}.
|
|
7
|
+
*/
|
|
8
|
+
export type BulkLine = string | Record<string, unknown>;
|
|
9
|
+
/**
|
|
10
|
+
* Read a newline-delimited `--file` and return one {@link BulkLine} per non-blank line.
|
|
11
|
+
*
|
|
12
|
+
* Line grammar:
|
|
13
|
+
* - a non-JSON line -> the raw line (a plain URL string)
|
|
14
|
+
* - a JSON object whose ONLY key is `url` -> the bare url string
|
|
15
|
+
* - a JSON object carrying other keys too -> the object verbatim (per-item config)
|
|
16
|
+
* - a JSON object with a missing/non-string url -> `InvalidArgumentError` (rejected locally)
|
|
17
|
+
* - a JSON non-object (number, array, string) -> the raw line
|
|
18
|
+
*
|
|
19
|
+
* The "only key is `url`" arm is what keeps the inverse case BYTE-IDENTICAL rather than merely
|
|
20
|
+
* equivalent: a file of url-only lines produces the exact same `{ urls, webhook_url }` body as
|
|
21
|
+
* before per-item config existed (#2596 E2).
|
|
22
|
+
*
|
|
23
|
+
* Rejecting a malformed `url` is a deliberate behavior change (#2596 DoD 8). The previous parser
|
|
24
|
+
* fell through to the raw line, so `{"url": 123}` was sent to the server AS a URL — a confusing
|
|
25
|
+
* 422 for what is a local input mistake. `InvalidArgumentError` surfaces through the same
|
|
26
|
+
* local-usage-error path as every argParser rejection (exit 1), before any request is issued.
|
|
27
|
+
*
|
|
28
|
+
* Throws on a missing file (ENOENT from readFileSync) — callers surface this via their
|
|
29
|
+
* handleError path.
|
|
30
|
+
*
|
|
31
|
+
* The rejection message for a malformed line (#2596 cr-1 / E12) names the 1-based PHYSICAL line
|
|
32
|
+
* NUMBER (i.e. the line's own position in the file, counting blank lines) and the line's own KEY
|
|
33
|
+
* NAMES only — never the raw line or any value. A `--file` line is where per-item `cookies`,
|
|
34
|
+
* `headers`, and auth-bearing `metadata` now live (#2596), so echoing the raw line back would
|
|
35
|
+
* leak session tokens and similar secrets to stderr, CI logs, and shell transcripts. Because the
|
|
36
|
+
* raw line is deliberately never echoed, the line number is the ONLY locator left in the message —
|
|
37
|
+
* it must be computed BEFORE blank lines are filtered out, or it silently mis-locates the bad
|
|
38
|
+
* entry whenever the file contains blank lines above it (#2596 cr-2).
|
|
39
|
+
*/
|
|
40
|
+
export declare function parseBulkLines(file: string): BulkLine[];
|
|
41
|
+
/**
|
|
42
|
+
* Promote every {@link BulkLine} to the `items[]` object shape used by `/v1/bulk` and
|
|
43
|
+
* `/v1/scrape/bulk`.
|
|
8
44
|
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
45
|
+
* Deliberately stamps NOTHING beyond `url`. `/v1/bulk` owns capture-kind precedence server-side
|
|
46
|
+
* (per-item value, then `defaults`, then the endpoint default) since #2641 made the per-item
|
|
47
|
+
* field optional; a CLI-side default would silently override a caller's `--defaults` and is
|
|
48
|
+
* grep-gated out (#2596 G7).
|
|
11
49
|
*/
|
|
12
|
-
export declare function
|
|
50
|
+
export declare function toBulkItems(lines: BulkLine[]): Array<Record<string, unknown>>;
|
package/dist/lib/url-lines.js
CHANGED
|
@@ -1,34 +1,73 @@
|
|
|
1
1
|
import { readFileSync } from 'node:fs';
|
|
2
|
+
import { InvalidArgumentError } from 'commander';
|
|
2
3
|
/**
|
|
3
|
-
* Read a newline-delimited file
|
|
4
|
+
* Read a newline-delimited `--file` and return one {@link BulkLine} per non-blank line.
|
|
4
5
|
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
6
|
+
* Line grammar:
|
|
7
|
+
* - a non-JSON line -> the raw line (a plain URL string)
|
|
8
|
+
* - a JSON object whose ONLY key is `url` -> the bare url string
|
|
9
|
+
* - a JSON object carrying other keys too -> the object verbatim (per-item config)
|
|
10
|
+
* - a JSON object with a missing/non-string url -> `InvalidArgumentError` (rejected locally)
|
|
11
|
+
* - a JSON non-object (number, array, string) -> the raw line
|
|
9
12
|
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
13
|
+
* The "only key is `url`" arm is what keeps the inverse case BYTE-IDENTICAL rather than merely
|
|
14
|
+
* equivalent: a file of url-only lines produces the exact same `{ urls, webhook_url }` body as
|
|
15
|
+
* before per-item config existed (#2596 E2).
|
|
16
|
+
*
|
|
17
|
+
* Rejecting a malformed `url` is a deliberate behavior change (#2596 DoD 8). The previous parser
|
|
18
|
+
* fell through to the raw line, so `{"url": 123}` was sent to the server AS a URL — a confusing
|
|
19
|
+
* 422 for what is a local input mistake. `InvalidArgumentError` surfaces through the same
|
|
20
|
+
* local-usage-error path as every argParser rejection (exit 1), before any request is issued.
|
|
21
|
+
*
|
|
22
|
+
* Throws on a missing file (ENOENT from readFileSync) — callers surface this via their
|
|
23
|
+
* handleError path.
|
|
24
|
+
*
|
|
25
|
+
* The rejection message for a malformed line (#2596 cr-1 / E12) names the 1-based PHYSICAL line
|
|
26
|
+
* NUMBER (i.e. the line's own position in the file, counting blank lines) and the line's own KEY
|
|
27
|
+
* NAMES only — never the raw line or any value. A `--file` line is where per-item `cookies`,
|
|
28
|
+
* `headers`, and auth-bearing `metadata` now live (#2596), so echoing the raw line back would
|
|
29
|
+
* leak session tokens and similar secrets to stderr, CI logs, and shell transcripts. Because the
|
|
30
|
+
* raw line is deliberately never echoed, the line number is the ONLY locator left in the message —
|
|
31
|
+
* it must be computed BEFORE blank lines are filtered out, or it silently mis-locates the bad
|
|
32
|
+
* entry whenever the file contains blank lines above it (#2596 cr-2).
|
|
12
33
|
*/
|
|
13
|
-
export function
|
|
34
|
+
export function parseBulkLines(file) {
|
|
14
35
|
return readFileSync(file, 'utf8')
|
|
15
36
|
.split('\n')
|
|
16
|
-
.map((
|
|
17
|
-
.filter((
|
|
18
|
-
.map((line) => {
|
|
37
|
+
.map((raw, index) => ({ line: raw.trim(), lineNumber: index + 1 }))
|
|
38
|
+
.filter(({ line }) => line.length > 0)
|
|
39
|
+
.map(({ line, lineNumber }) => {
|
|
40
|
+
let parsed;
|
|
19
41
|
try {
|
|
20
|
-
|
|
21
|
-
if (parsed !== null && typeof parsed === 'object' && 'url' in parsed) {
|
|
22
|
-
const url = parsed.url;
|
|
23
|
-
if (typeof url === 'string')
|
|
24
|
-
return url;
|
|
25
|
-
}
|
|
26
|
-
// Parsed as JSON but not a {url:string} object — fall back to the raw line.
|
|
27
|
-
return line;
|
|
42
|
+
parsed = JSON.parse(line);
|
|
28
43
|
}
|
|
29
44
|
catch {
|
|
30
45
|
// Not JSON — treat the whole line as a plain URL string.
|
|
31
46
|
return line;
|
|
32
47
|
}
|
|
48
|
+
if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
|
49
|
+
// Parsed as JSON but not an object — fall back to the raw line, as before.
|
|
50
|
+
return line;
|
|
51
|
+
}
|
|
52
|
+
const obj = parsed;
|
|
53
|
+
if (typeof obj['url'] !== 'string' || obj['url'].length === 0) {
|
|
54
|
+
const keys = Object.keys(obj).join(', ') || '(none)';
|
|
55
|
+
throw new InvalidArgumentError(`--file line ${lineNumber} must carry a non-empty string "url" (keys present: ${keys})`);
|
|
56
|
+
}
|
|
57
|
+
// A line whose only key is `url` is indistinguishable from a plain URL line — keep it a
|
|
58
|
+
// bare string so url-only files stay byte-identical on the wire.
|
|
59
|
+
return Object.keys(obj).length === 1 ? obj['url'] : obj;
|
|
33
60
|
});
|
|
34
61
|
}
|
|
62
|
+
/**
|
|
63
|
+
* Promote every {@link BulkLine} to the `items[]` object shape used by `/v1/bulk` and
|
|
64
|
+
* `/v1/scrape/bulk`.
|
|
65
|
+
*
|
|
66
|
+
* Deliberately stamps NOTHING beyond `url`. `/v1/bulk` owns capture-kind precedence server-side
|
|
67
|
+
* (per-item value, then `defaults`, then the endpoint default) since #2641 made the per-item
|
|
68
|
+
* field optional; a CLI-side default would silently override a caller's `--defaults` and is
|
|
69
|
+
* grep-gated out (#2596 G7).
|
|
70
|
+
*/
|
|
71
|
+
export function toBulkItems(lines) {
|
|
72
|
+
return lines.map((l) => (typeof l === 'string' ? { url: l } : l));
|
|
73
|
+
}
|
package/dist/output.d.ts
CHANGED
|
@@ -22,6 +22,24 @@ export interface OutputFlags {
|
|
|
22
22
|
raw?: boolean | undefined;
|
|
23
23
|
output?: string | undefined;
|
|
24
24
|
}
|
|
25
|
+
/**
|
|
26
|
+
* Serialize a value as the CLI's `--json` payload text. The single serializer EVERY `--json`
|
|
27
|
+
* branch in this module goes through, so no branch can regress to a raw `JSON.stringify`.
|
|
28
|
+
*
|
|
29
|
+
* `JSON.stringify` returns the VALUE `undefined` — not a string — for `undefined`, functions and
|
|
30
|
+
* symbols. Handed to `console.log` that prints the literal text `undefined`, which is NOT valid
|
|
31
|
+
* JSON and breaks every `| jq` pipeline; handed to `writeFileSync` it throws
|
|
32
|
+
* `ERR_INVALID_ARG_TYPE`. A `204 No Content` response resolves to `undefined` in the SDK
|
|
33
|
+
* (`monitors.delete` is typed `Promise<void>`), so this is the ORDINARY shape of
|
|
34
|
+
* `bytekit monitors delete <id> --json`, not an exotic edge case (#2544).
|
|
35
|
+
*
|
|
36
|
+
* The representation is JSON `null` — the canonical JSON spelling of "no content". Deliberately
|
|
37
|
+
* NOT a synthesized envelope such as `{"status":"deleted"}`: that is a shape the API never
|
|
38
|
+
* returned, and it would have to be re-invented (and kept consistent) per command. `null` is
|
|
39
|
+
* uniform across every 204-returning path, and `… --json | jq .` prints `null` and exits 0.
|
|
40
|
+
* The contract is documented in README (`Exit codes and output contracts`) and CHANGELOG.
|
|
41
|
+
*/
|
|
42
|
+
export declare function toJsonText(data: unknown): string;
|
|
25
43
|
/**
|
|
26
44
|
* Resolve the output mode from the flags, and enforce the `--json` / `--raw` exclusion.
|
|
27
45
|
*
|
|
@@ -75,10 +93,21 @@ interface QueuedEnvelope {
|
|
|
75
93
|
* The user asked for a content body that does not exist yet — but the job was accepted, so losing
|
|
76
94
|
* the handle (the old behaviour: throw `--raw: response contains no formats`, exit 1) is the real
|
|
77
95
|
* defect. Instead: emit the `sc_` id to STDOUT so `$(bytekit scrape create … --raw)` still captures
|
|
78
|
-
* a usable handle, print a human note to STDERR explaining it is queued and how to poll, and
|
|
79
|
-
*
|
|
96
|
+
* a usable handle, print a human note to STDERR explaining it is queued and how to poll, and signal
|
|
97
|
+
* {@link QUEUED_RAW_EXIT_CODE} — distinguishable from both success and error.
|
|
98
|
+
*
|
|
99
|
+
* The exit code is a SOFT `process.exitCode`, never a hard `process.exit` (#2544). Node documents
|
|
100
|
+
* `process.stdout` as ASYNCHRONOUS when it points at a pipe: a write that cannot be handed to the
|
|
101
|
+
* kernel in one go is queued in userspace and drained on a later event-loop turn, and
|
|
102
|
+
* `process.exit()` runs before any of those turns and DISCARDS the queue. That is exactly the
|
|
103
|
+
* `sc=$(bytekit scrape create … --async --raw)` case this branch exists to serve — the id would be
|
|
104
|
+
* truncated away whenever stdout is backlogged. Setting `process.exitCode` instead lets the process
|
|
105
|
+
* end naturally once stdout has drained, with the same observable exit code.
|
|
106
|
+
*
|
|
107
|
+
* Returns `void` (not `never`): callers MUST `return` after invoking it, since execution now
|
|
108
|
+
* continues past this function.
|
|
80
109
|
*/
|
|
81
|
-
export declare function emitQueuedRaw(env: QueuedEnvelope):
|
|
110
|
+
export declare function emitQueuedRaw(env: QueuedEnvelope): void;
|
|
82
111
|
/**
|
|
83
112
|
* Emit a raw body to the target.
|
|
84
113
|
*
|
|
@@ -101,18 +130,27 @@ export declare function emitRaw(body: string, target: OutputTarget): void;
|
|
|
101
130
|
* `--json | jq` pipelines stay byte-identical.
|
|
102
131
|
*/
|
|
103
132
|
export declare function emitResult(result: unknown, mode: OutputMode, target: OutputTarget, requestedFormat?: string): void;
|
|
133
|
+
/**
|
|
134
|
+
* Collect the `X-Fetch-*` response headers of a {@link FetchResult} into a Title-Cased object —
|
|
135
|
+
* the documented `--json` payload for a fetch command.
|
|
136
|
+
*/
|
|
137
|
+
export declare function collectFetchMetadata(headers: Headers): Record<string, string>;
|
|
104
138
|
/**
|
|
105
139
|
* Render a `/v1/fetch` content result.
|
|
106
140
|
*
|
|
107
|
-
* The
|
|
108
|
-
*
|
|
109
|
-
*
|
|
110
|
-
*
|
|
141
|
+
* The SDK's fetch resource resolves a {@link FetchResult} whose `bytes` ARE the fetched content and
|
|
142
|
+
* whose `headers` carry the `X-Fetch-*` metadata.
|
|
143
|
+
*
|
|
144
|
+
* - default — write the raw body BYTES verbatim (`process.stdout.write`, never `console.log`: no
|
|
145
|
+
* coercion, no appended newline, binary-safe). `bytes` is used rather than the `body` string
|
|
146
|
+
* because the string is a lossy UTF-8 decode for true binary passthrough (images, PDFs).
|
|
147
|
+
* - `--json` — print the `X-Fetch-*` metadata object instead of the body.
|
|
148
|
+
* - `target.file` — write the same bytes to that path instead of stdout, byte-for-byte.
|
|
111
149
|
*
|
|
112
|
-
* Falls back to {@link printResult} for any result that is not a fetch
|
|
150
|
+
* Falls back to {@link printResult} for any result that is not a fetch result (defensive — e.g. a
|
|
113
151
|
* bulk-fetch envelope routed here by mistake), so no path can silently emit nothing.
|
|
114
152
|
*/
|
|
115
|
-
export declare function emitFetchResult(result: unknown, jsonMode: boolean): void;
|
|
153
|
+
export declare function emitFetchResult(result: unknown, jsonMode: boolean, target?: OutputTarget): void;
|
|
116
154
|
/**
|
|
117
155
|
* Print a result to stdout.
|
|
118
156
|
*
|