@hunt-labs/bytekit-cli 0.5.0 → 0.6.0

package/dist/client.js

@@ -18,6 +18,24 @@ export const MIN_RETRY_AFTER_MS = 1_000;
 export const MAX_RETRY_AFTER_MS = 60_000;
 /** Exactly one retry. A second 429 is terminal — no loop. */
 export const MAX_RETRIES = 1;
+/**
+ * Cap on a best-effort, discard-only body drain (#2587 cr2-1). Independent of the per-attempt
+ * abort budget: the 429 drain below (`response.text().catch(() => {})`) exists ONLY so undici can
+ * return the socket to the connection pool — nothing downstream reads the drained bytes — but
+ * binding that read to the FULL per-attempt budget (120 s default, or, under `timeoutMs <= 0`,
+ * to no bound at all) let a stalled 429 body burn the whole budget, or hang forever, before a
+ * retry that needs nothing from that body. Short enough that a legitimately stalled 429 body costs
+ * the retry very little wall-clock time.
+ *
+ * #2587 cr3-2: past this cap, the read is not merely abandoned — {@link drainBestEffort} actively
+ * ABORTS its underlying request. Resolving the race without aborting anything (the earlier
+ * cr2-1 shape) let the retry proceed on schedule while the abandoned `response.text()` kept
+ * reading against a live socket forever, which — because the CLI exits via a soft
+ * `process.exitCode` (#2544) rather than `process.exit()` — kept the whole process alive
+ * indefinitely. `.catch`-guarding the read only made it safe to abandon the RESULT; it never made
+ * abandoning the underlying READ safe.
+ */
+export const DRAIN_TIMEOUT_MS = 2_000;
 /**
  * Default CLIENT-SIDE abort budget for a single fetch attempt, in ms.
  *
@@ -36,25 +54,44 @@ function timeoutReason(ms) {
     return reason;
 }
 /**
- * A per-attempt abort budget: an `AbortController` fired by a `setTimeout`.
+ * A per-attempt abort budget: an `AbortController`, optionally fired by a `setTimeout`.
  *
  * The budget spans the WHOLE attempt — time-to-headers **and** the response body read (#2546).
  * Arming it only until headers arrive left `response.text()` / `response.arrayBuffer()` unbounded,
  * so a server that answered with headers and then stalled its body hung the CLI forever.
  *
+ * The controller ALWAYS exists, even when `ms <= 0` — only its own timeout timer is conditional
+ * on `ms > 0`. `sendWithRetry`'s 429-drain cap ({@link DRAIN_TIMEOUT_MS}, via the returned
+ * `abort()`) needs a live controller to tear down an abandoned body read regardless of whether the
+ * OVERALL per-attempt budget is enabled — `ms <= 0` must still mean "no full-attempt timeout", not
+ * "no controller to abort a stalled drain with" (#2587 cr3-2: the earlier drain fix abandoned the
+ * read entirely rather than aborting it, so the socket — and the event loop — stayed alive
+ * indefinitely, worse than the pre-fix baseline it replaced).
+ *
  * Deliberately NOT `AbortSignal.timeout` (which the SDK prefers): that schedules on a timer the
  * test runner's fake timers cannot advance, so the transport's timeout would be untestable under
  * `vi.useFakeTimers()`. CLI requests are short-lived and single-shot, so the GC advantage
  * `AbortSignal.timeout` buys the long-running SDK does not apply here. The timer is `unref`'d so a
  * pending budget never keeps the process alive, and `cleanup()` clears it on every exit path.
+ *
+ * Exported so `lib/download-artifact.ts` can reuse the same budget for the artifact-download path
+ * (#2587) rather than growing a second timeout implementation — that path sits entirely outside
+ * `sendWithRetry` (it is not a JSON/raw API call), so it would otherwise never inherit this bound.
  */
-function createAttemptBudget(ms) {
+export function createAttemptBudget(ms) {
     const controller = new AbortController();
-    const timer = setTimeout(() => controller.abort(timeoutReason(ms)), ms);
-    timer.unref?.();
+    let timer;
+    if (ms > 0) {
+        timer = setTimeout(() => controller.abort(timeoutReason(ms)), ms);
+        timer.unref?.();
+    }
     return {
         signal: controller.signal,
-        cleanup: () => clearTimeout(timer),
+        cleanup: () => {
+            if (timer !== undefined)
+                clearTimeout(timer);
+        },
+        abort: (reason) => controller.abort(reason),
     };
 }
 /** The `name` of an error-shaped value, or `undefined` when it carries none. */
@@ -84,8 +121,11 @@ function abortNameOf(err) {
  * `timedOut` is true when this attempt's own budget fired the abort, which is the reliable
  * discriminator (undici rejects with the signal's reason — or a wrapper carrying it, see
  * {@link abortNameOf} — whose `name` we also inspect as a fallback for a caller-style AbortError).
+ *
+ * Exported so `lib/download-artifact.ts` reuses the same taxonomy for the artifact-download path
+ * (#2587) instead of hand-rolling a second error mapping.
  */
-function toConnectionError(err, method, path, timedOut) {
+export function toConnectionError(err, method, path, timedOut) {
     const name = abortNameOf(err) ?? '';
     let code;
     if (timedOut || name === 'TimeoutError') {
@@ -128,19 +168,84 @@ function sleep(ms) {
     });
 }
 /**
- * Shape a failed response into the SDK's OWN error class, using the SDK's own body parsing.
+ * Read (and discard) `body`, TEARING DOWN its underlying request after `DRAIN_TIMEOUT_MS` if it
+ * has not settled by then (#2587 cr3-2). `body` must already be guarded against rejection by its
+ * caller — this function only races it against a short local timer, it never itself decides what
+ * a rejection means.
+ *
+ * `abort` MUST be the abort function of the SAME `AbortController` the read's request was made
+ * with. Merely resolving this promise when the cap fires (the earlier, defective version of this
+ * function) let the retry proceed promptly while the abandoned `response.text()` kept running
+ * against a live socket forever — nothing else in the process held a reference to that read, but
+ * the open socket itself kept the event loop alive, so the CLI's soft `process.exitCode` exit
+ * (#2544) never actually fired. `response.body?.cancel()` cannot substitute for this: `text()` has
+ * already locked the stream by the time this function is called (property 4), so cancelling the
+ * body is not an option — aborting the ORIGINAL request's controller is the only way left to
+ * release a read already in flight. This is the same mechanism `sendWithRetry`'s headers+body
+ * budget already relies on to bound a normal successful read (#2546), so it is a proven pattern:
+ * undici rejects the pending read when its controller aborts, and tears down the socket rather
+ * than leaving it open.
+ *
+ * Deliberately a plain `setTimeout` (not `AbortSignal.timeout`), same reasoning as
+ * {@link createAttemptBudget}: a timer scheduled that way cannot be advanced by the test runner's
+ * fake timers, which would make this cap untestable under `vi.useFakeTimers()`. The cap timer is
+ * cleared as soon as `body` itself settles, so a fast drain never leaves a stray timer pending;
+ * conversely, if `body` settles only AFTER the cap fires, that later settlement is simply ignored
+ * — abandoning the VALUE is always safe because the caller already discards it either way (only
+ * the underlying READ itself must not be left running, which `abort` above now guarantees).
+ */
+function drainBestEffort(body, abort) {
+    return new Promise((resolve) => {
+        let settled = false;
+        const cap = setTimeout(() => {
+            if (!settled) {
+                settled = true;
+                abort(timeoutReason(DRAIN_TIMEOUT_MS));
+                resolve();
+            }
+        }, DRAIN_TIMEOUT_MS);
+        cap.unref?.();
+        void body.then(() => {
+            if (!settled) {
+                settled = true;
+                clearTimeout(cap);
+                resolve();
+            }
+        });
+    });
+}
+/**
+ * Shape a failed response into the SDK's OWN error class, mirroring the SDK's own `send()` body
+ * parsing (`packages/sdk-typescript/src/client.ts`) EXACTLY — precedence rule included — so
+ * `{status, code, message}` stays identical to what a direct SDK call would throw and
+ * `handleError()`'s `'status' in err` branch renders the same `Error: <message> (<code>, status:
+ * <n>)` either way.
  *
- * Reusing `ByteKitError` keeps `{status, code, message}` identical to what a direct SDK call
- * would throw, so `handleError()`'s `'status' in err` branch still renders
- * `Error: <message> (status: <n>)` and exits 1.
+ * Two error-body shapes exist across the API:
+ *   - the canonical unified envelope `{error: {code, message}}` — tested FIRST, so it always wins;
+ *   - a flat vendor-masked shape `{error: "<code>"}` (e.g. `/v1/search` 502:
+ *     `{"error":"search_provider_error"}`, docs/api/openapi.yaml §/v1/search 502) — the flat form
+ *     is strictly a FALLBACK and can never override an envelope (#2541, mirrored here for #2586:
+ *     the CLI's own copy of this parser had drifted from the SDK's and still only read the
+ *     envelope shape, so a flat 502 rendered `internal_error` instead of `search_provider_error`).
+ *
+ * The explicit `!== null` guard on the envelope check matters — `typeof null` is `'object'`.
  */
 async function toApiError(response) {
     let code = 'internal_error';
     let message = response.statusText;
     try {
         const json = (await response.json());
-        code = json.error?.code ?? code;
-        message = json.error?.message ?? message;
+        if (typeof json.error === 'object' && json.error !== null) {
+            code = json.error.code ?? code;
+            message = json.error.message ?? message;
+        }
+        else if (typeof json.error === 'string' && json.error.length > 0) {
+            // The string IS the error code. `message` falls back to it only when the response carried
+            // no statusText, so the thrown error is never message-empty.
+            code = json.error;
+            message = message || json.error;
+        }
     }
     catch {
         // leave defaults
@@ -189,11 +294,12 @@ async function sendWithRetry(apiKey, baseUrl, timeoutMs, method, path, body, rea
         if (body !== undefined) {
             init.body = JSON.stringify(body);
         }
-        // Per-attempt abort budget: a fresh timer for THIS attempt. A budget of 0 disables it.
-        const budget = timeoutMs > 0 ? createAttemptBudget(timeoutMs) : undefined;
-        if (budget !== undefined) {
-            init.signal = budget.signal;
-        }
+        // Per-attempt abort budget: a fresh controller for THIS attempt. `timeoutMs <= 0` disables
+        // the OVERALL attempt timer (unchanged), but the controller itself always exists — the 429
+        // drain below needs something to abort even when the full-attempt timeout is off (#2587
+        // cr3-2).
+        const budget = createAttemptBudget(timeoutMs);
+        init.signal = budget.signal;
         // Set only by the 429-retry branch below — the one path that falls out of the `try` rather
         // than returning or throwing. Left `undefined` on every other path, which never reaches the
         // `sleep` after the `finally`.
@@ -207,7 +313,7 @@ async function sendWithRetry(apiKey, baseUrl, timeoutMs, method, path, body, rea
                 // No HTTP response was produced (timeout, abort, socket/DNS/TLS). Re-throw as the SDK's
                 // own ByteKitConnectionError so a hung server can never hang the CLI and a network failure
                 // no longer surfaces as a bare `Error: fetch failed`.
-                throw toConnectionError(err, method, path, budget?.signal.aborted === true);
+                throw toConnectionError(err, method, path, budget.signal.aborted === true);
             }
             if (response.ok) {
                 try {
@@ -216,11 +322,44 @@ async function sendWithRetry(apiKey, baseUrl, timeoutMs, method, path, body, rea
                 catch (err) {
                     // A body read that never completes is aborted by the same budget; undici then rejects
                     // the stream (#2546). Map it into the same taxonomy the headers phase uses.
-                    throw toConnectionError(err, method, path, budget?.signal.aborted === true);
+                    throw toConnectionError(err, method, path, budget.signal.aborted === true);
                 }
             }
             if (response.status === 429 && attempt < MAX_RETRIES) {
                 retryAfterMs = parseRetryAfterMs(response.headers.get('Retry-After'));
+                // This 429 body is never read on the retry path (only the headers/Retry-After matter), so
+                // it must be fully CONSUMED before the retry — not merely cancelled — for undici to
+                // reclaim the underlying socket for connection-pool reuse (#2587 dod-4).
+                //
+                // #2587 cr-3 correction: an earlier version of this fix used `response.body?.cancel()`,
+                // reasoning that cancelling let undici reclaim the socket. That is backwards —
+                // `cancel()` ABORTS the fetch and DESTROYS the socket rather than returning it to the
+                // pool. Measured against a real `node:http` server and real undici (5x 429->200
+                // sequences, counting `connection` events): `cancel()` was inert for the gateway's actual
+                // tiny-JSON 429 body (2 conns, identical to no drain at all) and strictly WORSE than no
+                // drain for a large body (7 conns vs 6). A full read via `response.text()` is what
+                // actually lets undici return the connection to the pool (2 conns at either size). The
+                // `.catch(() => {})` guard stays: a real body read can still reject on an already-errored
+                // or locked stream, and that must never turn a recoverable 429 into a hard failure.
+                //
+                // #2587 cr2-1 correction: the read above was previously awaited directly, bounded only by
+                // the FULL per-attempt abort budget (120 s default) or, under `timeoutMs <= 0`, by nothing
+                // at all — a stalled 429 body could burn the whole budget (or hang forever) before a retry
+                // that needs nothing from that body (`Retry-After` is already parsed above).
+                // {@link drainBestEffort} bounds the read by its own short, independent cap
+                // ({@link DRAIN_TIMEOUT_MS}) instead, so a stalled drain can never delay the retry by more
+                // than that cap regardless of the attempt budget's size or absence.
+                //
+                // #2587 cr3-2 correction: cr2-1's `drainBestEffort` bounded only how long the RETRY
+                // waited — the cap resolved the returned promise but never touched the abandoned
+                // `response.text()` call itself, which kept reading against a live (never-aborted) socket
+                // indefinitely. Because every CLI success path exits via a soft `process.exitCode`
+                // (#2544) rather than `process.exit()`, that orphaned read then kept the event loop open
+                // forever: strictly WORSE than the pre-#2587 baseline (which at least exited once its own
+                // unbounded read finally settled or errored). `drainBestEffort` now takes this attempt's
+                // own `budget.abort` and calls it when the cap fires, so the abandoned read is actually
+                // torn down — not merely ignored — and the socket (and the event loop) is released.
+                await drainBestEffort(response.text().catch(() => { }), budget.abort);
             }
             else {
                 // `toApiError` reads the error body, so it too runs inside the budget.
@@ -230,7 +369,7 @@ async function sendWithRetry(apiKey, baseUrl, timeoutMs, method, path, body, rea
         finally {
             // Clear the timer on every exit path — success (body read included), HTTP error, or
             // abort — so a healthy request leaves no dangling 120s timer.
-            budget?.cleanup();
+            budget.cleanup();
         }
         // OUTSIDE the budget's `finally`: the backoff is not spent against any attempt's budget, and
         // the next attempt arms a fresh one. The `??` is unreachable in practice (only the branch
@@ -246,9 +385,21 @@ async function sendWithRetry(apiKey, baseUrl, timeoutMs, method, path, body, rea
  * property of the client (see {@link buildClient}) routes all JSON commands through it from one
  * seam. `/v1/fetch` does NOT ride this transport — it rides `requestRaw` (see
  * {@link createRetryingRequestRaw}); `/v1/fetch/bulk` is a JSON job envelope and does.
+ *
+ * DECISION (#2586 dod-4): the 4th `RequestOptions` parameter of the SDK's contract is accepted
+ * here — so the signature matches `ByteKit['request']` exactly, closing the gap where a caller-typo
+ * of a 5th positional argument would silently no-op instead of a compile error — but deliberately
+ * NOT forwarded into `sendWithRetry`. Two reasons: (1) no CLI command constructs a `RequestOptions`
+ * today (no `--timeout` flag exists yet, and `signal` has no CLI-level source), so the parameter is
+ * always `undefined` in practice; (2) the CLI already owns a per-attempt abort budget with its own
+ * semantics (`createAttemptBudget`, `timeoutMs` above) that would need explicit composition with a
+ * caller `signal` to honor `RequestOptions` correctly (the SDK's `composeSignals`) — inventing that
+ * unexercised composition here, with no caller able to reach it, would be undertested surface for no
+ * present behavior change. If a future CLI flag needs per-call overrides, wire it through
+ * `sendWithRetry`'s `timeoutMs` (already per-attempt) at that point, with its own tests.
  */
 export function createRetryingRequest(apiKey, baseUrl, timeoutMs = DEFAULT_TIMEOUT_MS) {
-    return async function request(method, path, body) {
+    return async function request(method, path, body, _options) {
         // The reader consumes the stream and nothing else, so the abort budget covers the read
         // (#2546) while the 204/empty guard and `JSON.parse` stay OUTSIDE the guarded region —
         // a malformed 200 body must still surface its own SyntaxError, never a connection error.
@@ -273,9 +424,14 @@ export function createRetryingRequest(apiKey, baseUrl, timeoutMs = DEFAULT_TIMEO
  * so `FetchResource` builds an ordinary `FetchResult` from it and the CLI never forks fetch
  * handling away from the SDK. `bytes` is byte-exact (binary-safe); `body` is the best-effort UTF-8
  * decode, matching the SDK's own semantics.
+ *
+ * The 4th `RequestOptions` parameter is accepted but not forwarded — see the DECISION note on
+ * {@link createRetryingRequest}, which applies here verbatim. `FetchResource.create`/`get` DO thread
+ * a `requestOptions` argument through to this seam (unlike every other resource), but no CLI `fetch`
+ * command constructs one today (no `--timeout` flag), so it is still always `undefined` in practice.
  */
 export function createRetryingRequestRaw(apiKey, baseUrl, timeoutMs = DEFAULT_TIMEOUT_MS) {
-    return async function requestRaw(method, path, body) {
+    return async function requestRaw(method, path, body, _options) {
         // As in `createRetryingRequest`: the reader only drains the stream (so the abort budget
         // covers it, #2546); the UTF-8 decode happens outside the guarded region.
         const { bytes, headers, status } = await sendWithRetry(apiKey, baseUrl, timeoutMs, method, path, body, async (response) => ({
@@ -291,6 +447,45 @@ export function createRetryingRequestRaw(apiKey, baseUrl, timeoutMs = DEFAULT_TI
         };
     };
 }
+/**
+ * Build the CLI's header-carrying JSON transport: the SDK's `requestWithResponse` contract
+ * (`{data, headers, status}`, #2547), plus the same bounded 429 retry every other CLI transport
+ * gets.
+ *
+ * Unused by any CLI command today — `screenshots.createWithResponse` / `getWithResponse` and
+ * `sitemap.getWithResponse` all exist on the SDK surface but no command builds on them yet. It is
+ * still installed (see {@link buildClient}) rather than left un-overridden: the transport-coverage
+ * contract test (#2586 dod-3) derives its method list from the real SDK prototype, and leaving a
+ * transport method un-overridden is exactly how #2535 shipped broken — the first command that DOES
+ * call a `*WithResponse` resource method must not silently ride the bare SDK transport and skip the
+ * CLI's retry/timeout/error taxonomy.
+ *
+ * Same body-read shape as {@link createRetryingRequest} (204/empty → `undefined`, `JSON.parse`
+ * otherwise), with `headers` and `status` threaded through from the successful response alongside
+ * the parsed `data`. The 4th `RequestOptions` parameter is accepted but not forwarded — see the
+ * DECISION note on {@link createRetryingRequest}.
+ */
+export function createRetryingRequestWithResponse(apiKey, baseUrl, timeoutMs = DEFAULT_TIMEOUT_MS) {
+    return async function requestWithResponse(method, path, body, _options) {
+        const { status, text, headers } = await sendWithRetry(apiKey, baseUrl, timeoutMs, method, path, body, async (response) => ({
+            status: response.status,
+            text: await response.text(),
+            headers: response.headers,
+        }));
+        const data = status === 204 || text.trim().length === 0 ? undefined : JSON.parse(text);
+        return { data, headers, status };
+    };
+}
+/**
+ * Strip a trailing slash (or run of them) from a base URL, mirroring the SDK's own constructor
+ * normalization verbatim (`packages/sdk-typescript/src/client.ts`, `/\/+$/`, #2481) — so
+ * `baseUrl + path` (every CLI-owned path already starts with `/`) never yields a double-slash URL.
+ * Only the trailing slash is removed; a path-carrying baseUrl (e.g. a reverse-proxy prefix
+ * `…/bytekit`) is preserved.
+ */
+function normalizeBaseUrl(url) {
+    return url.replace(/\/+$/, '');
+}
 /**
  * Build an authenticated SDK client from global CLI options.
  *
@@ -299,10 +494,19 @@ export function createRetryingRequestRaw(apiKey, baseUrl, timeoutMs = DEFAULT_TI
  *   baseUrl: --base-url flag → BYTEKIT_BASE_URL → DEFAULT_BASE_URL.
  * Exits with a descriptive error when no key is available.
  *
- * The returned client's `request` AND `requestRaw` are both replaced by the retrying transports, so
- * every command inherits the 429 retry — JSON commands via `request`, the `/v1/fetch` content
- * commands via `requestRaw`. Overriding only `request` is the #2535 defect: the SDK's fetch
- * resource rides `requestRaw`, so fetch bypassed the retry and the CLI's error taxonomy entirely.
+ * The returned client's `request`, `requestRaw`, AND `requestWithResponse` are all replaced by the
+ * retrying transports, so every command inherits the 429 retry — JSON commands via `request`, the
+ * `/v1/fetch` content commands via `requestRaw`, and the header-carrying `*WithResponse` surface via
+ * `requestWithResponse`. Overriding only `request` is the #2535 defect: the SDK's fetch resource
+ * rides `requestRaw`, so fetch bypassed the retry and the CLI's error taxonomy entirely; leaving
+ * `requestWithResponse` un-overridden is the same class of defect (#2586) — it is unused by any CLI
+ * command TODAY, but a future command built on `screenshots.createWithResponse` /
+ * `sitemap.getWithResponse` would otherwise silently ride the bare SDK transport.
+ *
+ * `baseUrl` is normalized ONCE, here, before it reaches either the SDK constructor or the CLI's own
+ * transport builders below — the CLI's transports build their request URL from this local variable
+ * directly (they do not call the SDK's own normalized `send()`), so a trailing-slash flag would
+ * otherwise double-slash every request the CLI makes (#2586).
  */
 export function buildClient(opts) {
     const apiKey = opts.key ?? process.env['BYTEKIT_API_KEY'];
@@ -310,9 +514,10 @@ export function buildClient(opts) {
         console.error('Error: API key required. Pass --key or set BYTEKIT_API_KEY.');
         process.exit(1);
     }
-    const baseUrl = opts.baseUrl ?? process.env['BYTEKIT_BASE_URL'] ?? DEFAULT_BASE_URL;
+    const baseUrl = normalizeBaseUrl(opts.baseUrl ?? process.env['BYTEKIT_BASE_URL'] ?? DEFAULT_BASE_URL);
     const client = new ByteKit({ apiKey, baseUrl });
     client.request = createRetryingRequest(apiKey, baseUrl);
     client.requestRaw = createRetryingRequestRaw(apiKey, baseUrl);
+    client.requestWithResponse = createRetryingRequestWithResponse(apiKey, baseUrl);
     return client;
 }

package/dist/commands/bulk.js

@@ -1,7 +1,8 @@
 import { Command } from 'commander';
 import { buildClient } from '../client.js';
 import { printResult, handleError } from '../output.js';
-import { parseUrlLines } from '../lib/url-lines.js';
+import { parseBulkLines, toBulkItems } from '../lib/url-lines.js';
+import { parseJsonFlag } from '../lib/parse-flags.js';
 export function makeBulkCommand(getGlobal) {
     const cmd = new Command('bulk').description('Bulk screenshot operations');
     const screenshotsCmd = new Command('screenshots').description('Bulk screenshot sub-resources');
@@ -23,14 +24,31 @@ export function makeBulkCommand(getGlobal) {
     cmd
         .command('create')
         .description('Create a bulk screenshots job')
-        .requiredOption('--file <path>', 'Path to newline-delimited file of URLs (plain URL or {"url":"..."} per line)')
+        .requiredOption('--file <path>', 'Path to a newline-delimited file. Each line is a plain URL, or a JSON object carrying "url" plus per-item config. A line whose only key is "url" behaves exactly as before (sent as a bare URL string); any line with additional keys switches the payload to the per-item shape.')
         .requiredOption('--webhook-url <url>', 'HTTPS webhook URL notified when the job completes')
+        .option('--webhook-secret <secret>', 'HMAC secret used to sign the completion webhook (omitted → unsigned)')
+        .option('--defaults <json>', 'Defaults applied to every entry, as a JSON object (omitted → none). Unknown keys are rejected by /v1/bulk (422) and ignored by scrape bulk / fetch bulk; fetch bulk additionally rejects browser-only keys (cookies, wait_until, wait_for_selector, delay_ms) with a 400.', (v) => parseJsonFlag('--defaults', v, 'object'))
+        .option('--metadata <json>', `Arbitrary metadata echoed on the job envelope, as a JSON object, e.g. '{"run":"7"}' (omitted → none)`, (v) => parseJsonFlag('--metadata', v, 'object'))
+        // Deliberately NO --custom: /v1/bulk is `.strict()` at the top level with no `custom`
+        // property (validators/bulk.ts:135), so the flag would 422 every call (#2596 drift D2).
         .action(async (opts) => {
         const g = getGlobal();
         const client = buildClient(g);
         try {
-            const urls = parseUrlLines(opts.file);
-            const result = await client.bulk.create({ urls, webhook_url: opts.webhookUrl });
+            const lines = parseBulkLines(opts.file);
+            // `urls` XOR `items` (validators/bulk.ts:136-155) — send exactly one. A file of only
+            // url-only lines keeps the pre-#2596 `{ urls, webhook_url }` body, key order included,
+            // so existing callers are byte-identical on the wire (E2).
+            const body = lines.every((l) => typeof l === 'string')
+                ? { urls: lines, webhook_url: opts.webhookUrl }
+                : { items: toBulkItems(lines), webhook_url: opts.webhookUrl };
+            if (opts.webhookSecret !== undefined)
+                body['webhook_secret'] = opts.webhookSecret;
+            if (opts.defaults !== undefined)
+                body['defaults'] = opts.defaults;
+            if (opts.metadata !== undefined)
+                body['metadata'] = opts.metadata;
+            const result = await client.bulk.create(body);
             printResult(result, g.json);
         }
         catch (err) {

package/dist/commands/fetch.js

@@ -1,11 +1,13 @@
 import { Command, InvalidArgumentError } from 'commander';
 import { buildClient } from '../client.js';
 import { printResult, emitFetchResult, handleError } from '../output.js';
-import { parseUrlLines } from '../lib/url-lines.js';
-import { parseIntFlag, parseEnum, parseCacheTtl } from '../lib/parse-flags.js';
+import { parseBulkLines } from '../lib/url-lines.js';
+import { parseIntFlag, parseEnum, parseCacheTtl, parseJsonFlag } from '../lib/parse-flags.js';
 const FETCH_FORMATS = ['markdown', 'html'];
 /** The `FetchRequest.markdown_mode` spec enum (create/POST only — excluded from the GET query). */
 const MARKDOWN_MODES = ['article', 'raw', 'llm'];
+/** The `markdown_links` / `markdown_images` render-mode enum (shared with `scrape create`). */
+const MARKDOWN_RENDER_MODES = ['inline', 'references', 'none', 'text'];
 /**
  * Validate `--format` locally, before any HTTP request is issued.
  *
@@ -28,14 +30,31 @@ export function makeFetchCommand(getGlobal) {
     bulkCmd
         .command('create')
         .description('Create a bulk fetch job')
-        .requiredOption('--file <path>', 'Path to newline-delimited file of URLs (plain URL or {"url":"..."} per line)')
+        .requiredOption('--file <path>', 'Path to a newline-delimited file. Each line is a plain URL, or a JSON object carrying "url" plus per-item config. A line whose only key is "url" behaves exactly as before (sent as a bare URL string); any line with additional keys switches the payload to the per-item shape.')
         .requiredOption('--webhook-url <url>', 'HTTPS webhook URL notified when the job completes')
+        .option('--webhook-secret <secret>', 'HMAC secret used to sign the completion webhook (omitted → unsigned)')
+        .option('--defaults <json>', 'Defaults applied to every entry, as a JSON object (omitted → none). Unknown keys are rejected by /v1/bulk (422) and ignored by scrape bulk / fetch bulk; fetch bulk additionally rejects browser-only keys (cookies, wait_until, wait_for_selector, delay_ms) with a 400.', (v) => parseJsonFlag('--defaults', v, 'object'))
+        .option('--metadata <json>', `Arbitrary metadata echoed on the job envelope, as a JSON object, e.g. '{"run":"7"}' (omitted → none)`, (v) => parseJsonFlag('--metadata', v, 'object'))
+        // Deliberately NO --custom: /v1/fetch/bulk has no top-level `custom` property
+        // (validators/fetch-bulk.ts:52-62), so the flag would be silently stripped (#2596 drift D2).
+        // FetchBulkBodySchema itself stays non-strict (issue #2648 only makes the per-item
+        // FetchBulkItemOverrideSchema reject unknown keys), so this top-level behavior is unchanged.
         .action(async (opts) => {
         const g = getGlobal();
         const client = buildClient(g);
         try {
-            const urls = parseUrlLines(opts.file);
-            const result = await client.fetch.bulk.create({ urls, webhook_url: opts.webhookUrl });
+            // /v1/fetch/bulk has NO `items` property — its `urls` is a
+            // `string | FetchBulkItemOverride` union (validators/fetch-bulk.ts:57), so both line
+            // shapes ride there directly and no promotion step is needed (#2596 drift D3).
+            const urls = parseBulkLines(opts.file);
+            const body = { urls, webhook_url: opts.webhookUrl };
+            if (opts.webhookSecret !== undefined)
+                body['webhook_secret'] = opts.webhookSecret;
+            if (opts.defaults !== undefined)
+                body['defaults'] = opts.defaults;
+            if (opts.metadata !== undefined)
+                body['metadata'] = opts.metadata;
+            const result = await client.fetch.bulk.create(body);
             printResult(result, g.json);
         }
         catch (err) {
@@ -65,23 +84,63 @@ export function makeFetchCommand(getGlobal) {
         .option('--country <cc>', 'Two-letter country code to route the request through (e.g. us, de)')
         .option('--timeout-ms <n>', 'Per-request timeout in milliseconds', (v) => parseIntFlag('--timeout-ms', v))
         .option('--cache-ttl <ttl>', 'Cache freshness: 0, Nh (hours), or Nd (days), e.g. 48h, 2d, 0 (omitted → server default)', (v) => parseCacheTtl('--cache-ttl', v))
-        // `--markdown-mode` is POST-only: FetchGetOpts excludes it, so adding it to `get` below would be
-        // a compile error — the create/get asymmetry is correct by construction.
+        // The markdown surface below is POST-only: FetchGetOpts excludes it, so adding any of it to
+        // `get` would be a compile error — the create/get asymmetry is correct by construction.
         .option('--markdown-mode <mode>', `Markdown mode: ${MARKDOWN_MODES.join(', ')}`, (v) => parseEnum('--markdown-mode', MARKDOWN_MODES, v))
+        .option('--markdown-query <q>', 'BM25 query string for relevance-ranked markdown filtering (omitted → disabled)')
+        .option('--markdown-links <mode>', `Link rendering: ${MARKDOWN_RENDER_MODES.join(', ')} (omitted → server default)`, (v) => parseEnum('--markdown-links', MARKDOWN_RENDER_MODES, v))
+        .option('--markdown-images <mode>', `Image retention: ${MARKDOWN_RENDER_MODES.join(', ')} (omitted → server default)`, (v) => parseEnum('--markdown-images', MARKDOWN_RENDER_MODES, v))
+        .option('--with-links-summary', 'Append a Links footer to the markdown output (omitted → off)')
+        .option('--with-images-summary', 'Append an Images footer to the markdown output (omitted → off)')
+        .option('--markdown-compact', 'Collapse excessive whitespace in the markdown output (omitted → off)')
+        .option('--markdown-filter-images', 'Filter low-signal images from the markdown output (omitted → off)')
+        .option('--markdown-include-media', 'Return rich link/image objects and a tables array (omitted → off)')
+        .option('--markdown-include-warnings', 'Include markdown-pipeline and tag-filter warnings (omitted → off)')
+        .option('--markdown-include-stats', 'Include a top-level stats object (chars, tokens, blocks) (omitted → off)')
+        .option('--custom <json>', `User payload echoed on the envelope as a JSON object, e.g. '{"job":"1"}' (omitted → none)`, (v) => parseJsonFlag('--custom', v, 'object'))
         .option('-o, --output <file>', 'Write the fetched body bytes to the given file path')
         .action(async (opts) => {
         const g = getGlobal();
         const client = buildClient(g);
+        // Omit-when-absent body, in the `body: Record<string, unknown>` form every other command
+        // file uses (scrape.ts:230-268 is the exemplar). The five pre-existing fields keep the
+        // exact same wire output they had as inline spreads; a boolean is written ONLY on
+        // `=== true`, so it never serializes `false` (#2480 E1).
+        const body = { url: opts.url };
+        if (opts.format !== undefined)
+            body['format'] = opts.format;
+        if (opts.country !== undefined)
+            body['country'] = opts.country;
+        if (opts.timeoutMs !== undefined)
+            body['timeout_ms'] = opts.timeoutMs;
+        if (opts.cacheTtl !== undefined)
+            body['cache_ttl'] = opts.cacheTtl;
+        if (opts.markdownMode !== undefined)
+            body['markdown_mode'] = opts.markdownMode;
+        if (opts.markdownQuery !== undefined)
+            body['markdown_query'] = opts.markdownQuery;
+        if (opts.markdownLinks !== undefined)
+            body['markdown_links'] = opts.markdownLinks;
+        if (opts.markdownImages !== undefined)
+            body['markdown_images'] = opts.markdownImages;
+        if (opts.withLinksSummary === true)
+            body['with_links_summary'] = true;
+        if (opts.withImagesSummary === true)
+            body['with_images_summary'] = true;
+        if (opts.markdownCompact === true)
+            body['markdown_compact'] = true;
+        if (opts.markdownFilterImages === true)
+            body['markdown_filter_images'] = true;
+        if (opts.markdownIncludeMedia === true)
+            body['markdown_include_media'] = true;
+        if (opts.markdownIncludeWarnings === true)
+            body['markdown_include_warnings'] = true;
+        if (opts.markdownIncludeStats === true)
+            body['markdown_include_stats'] = true;
+        if (opts.custom !== undefined)
+            body['custom'] = opts.custom;
         try {
-            const result = await client.fetch.create({
-                url: opts.url,
-                // Each optional field is spread ONLY when its flag is present (#2480 E1).
-                ...(opts.format !== undefined ? { format: opts.format } : {}),
-                ...(opts.country !== undefined ? { country: opts.country } : {}),
-                ...(opts.timeoutMs !== undefined ? { timeout_ms: opts.timeoutMs } : {}),
-                ...(opts.cacheTtl !== undefined ? { cache_ttl: opts.cacheTtl } : {}),
-                ...(opts.markdownMode !== undefined ? { markdown_mode: opts.markdownMode } : {}),
-            });
+            const result = await client.fetch.create(body);
             emitFetchResult(result, g.json, { file: opts.output });
         }
         catch (err) {

package/dist/commands/monitors.js

@@ -20,11 +20,24 @@ export function makeMonitorsCommand(getGlobal) {
     capturesCmd
         .command('list <id>')
         .description('List captures for a monitor')
-        .action(async (id) => {
+        .option('--limit <n>', 'Maximum number of captures to return', (v) => parseIntFlag('--limit', v))
+        .option('--cursor <cursor>', 'Pagination cursor from a previous page')
+        .action(async (id, opts) => {
         const g = getGlobal();
         const client = buildClient(g);
+        // Same omit-when-absent rule as `monitors list` (#2480): build the params object with
+        // ONLY the flags the caller passed, and call captures.list with a single argument (never
+        // captures.list(id, {}) / captures.list(id, undefined)) when no query flag is passed, so
+        // the inverse-case request-shape assertion stays byte-identical to today (#2594 DoD-4).
+        const params = {};
+        if (opts.limit !== undefined)
+            params.limit = opts.limit;
+        if (opts.cursor !== undefined)
+            params.cursor = opts.cursor;
         try {
-            const result = await client.monitors.captures.list(id);
+            const result = Object.keys(params).length > 0
+                ? await client.monitors.captures.list(id, params)
+                : await client.monitors.captures.list(id);
             printResult(result, g.json);
         }
         catch (err) {
@@ -48,6 +61,7 @@ export function makeMonitorsCommand(getGlobal) {
         .option('--webhook-headers <json>', `Extra webhook headers as a JSON object, e.g. '{"X-Sig":"abc"}' (omitted → none)`, (v) => parseJsonFlag('--webhook-headers', v, 'object'))
         .option('--options <json>', `Screenshot-monitor options as a JSON object (omitted → server defaults)`, (v) => parseJsonFlag('--options', v, 'object'))
         .option('--scrape-options <json>', `Scrape-monitor options as a JSON object (type=scrape only; omitted → server defaults)`, (v) => parseJsonFlag('--scrape-options', v, 'object'))
+        .option('--metadata <json>', `Arbitrary metadata echoed on the monitor envelope, as a JSON object, e.g. '{"run":"7"}' (omitted → none)`, (v) => parseJsonFlag('--metadata', v, 'object'))
         .action(async (opts) => {
         const g = getGlobal();
         const client = buildClient(g);
@@ -75,6 +89,8 @@ export function makeMonitorsCommand(getGlobal) {
             body['options'] = opts.options;
         if (opts.scrapeOptions !== undefined)
             body['scrape_options'] = opts.scrapeOptions;
+        if (opts.metadata !== undefined)
+            body['metadata'] = opts.metadata;
         try {
             const result = await client.monitors.create(body);
             printResult(result, g.json);
@@ -132,6 +148,7 @@ export function makeMonitorsCommand(getGlobal) {
     cmd
         .command('update <id>')
         .description('Update a monitor (including pause/resume via --status)')
+        .option('--url <url>', 'New URL to monitor (omitted → unchanged)')
         .option('--interval-type <type>', 'New check interval: hourly, daily, weekly, or cron')
         .option('--cron <expr>', 'New cron expression (when interval-type is cron, e.g. "0 * * * *")')
         .option('--webhook-url <url>', 'New HTTPS webhook URL')
@@ -144,10 +161,15 @@ export function makeMonitorsCommand(getGlobal) {
         .option('--webhook-headers <json>', `New webhook headers as a JSON object, e.g. '{"X-Sig":"abc"}'`, (v) => parseJsonFlag('--webhook-headers', v, 'object'))
         .option('--options <json>', `New screenshot-monitor options as a JSON object`, (v) => parseJsonFlag('--options', v, 'object'))
         .option('--scrape-options <json>', `New scrape-monitor options as a JSON object`, (v) => parseJsonFlag('--scrape-options', v, 'object'))
+        .option('--metadata <json>', `New metadata as a JSON object, e.g. '{"run":"7"}' (omitted → unchanged)`, (v) => parseJsonFlag('--metadata', v, 'object'))
         .action(async (id, opts) => {
         const g = getGlobal();
         const client = buildClient(g);
         const body = {};
+        // `url` IS settable on PATCH (openapi.yaml MonitorUpdateRequest) — unlike `type`, which
+        // stays immutable and deliberately has no flag (#1984).
+        if (opts.url !== undefined)
+            body['url'] = opts.url;
         if (opts.intervalType !== undefined)
             body['interval_type'] = opts.intervalType;
         if (opts.cron !== undefined)
@@ -168,6 +190,8 @@ export function makeMonitorsCommand(getGlobal) {
             body['options'] = opts.options;
         if (opts.scrapeOptions !== undefined)
             body['scrape_options'] = opts.scrapeOptions;
+        if (opts.metadata !== undefined)
+            body['metadata'] = opts.metadata;
         try {
             const result = await client.monitors.update(id, body);
             printResult(result, g.json);