@hunt-labs/bytekit-cli 0.5.0 → 0.6.0

package/dist/commands/scrape.js

@@ -1,7 +1,7 @@
 import { Command } from 'commander';
 import { buildClient } from '../client.js';
 import { printResult, handleError, resolveOutputMode, emitResult } from '../output.js';
-import { parseUrlLines } from '../lib/url-lines.js';
+import { parseBulkLines, toBulkItems } from '../lib/url-lines.js';
 import { parseIntFlag, parseEnum, parseCsv, parseCacheTtl, parseJsonFlag, collectFormats, } from '../lib/parse-flags.js';
 /** The `ScrapeRequest.markdown_mode` spec enum. */
 const MARKDOWN_MODES = ['article', 'raw', 'llm'];
@@ -45,15 +45,34 @@ export function makeScrapeCommand(getGlobal) {
     bulkCmd
         .command('create')
         .description('Create a bulk scrape job')
-        .requiredOption('--file <path>', 'Path to newline-delimited file of URLs (plain URL or {"url":"..."} per line)')
+        .requiredOption('--file <path>', 'Path to a newline-delimited file. Each line is a plain URL, or a JSON object carrying "url" plus per-item config. A line whose only key is "url" behaves exactly as before (sent as a bare URL string); any line with additional keys switches the payload to the per-item shape.')
         .requiredOption('--webhook-url <url>', 'HTTPS webhook URL notified when the job completes')
+        .option('--webhook-secret <secret>', 'HMAC secret used to sign the completion webhook (omitted → unsigned)')
+        .option('--defaults <json>', 'Defaults applied to every entry, as a JSON object (omitted → none). Unknown keys are rejected by /v1/bulk (422) and ignored by scrape bulk / fetch bulk; fetch bulk additionally rejects browser-only keys (cookies, wait_until, wait_for_selector, delay_ms) with a 400.', (v) => parseJsonFlag('--defaults', v, 'object'))
+        .option('--metadata <json>', `Arbitrary metadata echoed on the job envelope, as a JSON object, e.g. '{"run":"7"}' (omitted → none)`, (v) => parseJsonFlag('--metadata', v, 'object'))
+        // /v1/scrape/bulk is the ONE bulk endpoint with a top-level `custom`
+        // (validators/scrape-bulk.ts:88). Per-entry `custom` arrives through --file lines instead.
+        .option('--custom <json>', `User payload echoed on every scrape entry, as a JSON object, e.g. '{"job":"1"}' (omitted → none; a per-entry value in --file wins)`, (v) => parseJsonFlag('--custom', v, 'object'))
         .action(async (opts) => {
         const g = getGlobal();
         const client = buildClient(g);
         try {
             // readFileSync throws ENOENT on missing file — caught by handleError below
-            const urls = parseUrlLines(opts.file);
-            const result = await client.scrape.bulk.create({ urls, webhook_url: opts.webhookUrl });
+            const lines = parseBulkLines(opts.file);
+            // `urls` XOR `items` (validators/scrape-bulk.ts:91-98) — send exactly one, keeping the
+            // pre-#2596 `{ urls, webhook_url }` body byte-identical for url-only files (E2).
+            const body = lines.every((l) => typeof l === 'string')
+                ? { urls: lines, webhook_url: opts.webhookUrl }
+                : { items: toBulkItems(lines), webhook_url: opts.webhookUrl };
+            if (opts.webhookSecret !== undefined)
+                body['webhook_secret'] = opts.webhookSecret;
+            if (opts.defaults !== undefined)
+                body['defaults'] = opts.defaults;
+            if (opts.metadata !== undefined)
+                body['metadata'] = opts.metadata;
+            if (opts.custom !== undefined)
+                body['custom'] = opts.custom;
+            const result = await client.scrape.bulk.create(body);
             printResult(result, g.json);
         }
         catch (err) {
@@ -245,7 +264,7 @@ export function makeScrapeCommand(getGlobal) {
         .description('Get a scrape job by ID (use --raw/-o/--format to retrieve a completed body)')
         .option('--raw', 'Print the requested format body raw, with no JSON wrapper')
         .option('-o, --output <file>', 'Write the output to the given file path (implies --raw unless --json is set)')
-        .option('--format <format>', 'Which persisted format to emit raw when several are present')
+        .option('--format <format>', 'Which persisted format to emit — only takes effect with --raw/-o; ignored otherwise')
         .action(async (id, opts) => {
         const g = getGlobal();
         const client = buildClient(g);

package/dist/commands/screenshots.js

@@ -2,7 +2,7 @@ import { Command, InvalidArgumentError } from 'commander';
 import { buildClient } from '../client.js';
 import { printResult, handleError } from '../output.js';
 import { downloadArtifact } from '../lib/download-artifact.js';
-import { parseEnum, parseIntFlag } from '../lib/parse-flags.js';
+import { parseEnum, parseIntFlag, parseJsonFlag } from '../lib/parse-flags.js';
 /** `ScreenshotRequest` spec enums. */
 const DEVICES = ['desktop', 'mobile'];
 const FORMATS = ['jpeg', 'png'];
@@ -23,6 +23,24 @@ export function parseClip(value) {
         throw new InvalidArgumentError('--clip must be X,Y,WIDTH,HEIGHT (e.g. 0,0,800,600).');
     return { x: Number(m[1]), y: Number(m[2]), width: Number(m[3]), height: Number(m[4]) };
 }
+/**
+ * Exit code when a queued (202 async) envelope reaches human output (#2596).
+ *
+ * Mirrors `scrape create`'s own `QUEUED_EXIT_CODE` (`commands/scrape.ts:40`) EXACTLY, and is
+ * likewise a LOCAL constant rather than an import — the two commands agree on the value, not on a
+ * shared symbol, and the guard stays where the ticket pins it (the screenshots async branch)
+ * instead of migrating into the shared printer, where it would silently set exit 2 for every
+ * command that prints a result.
+ *
+ * Deliberately NOT 0 (a completed capture) and NOT 1 (a usage/API error): the job WAS accepted and
+ * billed, the artifact simply does not exist yet, and a distinct code lets a caller branch on
+ * "queued" without parsing stderr.
+ *
+ * BREAKING (#2596): `screenshots create --async` previously exited 0. See the README exit-code
+ * table and the CHANGELOG `[0.6.0]` note (this version's own section — `[Unreleased]` is kept
+ * deliberately empty per `packaging.test.ts:203`; see #2620).
+ */
+const QUEUED_EXIT_CODE = 2;
 /**
  * Extract a non-empty string `image_url` from a screenshot envelope, or throw an actionable error.
  *
@@ -53,6 +71,21 @@ function extractImageUrl(result) {
 async function emitScreenshot(result, opts) {
     if (opts.isAsync === true) {
         printResult(result, opts.json);
+        // Discriminate on the --async FLAG, never on the response body. The screenshot 202 reports a
+        // PENDING status (gateway src/routes/screenshots.ts:138) — not the queued status that
+        // `scrape create`'s envelope guard (commands/scrape.ts:49-53) tests for. Reusing that guard
+        // here would never fire in production and this branch would silently no-op. Grep gate G11
+        // (#2596) enforces the absence of a response-status discriminator in this file, which is why
+        // neither the symbol nor the comparison is spelled out above.
+        //
+        // The assignment sits INSIDE this branch, ahead of the `-o` return below, so `--async -o`
+        // carries the same exit code as bare `--async` (#2596 DoD 15 / E20). `--json` keeps exit 0:
+        // the envelope is a complete machine-readable payload, identical to scrape's split
+        // (scrape.ts:313-317). Soft `process.exitCode`, never `process.exit` — a hard exit runs
+        // before Node drains an async piped stdout and would truncate the envelope just written
+        // (#2544).
+        if (!opts.json)
+            process.exitCode = QUEUED_EXIT_CODE;
         return;
     }
     if (opts.output !== undefined) {
@@ -89,6 +122,9 @@ export function makeScreenshotsCommand(getGlobal) {
         .option('--include-html', 'Include the rendered HTML alongside the screenshot')
         .option('--country <cc>', 'Two-letter country code to route the request through (e.g. us, de)')
         .option('--language <lang>', 'Accept-Language override (e.g. en-US)')
+        .option('--headers <json>', `Extra request headers as a JSON object, e.g. '{"X-A":"b"}' (omitted → none)`, (v) => parseJsonFlag('--headers', v, 'object'))
+        .option('--cookies <json>', `Cookies as a JSON array of {name,value,domain}, e.g. '[{"name":"a","value":"b","domain":"example.com"}]' (omitted → none)`, (v) => parseJsonFlag('--cookies', v, 'array'))
+        .option('--metadata <json>', `Arbitrary metadata echoed on the screenshot envelope, as a JSON object, e.g. '{"run":"7"}' (omitted → none)`, (v) => parseJsonFlag('--metadata', v, 'object'))
         .option('--async', 'Queue the screenshot and return a job envelope immediately (no wait)')
         .option('-o, --output <file>', 'Download the captured image to this file path (use - for stdout). Not valid with --json.')
         .action(async (opts) => {
@@ -135,6 +171,14 @@ export function makeScreenshotsCommand(getGlobal) {
             body['country'] = opts.country;
         if (opts.language !== undefined)
             body['language'] = opts.language;
+        // The server defaults headers/cookies/metadata to {} / [] / {}; writing an empty
+        // container on a bare create would materialize those defaults (#2480 E1).
+        if (opts.headers !== undefined)
+            body['headers'] = opts.headers;
+        if (opts.cookies !== undefined)
+            body['cookies'] = opts.cookies;
+        if (opts.metadata !== undefined)
+            body['metadata'] = opts.metadata;
         try {
             // Reject --json + -o BEFORE issuing the request: -o downloads BINARY artifact bytes, so
             // it can never coexist with --json (which must stay a text envelope). Thrown here so no

package/dist/commands/sitemap.js

@@ -1,7 +1,7 @@
 import { Command } from 'commander';
 import { buildClient } from '../client.js';
 import { printResult, handleError } from '../output.js';
-import { parseEnum, parseIntFlag } from '../lib/parse-flags.js';
+import { parseEnum, parseIntFlag, parseCacheTtl, parseJsonFlag } from '../lib/parse-flags.js';
 /** The `SitemapRequest.strategy` spec enum. */
 const SITEMAP_STRATEGIES = ['quick', 'standard', 'deep'];
 export function makeSitemapCommand(getGlobal) {
@@ -13,6 +13,15 @@ export function makeSitemapCommand(getGlobal) {
         .option('--strategy <strategy>', `Crawl strategy: ${SITEMAP_STRATEGIES.join(', ')} (omitted → server default standard)`, (v) => parseEnum('--strategy', SITEMAP_STRATEGIES, v))
         .option('--max-depth <n>', 'Maximum crawl depth 1-5 (omitted → server default 2)', (v) => parseIntFlag('--max-depth', v))
         .option('--max-urls <n>', 'Maximum URLs to collect (omitted → server default 5000)', (v) => parseIntFlag('--max-urls', v))
+        .option('--webhook-url <url>', 'HTTPS webhook URL notified when the crawl completes (omitted → poll-only, no delivery)')
+        .option('--webhook-secret <secret>', 'HMAC secret used to sign the completion webhook (omitted → unsigned)')
+        // Grammar-only validation, deliberately: sitemap caps cache_ttl at 90 DAYS
+        // (validators/sitemap.ts MAX_CACHE_TTL_SECONDS), not the 7d used by scrape/fetch. Mirroring
+        // the cap client-side would fork a second source of truth for a number only the server knows.
+        .option('--cache-ttl <ttl>', 'Cache freshness: 0, Nh (hours), or Nd (days), e.g. 48h, 30d, 0 (omitted → server default 30d; max 90d, enforced server-side)', (v) => parseCacheTtl('--cache-ttl', v))
+        .option('--compact', 'Return a compact result payload (omitted → server default off)')
+        .option('--process <json>', `Post-crawl processing directive as a JSON object, e.g. '{"action":"summarize"}' (omitted → none)`, (v) => parseJsonFlag('--process', v, 'object'))
+        .option('--metadata <json>', `Arbitrary metadata echoed on the job envelope, as a JSON object, e.g. '{"run":"7"}' (omitted → none)`, (v) => parseJsonFlag('--metadata', v, 'object'))
         .action(async (opts) => {
         const g = getGlobal();
         const client = buildClient(g);
@@ -26,6 +35,20 @@ export function makeSitemapCommand(getGlobal) {
             body['max_depth'] = opts.maxDepth;
         if (opts.maxUrls !== undefined)
             body['max_urls'] = opts.maxUrls;
+        if (opts.webhookUrl !== undefined)
+            body['webhook_url'] = opts.webhookUrl;
+        if (opts.webhookSecret !== undefined)
+            body['webhook_secret'] = opts.webhookSecret;
+        if (opts.cacheTtl !== undefined)
+            body['cache_ttl'] = opts.cacheTtl;
+        // `compact` server-defaults to FALSE, so it is written ONLY on `=== true` — writing
+        // `false` on a bare create would materialize a default (#2480 E1).
+        if (opts.compact === true)
+            body['compact'] = true;
+        if (opts.process !== undefined)
+            body['process'] = opts.process;
+        if (opts.metadata !== undefined)
+            body['metadata'] = opts.metadata;
         try {
             const result = await client.sitemap.create(body);
             printResult(result, g.json);

package/dist/lib/download-artifact.d.ts

@@ -9,5 +9,34 @@
  * test that stubs `globalThis.fetch` in a `beforeEach` — after this module was imported — still
  * sees its stub (G-049). A non-2xx response throws with the HTTP status so `handleError` renders a
  * clean one-liner and exits 1.
+ *
+ * TIMEOUT (#2587): this path sat entirely outside the transport seam — no abort signal, no
+ * connection-error taxonomy — so a stalled artifact host (headers, then a body that never
+ * completes) hung the CLI forever, even though the 0.5.0 CHANGELOG's "a stalled body hung the CLI
+ * forever" note (#2546) reads as if every response body were covered. #2546 only covered API-call
+ * bodies read through `sendWithRetry`; this download never rode that seam. The fix REUSES
+ * `sendWithRetry`'s own per-attempt abort budget and connection-error taxonomy
+ * (`createAttemptBudget` / `toConnectionError`, exported from `../client.js`) rather than a second
+ * timeout implementation, and mirrors the same default (`DEFAULT_TIMEOUT_MS`, 120 s per attempt) —
+ * there is no retry here, since artifact GETs are not the API's 429 domain.
+ *
+ * PARTIAL-FILE POLICY (#2587 dod-1 / E1, decided + recorded, scoped to the DOWNLOAD phase only —
+ * cr-4): the response is buffered COMPLETELY in memory (`res.arrayBuffer()`) before either write
+ * path (`writeFileSync` / `process.stdout.write`) runs, so an ABORT — whether from the timeout or
+ * any other connection failure — always fails BEFORE a single byte is written; no partial file is
+ * ever left at `dest` from a download-side failure. This does NOT cover a failure DURING the write
+ * itself: `writeFileSync` opens `dest` with `O_TRUNC` and writes in a loop, so e.g. an `ENOSPC`
+ * partway through a large write leaves a created, truncated file at `dest` (any pre-existing file
+ * there already clobbered by the `O_TRUNC` open). That failure mode is accepted as-is for now (no
+ * write-to-temp-then-`renameSync` here) — the CLI still exits 1, and the write phase's tiny
+ * (post-buffering) failure window is a separate, unaddressed edge from the abort guarantee above.
+ *
+ * @param timeoutMs Per-attempt abort budget in ms; defaults to {@link DEFAULT_TIMEOUT_MS} (120 s).
+ *   `<= 0` DISABLES the budget entirely (no `AbortSignal` is attached, restoring the exact
+ *   unbounded-hang behavior this ticket fixes) rather than meaning "no override" — there is no
+ *   sentinel for that. #2587 cr-8: no CLI command reaches this parameter today
+ *   (`commands/screenshots.ts` always calls the 2-arg form), so a future caller — e.g. an
+ *   anticipated `--timeout` flag threaded through here — must NOT pass `0` to mean "use the
+ *   default"; it must omit the argument or pass `DEFAULT_TIMEOUT_MS` explicitly.
  */
-export declare function downloadArtifact(url: string, dest: string): Promise<void>;
+export declare function downloadArtifact(url: string, dest: string, timeoutMs?: number): Promise<void>;

package/dist/lib/download-artifact.js

@@ -1,4 +1,25 @@
 import { writeFileSync } from 'node:fs';
+import { createAttemptBudget, toConnectionError, DEFAULT_TIMEOUT_MS } from '../client.js';
+/**
+ * Redact a URL for error-message rendering: origin + pathname only, query string dropped.
+ *
+ * #2587 cr-2: `image_url` for a completed screenshot is an AWS SigV4 PRESIGNED URL
+ * (`X-Amz-Credential=…&X-Amz-Signature=…`, `gateway/src/lib/signed-url.ts`), unlike every other
+ * `toConnectionError` call site in `client.ts` which passes a bare relative API path with no
+ * credential material. Rendering the full URL in `handleError`'s unmodified `err.message` would
+ * print a live 600s-valid signature to stderr on every new failure path this diff adds. On a
+ * malformed URL (should not happen — `image_url` is always an absolute URL from the API) fall
+ * back to the original string rather than throwing out of an error-handling path.
+ */
+function redactedPath(url) {
+    try {
+        const u = new URL(url);
+        return `${u.origin}${u.pathname}`;
+    }
+    catch {
+        return url;
+    }
+}
 /**
  * Download a screenshot artifact from its `image_url` and write the raw bytes to `dest`.
  *
@@ -10,13 +31,76 @@ import { writeFileSync } from 'node:fs';
  * test that stubs `globalThis.fetch` in a `beforeEach` — after this module was imported — still
  * sees its stub (G-049). A non-2xx response throws with the HTTP status so `handleError` renders a
  * clean one-liner and exits 1.
+ *
+ * TIMEOUT (#2587): this path sat entirely outside the transport seam — no abort signal, no
+ * connection-error taxonomy — so a stalled artifact host (headers, then a body that never
+ * completes) hung the CLI forever, even though the 0.5.0 CHANGELOG's "a stalled body hung the CLI
+ * forever" note (#2546) reads as if every response body were covered. #2546 only covered API-call
+ * bodies read through `sendWithRetry`; this download never rode that seam. The fix REUSES
+ * `sendWithRetry`'s own per-attempt abort budget and connection-error taxonomy
+ * (`createAttemptBudget` / `toConnectionError`, exported from `../client.js`) rather than a second
+ * timeout implementation, and mirrors the same default (`DEFAULT_TIMEOUT_MS`, 120 s per attempt) —
+ * there is no retry here, since artifact GETs are not the API's 429 domain.
+ *
+ * PARTIAL-FILE POLICY (#2587 dod-1 / E1, decided + recorded, scoped to the DOWNLOAD phase only —
+ * cr-4): the response is buffered COMPLETELY in memory (`res.arrayBuffer()`) before either write
+ * path (`writeFileSync` / `process.stdout.write`) runs, so an ABORT — whether from the timeout or
+ * any other connection failure — always fails BEFORE a single byte is written; no partial file is
+ * ever left at `dest` from a download-side failure. This does NOT cover a failure DURING the write
+ * itself: `writeFileSync` opens `dest` with `O_TRUNC` and writes in a loop, so e.g. an `ENOSPC`
+ * partway through a large write leaves a created, truncated file at `dest` (any pre-existing file
+ * there already clobbered by the `O_TRUNC` open). That failure mode is accepted as-is for now (no
+ * write-to-temp-then-`renameSync` here) — the CLI still exits 1, and the write phase's tiny
+ * (post-buffering) failure window is a separate, unaddressed edge from the abort guarantee above.
+ *
+ * @param timeoutMs Per-attempt abort budget in ms; defaults to {@link DEFAULT_TIMEOUT_MS} (120 s).
+ *   `<= 0` DISABLES the budget entirely (no `AbortSignal` is attached, restoring the exact
+ *   unbounded-hang behavior this ticket fixes) rather than meaning "no override" — there is no
+ *   sentinel for that. #2587 cr-8: no CLI command reaches this parameter today
+ *   (`commands/screenshots.ts` always calls the 2-arg form), so a future caller — e.g. an
+ *   anticipated `--timeout` flag threaded through here — must NOT pass `0` to mean "use the
+ *   default"; it must omit the argument or pass `DEFAULT_TIMEOUT_MS` explicitly.
  */
-export async function downloadArtifact(url, dest) {
-    const res = await globalThis.fetch(url);
-    if (!res.ok) {
-        throw new Error(`artifact download failed: HTTP ${res.status}`);
+export async function downloadArtifact(url, dest, timeoutMs = DEFAULT_TIMEOUT_MS) {
+    const budget = timeoutMs > 0 ? createAttemptBudget(timeoutMs) : undefined;
+    const safePath = redactedPath(url);
+    let bytes;
+    try {
+        let res;
+        try {
+            res = await globalThis.fetch(url, budget ? { signal: budget.signal } : {});
+        }
+        catch (err) {
+            // No HTTP response was produced (timeout, abort, socket/DNS/TLS) — re-throw as the SDK's own
+            // ByteKitConnectionError, exactly like the API transport, so a hung/broken artifact host can
+            // never hang the CLI and `handleError` renders the same taxonomy either way. `safePath` (not
+            // `url`) is what gets rendered — see {@link redactedPath}.
+            throw toConnectionError(err, 'GET', safePath, budget?.signal.aborted === true);
+        }
+        if (!res.ok) {
+            // #2587 cr2-1: no drain here, unlike client.ts's 429 path. cr-11 originally added one
+            // (mirroring client.ts's corrected 429-drain mechanism, cr-3), reasoning that a full
+            // `text()` read lets undici return the connection to the pool. That reasoning does not
+            // apply on THIS path: `commands/screenshots.ts` throws this error straight into
+            // `handleError`, which `process.exit(1)`s immediately after — there is no next request in
+            // this process for a reclaimed socket to serve. A drain here was bounded only by the FULL
+            // per-attempt abort budget (or, under `timeoutMs <= 0`, by nothing at all) for zero benefit,
+            // so it is removed outright rather than merely capped.
+            throw new Error(`artifact download failed: HTTP ${res.status}`);
+        }
+        try {
+            bytes = Buffer.from(await res.arrayBuffer());
+        }
+        catch (err) {
+            // A body read that never completes is aborted by the same budget; undici then rejects the
+            // stream (mirrors #2546's body-read guard). Map it into the same taxonomy as above.
+            throw toConnectionError(err, 'GET', safePath, budget?.signal.aborted === true);
+        }
+    }
+    finally {
+        // Clear the timer on every exit path so a healthy download leaves no dangling budget timer.
+        budget?.cleanup();
     }
-    const bytes = Buffer.from(await res.arrayBuffer());
     if (dest === '-') {
         process.stdout.write(bytes);
         return;

package/dist/lib/url-lines.d.ts

@@ -1,12 +1,50 @@
 /**
- * Read a newline-delimited file of URLs and return a flat array of URL strings.
+ * One parsed `--file` line: a bare URL string, or a per-item config object carrying `url`.
  *
- * Each non-blank line is either a plain URL or a JSON object of the form
- * `{"url":"https://..."}`. Both forms normalize to the bare URL string, because
- * the v1 bulk endpoints (`/v1/bulk`, `/v1/scrape/bulk`, `/v1/fetch/bulk`) expect
- * `urls` to be an array of URL strings — NOT an array of objects.
+ * The two arms are not interchangeable on the wire. `/v1/bulk` and `/v1/scrape/bulk` accept
+ * `urls` (strings) XOR `items` (objects); `/v1/fetch/bulk` has no `items` property at all and
+ * takes a heterogeneous `urls` union. Callers pick the right shape — see {@link toBulkItems}.
+ */
+export type BulkLine = string | Record<string, unknown>;
+/**
+ * Read a newline-delimited `--file` and return one {@link BulkLine} per non-blank line.
+ *
+ * Line grammar:
+ * - a non-JSON line                            -> the raw line (a plain URL string)
+ * - a JSON object whose ONLY key is `url`      -> the bare url string
+ * - a JSON object carrying other keys too      -> the object verbatim (per-item config)
+ * - a JSON object with a missing/non-string url -> `InvalidArgumentError` (rejected locally)
+ * - a JSON non-object (number, array, string)  -> the raw line
+ *
+ * The "only key is `url`" arm is what keeps the inverse case BYTE-IDENTICAL rather than merely
+ * equivalent: a file of url-only lines produces the exact same `{ urls, webhook_url }` body as
+ * before per-item config existed (#2596 E2).
+ *
+ * Rejecting a malformed `url` is a deliberate behavior change (#2596 DoD 8). The previous parser
+ * fell through to the raw line, so `{"url": 123}` was sent to the server AS a URL — a confusing
+ * 422 for what is a local input mistake. `InvalidArgumentError` surfaces through the same
+ * local-usage-error path as every argParser rejection (exit 1), before any request is issued.
+ *
+ * Throws on a missing file (ENOENT from readFileSync) — callers surface this via their
+ * handleError path.
+ *
+ * The rejection message for a malformed line (#2596 cr-1 / E12) names the 1-based PHYSICAL line
+ * NUMBER (i.e. the line's own position in the file, counting blank lines) and the line's own KEY
+ * NAMES only — never the raw line or any value. A `--file` line is where per-item `cookies`,
+ * `headers`, and auth-bearing `metadata` now live (#2596), so echoing the raw line back would
+ * leak session tokens and similar secrets to stderr, CI logs, and shell transcripts. Because the
+ * raw line is deliberately never echoed, the line number is the ONLY locator left in the message —
+ * it must be computed BEFORE blank lines are filtered out, or it silently mis-locates the bad
+ * entry whenever the file contains blank lines above it (#2596 cr-2).
+ */
+export declare function parseBulkLines(file: string): BulkLine[];
+/**
+ * Promote every {@link BulkLine} to the `items[]` object shape used by `/v1/bulk` and
+ * `/v1/scrape/bulk`.
  *
- * Throws on a missing file (ENOENT from readFileSync) — callers surface this via
- * their handleError path.
+ * Deliberately stamps NOTHING beyond `url`. `/v1/bulk` owns capture-kind precedence server-side
+ * (per-item value, then `defaults`, then the endpoint default) since #2641 made the per-item
+ * field optional; a CLI-side default would silently override a caller's `--defaults` and is
+ * grep-gated out (#2596 G7).
  */
-export declare function parseUrlLines(file: string): string[];
+export declare function toBulkItems(lines: BulkLine[]): Array<Record<string, unknown>>;

package/dist/lib/url-lines.js

@@ -1,34 +1,73 @@
 import { readFileSync } from 'node:fs';
+import { InvalidArgumentError } from 'commander';
 /**
- * Read a newline-delimited file of URLs and return a flat array of URL strings.
+ * Read a newline-delimited `--file` and return one {@link BulkLine} per non-blank line.
  *
- * Each non-blank line is either a plain URL or a JSON object of the form
- * `{"url":"https://..."}`. Both forms normalize to the bare URL string, because
- * the v1 bulk endpoints (`/v1/bulk`, `/v1/scrape/bulk`, `/v1/fetch/bulk`) expect
- * `urls` to be an array of URL strings — NOT an array of objects.
+ * Line grammar:
+ * - a non-JSON line                            -> the raw line (a plain URL string)
+ * - a JSON object whose ONLY key is `url`      -> the bare url string
+ * - a JSON object carrying other keys too      -> the object verbatim (per-item config)
+ * - a JSON object with a missing/non-string url -> `InvalidArgumentError` (rejected locally)
+ * - a JSON non-object (number, array, string)  -> the raw line
  *
- * Throws on a missing file (ENOENT from readFileSync) — callers surface this via
- * their handleError path.
+ * The "only key is `url`" arm is what keeps the inverse case BYTE-IDENTICAL rather than merely
+ * equivalent: a file of url-only lines produces the exact same `{ urls, webhook_url }` body as
+ * before per-item config existed (#2596 E2).
+ *
+ * Rejecting a malformed `url` is a deliberate behavior change (#2596 DoD 8). The previous parser
+ * fell through to the raw line, so `{"url": 123}` was sent to the server AS a URL — a confusing
+ * 422 for what is a local input mistake. `InvalidArgumentError` surfaces through the same
+ * local-usage-error path as every argParser rejection (exit 1), before any request is issued.
+ *
+ * Throws on a missing file (ENOENT from readFileSync) — callers surface this via their
+ * handleError path.
+ *
+ * The rejection message for a malformed line (#2596 cr-1 / E12) names the 1-based PHYSICAL line
+ * NUMBER (i.e. the line's own position in the file, counting blank lines) and the line's own KEY
+ * NAMES only — never the raw line or any value. A `--file` line is where per-item `cookies`,
+ * `headers`, and auth-bearing `metadata` now live (#2596), so echoing the raw line back would
+ * leak session tokens and similar secrets to stderr, CI logs, and shell transcripts. Because the
+ * raw line is deliberately never echoed, the line number is the ONLY locator left in the message —
+ * it must be computed BEFORE blank lines are filtered out, or it silently mis-locates the bad
+ * entry whenever the file contains blank lines above it (#2596 cr-2).
  */
-export function parseUrlLines(file) {
+export function parseBulkLines(file) {
     return readFileSync(file, 'utf8')
         .split('\n')
-        .map((l) => l.trim())
-        .filter((l) => l.length > 0)
-        .map((line) => {
+        .map((raw, index) => ({ line: raw.trim(), lineNumber: index + 1 }))
+        .filter(({ line }) => line.length > 0)
+        .map(({ line, lineNumber }) => {
+        let parsed;
         try {
-            const parsed = JSON.parse(line);
-            if (parsed !== null && typeof parsed === 'object' && 'url' in parsed) {
-                const url = parsed.url;
-                if (typeof url === 'string')
-                    return url;
-            }
-            // Parsed as JSON but not a {url:string} object — fall back to the raw line.
-            return line;
+            parsed = JSON.parse(line);
         }
         catch {
             // Not JSON — treat the whole line as a plain URL string.
             return line;
         }
+        if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
+            // Parsed as JSON but not an object — fall back to the raw line, as before.
+            return line;
+        }
+        const obj = parsed;
+        if (typeof obj['url'] !== 'string' || obj['url'].length === 0) {
+            const keys = Object.keys(obj).join(', ') || '(none)';
+            throw new InvalidArgumentError(`--file line ${lineNumber} must carry a non-empty string "url" (keys present: ${keys})`);
+        }
+        // A line whose only key is `url` is indistinguishable from a plain URL line — keep it a
+        // bare string so url-only files stay byte-identical on the wire.
+        return Object.keys(obj).length === 1 ? obj['url'] : obj;
     });
 }
+/**
+ * Promote every {@link BulkLine} to the `items[]` object shape used by `/v1/bulk` and
+ * `/v1/scrape/bulk`.
+ *
+ * Deliberately stamps NOTHING beyond `url`. `/v1/bulk` owns capture-kind precedence server-side
+ * (per-item value, then `defaults`, then the endpoint default) since #2641 made the per-item
+ * field optional; a CLI-side default would silently override a caller's `--defaults` and is
+ * grep-gated out (#2596 G7).
+ */
+export function toBulkItems(lines) {
+    return lines.map((l) => (typeof l === 'string' ? { url: l } : l));
+}

package/dist/output.js

@@ -293,7 +293,7 @@ const BYTEKIT_ERROR_NAMES = new Set(['ByteKitError', 'ByteKitConnectionError']);
  * A non-empty API/connection error `code`, or undefined when absent.
  *
  * Scoped by `err.name` to the ByteKit taxonomy: plenty of OTHER errors also carry a string
- * `.code` — Node fs/system errors (ENOENT from `parseUrlLines`'s readFileSync, EACCES on a `-o`
+ * `.code` — Node fs/system errors (ENOENT from `parseBulkLines`'s readFileSync, EACCES on a `-o`
  * write) — whose message already embeds the code, so suffixing them would render the redundant
  * `Error: ENOENT: … (ENOENT)`. Only `ByteKitError` / `ByteKitConnectionError` codes qualify.
  */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hunt-labs/bytekit-cli",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "ByteKit CLI — wraps @hunt-labs/bytekit-sdk as shell subcommands",
   "license": "MIT",
   "type": "module",
@@ -41,10 +41,10 @@
   ],
   "dependencies": {
     "commander": "^12.0.0",
-    "@hunt-labs/bytekit-sdk": "^0.4.0"
+    "@hunt-labs/bytekit-sdk": "^0.5.0"
   },
   "scripts": {
-    "build": "tsc",
+    "build": "rm -rf dist tsconfig.tsbuildinfo && tsc",
     "typecheck": "tsc --noEmit",
     "test": "vitest run"
   }