@hunt-labs/bytekit-cli 0.5.0 → 0.7.0

package/dist/lib/url-lines.js

@@ -1,34 +1,73 @@
 import { readFileSync } from 'node:fs';
+import { InvalidArgumentError } from 'commander';
 /**
- * Read a newline-delimited file of URLs and return a flat array of URL strings.
+ * Read a newline-delimited `--file` and return one {@link BulkLine} per non-blank line.
  *
- * Each non-blank line is either a plain URL or a JSON object of the form
- * `{"url":"https://..."}`. Both forms normalize to the bare URL string, because
- * the v1 bulk endpoints (`/v1/bulk`, `/v1/scrape/bulk`, `/v1/fetch/bulk`) expect
- * `urls` to be an array of URL strings — NOT an array of objects.
+ * Line grammar:
+ * - a non-JSON line                            -> the raw line (a plain URL string)
+ * - a JSON object whose ONLY key is `url`      -> the bare url string
+ * - a JSON object carrying other keys too      -> the object verbatim (per-item config)
+ * - a JSON object with a missing/non-string url -> `InvalidArgumentError` (rejected locally)
+ * - a JSON non-object (number, array, string)  -> the raw line
  *
- * Throws on a missing file (ENOENT from readFileSync) — callers surface this via
- * their handleError path.
+ * The "only key is `url`" arm is what keeps the inverse case BYTE-IDENTICAL rather than merely
+ * equivalent: a file of url-only lines produces the exact same `{ urls, webhook_url }` body as
+ * before per-item config existed (#2596 E2).
+ *
+ * Rejecting a malformed `url` is a deliberate behavior change (#2596 DoD 8). The previous parser
+ * fell through to the raw line, so `{"url": 123}` was sent to the server AS a URL — a confusing
+ * 422 for what is a local input mistake. `InvalidArgumentError` surfaces through the same
+ * local-usage-error path as every argParser rejection (exit 1), before any request is issued.
+ *
+ * Throws on a missing file (ENOENT from readFileSync) — callers surface this via their
+ * handleError path.
+ *
+ * The rejection message for a malformed line (#2596 cr-1 / E12) names the 1-based PHYSICAL line
+ * NUMBER (i.e. the line's own position in the file, counting blank lines) and the line's own KEY
+ * NAMES only — never the raw line or any value. A `--file` line is where per-item `cookies`,
+ * `headers`, and auth-bearing `metadata` now live (#2596), so echoing the raw line back would
+ * leak session tokens and similar secrets to stderr, CI logs, and shell transcripts. Because the
+ * raw line is deliberately never echoed, the line number is the ONLY locator left in the message —
+ * it must be computed BEFORE blank lines are filtered out, or it silently mis-locates the bad
+ * entry whenever the file contains blank lines above it (#2596 cr-2).
  */
-export function parseUrlLines(file) {
+export function parseBulkLines(file) {
     return readFileSync(file, 'utf8')
         .split('\n')
-        .map((l) => l.trim())
-        .filter((l) => l.length > 0)
-        .map((line) => {
+        .map((raw, index) => ({ line: raw.trim(), lineNumber: index + 1 }))
+        .filter(({ line }) => line.length > 0)
+        .map(({ line, lineNumber }) => {
+        let parsed;
         try {
-            const parsed = JSON.parse(line);
-            if (parsed !== null && typeof parsed === 'object' && 'url' in parsed) {
-                const url = parsed.url;
-                if (typeof url === 'string')
-                    return url;
-            }
-            // Parsed as JSON but not a {url:string} object — fall back to the raw line.
-            return line;
+            parsed = JSON.parse(line);
         }
         catch {
             // Not JSON — treat the whole line as a plain URL string.
             return line;
         }
+        if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {
+            // Parsed as JSON but not an object — fall back to the raw line, as before.
+            return line;
+        }
+        const obj = parsed;
+        if (typeof obj['url'] !== 'string' || obj['url'].length === 0) {
+            const keys = Object.keys(obj).join(', ') || '(none)';
+            throw new InvalidArgumentError(`--file line ${lineNumber} must carry a non-empty string "url" (keys present: ${keys})`);
+        }
+        // A line whose only key is `url` is indistinguishable from a plain URL line — keep it a
+        // bare string so url-only files stay byte-identical on the wire.
+        return Object.keys(obj).length === 1 ? obj['url'] : obj;
     });
 }
+/**
+ * Promote every {@link BulkLine} to the `items[]` object shape used by `/v1/bulk` and
+ * `/v1/scrape/bulk`.
+ *
+ * Deliberately stamps NOTHING beyond `url`. `/v1/bulk` owns capture-kind precedence server-side
+ * (per-item value, then `defaults`, then the endpoint default) since #2641 made the per-item
+ * field optional; a CLI-side default would silently override a caller's `--defaults` and is
+ * grep-gated out (#2596 G7).
+ */
+export function toBulkItems(lines) {
+    return lines.map((l) => (typeof l === 'string' ? { url: l } : l));
+}

package/dist/output.d.ts

@@ -72,6 +72,20 @@ export declare function resolveOutputMode(flags: OutputFlags): OutputMode;
  *
  * Errors — rather than guessing or stringifying — whenever the request is ambiguous or the body is
  * not text, so `--raw` can never print `[object Object]` or a format the caller did not ask for.
+ *
+ * Three states have no body but are NOT "the response contains no formats" (#2686 item 2), and each
+ * gets its own diagnostic ahead of the generic ones, because the caller's next action differs:
+ *
+ * - a terminal-FAILED scrape (`ScrapeErrorEnvelope`) → the envelope's own `error.code` + message.
+ *   This is the scrape twin of the screenshots extractor's error-envelope branch (#2027). It is
+ *   checked FIRST so a failure envelope that also carries `formats: {}` still reports why it failed
+ *   rather than a format-selection problem the caller cannot act on.
+ * - a completed scrape with an EMPTY `formats: {}` bag and a warning explaining it (the #2675
+ *   `artifact_unavailable` shape) → the warning's code and message.
+ * - an empty bag with no warning → says the bag is empty. It is not "no formats" (the key IS
+ *   there), and `--format` cannot help, so the multi-format diagnostic would be a false lead.
+ *
+ * A queued envelope never reaches here — {@link emitResult} diverts it to {@link emitQueuedRaw}.
  */
 export declare function extractRawBody(response: unknown, requestedFormat?: string): string;
 /**
@@ -133,6 +147,10 @@ export declare function emitResult(result: unknown, mode: OutputMode, target: Ou
 /**
  * Collect the `X-Fetch-*` response headers of a {@link FetchResult} into a Title-Cased object —
  * the documented `--json` payload for a fetch command.
+ *
+ * Key names are the SPEC names: {@link FETCH_HEADER_CASING} first, {@link titleCaseHeader}
+ * otherwise. `fetch-metadata-keys.test.ts` pins the emitted set against the spec's own exhaustive
+ * header list. Values are passed through verbatim, and non-`X-Fetch-*` headers stay filtered out.
  */
 export declare function collectFetchMetadata(headers: Headers): Record<string, string>;
 /**
@@ -158,7 +176,20 @@ export declare function emitFetchResult(result: unknown, jsonMode: boolean, targ
  */
 export declare function printResult(data: unknown, jsonMode: boolean): void;
 /**
- * Handle a CLI command error: print to stderr and exit 1.
+ * Handle a CLI command error: print to stderr and signal exit 1.
+ *
+ * The signal is a SOFT `process.exitCode`, never a hard `process.exit` (#2686 item 4, restoring the
+ * #2544 policy this site contradicted). Node documents `process.stderr` as ASYNCHRONOUS when it
+ * points at a pipe: a write that cannot be handed to the kernel in one go is queued in userspace
+ * and drained on a later event-loop turn, and `process.exit()` runs before any of those turns and
+ * DISCARDS the queue. The one line a script greps was therefore the line most at risk of being
+ * thrown away. Setting `process.exitCode` lets the process end naturally once stderr has drained,
+ * with the same observable exit code.
+ *
+ * Returns `void` (not `never`): the hard exit used to provide an implicit stop, so every caller
+ * must be the LAST statement of its `catch` block, or `return` immediately after (#2686 E4). Every
+ * call site in this package is the final statement of an action's `catch`, so the action returns
+ * and nothing further runs.
  *
  * The message always begins `Error: ` (a prefix scripts grep on) followed by the error message and
  * an optional parenthesised suffix built from the two diagnostic axes the ByteKit errors carry:
@@ -174,5 +205,5 @@ export declare function printResult(data: unknown, jsonMode: boolean): void;
  * status is present but no code (a hand-built error), the suffix is `(status: <n>)` with no empty
  * placeholder — never `(, status: <n>)`.
  */
-export declare function handleError(err: unknown): never;
+export declare function handleError(err: unknown): void;
 export {};

package/dist/output.js

@@ -58,13 +58,77 @@ export function resolveOutputMode(flags) {
         return { kind: 'raw' };
     return { kind: 'human' };
 }
+/**
+ * The `error` of a spec `ScrapeErrorEnvelope` (`{status:'failed', id, error:{code, message, …}}`),
+ * or undefined when the response is not a failure envelope.
+ *
+ * Keyed on the presence of a well-formed `error` object rather than on `status === 'failed'`: the
+ * envelope is what a 200 GET returns for a terminally-failed job, and a producer that ships the
+ * error without the discriminator still owes the caller its reason.
+ */
+function scrapeErrorDetail(response) {
+    if (typeof response !== 'object' || response === null)
+        return undefined;
+    const err = response.error;
+    if (typeof err !== 'object' || err === null || Array.isArray(err))
+        return undefined;
+    const { code, message } = err;
+    const detail = {};
+    if (typeof code === 'string' && code.length > 0)
+        detail.code = code;
+    if (typeof message === 'string' && message.length > 0)
+        detail.message = message;
+    return detail;
+}
+/**
+ * The warning that best explains an empty `formats` bag — `artifact_unavailable` when present
+ * (the #2675 shape: a completed scrape whose persisted artifact could not be rehydrated),
+ * otherwise the first warning the envelope carries. Undefined when there are none.
+ */
+function emptyFormatsWarning(response) {
+    if (typeof response !== 'object' || response === null)
+        return undefined;
+    const raw = response.warnings;
+    if (!Array.isArray(raw))
+        return undefined;
+    const parsed = raw.flatMap((w) => {
+        if (typeof w !== 'object' || w === null)
+            return [];
+        const { code, message } = w;
+        if (typeof code !== 'string' || code.length === 0)
+            return [];
+        return typeof message === 'string' && message.length > 0 ? [{ code, message }] : [{ code }];
+    });
+    return parsed.find((w) => w.code === 'artifact_unavailable') ?? parsed[0];
+}
 /**
  * Narrow an SDK response (typed `unknown` today) down to the single format body to emit raw.
  *
  * Errors — rather than guessing or stringifying — whenever the request is ambiguous or the body is
  * not text, so `--raw` can never print `[object Object]` or a format the caller did not ask for.
+ *
+ * Three states have no body but are NOT "the response contains no formats" (#2686 item 2), and each
+ * gets its own diagnostic ahead of the generic ones, because the caller's next action differs:
+ *
+ * - a terminal-FAILED scrape (`ScrapeErrorEnvelope`) → the envelope's own `error.code` + message.
+ *   This is the scrape twin of the screenshots extractor's error-envelope branch (#2027). It is
+ *   checked FIRST so a failure envelope that also carries `formats: {}` still reports why it failed
+ *   rather than a format-selection problem the caller cannot act on.
+ * - a completed scrape with an EMPTY `formats: {}` bag and a warning explaining it (the #2675
+ *   `artifact_unavailable` shape) → the warning's code and message.
+ * - an empty bag with no warning → says the bag is empty. It is not "no formats" (the key IS
+ *   there), and `--format` cannot help, so the multi-format diagnostic would be a false lead.
+ *
+ * A queued envelope never reaches here — {@link emitResult} diverts it to {@link emitQueuedRaw}.
  */
 export function extractRawBody(response, requestedFormat) {
+    const failure = scrapeErrorDetail(response);
+    if (failure !== undefined) {
+        const reason = failure.message !== undefined && failure.code !== undefined
+            ? `${failure.message} (${failure.code})`
+            : (failure.message ?? failure.code ?? 'unknown error');
+        throw new Error(`--raw: scrape failed: ${reason}`);
+    }
     const formats = typeof response === 'object' && response !== null
         ? response.formats
         : undefined;
@@ -73,6 +137,13 @@ export function extractRawBody(response, requestedFormat) {
     }
     const bag = formats;
     const keys = Object.keys(bag);
+    if (keys.length === 0) {
+        const warning = emptyFormatsWarning(response);
+        throw new Error(warning === undefined
+            ? '--raw: the response carries an empty formats map — no body to emit'
+            : `--raw: the response carries an empty formats map — ${warning.code}` +
+                (warning.message !== undefined ? `: ${warning.message}` : ''));
+    }
     let key;
     if (requestedFormat !== undefined) {
         key = requestedFormat;
@@ -197,15 +268,39 @@ function titleCaseHeader(name) {
         .map((seg) => seg.charAt(0).toUpperCase() + seg.slice(1).toLowerCase())
         .join('-');
 }
+/**
+ * Spec header names that segment-wise Title-Casing cannot produce (#2686 item 3).
+ *
+ * `titleCaseHeader` lowercases every non-initial character of a segment, so `x-fetch-id` came out
+ * `X-Fetch-Id` and `x-fetch-url` came out `X-Fetch-Url` — while `docs/api/openapi.yaml` declares
+ * `X-Fetch-ID` and `X-Fetch-URL` (the gateway's CORS expose list, `lib/cors-config.ts`, mirrors the
+ * same spellings). A `jq '."X-Fetch-ID"'` filter written from the API reference therefore got
+ * `null` off `bytekit fetch … --json`.
+ *
+ * The exception lives HERE, in a lookup keyed by the lowercase wire name, and deliberately NOT
+ * inside `titleCaseHeader` (#2686 E3). That helper is a generic contract — an acronym rule baked
+ * into it would silently re-case names for any future caller, and would have to guess at segments
+ * like `Waf` (which the spec really does spell `X-Fetch-Waf-Vendor`, not `WAF`). Every header
+ * outside this map keeps the generic Title-Cased spelling it has always had.
+ */
+const FETCH_HEADER_CASING = {
+    'x-fetch-id': 'X-Fetch-ID',
+    'x-fetch-url': 'X-Fetch-URL',
+};
 /**
  * Collect the `X-Fetch-*` response headers of a {@link FetchResult} into a Title-Cased object —
  * the documented `--json` payload for a fetch command.
+ *
+ * Key names are the SPEC names: {@link FETCH_HEADER_CASING} first, {@link titleCaseHeader}
+ * otherwise. `fetch-metadata-keys.test.ts` pins the emitted set against the spec's own exhaustive
+ * header list. Values are passed through verbatim, and non-`X-Fetch-*` headers stay filtered out.
  */
 export function collectFetchMetadata(headers) {
     const meta = {};
     headers.forEach((value, key) => {
-        if (key.toLowerCase().startsWith('x-fetch-')) {
-            meta[titleCaseHeader(key)] = value;
+        const lower = key.toLowerCase();
+        if (lower.startsWith('x-fetch-')) {
+            meta[FETCH_HEADER_CASING[lower] ?? titleCaseHeader(key)] = value;
         }
     });
     return meta;
@@ -293,7 +388,7 @@ const BYTEKIT_ERROR_NAMES = new Set(['ByteKitError', 'ByteKitConnectionError']);
  * A non-empty API/connection error `code`, or undefined when absent.
  *
  * Scoped by `err.name` to the ByteKit taxonomy: plenty of OTHER errors also carry a string
- * `.code` — Node fs/system errors (ENOENT from `parseUrlLines`'s readFileSync, EACCES on a `-o`
+ * `.code` — Node fs/system errors (ENOENT from `parseBulkLines`'s readFileSync, EACCES on a `-o`
  * write) — whose message already embeds the code, so suffixing them would render the redundant
  * `Error: ENOENT: … (ENOENT)`. Only `ByteKitError` / `ByteKitConnectionError` codes qualify.
  */
@@ -304,7 +399,20 @@ function errorCode(err) {
     return typeof raw === 'string' && raw.length > 0 ? raw : undefined;
 }
 /**
- * Handle a CLI command error: print to stderr and exit 1.
+ * Handle a CLI command error: print to stderr and signal exit 1.
+ *
+ * The signal is a SOFT `process.exitCode`, never a hard `process.exit` (#2686 item 4, restoring the
+ * #2544 policy this site contradicted). Node documents `process.stderr` as ASYNCHRONOUS when it
+ * points at a pipe: a write that cannot be handed to the kernel in one go is queued in userspace
+ * and drained on a later event-loop turn, and `process.exit()` runs before any of those turns and
+ * DISCARDS the queue. The one line a script greps was therefore the line most at risk of being
+ * thrown away. Setting `process.exitCode` lets the process end naturally once stderr has drained,
+ * with the same observable exit code.
+ *
+ * Returns `void` (not `never`): the hard exit used to provide an implicit stop, so every caller
+ * must be the LAST statement of its `catch` block, or `return` immediately after (#2686 E4). Every
+ * call site in this package is the final statement of an action's `catch`, so the action returns
+ * and nothing further runs.
  *
  * The message always begins `Error: ` (a prefix scripts grep on) followed by the error message and
  * an optional parenthesised suffix built from the two diagnostic axes the ByteKit errors carry:
@@ -337,5 +445,5 @@ export function handleError(err) {
     else {
         console.error(`Error: ${String(err)}`);
     }
-    process.exit(1);
+    process.exitCode = 1;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hunt-labs/bytekit-cli",
-  "version": "0.5.0",
+  "version": "0.7.0",
   "description": "ByteKit CLI — wraps @hunt-labs/bytekit-sdk as shell subcommands",
   "license": "MIT",
   "type": "module",
@@ -41,10 +41,10 @@
   ],
   "dependencies": {
     "commander": "^12.0.0",
-    "@hunt-labs/bytekit-sdk": "^0.4.0"
+    "@hunt-labs/bytekit-sdk": "^0.6.0"
   },
   "scripts": {
-    "build": "tsc",
+    "build": "rm -rf dist tsconfig.tsbuildinfo && tsc",
     "typecheck": "tsc --noEmit",
     "test": "vitest run"
   }