@hunt-labs/bytekit-cli 0.3.0 → 0.6.0

package/dist/commands/scrape.js

@@ -1,10 +1,16 @@
 import { Command } from 'commander';
 import { buildClient } from '../client.js';
 import { printResult, handleError, resolveOutputMode, emitResult } from '../output.js';
-import { parseUrlLines } from '../lib/url-lines.js';
-import { parseIntFlag, parseEnum, parseCsv } from '../lib/parse-flags.js';
+import { parseBulkLines, toBulkItems } from '../lib/url-lines.js';
+import { parseIntFlag, parseEnum, parseCsv, parseCacheTtl, parseJsonFlag, collectFormats, } from '../lib/parse-flags.js';
 /** The `ScrapeRequest.markdown_mode` spec enum. */
 const MARKDOWN_MODES = ['article', 'raw', 'llm'];
+/** The `ScrapeRequest.events` item enum. */
+const SCRAPE_EVENTS = ['queued', 'completed', 'failed'];
+/** The `ScrapeRequest.token_encoding` spec enum. */
+const TOKEN_ENCODINGS = ['cl100k_base', 'o200k_base'];
+/** The `markdown_links` / `markdown_images` render-mode enum. */
+const MARKDOWN_RENDER_MODES = ['inline', 'references', 'none', 'text'];
 /**
  * Exit code when a queued (202 async) envelope reaches a non-JSON output mode.
  *
@@ -13,8 +19,11 @@ const MARKDOWN_MODES = ['article', 'raw', 'llm'];
  * that PR's symbols while the two guards still compose idempotently post-merge. Deliberately NOT 0
  * (a content-body success) and NOT 1 (a usage/API error): the job WAS accepted and billed, there
  * is simply no content body yet, and a distinct code lets a caller branch on "queued" without
- * parsing stderr. Human mode shares the same code (soft `process.exitCode`) so "queued, not
- * complete" is branchable there too, for consistency with the raw-mode signal.
+ * parsing stderr.
+ *
+ * BOTH modes carry it as a soft `process.exitCode` (#2544): raw mode used to hard-`process.exit`,
+ * which runs before Node drains an ASYNC piped stdout and could discard the id it had just
+ * written — the exact `$(…)` capture the raw branch exists for. Human mode was already soft.
  */
 const QUEUED_EXIT_CODE = 2;
 /**
@@ -36,15 +45,34 @@ export function makeScrapeCommand(getGlobal) {
     bulkCmd
         .command('create')
         .description('Create a bulk scrape job')
-        .requiredOption('--file <path>', 'Path to newline-delimited file of URLs (plain URL or {"url":"..."} per line)')
+        .requiredOption('--file <path>', 'Path to a newline-delimited file. Each line is a plain URL, or a JSON object carrying "url" plus per-item config. A line whose only key is "url" behaves exactly as before (sent as a bare URL string); any line with additional keys switches the payload to the per-item shape.')
         .requiredOption('--webhook-url <url>', 'HTTPS webhook URL notified when the job completes')
+        .option('--webhook-secret <secret>', 'HMAC secret used to sign the completion webhook (omitted → unsigned)')
+        .option('--defaults <json>', 'Defaults applied to every entry, as a JSON object (omitted → none). Unknown keys are rejected by /v1/bulk (422) and ignored by scrape bulk / fetch bulk; fetch bulk additionally rejects browser-only keys (cookies, wait_until, wait_for_selector, delay_ms) with a 400.', (v) => parseJsonFlag('--defaults', v, 'object'))
+        .option('--metadata <json>', `Arbitrary metadata echoed on the job envelope, as a JSON object, e.g. '{"run":"7"}' (omitted → none)`, (v) => parseJsonFlag('--metadata', v, 'object'))
+        // /v1/scrape/bulk is the ONE bulk endpoint with a top-level `custom`
+        // (validators/scrape-bulk.ts:88). Per-entry `custom` arrives through --file lines instead.
+        .option('--custom <json>', `User payload echoed on every scrape entry, as a JSON object, e.g. '{"job":"1"}' (omitted → none; a per-entry value in --file wins)`, (v) => parseJsonFlag('--custom', v, 'object'))
         .action(async (opts) => {
         const g = getGlobal();
         const client = buildClient(g);
         try {
             // readFileSync throws ENOENT on missing file — caught by handleError below
-            const urls = parseUrlLines(opts.file);
-            const result = await client.scrape.bulk.create({ urls, webhook_url: opts.webhookUrl });
+            const lines = parseBulkLines(opts.file);
+            // `urls` XOR `items` (validators/scrape-bulk.ts:91-98) — send exactly one, keeping the
+            // pre-#2596 `{ urls, webhook_url }` body byte-identical for url-only files (E2).
+            const body = lines.every((l) => typeof l === 'string')
+                ? { urls: lines, webhook_url: opts.webhookUrl }
+                : { items: toBulkItems(lines), webhook_url: opts.webhookUrl };
+            if (opts.webhookSecret !== undefined)
+                body['webhook_secret'] = opts.webhookSecret;
+            if (opts.defaults !== undefined)
+                body['defaults'] = opts.defaults;
+            if (opts.metadata !== undefined)
+                body['metadata'] = opts.metadata;
+            if (opts.custom !== undefined)
+                body['custom'] = opts.custom;
+            const result = await client.scrape.bulk.create(body);
             printResult(result, g.json);
         }
         catch (err) {
@@ -70,10 +98,34 @@ export function makeScrapeCommand(getGlobal) {
         .command('create')
         .description('Create a scrape job')
         .requiredOption('--url <url>', 'URL to scrape')
-        .option('--format <format>', 'Output format (e.g. markdown, html); sent as the formats array')
+        // Repeatable AND CSV-aware: `--format raw_html --format markdown` and `--format raw_html,markdown`
+        // both send formats: ['raw_html','markdown']. Sent as the `formats` array; omitted → server
+        // default. `--raw`/`-o` require EXACTLY ONE format (validated pre-request; see the action).
+        .option('--format <format>', 'Output format; repeat or comma-separate for multiple (e.g. --format raw_html --format markdown, or --format raw_html,markdown). Sent as the formats array; omitted → server default.', collectFormats)
         .option('--country <cc>', 'Two-letter country code to route the request through (e.g. us, de)')
         .option('--timeout-ms <n>', 'Per-request timeout in milliseconds', (v) => parseIntFlag('--timeout-ms', v))
         .option('--markdown-mode <mode>', `Markdown extraction mode: ${MARKDOWN_MODES.join(', ')}`, (v) => parseEnum('--markdown-mode', MARKDOWN_MODES, v))
+        .option('--markdown-query <q>', 'BM25 query string for relevance-ranked markdown filtering (omitted → disabled)')
+        .option('--cache-ttl <ttl>', 'Cache freshness: 0, Nh (hours), or Nd (days), e.g. 48h, 2d, 0 (omitted → server default)', (v) => parseCacheTtl('--cache-ttl', v))
+        .option('--cookies <json>', `Cookies as a JSON array, e.g. '[{"name":"a","value":"b"}]' (omitted → none)`, (v) => parseJsonFlag('--cookies', v, 'array'))
+        .option('--headers <json>', `Extra request headers as a JSON object, e.g. '{"X-A":"b"}' (omitted → none)`, (v) => parseJsonFlag('--headers', v, 'object'))
+        .option('--delay-ms <n>', 'Post-load delay in milliseconds before capture', (v) => parseIntFlag('--delay-ms', v))
+        .option('--events <csv>', `Webhook events (comma-separated): ${SCRAPE_EVENTS.join(', ')} (omitted → server default)`, (v) => parseCsv(v).map((e) => parseEnum('--events', SCRAPE_EVENTS, e)))
+        .option('--token-encoding <enc>', `Tokenizer for --token-budget: ${TOKEN_ENCODINGS.join(', ')} (omitted → server default)`, (v) => parseEnum('--token-encoding', TOKEN_ENCODINGS, v))
+        .option('--custom <json>', `User payload echoed on the envelope as a JSON object, e.g. '{"job":"1"}' (omitted → none)`, (v) => parseJsonFlag('--custom', v, 'object'))
+        // Server default is TRUE, so this is a tri-state pair: --remove-base64-images → true,
+        // --no-remove-base64-images → false, neither → key omitted (never materialize the default, E6).
+        .option('--remove-base64-images', 'Strip base64 data: images before conversion (server default)')
+        .option('--no-remove-base64-images', 'Preserve base64 data: images in the pipeline output')
+        .option('--markdown-links <mode>', `Link rendering: ${MARKDOWN_RENDER_MODES.join(', ')}`, (v) => parseEnum('--markdown-links', MARKDOWN_RENDER_MODES, v))
+        .option('--markdown-images <mode>', `Image retention: ${MARKDOWN_RENDER_MODES.join(', ')}`, (v) => parseEnum('--markdown-images', MARKDOWN_RENDER_MODES, v))
+        .option('--with-links-summary', 'Append a Links footer to the markdown output')
+        .option('--with-images-summary', 'Append an Images footer to the markdown output')
+        .option('--markdown-compact', 'Collapse excessive whitespace in the markdown output')
+        .option('--markdown-filter-images', 'Filter low-signal images from the markdown output')
+        .option('--markdown-include-media', 'Return rich link/image objects and a tables array')
+        .option('--markdown-include-warnings', 'Include markdown-pipeline and tag-filter warnings')
+        .option('--markdown-include-stats', 'Include a top-level stats object (chars, tokens, blocks)')
         .option('--async', 'Queue the scrape and return a job envelope immediately (no wait)')
         .option('--webhook-url <url>', 'HTTPS webhook URL notified when an async scrape completes')
         .option('--token-budget <n>', 'Maximum tokens of markdown to return', (v) => parseIntFlag('--token-budget', v))
@@ -87,11 +139,12 @@ export function makeScrapeCommand(getGlobal) {
         const g = getGlobal();
         const client = buildClient(g);
         const body = { url: opts.url };
-        // The v1 scrape validator expects `formats` (array), not `format` (string).
-        // Omit the key entirely when --format is not provided so we never send
-        // formats: ["undefined"].
-        if (opts.format !== undefined)
-            body['formats'] = [opts.format];
+        // The v1 scrape validator expects `formats` (array). The --format collector already
+        // accumulates repeated/CSV occurrences into an array; write it verbatim, omitting the key
+        // entirely when --format is absent so we never send formats: [] or ["undefined"].
+        const formats = opts.format;
+        if (formats !== undefined)
+            body['formats'] = formats;
         // Each optional flag maps 1:1 to its snake_case ScrapeRequest field and is written ONLY
         // when the flag is present — a boolean must never serialize `false` (#2480 E1).
         if (opts.country !== undefined)
@@ -100,6 +153,44 @@ export function makeScrapeCommand(getGlobal) {
             body['timeout_ms'] = opts.timeoutMs;
         if (opts.markdownMode !== undefined)
             body['markdown_mode'] = opts.markdownMode;
+        if (opts.markdownQuery !== undefined)
+            body['markdown_query'] = opts.markdownQuery;
+        if (opts.cacheTtl !== undefined)
+            body['cache_ttl'] = opts.cacheTtl;
+        if (opts.cookies !== undefined)
+            body['cookies'] = opts.cookies;
+        if (opts.headers !== undefined)
+            body['headers'] = opts.headers;
+        if (opts.delayMs !== undefined)
+            body['delay_ms'] = opts.delayMs;
+        if (opts.events !== undefined)
+            body['events'] = opts.events;
+        if (opts.tokenEncoding !== undefined)
+            body['token_encoding'] = opts.tokenEncoding;
+        if (opts.custom !== undefined)
+            body['custom'] = opts.custom;
+        // Tri-state boolean: commander yields `undefined` when neither --remove-base64-images nor
+        // --no-remove-base64-images is passed, so the key is omitted and the server default holds.
+        if (opts.removeBase64Images !== undefined)
+            body['remove_base64_images'] = opts.removeBase64Images;
+        if (opts.markdownLinks !== undefined)
+            body['markdown_links'] = opts.markdownLinks;
+        if (opts.markdownImages !== undefined)
+            body['markdown_images'] = opts.markdownImages;
+        if (opts.withLinksSummary === true)
+            body['with_links_summary'] = true;
+        if (opts.withImagesSummary === true)
+            body['with_images_summary'] = true;
+        if (opts.markdownCompact === true)
+            body['markdown_compact'] = true;
+        if (opts.markdownFilterImages === true)
+            body['markdown_filter_images'] = true;
+        if (opts.markdownIncludeMedia === true)
+            body['markdown_include_media'] = true;
+        if (opts.markdownIncludeWarnings === true)
+            body['markdown_include_warnings'] = true;
+        if (opts.markdownIncludeStats === true)
+            body['markdown_include_stats'] = true;
         if (opts.async === true)
             body['async'] = true;
         if (opts.webhookUrl !== undefined)
@@ -122,6 +213,12 @@ export function makeScrapeCommand(getGlobal) {
             // handleError()'s clean one-liner. buildClient() does no network I/O, so rejecting the
             // --json/--raw conflict here still happens before any request is issued.
             const mode = resolveOutputMode({ json: g.json, raw: opts.raw, output: opts.output });
+            // Raw mode (--raw or a bare -o) emits a SINGLE format body; more than one format is
+            // ambiguous, so reject before the request (E2). Thrown inside the try so handleError()
+            // prints the clean one-liner and no scrape.create call is ever recorded.
+            if (mode.kind === 'raw' && formats !== undefined && formats.length > 1) {
+                throw new Error('--raw requires exactly one --format');
+            }
             const result = await client.scrape.create(body);
             // Queued-envelope precedence (#2480 E4s): a `--async` scrape returns a 202
             // `{ id, status: 'queued' }` envelope with NO formats map. Detect that shape here —
@@ -131,8 +228,10 @@ export function makeScrapeCommand(getGlobal) {
             // indistinguishable from that one):
             //   - raw mode (`--raw` or a bare `-o`): the bare sc_ id on STDOUT (a usable
             //     `$(bytekit scrape create … --raw)` handle), a human poll note on STDERR, exit
-            //     QUEUED_EXIT_CODE. The `-o` file is deliberately NEVER written — there is no
-            //     content body yet, and a file holding a bare id would masquerade as content.
+            //     QUEUED_EXIT_CODE via soft `process.exitCode` (#2544 — a hard `process.exit` runs
+            //     before Node drains an ASYNC piped stdout and would discard the very id this
+            //     branch exists to deliver). The `-o` file is deliberately NEVER written — there is
+            //     no content body yet, and a file holding a bare id would masquerade as content.
             //   - json mode (`--json`, with or without `-o`): OUTSIDE the raw rule — fall through
             //     to emitResult, which emits the envelope verbatim (exit 0).
             //   - human default: keep the readable id/status dump, but signal queued-not-complete
@@ -143,7 +242,8 @@ export function makeScrapeCommand(getGlobal) {
                     console.error(`Note: scrape ${result.id} was queued (async) — no content body is available yet. ` +
                         `Poll it with: bytekit scrape get ${result.id}`);
                     process.stdout.write(`${result.id}\n`);
-                    process.exit(QUEUED_EXIT_CODE);
+                    process.exitCode = QUEUED_EXIT_CODE;
+                    return;
                 }
                 if (mode.kind === 'human') {
                     printResult(result, false);
@@ -151,7 +251,9 @@ export function makeScrapeCommand(getGlobal) {
                     return;
                 }
             }
-            emitResult(result, mode, { file: opts.output }, opts.format);
+            // At this point raw mode has at most one format (E2 guard above), so formats[0] is the
+            // single requested body selector; json/human modes ignore the argument.
+            emitResult(result, mode, { file: opts.output }, formats?.[0]);
         }
         catch (err) {
             handleError(err);
@@ -159,13 +261,22 @@ export function makeScrapeCommand(getGlobal) {
     });
     cmd
         .command('get <id>')
-        .description('Get a scrape job by ID')
-        .action(async (id) => {
+        .description('Get a scrape job by ID (use --raw/-o/--format to retrieve a completed body)')
+        .option('--raw', 'Print the requested format body raw, with no JSON wrapper')
+        .option('-o, --output <file>', 'Write the output to the given file path (implies --raw unless --json is set)')
+        .option('--format <format>', 'Which persisted format to emit — only takes effect with --raw/-o; ignored otherwise')
+        .action(async (id, opts) => {
         const g = getGlobal();
         const client = buildClient(g);
         try {
+            // Mirror `scrape create`'s output path: resolveOutputMode FIRST inside the try (so a
+            // --json/--raw conflict rejects before the request), then emitResult. The request body is
+            // unchanged — scrape.get(id) takes no options — so the inverse-case arg-length assertion in
+            // request-shapes.test.ts stays green. A still-queued job flows through emitResult's
+            // queued-envelope branch (id + soft exit 2, #2508/#2544); a completed job emits its body.
+            const mode = resolveOutputMode({ json: g.json, raw: opts.raw, output: opts.output });
             const result = await client.scrape.get(id);
-            printResult(result, g.json);
+            emitResult(result, mode, { file: opts.output }, opts.format);
         }
         catch (err) {
             handleError(err);

package/dist/commands/screenshots.js

@@ -2,11 +2,13 @@ import { Command, InvalidArgumentError } from 'commander';
 import { buildClient } from '../client.js';
 import { printResult, handleError } from '../output.js';
 import { downloadArtifact } from '../lib/download-artifact.js';
-import { parseEnum, parseIntFlag } from '../lib/parse-flags.js';
+import { parseEnum, parseIntFlag, parseJsonFlag } from '../lib/parse-flags.js';
 /** `ScreenshotRequest` spec enums. */
 const DEVICES = ['desktop', 'mobile'];
 const FORMATS = ['jpeg', 'png'];
 const WAIT_UNTILS = ['load', 'domcontentloaded', 'networkidle'];
+/** `device_scale_factor` accepts only 1 or 2 (spec enum [1, 2, null]); parsed as a string, sent numeric. */
+const DEVICE_SCALE_FACTORS = ['1', '2'];
 /** Parse `--viewport WxH` (e.g. `1280x720`) into `{ width, height }`; reject malformed input. */
 export function parseViewport(value) {
     const m = /^(\d+)x(\d+)$/.exec(value);
@@ -21,6 +23,24 @@ export function parseClip(value) {
         throw new InvalidArgumentError('--clip must be X,Y,WIDTH,HEIGHT (e.g. 0,0,800,600).');
     return { x: Number(m[1]), y: Number(m[2]), width: Number(m[3]), height: Number(m[4]) };
 }
+/**
+ * Exit code when a queued (202 async) envelope reaches human output (#2596).
+ *
+ * Mirrors `scrape create`'s own `QUEUED_EXIT_CODE` (`commands/scrape.ts:40`) EXACTLY, and is
+ * likewise a LOCAL constant rather than an import — the two commands agree on the value, not on a
+ * shared symbol, and the guard stays where the ticket pins it (the screenshots async branch)
+ * instead of migrating into the shared printer, where it would silently set exit 2 for every
+ * command that prints a result.
+ *
+ * Deliberately NOT 0 (a completed capture) and NOT 1 (a usage/API error): the job WAS accepted and
+ * billed, the artifact simply does not exist yet, and a distinct code lets a caller branch on
+ * "queued" without parsing stderr.
+ *
+ * BREAKING (#2596): `screenshots create --async` previously exited 0. See the README exit-code
+ * table and the CHANGELOG `[0.6.0]` note (this version's own section — `[Unreleased]` is kept
+ * deliberately empty per `packaging.test.ts:203`; see #2620).
+ */
+const QUEUED_EXIT_CODE = 2;
 /**
  * Extract a non-empty string `image_url` from a screenshot envelope, or throw an actionable error.
  *
@@ -51,6 +71,21 @@ function extractImageUrl(result) {
 async function emitScreenshot(result, opts) {
     if (opts.isAsync === true) {
         printResult(result, opts.json);
+        // Discriminate on the --async FLAG, never on the response body. The screenshot 202 reports a
+        // PENDING status (gateway src/routes/screenshots.ts:138) — not the queued status that
+        // `scrape create`'s envelope guard (commands/scrape.ts:49-53) tests for. Reusing that guard
+        // here would never fire in production and this branch would silently no-op. Grep gate G11
+        // (#2596) enforces the absence of a response-status discriminator in this file, which is why
+        // neither the symbol nor the comparison is spelled out above.
+        //
+        // The assignment sits INSIDE this branch, ahead of the `-o` return below, so `--async -o`
+        // carries the same exit code as bare `--async` (#2596 DoD 15 / E20). `--json` keeps exit 0:
+        // the envelope is a complete machine-readable payload, identical to scrape's split
+        // (scrape.ts:313-317). Soft `process.exitCode`, never `process.exit` — a hard exit runs
+        // before Node drains an async piped stdout and would truncate the envelope just written
+        // (#2544).
+        if (!opts.json)
+            process.exitCode = QUEUED_EXIT_CODE;
         return;
     }
     if (opts.output !== undefined) {
@@ -72,6 +107,24 @@ export function makeScreenshotsCommand(getGlobal) {
         .option('--quality <n>', 'JPEG quality (1-100)', (v) => parseIntFlag('--quality', v))
         .option('--wait-until <event>', `Wait condition: ${WAIT_UNTILS.join(', ')}`, (v) => parseEnum('--wait-until', WAIT_UNTILS, v))
         .option('--clip <x,y,w,h>', 'Clip region X,Y,WIDTH,HEIGHT (e.g. 0,0,800,600)', parseClip)
+        // Tri-state pairs — server default is TRUE, so define BOTH --x and --no-x: commander yields
+        // `undefined` when neither is passed, and the key is omitted (never materialize the default, E6).
+        .option('--block-ads', 'Block ads before capture (server default)')
+        .option('--no-block-ads', 'Do not block ads')
+        .option('--block-cookie-banners', 'Block cookie-consent banners before capture (server default)')
+        .option('--no-block-cookie-banners', 'Do not block cookie-consent banners')
+        .option('--scroll', 'Scroll the page before capture to trigger lazy content (server default)')
+        .option('--no-scroll', 'Do not scroll the page before capture')
+        .option('--wait-for-selector <sel>', 'Wait until this CSS selector is present before capture')
+        .option('--delay-ms <n>', 'Post-load delay in milliseconds before capture', (v) => parseIntFlag('--delay-ms', v))
+        .option('--dark-mode', 'Emulate prefers-color-scheme: dark')
+        .option('--device-scale-factor <n>', `Device pixel ratio: ${DEVICE_SCALE_FACTORS.join(', ')}`, (v) => parseEnum('--device-scale-factor', DEVICE_SCALE_FACTORS, v))
+        .option('--include-html', 'Include the rendered HTML alongside the screenshot')
+        .option('--country <cc>', 'Two-letter country code to route the request through (e.g. us, de)')
+        .option('--language <lang>', 'Accept-Language override (e.g. en-US)')
+        .option('--headers <json>', `Extra request headers as a JSON object, e.g. '{"X-A":"b"}' (omitted → none)`, (v) => parseJsonFlag('--headers', v, 'object'))
+        .option('--cookies <json>', `Cookies as a JSON array of {name,value,domain}, e.g. '[{"name":"a","value":"b","domain":"example.com"}]' (omitted → none)`, (v) => parseJsonFlag('--cookies', v, 'array'))
+        .option('--metadata <json>', `Arbitrary metadata echoed on the screenshot envelope, as a JSON object, e.g. '{"run":"7"}' (omitted → none)`, (v) => parseJsonFlag('--metadata', v, 'object'))
         .option('--async', 'Queue the screenshot and return a job envelope immediately (no wait)')
         .option('-o, --output <file>', 'Download the captured image to this file path (use - for stdout). Not valid with --json.')
         .action(async (opts) => {
@@ -94,6 +147,38 @@ export function makeScreenshotsCommand(getGlobal) {
             body['wait_until'] = opts.waitUntil;
         if (opts.clip !== undefined)
             body['clip'] = opts.clip;
+        // Tri-state booleans: commander yields `undefined` when neither --x nor --no-x is passed,
+        // so the key is omitted and the server default (true) holds (E6).
+        if (opts.blockAds !== undefined)
+            body['block_ads'] = opts.blockAds;
+        if (opts.blockCookieBanners !== undefined)
+            body['block_cookie_banners'] = opts.blockCookieBanners;
+        if (opts.scroll !== undefined)
+            body['scroll'] = opts.scroll;
+        if (opts.waitForSelector !== undefined)
+            body['wait_for_selector'] = opts.waitForSelector;
+        if (opts.delayMs !== undefined)
+            body['delay_ms'] = opts.delayMs;
+        if (opts.darkMode === true)
+            body['dark_mode'] = true;
+        // device_scale_factor is a spec integer enum [1, 2]; the parser validates the string, and
+        // the wire value is numeric.
+        if (opts.deviceScaleFactor !== undefined)
+            body['device_scale_factor'] = Number(opts.deviceScaleFactor);
+        if (opts.includeHtml === true)
+            body['include_html'] = true;
+        if (opts.country !== undefined)
+            body['country'] = opts.country;
+        if (opts.language !== undefined)
+            body['language'] = opts.language;
+        // The server defaults headers/cookies/metadata to {} / [] / {}; writing an empty
+        // container on a bare create would materialize those defaults (#2480 E1).
+        if (opts.headers !== undefined)
+            body['headers'] = opts.headers;
+        if (opts.cookies !== undefined)
+            body['cookies'] = opts.cookies;
+        if (opts.metadata !== undefined)
+            body['metadata'] = opts.metadata;
         try {
             // Reject --json + -o BEFORE issuing the request: -o downloads BINARY artifact bytes, so
             // it can never coexist with --json (which must stay a text envelope). Thrown here so no

package/dist/commands/sitemap.js

@@ -1,22 +1,54 @@
 import { Command } from 'commander';
 import { buildClient } from '../client.js';
 import { printResult, handleError } from '../output.js';
+import { parseEnum, parseIntFlag, parseCacheTtl, parseJsonFlag } from '../lib/parse-flags.js';
+/** The `SitemapRequest.strategy` spec enum. */
+const SITEMAP_STRATEGIES = ['quick', 'standard', 'deep'];
 export function makeSitemapCommand(getGlobal) {
     const cmd = new Command('sitemap').description('Sitemap crawl operations');
     cmd
         .command('create')
         .description('Start a sitemap crawl')
         .requiredOption('--url <url>', 'Root URL to crawl')
+        .option('--strategy <strategy>', `Crawl strategy: ${SITEMAP_STRATEGIES.join(', ')} (omitted → server default standard)`, (v) => parseEnum('--strategy', SITEMAP_STRATEGIES, v))
+        .option('--max-depth <n>', 'Maximum crawl depth 1-5 (omitted → server default 2)', (v) => parseIntFlag('--max-depth', v))
+        .option('--max-urls <n>', 'Maximum URLs to collect (omitted → server default 5000)', (v) => parseIntFlag('--max-urls', v))
+        .option('--webhook-url <url>', 'HTTPS webhook URL notified when the crawl completes (omitted → poll-only, no delivery)')
+        .option('--webhook-secret <secret>', 'HMAC secret used to sign the completion webhook (omitted → unsigned)')
+        // Grammar-only validation, deliberately: sitemap caps cache_ttl at 90 DAYS
+        // (validators/sitemap.ts MAX_CACHE_TTL_SECONDS), not the 7d used by scrape/fetch. Mirroring
+        // the cap client-side would fork a second source of truth for a number only the server knows.
+        .option('--cache-ttl <ttl>', 'Cache freshness: 0, Nh (hours), or Nd (days), e.g. 48h, 30d, 0 (omitted → server default 30d; max 90d, enforced server-side)', (v) => parseCacheTtl('--cache-ttl', v))
+        .option('--compact', 'Return a compact result payload (omitted → server default off)')
+        .option('--process <json>', `Post-crawl processing directive as a JSON object, e.g. '{"action":"summarize"}' (omitted → none)`, (v) => parseJsonFlag('--process', v, 'object'))
+        .option('--metadata <json>', `Arbitrary metadata echoed on the job envelope, as a JSON object, e.g. '{"run":"7"}' (omitted → none)`, (v) => parseJsonFlag('--metadata', v, 'object'))
         .action(async (opts) => {
         const g = getGlobal();
         const client = buildClient(g);
-        // Key-checked subset of the SDK's SitemapCreateRequest (same pattern as monitors.ts).
-        // The CLI intentionally sends only `url` and relies on the server-side defaults the spec
-        // declares for the remaining SitemapRequest fields (strategy, max_depth, max_urls, …).
-        // The generated request type marks those fields required, so the final call needs an
-        // explicit assertion — type-level only; the wire body is unchanged (pinned by
-        // __tests__/request-shapes.test.ts).
+        // Omit-when-absent body; the remaining SitemapRequest fields keep their server defaults.
+        // The generated request type marks defaulted fields required, so the call needs an explicit
+        // assertion — type-level only; the wire body is pinned by request-shapes.test.ts.
         const body = { url: opts.url };
+        if (opts.strategy !== undefined)
+            body['strategy'] = opts.strategy;
+        if (opts.maxDepth !== undefined)
+            body['max_depth'] = opts.maxDepth;
+        if (opts.maxUrls !== undefined)
+            body['max_urls'] = opts.maxUrls;
+        if (opts.webhookUrl !== undefined)
+            body['webhook_url'] = opts.webhookUrl;
+        if (opts.webhookSecret !== undefined)
+            body['webhook_secret'] = opts.webhookSecret;
+        if (opts.cacheTtl !== undefined)
+            body['cache_ttl'] = opts.cacheTtl;
+        // `compact` server-defaults to FALSE, so it is written ONLY on `=== true` — writing
+        // `false` on a bare create would materialize a default (#2480 E1).
+        if (opts.compact === true)
+            body['compact'] = true;
+        if (opts.process !== undefined)
+            body['process'] = opts.process;
+        if (opts.metadata !== undefined)
+            body['metadata'] = opts.metadata;
         try {
             const result = await client.sitemap.create(body);
             printResult(result, g.json);

package/dist/commands/usage.d.ts

@@ -0,0 +1,13 @@
+import { Command } from 'commander';
+import { type GlobalOptions } from '../client.js';
+/**
+ * The `usage` command group (#2545) — the `x-sdk-scope: v0.1` usage endpoints, previously curl-only.
+ *
+ * `daily` / `by-endpoint` accept the same three optional query flags (`--from`, `--to`,
+ * `--api-key`). Each maps 1:1 to its snake_case query param and is written ONLY when present; when
+ * NO flag is passed the SDK method is invoked with ZERO arguments (the `monitors list` pattern), so
+ * the bare `/v1/usage/daily` URL is serialized unchanged.
+ */
+export declare function makeUsageCommand(getGlobal: () => GlobalOptions & {
+    json: boolean;
+}): Command;

package/dist/commands/usage.js

@@ -0,0 +1,81 @@
+import { Command } from 'commander';
+import { buildClient } from '../client.js';
+import { printResult, handleError } from '../output.js';
+/**
+ * The `usage` command group (#2545) — the `x-sdk-scope: v0.1` usage endpoints, previously curl-only.
+ *
+ * `daily` / `by-endpoint` accept the same three optional query flags (`--from`, `--to`,
+ * `--api-key`). Each maps 1:1 to its snake_case query param and is written ONLY when present; when
+ * NO flag is passed the SDK method is invoked with ZERO arguments (the `monitors list` pattern), so
+ * the bare `/v1/usage/daily` URL is serialized unchanged.
+ */
+export function makeUsageCommand(getGlobal) {
+    const cmd = new Command('usage').description('Account usage and billing-period reporting');
+    cmd
+        .command('get')
+        .description('Get current billing-period usage summary')
+        .action(async () => {
+        const g = getGlobal();
+        const client = buildClient(g);
+        try {
+            const result = await client.usage.get();
+            printResult(result, g.json);
+        }
+        catch (err) {
+            handleError(err);
+        }
+    });
+    cmd
+        .command('daily')
+        .description('Get daily usage breakdown (default last 30 days; max 366)')
+        .option('--from <date>', 'Start date YYYY-MM-DD (omitted → server default, 30 days ago)')
+        .option('--to <date>', 'End date YYYY-MM-DD (omitted → server default, today)')
+        .option('--api-key <ak_id>', 'Filter by API key prefixed ID (omitted → all keys)')
+        .action(async (opts) => {
+        const g = getGlobal();
+        const client = buildClient(g);
+        const params = {};
+        if (opts.from !== undefined)
+            params.from = opts.from;
+        if (opts.to !== undefined)
+            params.to = opts.to;
+        if (opts.apiKey !== undefined)
+            params.api_key = opts.apiKey;
+        try {
+            const result = Object.keys(params).length > 0
+                ? await client.usage.daily(params)
+                : await client.usage.daily();
+            printResult(result, g.json);
+        }
+        catch (err) {
+            handleError(err);
+        }
+    });
+    cmd
+        .command('by-endpoint')
+        .description('Get per-endpoint bandwidth breakdown (default last 30 days; max 366)')
+        .option('--from <date>', 'Start date YYYY-MM-DD (omitted → server default, 30 days ago)')
+        .option('--to <date>', 'End date YYYY-MM-DD (omitted → server default, today)')
+        .option('--api-key <ak_id>', 'Filter by API key prefixed ID (omitted → all keys)')
+        .action(async (opts) => {
+        const g = getGlobal();
+        const client = buildClient(g);
+        const params = {};
+        if (opts.from !== undefined)
+            params.from = opts.from;
+        if (opts.to !== undefined)
+            params.to = opts.to;
+        if (opts.apiKey !== undefined)
+            params.api_key = opts.apiKey;
+        try {
+            const result = Object.keys(params).length > 0
+                ? await client.usage.byEndpoint(params)
+                : await client.usage.byEndpoint();
+            printResult(result, g.json);
+        }
+        catch (err) {
+            handleError(err);
+        }
+    });
+    return cmd;
+}

package/dist/commands/webhook-deliveries.d.ts

@@ -0,0 +1,11 @@
+import { Command } from 'commander';
+import { type GlobalOptions } from '../client.js';
+/**
+ * The `webhook-deliveries` command group (#2545) — inspect and requeue webhook delivery attempts.
+ *
+ * `list` flags map 1:1 to their query params and are omitted-when-absent; a bare `list` calls the
+ * SDK with ZERO args so the plain `/v1/webhook-deliveries` URL is serialized unchanged.
+ */
+export declare function makeWebhookDeliveriesCommand(getGlobal: () => GlobalOptions & {
+    json: boolean;
+}): Command;

package/dist/commands/webhook-deliveries.js

@@ -0,0 +1,59 @@
+import { Command } from 'commander';
+import { buildClient } from '../client.js';
+import { printResult, handleError } from '../output.js';
+import { parseEnum, parseIntFlag } from '../lib/parse-flags.js';
+/** The `listWebhookDeliveries` `status` query enum (DISTINCT from the monitor status enum). */
+const DELIVERY_STATUSES = ['pending', 'delivered', 'failed', 'exhausted'];
+/**
+ * The `webhook-deliveries` command group (#2545) — inspect and requeue webhook delivery attempts.
+ *
+ * `list` flags map 1:1 to their query params and are omitted-when-absent; a bare `list` calls the
+ * SDK with ZERO args so the plain `/v1/webhook-deliveries` URL is serialized unchanged.
+ */
+export function makeWebhookDeliveriesCommand(getGlobal) {
+    const cmd = new Command('webhook-deliveries').description('Webhook delivery operations');
+    cmd
+        .command('list')
+        .description('List webhook deliveries')
+        .option('--limit <n>', 'Maximum number of deliveries to return', (v) => parseIntFlag('--limit', v))
+        .option('--cursor <cursor>', 'Pagination cursor from a previous page')
+        .option('--status <status>', `Filter by delivery status: ${DELIVERY_STATUSES.join(', ')}`, (v) => parseEnum('--status', DELIVERY_STATUSES, v))
+        .option('--since <iso>', 'Only deliveries created after this ISO date-time')
+        .action(async (opts) => {
+        const g = getGlobal();
+        const client = buildClient(g);
+        const params = {};
+        if (opts.limit !== undefined)
+            params.limit = opts.limit;
+        if (opts.cursor !== undefined)
+            params.cursor = opts.cursor;
+        if (opts.status !== undefined)
+            params.status = opts.status;
+        if (opts.since !== undefined)
+            params.since = opts.since;
+        try {
+            const result = Object.keys(params).length > 0
+                ? await client.webhooks.list(params)
+                : await client.webhooks.list();
+            printResult(result, g.json);
+        }
+        catch (err) {
+            handleError(err);
+        }
+    });
+    cmd
+        .command('retry <id>')
+        .description('Requeue a failed or exhausted webhook delivery')
+        .action(async (id) => {
+        const g = getGlobal();
+        const client = buildClient(g);
+        try {
+            const result = await client.webhooks.retry(id);
+            printResult(result, g.json);
+        }
+        catch (err) {
+            handleError(err);
+        }
+    });
+    return cmd;
+}

package/dist/index.js

@@ -11,6 +11,8 @@ import { makeMonitorsCommand } from './commands/monitors.js';
 import { makeSitemapCommand } from './commands/sitemap.js';
 import { makeSearchCommand } from './commands/search.js';
 import { makeAccountCommand } from './commands/account.js';
+import { makeUsageCommand } from './commands/usage.js';
+import { makeWebhookDeliveriesCommand } from './commands/webhook-deliveries.js';
 // Resolve version from package.json at runtime (supports compiled output).
 const require = createRequire(import.meta.url);
 // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
@@ -44,6 +46,8 @@ export function buildProgram() {
     program.addCommand(makeSitemapCommand(getGlobal));
     program.addCommand(makeSearchCommand(getGlobal));
     program.addCommand(makeAccountCommand(getGlobal));
+    program.addCommand(makeUsageCommand(getGlobal));
+    program.addCommand(makeWebhookDeliveriesCommand(getGlobal));
     return program;
 }
 /** Resolve symlinks and `..` segments, falling back to the raw path if it cannot be resolved. */