@hunt-labs/bytekit-cli 0.9.2 → 0.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/README.md +31 -31
  3. package/dist/client.js +8 -1
  4. package/dist/commands/scrape.js +3 -3
  5. package/dist/lib/parse-flags.d.ts +2 -2
  6. package/dist/lib/parse-flags.js +2 -2
  7. package/dist/output.js +1 -1
  8. package/package.json +2 -2
package/CHANGELOG.md CHANGED
@@ -10,6 +10,24 @@ today, see [README.md](./README.md), which remains the authoritative usage guide
10
10
 
11
11
  ## [Unreleased]
12
12
 
13
+ ## [0.9.3] - 2026-09-03
14
+
15
+ ### Changed
16
+
17
+ - **Breaking (API):** `--format` now takes `raw | markdown | links | images` (#3775). `raw` is
18
+ the renamed unprocessed-source format, and the cleaned-article-HTML format is gone; the
19
+ server rejects both retired values rather than translating them, so
20
+ `--format raw --format markdown` replaces the old spelling. `--raw`/`-o` accept the two
21
+ textual formats, `markdown` and `raw`.
22
+
23
+ ### Fixed
24
+
25
+ - A failed request's response headers now survive on the thrown `ByteKitError` (#3784). The CLI's
26
+ own copy of the SDK error parser dropped the response after reading its body, so `Retry-After`
27
+ and the `X-Quota-*` trio were unreadable on an error even though the gateway sent them. The SDK
28
+ has passed them into the fifth constructor argument since #2957; this copy now matches, and the
29
+ parity suite pins the two parsers against each other on a shared header corpus.
30
+
13
31
  ## [0.9.2] - 2026-08-31
14
32
 
15
33
  ### Changed
package/README.md CHANGED
@@ -100,37 +100,37 @@ bytekit scrape bulk create --file urls.txt --webhook-url https://your.app/hooks/
100
100
  absent (a boolean flag never sends `false`, so a server default is never
101
101
  materialized):
102
102
 
103
- | Flag | Effect |
104
- | ------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
105
- | `--format <fmt>` | Output format. **Repeatable and comma-separated** — `--format raw_html --format markdown` and `--format raw_html,markdown` both send `formats: ["raw_html","markdown"]`. Omitted → server default. |
106
- | `--country <cc>` | Route through a country (e.g. `us`, `de`). |
107
- | `--timeout-ms <n>` | Per-request timeout in milliseconds. |
108
- | `--cache-ttl <ttl>` | Cache freshness: `0`, `Nh` (hours), or `Nd` (days) — e.g. `48h`, `2d`, `0`. Omitted → server default. |
109
- | `--markdown-mode <mode>` | Markdown extraction mode: `article`, `raw`, `llm`. |
110
- | `--markdown-query <q>` | BM25 query for relevance-ranked markdown filtering. |
111
- | `--cookies <json>` | Cookies as a JSON array, e.g. `'[{"name":"a","value":"b"}]'`. |
112
- | `--headers <json>` | Extra request headers as a JSON object, e.g. `'{"X-A":"b"}'`. |
113
- | `--delay-ms <n>` | Post-load delay before capture, in milliseconds. |
114
- | `--events <csv>` | Webhook events (comma-separated): `queued`, `completed`, `failed`. |
115
- | `--token-budget <n>` | Cap the markdown returned to `n` tokens. |
116
- | `--token-encoding <enc>` | Tokenizer for `--token-budget`: `cl100k_base`, `o200k_base`. |
117
- | `--custom <json>` | User payload echoed on the envelope, as a JSON object. |
118
- | `--remove-base64-images` / `--no-remove-base64-images` | Tri-state: strip / preserve base64 `data:` images. Omitted → server default (strip). |
119
- | `--markdown-links <mode>` | Link rendering: `inline`, `references`, `none`, `text`. |
120
- | `--markdown-images <mode>` | Image retention: `inline`, `references`, `none`, `text`. |
121
- | `--with-links-summary` | Append a Links footer to the markdown. |
122
- | `--with-images-summary` | Append an Images footer to the markdown. |
123
- | `--markdown-compact` | Collapse excessive whitespace. |
124
- | `--markdown-filter-images` | Filter low-signal images. |
125
- | `--markdown-include-media` | Return rich link/image objects and a tables array. |
126
- | `--markdown-include-warnings` | Include markdown-pipeline and tag-filter warnings. |
127
- | `--markdown-include-stats` | Include a top-level stats object (chars, tokens, blocks). |
128
- | `--clean-markdown` | LLM-clean the markdown (billed 3x). |
129
- | `--include-tags <csv>` | Comma-separated HTML tags to keep. |
130
- | `--exclude-tags <csv>` | Comma-separated HTML tags to drop. |
131
- | `--mobile` | Render with a mobile viewport/user-agent. |
132
- | `--async` | Queue the scrape and return a job envelope immediately. |
133
- | `--webhook-url <url>` | Webhook notified when an async scrape completes. |
103
+ | Flag | Effect |
104
+ | ------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
105
+ | `--format <fmt>` | Output format. **Repeatable and comma-separated** — `--format raw --format markdown` and `--format raw,markdown` both send `formats: ["raw","markdown"]`. Omitted → server default. |
106
+ | `--country <cc>` | Route through a country (e.g. `us`, `de`). |
107
+ | `--timeout-ms <n>` | Per-request timeout in milliseconds. |
108
+ | `--cache-ttl <ttl>` | Cache freshness: `0`, `Nh` (hours), or `Nd` (days) — e.g. `48h`, `2d`, `0`. Omitted → server default. |
109
+ | `--markdown-mode <mode>` | Markdown extraction mode: `article`, `raw`, `llm`. |
110
+ | `--markdown-query <q>` | BM25 query for relevance-ranked markdown filtering. |
111
+ | `--cookies <json>` | Cookies as a JSON array, e.g. `'[{"name":"a","value":"b"}]'`. |
112
+ | `--headers <json>` | Extra request headers as a JSON object, e.g. `'{"X-A":"b"}'`. |
113
+ | `--delay-ms <n>` | Post-load delay before capture, in milliseconds. |
114
+ | `--events <csv>` | Webhook events (comma-separated): `queued`, `completed`, `failed`. |
115
+ | `--token-budget <n>` | Cap the markdown returned to `n` tokens. |
116
+ | `--token-encoding <enc>` | Tokenizer for `--token-budget`: `cl100k_base`, `o200k_base`. |
117
+ | `--custom <json>` | User payload echoed on the envelope, as a JSON object. |
118
+ | `--remove-base64-images` / `--no-remove-base64-images` | Tri-state: strip / preserve base64 `data:` images. Omitted → server default (strip). |
119
+ | `--markdown-links <mode>` | Link rendering: `inline`, `references`, `none`, `text`. |
120
+ | `--markdown-images <mode>` | Image retention: `inline`, `references`, `none`, `text`. |
121
+ | `--with-links-summary` | Append a Links footer to the markdown. |
122
+ | `--with-images-summary` | Append an Images footer to the markdown. |
123
+ | `--markdown-compact` | Collapse excessive whitespace. |
124
+ | `--markdown-filter-images` | Filter low-signal images. |
125
+ | `--markdown-include-media` | Return rich link/image objects and a tables array. |
126
+ | `--markdown-include-warnings` | Include markdown-pipeline and tag-filter warnings. |
127
+ | `--markdown-include-stats` | Include a top-level stats object (chars, tokens, blocks). |
128
+ | `--clean-markdown` | LLM-clean the markdown (billed 3x). |
129
+ | `--include-tags <csv>` | Comma-separated HTML tags to keep. |
130
+ | `--exclude-tags <csv>` | Comma-separated HTML tags to drop. |
131
+ | `--mobile` | Render with a mobile viewport/user-agent. |
132
+ | `--async` | Queue the scrape and return a job envelope immediately. |
133
+ | `--webhook-url <url>` | Webhook notified when an async scrape completes. |
134
134
 
135
135
  `scrape get` accepts the same output flags as `scrape create` —
136
136
  `--raw`, `-o <file>`, and `--format <fmt>` — so a job queued with `--async --raw`
package/dist/client.js CHANGED
@@ -240,6 +240,10 @@ function drainBestEffort(body, abort) {
240
240
  * phantom `details`, and `ByteKitErrorDetails` promises something indexable, so an array or a
241
241
  * scalar is dropped rather than surfaced. `error-parser-parity.test.ts` compares this field against
242
242
  * the real SDK's parser on a shared corpus.
243
+ *
244
+ * The response's headers are mirrored for the same reason (#3784 E4): the SDK grew that fifth
245
+ * constructor argument in #2957 and this copy did not follow, so `Retry-After` and the `X-Quota-*`
246
+ * trio were parsed away before any caller could read them off the rejection.
243
247
  */
244
248
  async function toApiError(response) {
245
249
  let code = 'internal_error';
@@ -265,7 +269,10 @@ async function toApiError(response) {
265
269
  catch {
266
270
  // leave defaults
267
271
  }
268
- return new ByteKitError(response.status, code, message, details);
272
+ // The failing response's OWN headers ride along as the SDK's fifth argument (#2957, mirrored
273
+ // here for #3784 E4). Without them a caught CLI error could not tell the caller how long to wait
274
+ // (`Retry-After`) or how much budget is left (`X-Quota-*`), even though the gateway sent both.
275
+ return new ByteKitError(response.status, code, message, details, response.headers);
269
276
  }
270
277
  /**
271
278
  * Perform the authenticated round-trip with a single bounded retry on 429, returning whatever the
package/dist/commands/scrape.js CHANGED
@@ -100,8 +100,8 @@ export function makeScrapeCommand(getGlobal) {
100
100
  .command('create')
101
101
  .description('Create a scrape job')
102
102
  .requiredOption('--url <url>', 'URL to scrape')
103
- // Repeatable AND CSV-aware: `--format raw_html --format markdown` and `--format raw_html,markdown`
104
- // both send formats: ['raw_html','markdown']. Sent as the `formats` array; omitted → server
103
+ // Repeatable AND CSV-aware: `--format raw --format markdown` and `--format raw,markdown`
104
+ // both send formats: ['raw','markdown']. Sent as the `formats` array; omitted → server
105
105
  // default. `--raw`/`-o` require EXACTLY ONE format (validated pre-request; see the action).
106
106
  //
107
107
  // The format VALUES travel to the server unvalidated — deliberate, and recorded here for
@@ -109,7 +109,7 @@ export function makeScrapeCommand(getGlobal) {
109
109
  // `--type`/`--date-range`). The persisted-format vocabulary grows with the spec, and a local
110
110
  // enum list is a second source of truth that drifts silently: the CLI would reject a format
111
111
  // the API has begun serving, and the error would come from a stale copy rather than the API.
112
- .option('--format <format>', 'Output format; repeat or comma-separate for multiple (e.g. --format raw_html --format markdown, or --format raw_html,markdown). Sent as the formats array; omitted → server default.', collectFormats)
112
+ .option('--format <format>', 'Output format; repeat or comma-separate for multiple (e.g. --format raw --format markdown, or --format raw,markdown). Sent as the formats array; omitted → server default.', collectFormats)
113
113
  .option('--country <cc>', 'Two-letter country code to route the request through (e.g. us, de)')
114
114
  .option('--timeout-ms <n>', 'Per-request timeout in milliseconds', (v) => parseIntFlag('--timeout-ms', v))
115
115
  .option('--markdown-mode <mode>', `Markdown extraction mode: ${MARKDOWN_MODES.join(', ')}`, (v) => parseEnum('--markdown-mode', MARKDOWN_MODES, v))
package/dist/lib/parse-flags.d.ts CHANGED
@@ -39,8 +39,8 @@ export declare function parseCacheTtl(flag: string, value: string): string;
39
39
  export declare function parseJsonFlag(flag: string, value: string, expect: 'object' | 'array'): unknown;
40
40
  /**
41
41
  * Commander collector for a repeatable, CSV-aware `--format` flag (#2545). Each occurrence is
42
- * CSV-split and appended to the accumulator, so `--format raw_html --format markdown` and
43
- * `--format raw_html,markdown` both yield `['raw_html','markdown']`. Commander seeds `prev` with
42
+ * CSV-split and appended to the accumulator, so `--format raw --format markdown` and
43
+ * `--format raw,markdown` both yield `['raw','markdown']`. Commander seeds `prev` with
44
44
  * the option default (or the previous return) on each occurrence.
45
45
  */
46
46
  export declare function collectFormats(value: string, prev: string[] | undefined): string[];
package/dist/lib/parse-flags.js CHANGED
@@ -83,8 +83,8 @@ export function parseJsonFlag(flag, value, expect) {
83
83
  }
84
84
  /**
85
85
  * Commander collector for a repeatable, CSV-aware `--format` flag (#2545). Each occurrence is
86
- * CSV-split and appended to the accumulator, so `--format raw_html --format markdown` and
87
- * `--format raw_html,markdown` both yield `['raw_html','markdown']`. Commander seeds `prev` with
86
+ * CSV-split and appended to the accumulator, so `--format raw --format markdown` and
87
+ * `--format raw,markdown` both yield `['raw','markdown']`. Commander seeds `prev` with
88
88
  * the option default (or the previous return) on each occurrence.
89
89
  */
90
90
  export function collectFormats(value, prev) {
package/dist/output.js CHANGED
@@ -1,6 +1,6 @@
1
1
  import { writeFileSync } from 'node:fs';
2
2
  /** Text formats whose bodies are strings and can therefore be emitted raw. */
3
- const TEXT_FORMATS = 'markdown, html, raw_html';
3
+ const TEXT_FORMATS = 'markdown, raw';
4
4
  /**
5
5
  * Serialize a value as the CLI's `--json` payload text. The single serializer EVERY `--json`
6
6
  * branch in this module goes through, so no branch can regress to a raw `JSON.stringify`.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@hunt-labs/bytekit-cli",
3
- "version": "0.9.2",
3
+ "version": "0.9.3",
4
4
  "description": "ByteKit CLI — wraps @hunt-labs/bytekit-sdk as shell subcommands",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -41,7 +41,7 @@
41
41
  ],
42
42
  "dependencies": {
43
43
  "commander": "^12.0.0",
44
- "@hunt-labs/bytekit-sdk": "^0.9.1"
44
+ "@hunt-labs/bytekit-sdk": "^0.9.2"
45
45
  },
46
46
  "scripts": {
47
47
  "build": "rm -rf dist tsconfig.tsbuildinfo && tsc",