@hunt-labs/bytekit-cli 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +51 -0
- package/README.md +53 -11
- package/dist/client.d.ts +19 -9
- package/dist/client.js +140 -18
- package/dist/commands/account.js +49 -0
- package/dist/commands/fetch.d.ts +14 -0
- package/dist/commands/fetch.js +23 -6
- package/dist/commands/monitors.js +26 -3
- package/dist/commands/scrape.js +88 -0
- package/dist/commands/screenshots.d.ts +12 -0
- package/dist/commands/screenshots.js +105 -6
- package/dist/commands/search.js +5 -1
- package/dist/commands/sitemap.js +8 -1
- package/dist/lib/download-artifact.d.ts +13 -0
- package/dist/lib/download-artifact.js +25 -0
- package/dist/lib/fetch-body.d.ts +18 -0
- package/dist/lib/fetch-body.js +6 -0
- package/dist/lib/parse-flags.d.ts +16 -0
- package/dist/lib/parse-flags.js +32 -0
- package/dist/output.d.ts +50 -0
- package/dist/output.js +99 -5
- package/package.json +2 -2
package/dist/commands/scrape.js
CHANGED
|
@@ -2,6 +2,34 @@ import { Command } from 'commander';
|
|
|
2
2
|
import { buildClient } from '../client.js';
|
|
3
3
|
import { printResult, handleError, resolveOutputMode, emitResult } from '../output.js';
|
|
4
4
|
import { parseUrlLines } from '../lib/url-lines.js';
|
|
5
|
+
import { parseIntFlag, parseEnum, parseCsv } from '../lib/parse-flags.js';
|
|
6
|
+
/** The `ScrapeRequest.markdown_mode` spec enum. */
|
|
7
|
+
const MARKDOWN_MODES = ['article', 'raw', 'llm'];
|
|
8
|
+
/**
|
|
9
|
+
* Exit code when a queued (202 async) envelope reaches a non-JSON output mode.
|
|
10
|
+
*
|
|
11
|
+
* Mirrors the transport-hardening ticket's `QUEUED_RAW_EXIT_CODE` in `output.ts` (PR #2508)
|
|
12
|
+
* EXACTLY — deliberately a local constant, not an import, so this branch carries no dependency on
|
|
13
|
+
* that PR's symbols while the two guards still compose idempotently post-merge. Deliberately NOT 0
|
|
14
|
+
* (a content-body success) and NOT 1 (a usage/API error): the job WAS accepted and billed, there
|
|
15
|
+
* is simply no content body yet, and a distinct code lets a caller branch on "queued" without
|
|
16
|
+
* parsing stderr. Human mode shares the same code (soft `process.exitCode`) so "queued, not
|
|
17
|
+
* complete" is branchable there too, for consistency with the raw-mode signal.
|
|
18
|
+
*/
|
|
19
|
+
const QUEUED_EXIT_CODE = 2;
|
|
20
|
+
/**
|
|
21
|
+
* A 202 ScrapeQueuedEnvelope: `status: 'queued'` plus a stable `sc_` id (openapi.yaml).
|
|
22
|
+
*
|
|
23
|
+
* The strict discriminator — identical to #2508's `output.ts` guard, so both guards agree on
|
|
24
|
+
* every input post-merge. A malformed queued-ish shape (no id, or a non-'queued' status) falls
|
|
25
|
+
* through to the normal output path and surfaces `output.ts`'s own diagnostics (#2480 E4s).
|
|
26
|
+
*/
|
|
27
|
+
function isQueuedEnvelope(result) {
|
|
28
|
+
if (typeof result !== 'object' || result === null)
|
|
29
|
+
return false;
|
|
30
|
+
const obj = result;
|
|
31
|
+
return obj.status === 'queued' && typeof obj.id === 'string' && obj.id.length > 0;
|
|
32
|
+
}
|
|
5
33
|
export function makeScrapeCommand(getGlobal) {
|
|
6
34
|
const cmd = new Command('scrape').description('Scrape URL content');
|
|
7
35
|
const bulkCmd = new Command('bulk').description('Bulk scrape operations');
|
|
@@ -43,6 +71,16 @@ export function makeScrapeCommand(getGlobal) {
|
|
|
43
71
|
.description('Create a scrape job')
|
|
44
72
|
.requiredOption('--url <url>', 'URL to scrape')
|
|
45
73
|
.option('--format <format>', 'Output format (e.g. markdown, html); sent as the formats array')
|
|
74
|
+
.option('--country <cc>', 'Two-letter country code to route the request through (e.g. us, de)')
|
|
75
|
+
.option('--timeout-ms <n>', 'Per-request timeout in milliseconds', (v) => parseIntFlag('--timeout-ms', v))
|
|
76
|
+
.option('--markdown-mode <mode>', `Markdown extraction mode: ${MARKDOWN_MODES.join(', ')}`, (v) => parseEnum('--markdown-mode', MARKDOWN_MODES, v))
|
|
77
|
+
.option('--async', 'Queue the scrape and return a job envelope immediately (no wait)')
|
|
78
|
+
.option('--webhook-url <url>', 'HTTPS webhook URL notified when an async scrape completes')
|
|
79
|
+
.option('--token-budget <n>', 'Maximum tokens of markdown to return', (v) => parseIntFlag('--token-budget', v))
|
|
80
|
+
.option('--clean-markdown', 'Post-process markdown with the LLM cleaner (billed 3x)')
|
|
81
|
+
.option('--include-tags <csv>', 'Comma-separated HTML tags to keep (e.g. article,main)')
|
|
82
|
+
.option('--exclude-tags <csv>', 'Comma-separated HTML tags to drop (e.g. nav,footer)')
|
|
83
|
+
.option('--mobile', 'Render with a mobile viewport/user-agent')
|
|
46
84
|
.option('--raw', 'Print the requested format body raw, with no JSON wrapper')
|
|
47
85
|
.option('-o, --output <file>', 'Write the output to the given file path (implies --raw unless --json is set)')
|
|
48
86
|
.action(async (opts) => {
|
|
@@ -54,6 +92,28 @@ export function makeScrapeCommand(getGlobal) {
|
|
|
54
92
|
// formats: ["undefined"].
|
|
55
93
|
if (opts.format !== undefined)
|
|
56
94
|
body['formats'] = [opts.format];
|
|
95
|
+
// Each optional flag maps 1:1 to its snake_case ScrapeRequest field and is written ONLY
|
|
96
|
+
// when the flag is present — a boolean must never serialize `false` (#2480 E1).
|
|
97
|
+
if (opts.country !== undefined)
|
|
98
|
+
body['country'] = opts.country;
|
|
99
|
+
if (opts.timeoutMs !== undefined)
|
|
100
|
+
body['timeout_ms'] = opts.timeoutMs;
|
|
101
|
+
if (opts.markdownMode !== undefined)
|
|
102
|
+
body['markdown_mode'] = opts.markdownMode;
|
|
103
|
+
if (opts.async === true)
|
|
104
|
+
body['async'] = true;
|
|
105
|
+
if (opts.webhookUrl !== undefined)
|
|
106
|
+
body['webhook_url'] = opts.webhookUrl;
|
|
107
|
+
if (opts.tokenBudget !== undefined)
|
|
108
|
+
body['token_budget'] = opts.tokenBudget;
|
|
109
|
+
if (opts.cleanMarkdown === true)
|
|
110
|
+
body['clean_markdown'] = true;
|
|
111
|
+
if (opts.includeTags !== undefined)
|
|
112
|
+
body['include_tags'] = parseCsv(opts.includeTags);
|
|
113
|
+
if (opts.excludeTags !== undefined)
|
|
114
|
+
body['exclude_tags'] = parseCsv(opts.excludeTags);
|
|
115
|
+
if (opts.mobile === true)
|
|
116
|
+
body['mobile'] = true;
|
|
57
117
|
try {
|
|
58
118
|
// FIRST statement inside the try — deliberately NOT hoisted next to buildClient() where
|
|
59
119
|
// the other handlers put pre-request logic. index.ts drives commander with .parse(), not
|
|
@@ -63,6 +123,34 @@ export function makeScrapeCommand(getGlobal) {
|
|
|
63
123
|
// --json/--raw conflict here still happens before any request is issued.
|
|
64
124
|
const mode = resolveOutputMode({ json: g.json, raw: opts.raw, output: opts.output });
|
|
65
125
|
const result = await client.scrape.create(body);
|
|
126
|
+
// Queued-envelope precedence (#2480 E4s): a `--async` scrape returns a 202
|
|
127
|
+
// `{ id, status: 'queued' }` envelope with NO formats map. Detect that shape here —
|
|
128
|
+
// AFTER the request returns, BEFORE raw-format extraction — per the contract the
|
|
129
|
+
// transport-hardening ticket (#2508) establishes in `output.ts`, mirrored verbatim so
|
|
130
|
+
// the two guards compose idempotently post-merge (this one runs first and must be
|
|
131
|
+
// indistinguishable from that one):
|
|
132
|
+
// - raw mode (`--raw` or a bare `-o`): the bare sc_ id on STDOUT (a usable
|
|
133
|
+
// `$(bytekit scrape create … --raw)` handle), a human poll note on STDERR, exit
|
|
134
|
+
// QUEUED_EXIT_CODE. The `-o` file is deliberately NEVER written — there is no
|
|
135
|
+
// content body yet, and a file holding a bare id would masquerade as content.
|
|
136
|
+
// - json mode (`--json`, with or without `-o`): OUTSIDE the raw rule — fall through
|
|
137
|
+
// to emitResult, which emits the envelope verbatim (exit 0).
|
|
138
|
+
// - human default: keep the readable id/status dump, but signal queued-not-complete
|
|
139
|
+
// with the same exit code 2 — via soft `process.exitCode` (not `process.exit`) so
|
|
140
|
+
// the dump is guaranteed to flush before the process ends naturally.
|
|
141
|
+
if (isQueuedEnvelope(result)) {
|
|
142
|
+
if (mode.kind === 'raw') {
|
|
143
|
+
console.error(`Note: scrape ${result.id} was queued (async) — no content body is available yet. ` +
|
|
144
|
+
`Poll it with: bytekit scrape get ${result.id}`);
|
|
145
|
+
process.stdout.write(`${result.id}\n`);
|
|
146
|
+
process.exit(QUEUED_EXIT_CODE);
|
|
147
|
+
}
|
|
148
|
+
if (mode.kind === 'human') {
|
|
149
|
+
printResult(result, false);
|
|
150
|
+
process.exitCode = QUEUED_EXIT_CODE;
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
66
154
|
emitResult(result, mode, { file: opts.output }, opts.format);
|
|
67
155
|
}
|
|
68
156
|
catch (err) {
|
|
@@ -1,5 +1,17 @@
|
|
|
1
1
|
import { Command } from 'commander';
|
|
2
2
|
import { type GlobalOptions } from '../client.js';
|
|
3
|
+
/** Parse `--viewport WxH` (e.g. `1280x720`) into `{ width, height }`; reject malformed input. */
|
|
4
|
+
export declare function parseViewport(value: string): {
|
|
5
|
+
width: number;
|
|
6
|
+
height: number;
|
|
7
|
+
};
|
|
8
|
+
/** Parse `--clip x,y,w,h` into `{ x, y, width, height }`; reject malformed input. */
|
|
9
|
+
export declare function parseClip(value: string): {
|
|
10
|
+
x: number;
|
|
11
|
+
y: number;
|
|
12
|
+
width: number;
|
|
13
|
+
height: number;
|
|
14
|
+
};
|
|
3
15
|
export declare function makeScreenshotsCommand(getGlobal: () => GlobalOptions & {
|
|
4
16
|
json: boolean;
|
|
5
17
|
}): Command;
|
|
@@ -1,6 +1,64 @@
|
|
|
1
|
-
import { Command } from 'commander';
|
|
1
|
+
import { Command, InvalidArgumentError } from 'commander';
|
|
2
2
|
import { buildClient } from '../client.js';
|
|
3
3
|
import { printResult, handleError } from '../output.js';
|
|
4
|
+
import { downloadArtifact } from '../lib/download-artifact.js';
|
|
5
|
+
import { parseEnum, parseIntFlag } from '../lib/parse-flags.js';
|
|
6
|
+
/** `ScreenshotRequest` spec enums. */
|
|
7
|
+
const DEVICES = ['desktop', 'mobile'];
|
|
8
|
+
const FORMATS = ['jpeg', 'png'];
|
|
9
|
+
const WAIT_UNTILS = ['load', 'domcontentloaded', 'networkidle'];
|
|
10
|
+
/** Parse `--viewport WxH` (e.g. `1280x720`) into `{ width, height }`; reject malformed input. */
|
|
11
|
+
export function parseViewport(value) {
|
|
12
|
+
const m = /^(\d+)x(\d+)$/.exec(value);
|
|
13
|
+
if (!m)
|
|
14
|
+
throw new InvalidArgumentError('--viewport must be WIDTHxHEIGHT (e.g. 1280x720).');
|
|
15
|
+
return { width: Number(m[1]), height: Number(m[2]) };
|
|
16
|
+
}
|
|
17
|
+
/** Parse `--clip x,y,w,h` into `{ x, y, width, height }`; reject malformed input. */
|
|
18
|
+
export function parseClip(value) {
|
|
19
|
+
const m = /^(\d+),(\d+),(\d+),(\d+)$/.exec(value);
|
|
20
|
+
if (!m)
|
|
21
|
+
throw new InvalidArgumentError('--clip must be X,Y,WIDTH,HEIGHT (e.g. 0,0,800,600).');
|
|
22
|
+
return { x: Number(m[1]), y: Number(m[2]), width: Number(m[3]), height: Number(m[4]) };
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Extract a non-empty string `image_url` from a screenshot envelope, or throw an actionable error.
|
|
26
|
+
*
|
|
27
|
+
* A pending/processing envelope has no `image_url` yet; a terminal-`failed` screenshot returns the
|
|
28
|
+
* unified `Error` envelope on a 200 (#2027). Either way the artifact was never materialized, so we
|
|
29
|
+
* throw rather than fetching `undefined` (#2480 E5).
|
|
30
|
+
*/
|
|
31
|
+
function extractImageUrl(result) {
|
|
32
|
+
if (typeof result === 'object' && result !== null) {
|
|
33
|
+
const obj = result;
|
|
34
|
+
if (obj.error) {
|
|
35
|
+
throw new Error(`screenshot failed: ${obj.error.message ?? obj.error.code ?? 'unknown error'}`);
|
|
36
|
+
}
|
|
37
|
+
if (typeof obj.image_url === 'string' && obj.image_url.length > 0) {
|
|
38
|
+
return obj.image_url;
|
|
39
|
+
}
|
|
40
|
+
const status = typeof obj.status === 'string' ? obj.status : 'unknown';
|
|
41
|
+
throw new Error(`not ready yet: screenshot is ${status}`);
|
|
42
|
+
}
|
|
43
|
+
throw new Error('not ready yet: screenshot is unknown');
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Render a screenshot result: with `-o` download the `image_url` artifact bytes (to a file, or to
|
|
47
|
+
* stdout with `-o -`); otherwise print the envelope. `--async` + `-o` prints the queued envelope
|
|
48
|
+
* and skips the download (queued-envelope precedence, #2480 E4). `--json` + `-o` is rejected by the
|
|
49
|
+
* caller BEFORE any request so `--json` can never emit binary (#2480 E2).
|
|
50
|
+
*/
|
|
51
|
+
async function emitScreenshot(result, opts) {
|
|
52
|
+
if (opts.isAsync === true) {
|
|
53
|
+
printResult(result, opts.json);
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
if (opts.output !== undefined) {
|
|
57
|
+
await downloadArtifact(extractImageUrl(result), opts.output);
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
printResult(result, opts.json);
|
|
61
|
+
}
|
|
4
62
|
export function makeScreenshotsCommand(getGlobal) {
|
|
5
63
|
const cmd = new Command('screenshots').description('Screenshot operations');
|
|
6
64
|
cmd
|
|
@@ -8,15 +66,52 @@ export function makeScreenshotsCommand(getGlobal) {
|
|
|
8
66
|
.description('Capture a screenshot')
|
|
9
67
|
.requiredOption('--url <url>', 'URL to screenshot')
|
|
10
68
|
.option('--full-page', 'Capture the full page (default: viewport only)')
|
|
69
|
+
.option('--device <device>', `Device profile: ${DEVICES.join(', ')}`, (v) => parseEnum('--device', DEVICES, v))
|
|
70
|
+
.option('--viewport <WxH>', 'Viewport size WIDTHxHEIGHT (e.g. 1280x720)', parseViewport)
|
|
71
|
+
.option('--format <format>', `Image format: ${FORMATS.join(', ')}`, (v) => parseEnum('--format', FORMATS, v))
|
|
72
|
+
.option('--quality <n>', 'JPEG quality (1-100)', (v) => parseIntFlag('--quality', v))
|
|
73
|
+
.option('--wait-until <event>', `Wait condition: ${WAIT_UNTILS.join(', ')}`, (v) => parseEnum('--wait-until', WAIT_UNTILS, v))
|
|
74
|
+
.option('--clip <x,y,w,h>', 'Clip region X,Y,WIDTH,HEIGHT (e.g. 0,0,800,600)', parseClip)
|
|
75
|
+
.option('--async', 'Queue the screenshot and return a job envelope immediately (no wait)')
|
|
76
|
+
.option('-o, --output <file>', 'Download the captured image to this file path (use - for stdout). Not valid with --json.')
|
|
11
77
|
.action(async (opts) => {
|
|
12
78
|
const g = getGlobal();
|
|
13
79
|
const client = buildClient(g);
|
|
14
80
|
const body = { url: opts.url };
|
|
81
|
+
// Each optional flag maps 1:1 to its snake_case ScreenshotRequest field, written ONLY when
|
|
82
|
+
// present — a boolean never serializes `false` (#2480 E1).
|
|
15
83
|
if (opts.fullPage === true)
|
|
16
|
-
body
|
|
84
|
+
body['full_page'] = true;
|
|
85
|
+
if (opts.device !== undefined)
|
|
86
|
+
body['device'] = opts.device;
|
|
87
|
+
if (opts.viewport !== undefined)
|
|
88
|
+
body['viewport'] = opts.viewport;
|
|
89
|
+
if (opts.format !== undefined)
|
|
90
|
+
body['format'] = opts.format;
|
|
91
|
+
if (opts.quality !== undefined)
|
|
92
|
+
body['quality'] = opts.quality;
|
|
93
|
+
if (opts.waitUntil !== undefined)
|
|
94
|
+
body['wait_until'] = opts.waitUntil;
|
|
95
|
+
if (opts.clip !== undefined)
|
|
96
|
+
body['clip'] = opts.clip;
|
|
17
97
|
try {
|
|
18
|
-
|
|
19
|
-
|
|
98
|
+
// Reject --json + -o BEFORE issuing the request: -o downloads BINARY artifact bytes, so
|
|
99
|
+
// it can never coexist with --json (which must stay a text envelope). Thrown here so no
|
|
100
|
+
// request — and therefore no download fetch — is ever issued (#2480 E2).
|
|
101
|
+
if (g.json && opts.output !== undefined) {
|
|
102
|
+
throw new Error('-o cannot be used with --json');
|
|
103
|
+
}
|
|
104
|
+
const typedBody = body;
|
|
105
|
+
// --async is a QUERY param; pass the second arg ONLY when present so existing single-arg
|
|
106
|
+
// callers stay byte-identical (#2480 inverse case).
|
|
107
|
+
const result = opts.async === true
|
|
108
|
+
? await client.screenshots.create(typedBody, { async: true })
|
|
109
|
+
: await client.screenshots.create(typedBody);
|
|
110
|
+
await emitScreenshot(result, {
|
|
111
|
+
output: opts.output,
|
|
112
|
+
json: g.json,
|
|
113
|
+
isAsync: opts.async === true,
|
|
114
|
+
});
|
|
20
115
|
}
|
|
21
116
|
catch (err) {
|
|
22
117
|
handleError(err);
|
|
@@ -25,12 +120,16 @@ export function makeScreenshotsCommand(getGlobal) {
|
|
|
25
120
|
cmd
|
|
26
121
|
.command('get <id>')
|
|
27
122
|
.description('Get a screenshot by ID')
|
|
28
|
-
.
|
|
123
|
+
.option('-o, --output <file>', 'Download the captured image to this file path (use - for stdout). Not valid with --json.')
|
|
124
|
+
.action(async (id, opts) => {
|
|
29
125
|
const g = getGlobal();
|
|
30
126
|
const client = buildClient(g);
|
|
31
127
|
try {
|
|
128
|
+
if (g.json && opts.output !== undefined) {
|
|
129
|
+
throw new Error('-o cannot be used with --json');
|
|
130
|
+
}
|
|
32
131
|
const result = await client.screenshots.get(id);
|
|
33
|
-
|
|
132
|
+
await emitScreenshot(result, { output: opts.output, json: g.json });
|
|
34
133
|
}
|
|
35
134
|
catch (err) {
|
|
36
135
|
handleError(err);
|
package/dist/commands/search.js
CHANGED
|
@@ -11,7 +11,11 @@ function renderTable(response) {
|
|
|
11
11
|
for (const result of response.results) {
|
|
12
12
|
console.log(`${result.position}. ${result.title}`);
|
|
13
13
|
console.log(` ${result.url}`);
|
|
14
|
-
|
|
14
|
+
// `snippet` is typed as required but is OMITTED for `--type images` results (openapi.yaml),
|
|
15
|
+
// so guard against printing a literal `undefined` line when it is absent or empty.
|
|
16
|
+
if (typeof result.snippet === 'string' && result.snippet.length > 0) {
|
|
17
|
+
console.log(` ${result.snippet}`);
|
|
18
|
+
}
|
|
15
19
|
}
|
|
16
20
|
}
|
|
17
21
|
const LIMIT_MIN = 1;
|
package/dist/commands/sitemap.js
CHANGED
|
@@ -10,8 +10,15 @@ export function makeSitemapCommand(getGlobal) {
|
|
|
10
10
|
.action(async (opts) => {
|
|
11
11
|
const g = getGlobal();
|
|
12
12
|
const client = buildClient(g);
|
|
13
|
+
// Key-checked subset of the SDK's SitemapCreateRequest (same pattern as monitors.ts).
|
|
14
|
+
// The CLI intentionally sends only `url` and relies on the server-side defaults the spec
|
|
15
|
+
// declares for the remaining SitemapRequest fields (strategy, max_depth, max_urls, …).
|
|
16
|
+
// The generated request type marks those fields required, so the final call needs an
|
|
17
|
+
// explicit assertion — type-level only; the wire body is unchanged (pinned by
|
|
18
|
+
// __tests__/request-shapes.test.ts).
|
|
19
|
+
const body = { url: opts.url };
|
|
13
20
|
try {
|
|
14
|
-
const result = await client.sitemap.create(
|
|
21
|
+
const result = await client.sitemap.create(body);
|
|
15
22
|
printResult(result, g.json);
|
|
16
23
|
}
|
|
17
24
|
catch (err) {
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Download a screenshot artifact from its `image_url` and write the raw bytes to `dest`.
|
|
3
|
+
*
|
|
4
|
+
* - `dest === '-'` writes the bytes to stdout (the conventional dash-is-stdout convention),
|
|
5
|
+
* never with `console.log` — binary must not be coerced or newline-appended.
|
|
6
|
+
* - any other `dest` is a file path written verbatim.
|
|
7
|
+
*
|
|
8
|
+
* `globalThis.fetch` is resolved at CALL time (never captured at module/construction load) so a
|
|
9
|
+
* test that stubs `globalThis.fetch` in a `beforeEach` — after this module was imported — still
|
|
10
|
+
* sees its stub (G-049). A non-2xx response throws with the HTTP status so `handleError` renders a
|
|
11
|
+
* clean one-liner and exits 1.
|
|
12
|
+
*/
|
|
13
|
+
export declare function downloadArtifact(url: string, dest: string): Promise<void>;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { writeFileSync } from 'node:fs';
|
|
2
|
+
/**
|
|
3
|
+
* Download a screenshot artifact from its `image_url` and write the raw bytes to `dest`.
|
|
4
|
+
*
|
|
5
|
+
* - `dest === '-'` writes the bytes to stdout (the conventional dash-is-stdout convention),
|
|
6
|
+
* never with `console.log` — binary must not be coerced or newline-appended.
|
|
7
|
+
* - any other `dest` is a file path written verbatim.
|
|
8
|
+
*
|
|
9
|
+
* `globalThis.fetch` is resolved at CALL time (never captured at module/construction load) so a
|
|
10
|
+
* test that stubs `globalThis.fetch` in a `beforeEach` — after this module was imported — still
|
|
11
|
+
* sees its stub (G-049). A non-2xx response throws with the HTTP status so `handleError` renders a
|
|
12
|
+
* clean one-liner and exits 1.
|
|
13
|
+
*/
|
|
14
|
+
export async function downloadArtifact(url, dest) {
|
|
15
|
+
const res = await globalThis.fetch(url);
|
|
16
|
+
if (!res.ok) {
|
|
17
|
+
throw new Error(`artifact download failed: HTTP ${res.status}`);
|
|
18
|
+
}
|
|
19
|
+
const bytes = Buffer.from(await res.arrayBuffer());
|
|
20
|
+
if (dest === '-') {
|
|
21
|
+
process.stdout.write(bytes);
|
|
22
|
+
return;
|
|
23
|
+
}
|
|
24
|
+
writeFileSync(dest, bytes);
|
|
25
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A raw (non-JSON) response body from a `/v1/fetch` content request, plus its `X-Fetch-*` metadata.
|
|
3
|
+
*
|
|
4
|
+
* The `/v1/fetch` content endpoints (POST and GET) return the fetched bytes AS the body — HTML,
|
|
5
|
+
* markdown, or arbitrary binary — with all metadata in `X-Fetch-*` response headers. Every OTHER
|
|
6
|
+
* endpoint (including `/v1/fetch/bulk`) returns a JSON envelope. The CLI transport tags a content
|
|
7
|
+
* result with this shape so the `fetch` command can print the body verbatim (default) or the
|
|
8
|
+
* metadata as JSON (`--json`), instead of JSON-parsing a body that was never JSON to begin with.
|
|
9
|
+
*/
|
|
10
|
+
export interface FetchBodyResult {
|
|
11
|
+
readonly __bytekitFetchBody: true;
|
|
12
|
+
/** The raw response body bytes, uncorrupted — safe for binary content (no string coercion). */
|
|
13
|
+
readonly body: Buffer;
|
|
14
|
+
/** The `X-Fetch-*` response headers, Title-Cased (`x-fetch-status` → `X-Fetch-Status`). */
|
|
15
|
+
readonly metadata: Record<string, string>;
|
|
16
|
+
}
|
|
17
|
+
/** Narrow an arbitrary transport result to a {@link FetchBodyResult}. */
|
|
18
|
+
export declare function isFetchBodyResult(value: unknown): value is FetchBodyResult;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared commander argParser helpers for CLI flag values (#2480 cr-3). One home for the parser
|
|
3
|
+
* family that scrape.ts, screenshots.ts, and monitors.ts previously each held private copies of —
|
|
4
|
+
* every off-spec value is rejected locally, at parse time, with an actionable message, before any
|
|
5
|
+
* HTTP request is issued.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Parse a whole-number flag value. The strict `/^\d+$/` test deterministically rejects `abc`, ``,
|
|
9
|
+
* `1.5`, `-5`, `1e2`, so an off-spec value fails at parse time with an actionable message instead
|
|
10
|
+
* of a server 422 or a NaN on the wire.
|
|
11
|
+
*/
|
|
12
|
+
export declare function parseIntFlag(flag: string, value: string): number;
|
|
13
|
+
/** Reject an off-spec enum value at parse time with the allowed set in the message. */
|
|
14
|
+
export declare function parseEnum<T extends string>(flag: string, allowed: readonly T[], value: string): T;
|
|
15
|
+
/** Split a comma-separated flag value into a trimmed, non-empty string array. */
|
|
16
|
+
export declare function parseCsv(value: string): string[];
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { InvalidArgumentError } from 'commander';
|
|
2
|
+
/**
|
|
3
|
+
* Shared commander argParser helpers for CLI flag values (#2480 cr-3). One home for the parser
|
|
4
|
+
* family that scrape.ts, screenshots.ts, and monitors.ts previously each held private copies of —
|
|
5
|
+
* every off-spec value is rejected locally, at parse time, with an actionable message, before any
|
|
6
|
+
* HTTP request is issued.
|
|
7
|
+
*/
|
|
8
|
+
/**
|
|
9
|
+
* Parse a whole-number flag value. The strict `/^\d+$/` test deterministically rejects `abc`, ``,
|
|
10
|
+
* `1.5`, `-5`, `1e2`, so an off-spec value fails at parse time with an actionable message instead
|
|
11
|
+
* of a server 422 or a NaN on the wire.
|
|
12
|
+
*/
|
|
13
|
+
export function parseIntFlag(flag, value) {
|
|
14
|
+
if (!/^\d+$/.test(value)) {
|
|
15
|
+
throw new InvalidArgumentError(`${flag} must be a whole number.`);
|
|
16
|
+
}
|
|
17
|
+
return Number(value);
|
|
18
|
+
}
|
|
19
|
+
/** Reject an off-spec enum value at parse time with the allowed set in the message. */
|
|
20
|
+
export function parseEnum(flag, allowed, value) {
|
|
21
|
+
if (!allowed.includes(value)) {
|
|
22
|
+
throw new InvalidArgumentError(`${flag} must be one of: ${allowed.join(', ')}.`);
|
|
23
|
+
}
|
|
24
|
+
return value;
|
|
25
|
+
}
|
|
26
|
+
/** Split a comma-separated flag value into a trimmed, non-empty string array. */
|
|
27
|
+
export function parseCsv(value) {
|
|
28
|
+
return value
|
|
29
|
+
.split(',')
|
|
30
|
+
.map((s) => s.trim())
|
|
31
|
+
.filter((s) => s.length > 0);
|
|
32
|
+
}
|
package/dist/output.d.ts
CHANGED
|
@@ -56,6 +56,29 @@ export declare function resolveOutputMode(flags: OutputFlags): OutputMode;
|
|
|
56
56
|
* not text, so `--raw` can never print `[object Object]` or a format the caller did not ask for.
|
|
57
57
|
*/
|
|
58
58
|
export declare function extractRawBody(response: unknown, requestedFormat?: string): string;
|
|
59
|
+
/**
|
|
60
|
+
* Exit code for `--raw`/`-o` against a 202 queued (async) scrape.
|
|
61
|
+
*
|
|
62
|
+
* Deliberately NOT 0 (a full raw-body success) and NOT 1 (a usage/API error): the job WAS accepted
|
|
63
|
+
* and billed, there is simply no content body to emit yet. A distinct code lets a caller branch on
|
|
64
|
+
* "queued" without parsing stderr.
|
|
65
|
+
*/
|
|
66
|
+
export declare const QUEUED_RAW_EXIT_CODE = 2;
|
|
67
|
+
/** A 202 ScrapeQueuedEnvelope: `status: 'queued'` plus a stable `sc_` id (openapi.yaml). */
|
|
68
|
+
interface QueuedEnvelope {
|
|
69
|
+
status: 'queued';
|
|
70
|
+
id: string;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Handle `--raw`/`-o` on a 202 queued envelope.
|
|
74
|
+
*
|
|
75
|
+
* The user asked for a content body that does not exist yet — but the job was accepted, so losing
|
|
76
|
+
* the handle (the old behaviour: throw `--raw: response contains no formats`, exit 1) is the real
|
|
77
|
+
* defect. Instead: emit the `sc_` id to STDOUT so `$(bytekit scrape create … --raw)` still captures
|
|
78
|
+
* a usable handle, print a human note to STDERR explaining it is queued and how to poll, and exit
|
|
79
|
+
* with {@link QUEUED_RAW_EXIT_CODE} — distinguishable from both success and error.
|
|
80
|
+
*/
|
|
81
|
+
export declare function emitQueuedRaw(env: QueuedEnvelope): never;
|
|
59
82
|
/**
|
|
60
83
|
* Emit a raw body to the target.
|
|
61
84
|
*
|
|
@@ -78,6 +101,18 @@ export declare function emitRaw(body: string, target: OutputTarget): void;
|
|
|
78
101
|
* `--json | jq` pipelines stay byte-identical.
|
|
79
102
|
*/
|
|
80
103
|
export declare function emitResult(result: unknown, mode: OutputMode, target: OutputTarget, requestedFormat?: string): void;
|
|
104
|
+
/**
|
|
105
|
+
* Render a `/v1/fetch` content result.
|
|
106
|
+
*
|
|
107
|
+
* The transport hands back a {@link FetchBodyResult} whose body IS the fetched content and whose
|
|
108
|
+
* metadata are the `X-Fetch-*` headers. Default: write the raw body bytes to stdout verbatim
|
|
109
|
+
* (`process.stdout.write`, never `console.log` — no coercion, no appended newline, binary-safe).
|
|
110
|
+
* With `--json`: print the `X-Fetch-*` metadata as JSON instead of the body.
|
|
111
|
+
*
|
|
112
|
+
* Falls back to {@link printResult} for any result that is not a fetch body (defensive — e.g. a
|
|
113
|
+
* bulk-fetch envelope routed here by mistake), so no path can silently emit nothing.
|
|
114
|
+
*/
|
|
115
|
+
export declare function emitFetchResult(result: unknown, jsonMode: boolean): void;
|
|
81
116
|
/**
|
|
82
117
|
* Print a result to stdout.
|
|
83
118
|
*
|
|
@@ -86,5 +121,20 @@ export declare function emitResult(result: unknown, mode: OutputMode, target: Ou
|
|
|
86
121
|
export declare function printResult(data: unknown, jsonMode: boolean): void;
|
|
87
122
|
/**
|
|
88
123
|
* Handle a CLI command error: print to stderr and exit 1.
|
|
124
|
+
*
|
|
125
|
+
* The message always begins `Error: ` (a prefix scripts grep on) followed by the error message and
|
|
126
|
+
* an optional parenthesised suffix built from the two diagnostic axes the ByteKit errors carry:
|
|
127
|
+
*
|
|
128
|
+
* - a `ByteKitError` from the transport carries both a numeric `status` and an API `code`
|
|
129
|
+
* (`rate_limited`, `invalid_url`, …) → `Error: <message> (<code>, status: <n>)`;
|
|
130
|
+
* - a `ByteKitConnectionError` (client-side timeout / abort / socket failure) carries a `code`
|
|
131
|
+
* (`timeout` | `aborted` | `connection`) but NO status → `Error: <message> (<code>)`;
|
|
132
|
+
* - a plain error with neither → `Error: <message>`.
|
|
133
|
+
*
|
|
134
|
+
* The `code` axis is read only off the two ByteKit error classes (see {@link errorCode}) — a plain
|
|
135
|
+
* error carrying an unrelated string `.code` (a Node errno error) gets NO code suffix. When a
|
|
136
|
+
* status is present but no code (a hand-built error), the suffix is `(status: <n>)` with no empty
|
|
137
|
+
* placeholder — never `(, status: <n>)`.
|
|
89
138
|
*/
|
|
90
139
|
export declare function handleError(err: unknown): never;
|
|
140
|
+
export {};
|
package/dist/output.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { writeFileSync } from 'node:fs';
|
|
2
|
+
import { isFetchBodyResult } from './lib/fetch-body.js';
|
|
2
3
|
/** Text formats whose bodies are strings and can therefore be emitted raw. */
|
|
3
4
|
const TEXT_FORMATS = 'markdown, html, raw_html';
|
|
4
5
|
/**
|
|
@@ -71,6 +72,36 @@ export function extractRawBody(response, requestedFormat) {
|
|
|
71
72
|
}
|
|
72
73
|
return value;
|
|
73
74
|
}
|
|
75
|
+
/**
|
|
76
|
+
* Exit code for `--raw`/`-o` against a 202 queued (async) scrape.
|
|
77
|
+
*
|
|
78
|
+
* Deliberately NOT 0 (a full raw-body success) and NOT 1 (a usage/API error): the job WAS accepted
|
|
79
|
+
* and billed, there is simply no content body to emit yet. A distinct code lets a caller branch on
|
|
80
|
+
* "queued" without parsing stderr.
|
|
81
|
+
*/
|
|
82
|
+
export const QUEUED_RAW_EXIT_CODE = 2;
|
|
83
|
+
/** Is `result` a 202 queued scrape envelope carrying a job id? */
|
|
84
|
+
function isQueuedEnvelope(result) {
|
|
85
|
+
if (typeof result !== 'object' || result === null)
|
|
86
|
+
return false;
|
|
87
|
+
const obj = result;
|
|
88
|
+
return obj.status === 'queued' && typeof obj.id === 'string' && obj.id.length > 0;
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Handle `--raw`/`-o` on a 202 queued envelope.
|
|
92
|
+
*
|
|
93
|
+
* The user asked for a content body that does not exist yet — but the job was accepted, so losing
|
|
94
|
+
* the handle (the old behaviour: throw `--raw: response contains no formats`, exit 1) is the real
|
|
95
|
+
* defect. Instead: emit the `sc_` id to STDOUT so `$(bytekit scrape create … --raw)` still captures
|
|
96
|
+
* a usable handle, print a human note to STDERR explaining it is queued and how to poll, and exit
|
|
97
|
+
* with {@link QUEUED_RAW_EXIT_CODE} — distinguishable from both success and error.
|
|
98
|
+
*/
|
|
99
|
+
export function emitQueuedRaw(env) {
|
|
100
|
+
console.error(`Note: scrape ${env.id} was queued (async) — no content body is available yet. ` +
|
|
101
|
+
`Poll it with: bytekit scrape get ${env.id}`);
|
|
102
|
+
process.stdout.write(`${env.id}\n`);
|
|
103
|
+
process.exit(QUEUED_RAW_EXIT_CODE);
|
|
104
|
+
}
|
|
74
105
|
/**
|
|
75
106
|
* Emit a raw body to the target.
|
|
76
107
|
*
|
|
@@ -101,6 +132,14 @@ export function emitRaw(body, target) {
|
|
|
101
132
|
*/
|
|
102
133
|
export function emitResult(result, mode, target, requestedFormat) {
|
|
103
134
|
if (mode.kind === 'raw') {
|
|
135
|
+
// A 202 queued scrape has no format body to print — surface its `sc_` handle instead of
|
|
136
|
+
// throwing away the accepted (and billed) job. Scoped to raw mode only: json/human rendering
|
|
137
|
+
// of a queued envelope stays unchanged (falls through to printResult below). A `-o <file>`
|
|
138
|
+
// target is deliberately ignored here (id always on STDOUT, never written to the file): there
|
|
139
|
+
// is no content body yet, and a file holding a bare `sc_` id would masquerade as content.
|
|
140
|
+
if (isQueuedEnvelope(result)) {
|
|
141
|
+
emitQueuedRaw(result);
|
|
142
|
+
}
|
|
104
143
|
emitRaw(extractRawBody(result, requestedFormat), target);
|
|
105
144
|
return;
|
|
106
145
|
}
|
|
@@ -115,6 +154,28 @@ export function emitResult(result, mode, target, requestedFormat) {
|
|
|
115
154
|
// human — unreachable with a file destination (resolveOutputMode never yields it alongside one).
|
|
116
155
|
printResult(result, false);
|
|
117
156
|
}
|
|
157
|
+
/**
|
|
158
|
+
* Render a `/v1/fetch` content result.
|
|
159
|
+
*
|
|
160
|
+
* The transport hands back a {@link FetchBodyResult} whose body IS the fetched content and whose
|
|
161
|
+
* metadata are the `X-Fetch-*` headers. Default: write the raw body bytes to stdout verbatim
|
|
162
|
+
* (`process.stdout.write`, never `console.log` — no coercion, no appended newline, binary-safe).
|
|
163
|
+
* With `--json`: print the `X-Fetch-*` metadata as JSON instead of the body.
|
|
164
|
+
*
|
|
165
|
+
* Falls back to {@link printResult} for any result that is not a fetch body (defensive — e.g. a
|
|
166
|
+
* bulk-fetch envelope routed here by mistake), so no path can silently emit nothing.
|
|
167
|
+
*/
|
|
168
|
+
export function emitFetchResult(result, jsonMode) {
|
|
169
|
+
if (isFetchBodyResult(result)) {
|
|
170
|
+
if (jsonMode) {
|
|
171
|
+
console.log(JSON.stringify(result.metadata, null, 2));
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
process.stdout.write(result.body);
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
printResult(result, jsonMode);
|
|
178
|
+
}
|
|
118
179
|
/**
|
|
119
180
|
* Print a result to stdout.
|
|
120
181
|
*
|
|
@@ -139,19 +200,52 @@ export function printResult(data, jsonMode) {
|
|
|
139
200
|
}
|
|
140
201
|
console.log(JSON.stringify(data, null, 2));
|
|
141
202
|
}
|
|
203
|
+
/** The SDK error classes whose `code` is a ByteKit diagnostic worth surfacing as a suffix. */
|
|
204
|
+
const BYTEKIT_ERROR_NAMES = new Set(['ByteKitError', 'ByteKitConnectionError']);
|
|
205
|
+
/**
|
|
206
|
+
* A non-empty API/connection error `code`, or undefined when absent.
|
|
207
|
+
*
|
|
208
|
+
* Scoped by `err.name` to the ByteKit taxonomy: plenty of OTHER errors also carry a string
|
|
209
|
+
* `.code` — Node fs/system errors (ENOENT from `parseUrlLines`'s readFileSync, EACCES on a `-o`
|
|
210
|
+
* write) — whose message already embeds the code, so suffixing them would render the redundant
|
|
211
|
+
* `Error: ENOENT: … (ENOENT)`. Only `ByteKitError` / `ByteKitConnectionError` codes qualify.
|
|
212
|
+
*/
|
|
213
|
+
function errorCode(err) {
|
|
214
|
+
if (!BYTEKIT_ERROR_NAMES.has(err.name))
|
|
215
|
+
return undefined;
|
|
216
|
+
const raw = err.code;
|
|
217
|
+
return typeof raw === 'string' && raw.length > 0 ? raw : undefined;
|
|
218
|
+
}
|
|
142
219
|
/**
|
|
143
220
|
* Handle a CLI command error: print to stderr and exit 1.
|
|
221
|
+
*
|
|
222
|
+
* The message always begins `Error: ` (a prefix scripts grep on) followed by the error message and
|
|
223
|
+
* an optional parenthesised suffix built from the two diagnostic axes the ByteKit errors carry:
|
|
224
|
+
*
|
|
225
|
+
* - a `ByteKitError` from the transport carries both a numeric `status` and an API `code`
|
|
226
|
+
* (`rate_limited`, `invalid_url`, …) → `Error: <message> (<code>, status: <n>)`;
|
|
227
|
+
* - a `ByteKitConnectionError` (client-side timeout / abort / socket failure) carries a `code`
|
|
228
|
+
* (`timeout` | `aborted` | `connection`) but NO status → `Error: <message> (<code>)`;
|
|
229
|
+
* - a plain error with neither → `Error: <message>`.
|
|
230
|
+
*
|
|
231
|
+
* The `code` axis is read only off the two ByteKit error classes (see {@link errorCode}) — a plain
|
|
232
|
+
* error carrying an unrelated string `.code` (a Node errno error) gets NO code suffix. When a
|
|
233
|
+
* status is present but no code (a hand-built error), the suffix is `(status: <n>)` with no empty
|
|
234
|
+
* placeholder — never `(, status: <n>)`.
|
|
144
235
|
*/
|
|
145
236
|
export function handleError(err) {
|
|
146
237
|
if (err instanceof Error) {
|
|
147
|
-
const
|
|
238
|
+
const status = err.status;
|
|
239
|
+
const hasStatus = typeof status === 'number';
|
|
240
|
+
const code = errorCode(err);
|
|
241
|
+
let suffix = '';
|
|
148
242
|
if (hasStatus) {
|
|
149
|
-
|
|
150
|
-
console.error(`Error: ${apiErr.message} (status: ${apiErr.status})`);
|
|
243
|
+
suffix = code !== undefined ? ` (${code}, status: ${status})` : ` (status: ${status})`;
|
|
151
244
|
}
|
|
152
|
-
else {
|
|
153
|
-
|
|
245
|
+
else if (code !== undefined) {
|
|
246
|
+
suffix = ` (${code})`;
|
|
154
247
|
}
|
|
248
|
+
console.error(`Error: ${err.message}${suffix}`);
|
|
155
249
|
}
|
|
156
250
|
else {
|
|
157
251
|
console.error(`Error: ${String(err)}`);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hunt-labs/bytekit-cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "ByteKit CLI — wraps @hunt-labs/bytekit-sdk as shell subcommands",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -41,7 +41,7 @@
|
|
|
41
41
|
],
|
|
42
42
|
"dependencies": {
|
|
43
43
|
"commander": "^12.0.0",
|
|
44
|
-
"@hunt-labs/bytekit-sdk": "^0.
|
|
44
|
+
"@hunt-labs/bytekit-sdk": "^0.3.0"
|
|
45
45
|
},
|
|
46
46
|
"scripts": {
|
|
47
47
|
"build": "tsc",
|