@hunt-labs/bytekit-cli 0.3.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +230 -0
- package/README.md +187 -29
- package/dist/client.d.ts +128 -11
- package/dist/client.js +369 -95
- package/dist/commands/account.d.ts +11 -0
- package/dist/commands/account.js +10 -127
- package/dist/commands/bulk.js +36 -4
- package/dist/commands/fetch.js +91 -10
- package/dist/commands/monitors.js +92 -11
- package/dist/commands/scrape.js +131 -20
- package/dist/commands/screenshots.js +86 -1
- package/dist/commands/sitemap.js +38 -6
- package/dist/commands/usage.d.ts +13 -0
- package/dist/commands/usage.js +81 -0
- package/dist/commands/webhook-deliveries.d.ts +11 -0
- package/dist/commands/webhook-deliveries.js +59 -0
- package/dist/index.js +4 -0
- package/dist/lib/download-artifact.d.ts +30 -1
- package/dist/lib/download-artifact.js +89 -5
- package/dist/lib/parse-flags.d.ts +30 -0
- package/dist/lib/parse-flags.js +58 -0
- package/dist/lib/url-lines.d.ts +46 -8
- package/dist/lib/url-lines.js +58 -19
- package/dist/output.d.ts +47 -9
- package/dist/output.js +103 -16
- package/package.json +3 -3
- package/dist/lib/fetch-body.d.ts +0 -18
- package/dist/lib/fetch-body.js +0 -6
package/dist/output.js
CHANGED
|
@@ -1,7 +1,27 @@
|
|
|
1
1
|
import { writeFileSync } from 'node:fs';
|
|
2
|
-
import { isFetchBodyResult } from './lib/fetch-body.js';
|
|
3
2
|
/** Text formats whose bodies are strings and can therefore be emitted raw. */
|
|
4
3
|
const TEXT_FORMATS = 'markdown, html, raw_html';
|
|
4
|
+
/**
|
|
5
|
+
* Serialize a value as the CLI's `--json` payload text. The single serializer EVERY `--json`
|
|
6
|
+
* branch in this module goes through, so no branch can regress to a raw `JSON.stringify`.
|
|
7
|
+
*
|
|
8
|
+
* `JSON.stringify` returns the VALUE `undefined` — not a string — for `undefined`, functions and
|
|
9
|
+
* symbols. Handed to `console.log` that prints the literal text `undefined`, which is NOT valid
|
|
10
|
+
* JSON and breaks every `| jq` pipeline; handed to `writeFileSync` it throws
|
|
11
|
+
* `ERR_INVALID_ARG_TYPE`. A `204 No Content` response resolves to `undefined` in the SDK
|
|
12
|
+
* (`monitors.delete` is typed `Promise<void>`), so this is the ORDINARY shape of
|
|
13
|
+
* `bytekit monitors delete <id> --json`, not an exotic edge case (#2544).
|
|
14
|
+
*
|
|
15
|
+
* The representation is JSON `null` — the canonical JSON spelling of "no content". Deliberately
|
|
16
|
+
* NOT a synthesized envelope such as `{"status":"deleted"}`: that is a shape the API never
|
|
17
|
+
* returned, and it would have to be re-invented (and kept consistent) per command. `null` is
|
|
18
|
+
* uniform across every 204-returning path, and `… --json | jq .` prints `null` and exits 0.
|
|
19
|
+
* The contract is documented in README (`Exit codes and output contracts`) and CHANGELOG.
|
|
20
|
+
*/
|
|
21
|
+
export function toJsonText(data) {
|
|
22
|
+
const text = JSON.stringify(data, null, 2);
|
|
23
|
+
return text === undefined ? 'null' : text;
|
|
24
|
+
}
|
|
5
25
|
/**
|
|
6
26
|
* Resolve the output mode from the flags, and enforce the `--json` / `--raw` exclusion.
|
|
7
27
|
*
|
|
@@ -93,14 +113,25 @@ function isQueuedEnvelope(result) {
|
|
|
93
113
|
* The user asked for a content body that does not exist yet — but the job was accepted, so losing
|
|
94
114
|
* the handle (the old behaviour: throw `--raw: response contains no formats`, exit 1) is the real
|
|
95
115
|
* defect. Instead: emit the `sc_` id to STDOUT so `$(bytekit scrape create … --raw)` still captures
|
|
96
|
-
* a usable handle, print a human note to STDERR explaining it is queued and how to poll, and
|
|
97
|
-
*
|
|
116
|
+
* a usable handle, print a human note to STDERR explaining it is queued and how to poll, and signal
|
|
117
|
+
* {@link QUEUED_RAW_EXIT_CODE} — distinguishable from both success and error.
|
|
118
|
+
*
|
|
119
|
+
* The exit code is a SOFT `process.exitCode`, never a hard `process.exit` (#2544). Node documents
|
|
120
|
+
* `process.stdout` as ASYNCHRONOUS when it points at a pipe: a write that cannot be handed to the
|
|
121
|
+
* kernel in one go is queued in userspace and drained on a later event-loop turn, and
|
|
122
|
+
* `process.exit()` runs before any of those turns and DISCARDS the queue. That is exactly the
|
|
123
|
+
* `sc=$(bytekit scrape create … --async --raw)` case this branch exists to serve — the id would be
|
|
124
|
+
* truncated away whenever stdout is backlogged. Setting `process.exitCode` instead lets the process
|
|
125
|
+
* end naturally once stdout has drained, with the same observable exit code.
|
|
126
|
+
*
|
|
127
|
+
* Returns `void` (not `never`): callers MUST `return` after invoking it, since execution now
|
|
128
|
+
* continues past this function.
|
|
98
129
|
*/
|
|
99
130
|
export function emitQueuedRaw(env) {
|
|
100
131
|
console.error(`Note: scrape ${env.id} was queued (async) — no content body is available yet. ` +
|
|
101
132
|
`Poll it with: bytekit scrape get ${env.id}`);
|
|
102
133
|
process.stdout.write(`${env.id}\n`);
|
|
103
|
-
process.
|
|
134
|
+
process.exitCode = QUEUED_RAW_EXIT_CODE;
|
|
104
135
|
}
|
|
105
136
|
/**
|
|
106
137
|
* Emit a raw body to the target.
|
|
@@ -137,15 +168,20 @@ export function emitResult(result, mode, target, requestedFormat) {
|
|
|
137
168
|
// of a queued envelope stays unchanged (falls through to printResult below). A `-o <file>`
|
|
138
169
|
// target is deliberately ignored here (id always on STDOUT, never written to the file): there
|
|
139
170
|
// is no content body yet, and a file holding a bare `sc_` id would masquerade as content.
|
|
171
|
+
// `emitQueuedRaw` signals via a soft `process.exitCode` and RETURNS (#2544), so this branch
|
|
172
|
+
// must stop explicitly. Falling through would run `extractRawBody` on a formats-less queued
|
|
173
|
+
// envelope, throwing `--raw: response contains no formats` and clobbering the queued signal
|
|
174
|
+
// with the generic exit 1.
|
|
140
175
|
if (isQueuedEnvelope(result)) {
|
|
141
176
|
emitQueuedRaw(result);
|
|
177
|
+
return;
|
|
142
178
|
}
|
|
143
179
|
emitRaw(extractRawBody(result, requestedFormat), target);
|
|
144
180
|
return;
|
|
145
181
|
}
|
|
146
182
|
if (mode.kind === 'json') {
|
|
147
183
|
if (target.file !== undefined) {
|
|
148
|
-
writeFileSync(target.file,
|
|
184
|
+
writeFileSync(target.file, toJsonText(result));
|
|
149
185
|
return;
|
|
150
186
|
}
|
|
151
187
|
printResult(result, true);
|
|
@@ -154,24 +190,73 @@ export function emitResult(result, mode, target, requestedFormat) {
|
|
|
154
190
|
// human — unreachable with a file destination (resolveOutputMode never yields it alongside one).
|
|
155
191
|
printResult(result, false);
|
|
156
192
|
}
|
|
193
|
+
/** Title-Case an HTTP header name segment-by-segment: `x-fetch-status` → `X-Fetch-Status`. */
|
|
194
|
+
function titleCaseHeader(name) {
|
|
195
|
+
return name
|
|
196
|
+
.split('-')
|
|
197
|
+
.map((seg) => seg.charAt(0).toUpperCase() + seg.slice(1).toLowerCase())
|
|
198
|
+
.join('-');
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* Collect the `X-Fetch-*` response headers of a {@link FetchResult} into a Title-Cased object —
|
|
202
|
+
* the documented `--json` payload for a fetch command.
|
|
203
|
+
*/
|
|
204
|
+
export function collectFetchMetadata(headers) {
|
|
205
|
+
const meta = {};
|
|
206
|
+
headers.forEach((value, key) => {
|
|
207
|
+
if (key.toLowerCase().startsWith('x-fetch-')) {
|
|
208
|
+
meta[titleCaseHeader(key)] = value;
|
|
209
|
+
}
|
|
210
|
+
});
|
|
211
|
+
return meta;
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* Narrow an SDK response to the `FetchResult` the fetch resource resolves.
|
|
215
|
+
*
|
|
216
|
+
* Structural, not a branded tag: the shape is the SDK's PUBLIC contract (`bytes` + `headers`), so
|
|
217
|
+
* the guard stays true across SDK transport refactors. The CLI previously narrowed on a CLI-owned
|
|
218
|
+
* `__bytekitFetchBody` brand its own transport stamped — which stopped matching the moment the SDK
|
|
219
|
+
* moved `/v1/fetch` onto `requestRaw` (#2535), silently demoting every fetch to the key/value
|
|
220
|
+
* printer. `fetch-contract.test.ts` pins this guard against the real SDK.
|
|
221
|
+
*/
|
|
222
|
+
function isFetchResult(value) {
|
|
223
|
+
if (typeof value !== 'object' || value === null)
|
|
224
|
+
return false;
|
|
225
|
+
const v = value;
|
|
226
|
+
return v.bytes instanceof Uint8Array && v.headers instanceof Headers;
|
|
227
|
+
}
|
|
157
228
|
/**
|
|
158
229
|
* Render a `/v1/fetch` content result.
|
|
159
230
|
*
|
|
160
|
-
* The
|
|
161
|
-
*
|
|
162
|
-
*
|
|
163
|
-
*
|
|
231
|
+
* The SDK's fetch resource resolves a {@link FetchResult} whose `bytes` ARE the fetched content and
|
|
232
|
+
* whose `headers` carry the `X-Fetch-*` metadata.
|
|
233
|
+
*
|
|
234
|
+
* - default — write the raw body BYTES verbatim (`process.stdout.write`, never `console.log`: no
|
|
235
|
+
* coercion, no appended newline, binary-safe). `bytes` is used rather than the `body` string
|
|
236
|
+
* because the string is a lossy UTF-8 decode for true binary passthrough (images, PDFs).
|
|
237
|
+
* - `--json` — print the `X-Fetch-*` metadata object instead of the body.
|
|
238
|
+
* - `target.file` — write the same bytes to that path instead of stdout, byte-for-byte.
|
|
164
239
|
*
|
|
165
|
-
* Falls back to {@link printResult} for any result that is not a fetch
|
|
240
|
+
* Falls back to {@link printResult} for any result that is not a fetch result (defensive — e.g. a
|
|
166
241
|
* bulk-fetch envelope routed here by mistake), so no path can silently emit nothing.
|
|
167
242
|
*/
|
|
168
|
-
export function emitFetchResult(result, jsonMode) {
|
|
169
|
-
if (
|
|
243
|
+
export function emitFetchResult(result, jsonMode, target = {}) {
|
|
244
|
+
if (isFetchResult(result)) {
|
|
170
245
|
if (jsonMode) {
|
|
171
|
-
|
|
246
|
+
const metadata = toJsonText(collectFetchMetadata(result.headers));
|
|
247
|
+
if (target.file !== undefined) {
|
|
248
|
+
writeFileSync(target.file, metadata);
|
|
249
|
+
return;
|
|
250
|
+
}
|
|
251
|
+
console.log(metadata);
|
|
252
|
+
return;
|
|
253
|
+
}
|
|
254
|
+
const bytes = Buffer.from(result.bytes);
|
|
255
|
+
if (target.file !== undefined) {
|
|
256
|
+
writeFileSync(target.file, bytes);
|
|
172
257
|
return;
|
|
173
258
|
}
|
|
174
|
-
process.stdout.write(
|
|
259
|
+
process.stdout.write(bytes);
|
|
175
260
|
return;
|
|
176
261
|
}
|
|
177
262
|
printResult(result, jsonMode);
|
|
@@ -183,7 +268,9 @@ export function emitFetchResult(result, jsonMode) {
|
|
|
183
268
|
*/
|
|
184
269
|
export function printResult(data, jsonMode) {
|
|
185
270
|
if (jsonMode) {
|
|
186
|
-
|
|
271
|
+
// Always via toJsonText: a 204 hands us `undefined`, whose raw JSON.stringify is the VALUE
|
|
272
|
+
// undefined and would print the literal, unparseable text `undefined` (#2544).
|
|
273
|
+
console.log(toJsonText(data));
|
|
187
274
|
return;
|
|
188
275
|
}
|
|
189
276
|
if (data === null || data === undefined) {
|
|
@@ -206,7 +293,7 @@ const BYTEKIT_ERROR_NAMES = new Set(['ByteKitError', 'ByteKitConnectionError']);
|
|
|
206
293
|
* A non-empty API/connection error `code`, or undefined when absent.
|
|
207
294
|
*
|
|
208
295
|
* Scoped by `err.name` to the ByteKit taxonomy: plenty of OTHER errors also carry a string
|
|
209
|
-
* `.code` — Node fs/system errors (ENOENT from `
|
|
296
|
+
* `.code` — Node fs/system errors (ENOENT from `parseBulkLines`'s readFileSync, EACCES on a `-o`
|
|
210
297
|
* write) — whose message already embeds the code, so suffixing them would render the redundant
|
|
211
298
|
* `Error: ENOENT: … (ENOENT)`. Only `ByteKitError` / `ByteKitConnectionError` codes qualify.
|
|
212
299
|
*/
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hunt-labs/bytekit-cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"description": "ByteKit CLI — wraps @hunt-labs/bytekit-sdk as shell subcommands",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -41,10 +41,10 @@
|
|
|
41
41
|
],
|
|
42
42
|
"dependencies": {
|
|
43
43
|
"commander": "^12.0.0",
|
|
44
|
-
"@hunt-labs/bytekit-sdk": "^0.
|
|
44
|
+
"@hunt-labs/bytekit-sdk": "^0.5.0"
|
|
45
45
|
},
|
|
46
46
|
"scripts": {
|
|
47
|
-
"build": "tsc",
|
|
47
|
+
"build": "rm -rf dist tsconfig.tsbuildinfo && tsc",
|
|
48
48
|
"typecheck": "tsc --noEmit",
|
|
49
49
|
"test": "vitest run"
|
|
50
50
|
}
|
package/dist/lib/fetch-body.d.ts
DELETED
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* A raw (non-JSON) response body from a `/v1/fetch` content request, plus its `X-Fetch-*` metadata.
|
|
3
|
-
*
|
|
4
|
-
* The `/v1/fetch` content endpoints (POST and GET) return the fetched bytes AS the body — HTML,
|
|
5
|
-
* markdown, or arbitrary binary — with all metadata in `X-Fetch-*` response headers. Every OTHER
|
|
6
|
-
* endpoint (including `/v1/fetch/bulk`) returns a JSON envelope. The CLI transport tags a content
|
|
7
|
-
* result with this shape so the `fetch` command can print the body verbatim (default) or the
|
|
8
|
-
* metadata as JSON (`--json`), instead of JSON-parsing a body that was never JSON to begin with.
|
|
9
|
-
*/
|
|
10
|
-
export interface FetchBodyResult {
|
|
11
|
-
readonly __bytekitFetchBody: true;
|
|
12
|
-
/** The raw response body bytes, uncorrupted — safe for binary content (no string coercion). */
|
|
13
|
-
readonly body: Buffer;
|
|
14
|
-
/** The `X-Fetch-*` response headers, Title-Cased (`x-fetch-status` → `X-Fetch-Status`). */
|
|
15
|
-
readonly metadata: Record<string, string>;
|
|
16
|
-
}
|
|
17
|
-
/** Narrow an arbitrary transport result to a {@link FetchBodyResult}. */
|
|
18
|
-
export declare function isFetchBodyResult(value: unknown): value is FetchBodyResult;
|