@hunt-labs/bytekit-cli 0.3.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +230 -0
- package/README.md +187 -29
- package/dist/client.d.ts +128 -11
- package/dist/client.js +369 -95
- package/dist/commands/account.d.ts +11 -0
- package/dist/commands/account.js +10 -127
- package/dist/commands/bulk.js +36 -4
- package/dist/commands/fetch.js +91 -10
- package/dist/commands/monitors.js +92 -11
- package/dist/commands/scrape.js +131 -20
- package/dist/commands/screenshots.js +86 -1
- package/dist/commands/sitemap.js +38 -6
- package/dist/commands/usage.d.ts +13 -0
- package/dist/commands/usage.js +81 -0
- package/dist/commands/webhook-deliveries.d.ts +11 -0
- package/dist/commands/webhook-deliveries.js +59 -0
- package/dist/index.js +4 -0
- package/dist/lib/download-artifact.d.ts +30 -1
- package/dist/lib/download-artifact.js +89 -5
- package/dist/lib/parse-flags.d.ts +30 -0
- package/dist/lib/parse-flags.js +58 -0
- package/dist/lib/url-lines.d.ts +46 -8
- package/dist/lib/url-lines.js +58 -19
- package/dist/output.d.ts +47 -9
- package/dist/output.js +103 -16
- package/package.json +3 -3
- package/dist/lib/fetch-body.d.ts +0 -18
- package/dist/lib/fetch-body.js +0 -6
package/dist/client.js
CHANGED
|
@@ -18,6 +18,24 @@ export const MIN_RETRY_AFTER_MS = 1_000;
|
|
|
18
18
|
export const MAX_RETRY_AFTER_MS = 60_000;
|
|
19
19
|
/** Exactly one retry. A second 429 is terminal — no loop. */
|
|
20
20
|
export const MAX_RETRIES = 1;
|
|
21
|
+
/**
|
|
22
|
+
* Cap on a best-effort, discard-only body drain (#2587 cr2-1). Independent of the per-attempt
|
|
23
|
+
* abort budget: the 429 drain below (`response.text().catch(() => {})`) exists ONLY so undici can
|
|
24
|
+
* return the socket to the connection pool — nothing downstream reads the drained bytes — but
|
|
25
|
+
* binding that read to the FULL per-attempt budget (120 s default, or, under `timeoutMs <= 0`,
|
|
26
|
+
* to no bound at all) let a stalled 429 body burn the whole budget, or hang forever, before a
|
|
27
|
+
* retry that needs nothing from that body. Short enough that a legitimately stalled 429 body costs
|
|
28
|
+
* the retry very little wall-clock time.
|
|
29
|
+
*
|
|
30
|
+
* #2587 cr3-2: past this cap, the read is not merely abandoned — {@link drainBestEffort} actively
|
|
31
|
+
* ABORTS its underlying request. Resolving the race without aborting anything (the earlier
|
|
32
|
+
* cr2-1 shape) let the retry proceed on schedule while the abandoned `response.text()` kept
|
|
33
|
+
* reading against a live socket forever, which — because the CLI exits via a soft
|
|
34
|
+
* `process.exitCode` (#2544) rather than `process.exit()` — kept the whole process alive
|
|
35
|
+
* indefinitely. `.catch`-guarding the read only made it safe to abandon the RESULT; it never made
|
|
36
|
+
* abandoning the underlying READ safe.
|
|
37
|
+
*/
|
|
38
|
+
export const DRAIN_TIMEOUT_MS = 2_000;
|
|
21
39
|
/**
|
|
22
40
|
* Default CLIENT-SIDE abort budget for a single fetch attempt, in ms.
|
|
23
41
|
*
|
|
@@ -36,34 +54,79 @@ function timeoutReason(ms) {
|
|
|
36
54
|
return reason;
|
|
37
55
|
}
|
|
38
56
|
/**
|
|
39
|
-
* A per-attempt abort budget: an `AbortController
|
|
57
|
+
* A per-attempt abort budget: an `AbortController`, optionally fired by a `setTimeout`.
|
|
58
|
+
*
|
|
59
|
+
* The budget spans the WHOLE attempt — time-to-headers **and** the response body read (#2546).
|
|
60
|
+
* Arming it only until headers arrive left `response.text()` / `response.arrayBuffer()` unbounded,
|
|
61
|
+
* so a server that answered with headers and then stalled its body hung the CLI forever.
|
|
62
|
+
*
|
|
63
|
+
* The controller ALWAYS exists, even when `ms <= 0` — only its own timeout timer is conditional
|
|
64
|
+
* on `ms > 0`. `sendWithRetry`'s 429-drain cap ({@link DRAIN_TIMEOUT_MS}, via the returned
|
|
65
|
+
* `abort()`) needs a live controller to tear down an abandoned body read regardless of whether the
|
|
66
|
+
* OVERALL per-attempt budget is enabled — `ms <= 0` must still mean "no full-attempt timeout", not
|
|
67
|
+
* "no controller to abort a stalled drain with" (#2587 cr3-2: the earlier drain fix abandoned the
|
|
68
|
+
* read entirely rather than aborting it, so the socket — and the event loop — stayed alive
|
|
69
|
+
* indefinitely, worse than the pre-fix baseline it replaced).
|
|
40
70
|
*
|
|
41
71
|
* Deliberately NOT `AbortSignal.timeout` (which the SDK prefers): that schedules on a timer the
|
|
42
72
|
* test runner's fake timers cannot advance, so the transport's timeout would be untestable under
|
|
43
73
|
* `vi.useFakeTimers()`. CLI requests are short-lived and single-shot, so the GC advantage
|
|
44
74
|
* `AbortSignal.timeout` buys the long-running SDK does not apply here. The timer is `unref`'d so a
|
|
45
75
|
* pending budget never keeps the process alive, and `cleanup()` clears it on every exit path.
|
|
76
|
+
*
|
|
77
|
+
* Exported so `lib/download-artifact.ts` can reuse the same budget for the artifact-download path
|
|
78
|
+
* (#2587) rather than growing a second timeout implementation — that path sits entirely outside
|
|
79
|
+
* `sendWithRetry` (it is not a JSON/raw API call), so it would otherwise never inherit this bound.
|
|
46
80
|
*/
|
|
47
|
-
function createAttemptBudget(ms) {
|
|
81
|
+
export function createAttemptBudget(ms) {
|
|
48
82
|
const controller = new AbortController();
|
|
49
|
-
|
|
50
|
-
|
|
83
|
+
let timer;
|
|
84
|
+
if (ms > 0) {
|
|
85
|
+
timer = setTimeout(() => controller.abort(timeoutReason(ms)), ms);
|
|
86
|
+
timer.unref?.();
|
|
87
|
+
}
|
|
51
88
|
return {
|
|
52
89
|
signal: controller.signal,
|
|
53
|
-
cleanup: () =>
|
|
90
|
+
cleanup: () => {
|
|
91
|
+
if (timer !== undefined)
|
|
92
|
+
clearTimeout(timer);
|
|
93
|
+
},
|
|
94
|
+
abort: (reason) => controller.abort(reason),
|
|
54
95
|
};
|
|
55
96
|
}
|
|
97
|
+
/** The `name` of an error-shaped value, or `undefined` when it carries none. */
|
|
98
|
+
const nameOf = (e) => typeof e === 'object' && e !== null && 'name' in e ? String(e.name) : undefined;
|
|
99
|
+
/** The `cause` of an error-shaped value, or `undefined` when it carries none. */
|
|
100
|
+
const causeOf = (e) => typeof e === 'object' && e !== null && 'cause' in e ? e.cause : undefined;
|
|
56
101
|
/**
|
|
57
|
-
*
|
|
58
|
-
*
|
|
102
|
+
* The abort-taxonomy name of a rejection, seeing through undici's mid-body wrapper.
|
|
103
|
+
*
|
|
104
|
+
* When a request is aborted while its BODY is streaming, undici does not reject with the abort
|
|
105
|
+
* reason itself — it rejects with `TypeError: terminated` carrying the reason on `.cause` (#2546).
|
|
106
|
+
* A name-only check would read that as a plain connection failure. Nothing that reached this
|
|
107
|
+
* function before #2546 carried a `.cause`, so every previously-asserted classification is
|
|
108
|
+
* unchanged; a `.cause` whose own name is not abort-shaped still falls through to `'connection'`.
|
|
109
|
+
*/
|
|
110
|
+
function abortNameOf(err) {
|
|
111
|
+
const top = nameOf(err);
|
|
112
|
+
if (top === 'TimeoutError' || top === 'AbortError')
|
|
113
|
+
return top;
|
|
114
|
+
return nameOf(causeOf(err));
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Map a `fetch` or body-read rejection (no usable HTTP payload was produced) onto the SDK's
|
|
118
|
+
* `ByteKitConnectionError` taxonomy, so a CLI failure carries the same `{name, code, cause}` a
|
|
119
|
+
* direct SDK call would.
|
|
59
120
|
*
|
|
60
121
|
* `timedOut` is true when this attempt's own budget fired the abort, which is the reliable
|
|
61
|
-
* discriminator (undici rejects with the signal's reason
|
|
62
|
-
* fallback for a caller-style AbortError).
|
|
122
|
+
* discriminator (undici rejects with the signal's reason — or a wrapper carrying it, see
|
|
123
|
+
* {@link abortNameOf} — whose `name` we also inspect as a fallback for a caller-style AbortError).
|
|
124
|
+
*
|
|
125
|
+
* Exported so `lib/download-artifact.ts` reuses the same taxonomy for the artifact-download path
|
|
126
|
+
* (#2587) instead of hand-rolling a second error mapping.
|
|
63
127
|
*/
|
|
64
|
-
function toConnectionError(err, method, path, timedOut) {
|
|
65
|
-
const
|
|
66
|
-
const name = typeof rawName === 'string' ? rawName : '';
|
|
128
|
+
export function toConnectionError(err, method, path, timedOut) {
|
|
129
|
+
const name = abortNameOf(err) ?? '';
|
|
67
130
|
let code;
|
|
68
131
|
if (timedOut || name === 'TimeoutError') {
|
|
69
132
|
code = 'timeout';
|
|
@@ -105,19 +168,84 @@ function sleep(ms) {
|
|
|
105
168
|
});
|
|
106
169
|
}
|
|
107
170
|
/**
|
|
108
|
-
*
|
|
171
|
+
* Read (and discard) `body`, TEARING DOWN its underlying request after `DRAIN_TIMEOUT_MS` if it
|
|
172
|
+
* has not settled by then (#2587 cr3-2). `body` must already be guarded against rejection by its
|
|
173
|
+
* caller — this function only races it against a short local timer, it never itself decides what
|
|
174
|
+
* a rejection means.
|
|
175
|
+
*
|
|
176
|
+
* `abort` MUST be the abort function of the SAME `AbortController` the read's request was made
|
|
177
|
+
* with. Merely resolving this promise when the cap fires (the earlier, defective version of this
|
|
178
|
+
* function) let the retry proceed promptly while the abandoned `response.text()` kept running
|
|
179
|
+
* against a live socket forever — nothing else in the process held a reference to that read, but
|
|
180
|
+
* the open socket itself kept the event loop alive, so the CLI's soft `process.exitCode` exit
|
|
181
|
+
* (#2544) never actually fired. `response.body?.cancel()` cannot substitute for this: `text()` has
|
|
182
|
+
* already locked the stream by the time this function is called (property 4), so cancelling the
|
|
183
|
+
* body is not an option — aborting the ORIGINAL request's controller is the only way left to
|
|
184
|
+
* release a read already in flight. This is the same mechanism `sendWithRetry`'s headers+body
|
|
185
|
+
* budget already relies on to bound a normal successful read (#2546), so it is a proven pattern:
|
|
186
|
+
* undici rejects the pending read when its controller aborts, and tears down the socket rather
|
|
187
|
+
* than leaving it open.
|
|
188
|
+
*
|
|
189
|
+
* Deliberately a plain `setTimeout` (not `AbortSignal.timeout`), same reasoning as
|
|
190
|
+
* {@link createAttemptBudget}: a timer scheduled that way cannot be advanced by the test runner's
|
|
191
|
+
* fake timers, which would make this cap untestable under `vi.useFakeTimers()`. The cap timer is
|
|
192
|
+
* cleared as soon as `body` itself settles, so a fast drain never leaves a stray timer pending;
|
|
193
|
+
* conversely, if `body` settles only AFTER the cap fires, that later settlement is simply ignored
|
|
194
|
+
* — abandoning the VALUE is always safe because the caller already discards it either way (only
|
|
195
|
+
* the underlying READ itself must not be left running, which `abort` above now guarantees).
|
|
196
|
+
*/
|
|
197
|
+
function drainBestEffort(body, abort) {
|
|
198
|
+
return new Promise((resolve) => {
|
|
199
|
+
let settled = false;
|
|
200
|
+
const cap = setTimeout(() => {
|
|
201
|
+
if (!settled) {
|
|
202
|
+
settled = true;
|
|
203
|
+
abort(timeoutReason(DRAIN_TIMEOUT_MS));
|
|
204
|
+
resolve();
|
|
205
|
+
}
|
|
206
|
+
}, DRAIN_TIMEOUT_MS);
|
|
207
|
+
cap.unref?.();
|
|
208
|
+
void body.then(() => {
|
|
209
|
+
if (!settled) {
|
|
210
|
+
settled = true;
|
|
211
|
+
clearTimeout(cap);
|
|
212
|
+
resolve();
|
|
213
|
+
}
|
|
214
|
+
});
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* Shape a failed response into the SDK's OWN error class, mirroring the SDK's own `send()` body
|
|
219
|
+
* parsing (`packages/sdk-typescript/src/client.ts`) EXACTLY — precedence rule included — so
|
|
220
|
+
* `{status, code, message}` stays identical to what a direct SDK call would throw and
|
|
221
|
+
* `handleError()`'s `'status' in err` branch renders the same `Error: <message> (<code>, status:
|
|
222
|
+
* <n>)` either way.
|
|
223
|
+
*
|
|
224
|
+
* Two error-body shapes exist across the API:
|
|
225
|
+
* - the canonical unified envelope `{error: {code, message}}` — tested FIRST, so it always wins;
|
|
226
|
+
* - a flat vendor-masked shape `{error: "<code>"}` (e.g. `/v1/search` 502:
|
|
227
|
+
* `{"error":"search_provider_error"}`, docs/api/openapi.yaml §/v1/search 502) — the flat form
|
|
228
|
+
* is strictly a FALLBACK and can never override an envelope (#2541, mirrored here for #2586:
|
|
229
|
+
* the CLI's own copy of this parser had drifted from the SDK's and still only read the
|
|
230
|
+
* envelope shape, so a flat 502 rendered `internal_error` instead of `search_provider_error`).
|
|
109
231
|
*
|
|
110
|
-
*
|
|
111
|
-
* would throw, so `handleError()`'s `'status' in err` branch still renders
|
|
112
|
-
* `Error: <message> (status: <n>)` and exits 1.
|
|
232
|
+
* The explicit `!== null` guard on the envelope check matters — `typeof null` is `'object'`.
|
|
113
233
|
*/
|
|
114
234
|
async function toApiError(response) {
|
|
115
235
|
let code = 'internal_error';
|
|
116
236
|
let message = response.statusText;
|
|
117
237
|
try {
|
|
118
238
|
const json = (await response.json());
|
|
119
|
-
|
|
120
|
-
|
|
239
|
+
if (typeof json.error === 'object' && json.error !== null) {
|
|
240
|
+
code = json.error.code ?? code;
|
|
241
|
+
message = json.error.message ?? message;
|
|
242
|
+
}
|
|
243
|
+
else if (typeof json.error === 'string' && json.error.length > 0) {
|
|
244
|
+
// The string IS the error code. `message` falls back to it only when the response carried
|
|
245
|
+
// no statusText, so the thrown error is never message-empty.
|
|
246
|
+
code = json.error;
|
|
247
|
+
message = message || json.error;
|
|
248
|
+
}
|
|
121
249
|
}
|
|
122
250
|
catch {
|
|
123
251
|
// leave defaults
|
|
@@ -125,64 +253,58 @@ async function toApiError(response) {
|
|
|
125
253
|
return new ByteKitError(response.status, code, message);
|
|
126
254
|
}
|
|
127
255
|
/**
|
|
128
|
-
*
|
|
129
|
-
*
|
|
256
|
+
* Perform the authenticated round-trip with a single bounded retry on 429, returning whatever the
|
|
257
|
+
* caller's `readBody` continuation makes of the successful response.
|
|
130
258
|
*
|
|
131
|
-
*
|
|
132
|
-
*
|
|
133
|
-
*
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
/** Collect the `X-Fetch-*` response headers into a Title-Cased metadata object. */
|
|
146
|
-
function collectFetchMetadata(headers) {
|
|
147
|
-
const meta = {};
|
|
148
|
-
headers.forEach((value, key) => {
|
|
149
|
-
if (key.toLowerCase().startsWith('x-fetch-')) {
|
|
150
|
-
meta[titleCaseHeader(key)] = value;
|
|
151
|
-
}
|
|
152
|
-
});
|
|
153
|
-
return meta;
|
|
154
|
-
}
|
|
155
|
-
/**
|
|
156
|
-
* Build the CLI's HTTP transport: the SDK's `request` contract, plus a single bounded retry on 429.
|
|
259
|
+
* This is the CLI's one transport core: it owns auth, the per-attempt abort budget, the 429 retry,
|
|
260
|
+
* and the error taxonomy — mirroring the SDK's own private `send()`. Taking the body reader as a
|
|
261
|
+
* continuation (rather than handing the `Response` back with its body unread) is what lets the two
|
|
262
|
+
* public transports built on it ({@link createRetryingRequest} and {@link createRetryingRequestRaw})
|
|
263
|
+
* differ ONLY in how they decode a successful body, so no endpoint can ever ride one behavior and
|
|
264
|
+
* miss the other (#2535: `/v1/fetch` silently bypassed the retry because only `request` was
|
|
265
|
+
* overridden) — while keeping the abort budget armed ACROSS that read (#2546).
|
|
266
|
+
*
|
|
267
|
+
* The budget spans headers **and** body. Returning the `Response` used to end the guarded region
|
|
268
|
+
* at the headers boundary, leaving `response.text()` / `response.arrayBuffer()` — and
|
|
269
|
+
* `toApiError`'s error-body `response.json()` — unbounded: a server that sent headers and then
|
|
270
|
+
* stalled its body hung the CLI forever. `readBody` therefore MUST only consume the stream;
|
|
271
|
+
* `JSON.parse` / `TextDecoder` run in the callers, OUTSIDE the guard, so a malformed body still
|
|
272
|
+
* surfaces its own parse error instead of being remapped to a connection error.
|
|
157
273
|
*
|
|
158
|
-
*
|
|
274
|
+
* The budget stays PER ATTEMPT: `cleanup()` runs in an attempt-wide `finally`, and the 429 retry
|
|
275
|
+
* sleep happens after that `finally` — so a retry can never be cut short by time already spent on
|
|
276
|
+
* the first attempt.
|
|
277
|
+
*
|
|
278
|
+
* Why the CLI owns the `fetch` instead of wrapping the SDK's transports: the SDK throws a
|
|
159
279
|
* header-less `ByteKitError` and discards the `Response`, so a delegating wrapper can see
|
|
160
280
|
* `status === 429` but can never read `Retry-After`. Owning the fetch is the only way to honor the
|
|
161
281
|
* server's advertised backoff without editing the SDK.
|
|
162
282
|
*
|
|
163
|
-
* Every SDK resource dereferences `this.client.request` at call time, so installing this as an own
|
|
164
|
-
* property of the client (see {@link buildClient}) routes *all* commands through it from one seam.
|
|
165
|
-
*
|
|
166
283
|
* Retries apply to 429 only. Non-429 statuses and network rejections propagate untouched.
|
|
167
284
|
*/
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
285
|
+
async function sendWithRetry(apiKey, baseUrl, timeoutMs, method, path, body, readBody) {
|
|
286
|
+
for (let attempt = 0;; attempt++) {
|
|
287
|
+
const init = {
|
|
288
|
+
method,
|
|
289
|
+
headers: {
|
|
290
|
+
Authorization: `Bearer ${apiKey}`,
|
|
291
|
+
'Content-Type': 'application/json',
|
|
292
|
+
},
|
|
293
|
+
};
|
|
294
|
+
if (body !== undefined) {
|
|
295
|
+
init.body = JSON.stringify(body);
|
|
296
|
+
}
|
|
297
|
+
// Per-attempt abort budget: a fresh controller for THIS attempt. `timeoutMs <= 0` disables
|
|
298
|
+
// the OVERALL attempt timer (unchanged), but the controller itself always exists — the 429
|
|
299
|
+
// drain below needs something to abort even when the full-attempt timeout is off (#2587
|
|
300
|
+
// cr3-2).
|
|
301
|
+
const budget = createAttemptBudget(timeoutMs);
|
|
302
|
+
init.signal = budget.signal;
|
|
303
|
+
// Set only by the 429-retry branch below — the one path that falls out of the `try` rather
|
|
304
|
+
// than returning or throwing. Left `undefined` on every other path, which never reaches the
|
|
305
|
+
// `sleep` after the `finally`.
|
|
306
|
+
let retryAfterMs;
|
|
307
|
+
try {
|
|
186
308
|
let response;
|
|
187
309
|
try {
|
|
188
310
|
response = await fetch(`${baseUrl}${path}`, init);
|
|
@@ -191,40 +313,179 @@ export function createRetryingRequest(apiKey, baseUrl, timeoutMs = DEFAULT_TIMEO
|
|
|
191
313
|
// No HTTP response was produced (timeout, abort, socket/DNS/TLS). Re-throw as the SDK's
|
|
192
314
|
// own ByteKitConnectionError so a hung server can never hang the CLI and a network failure
|
|
193
315
|
// no longer surfaces as a bare `Error: fetch failed`.
|
|
194
|
-
throw toConnectionError(err, method, path, budget
|
|
195
|
-
}
|
|
196
|
-
finally {
|
|
197
|
-
// Clear the timer on every exit path — success, HTTP error, or abort — so a healthy
|
|
198
|
-
// request leaves no dangling 120s timer.
|
|
199
|
-
budget?.cleanup();
|
|
316
|
+
throw toConnectionError(err, method, path, budget.signal.aborted === true);
|
|
200
317
|
}
|
|
201
318
|
if (response.ok) {
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
// #2467 DOA crash. Every other endpoint returns a JSON envelope, parsed below.
|
|
205
|
-
if (isFetchContentPath(path)) {
|
|
206
|
-
const bodyBytes = Buffer.from(await response.arrayBuffer());
|
|
207
|
-
const fetchResult = {
|
|
208
|
-
__bytekitFetchBody: true,
|
|
209
|
-
body: bodyBytes,
|
|
210
|
-
metadata: collectFetchMetadata(response.headers),
|
|
211
|
-
};
|
|
212
|
-
return fetchResult;
|
|
319
|
+
try {
|
|
320
|
+
return await readBody(response);
|
|
213
321
|
}
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
322
|
+
catch (err) {
|
|
323
|
+
// A body read that never completes is aborted by the same budget; undici then rejects
|
|
324
|
+
// the stream (#2546). Map it into the same taxonomy the headers phase uses.
|
|
325
|
+
throw toConnectionError(err, method, path, budget.signal.aborted === true);
|
|
217
326
|
}
|
|
218
|
-
return JSON.parse(text);
|
|
219
327
|
}
|
|
220
328
|
if (response.status === 429 && attempt < MAX_RETRIES) {
|
|
221
|
-
|
|
222
|
-
|
|
329
|
+
retryAfterMs = parseRetryAfterMs(response.headers.get('Retry-After'));
|
|
330
|
+
// This 429 body is never read on the retry path (only the headers/Retry-After matter), so
|
|
331
|
+
// it must be fully CONSUMED before the retry — not merely cancelled — for undici to
|
|
332
|
+
// reclaim the underlying socket for connection-pool reuse (#2587 dod-4).
|
|
333
|
+
//
|
|
334
|
+
// #2587 cr-3 correction: an earlier version of this fix used `response.body?.cancel()`,
|
|
335
|
+
// reasoning that cancelling let undici reclaim the socket. That is backwards —
|
|
336
|
+
// `cancel()` ABORTS the fetch and DESTROYS the socket rather than returning it to the
|
|
337
|
+
// pool. Measured against a real `node:http` server and real undici (5x 429->200
|
|
338
|
+
// sequences, counting `connection` events): `cancel()` was inert for the gateway's actual
|
|
339
|
+
// tiny-JSON 429 body (2 conns, identical to no drain at all) and strictly WORSE than no
|
|
340
|
+
// drain for a large body (7 conns vs 6). A full read via `response.text()` is what
|
|
341
|
+
// actually lets undici return the connection to the pool (2 conns at either size). The
|
|
342
|
+
// `.catch(() => {})` guard stays: a real body read can still reject on an already-errored
|
|
343
|
+
// or locked stream, and that must never turn a recoverable 429 into a hard failure.
|
|
344
|
+
//
|
|
345
|
+
// #2587 cr2-1 correction: the read above was previously awaited directly, bounded only by
|
|
346
|
+
// the FULL per-attempt abort budget (120 s default) or, under `timeoutMs <= 0`, by nothing
|
|
347
|
+
// at all — a stalled 429 body could burn the whole budget (or hang forever) before a retry
|
|
348
|
+
// that needs nothing from that body (`Retry-After` is already parsed above).
|
|
349
|
+
// {@link drainBestEffort} bounds the read by its own short, independent cap
|
|
350
|
+
// ({@link DRAIN_TIMEOUT_MS}) instead, so a stalled drain can never delay the retry by more
|
|
351
|
+
// than that cap regardless of the attempt budget's size or absence.
|
|
352
|
+
//
|
|
353
|
+
// #2587 cr3-2 correction: cr2-1's `drainBestEffort` bounded only how long the RETRY
|
|
354
|
+
// waited — the cap resolved the returned promise but never touched the abandoned
|
|
355
|
+
// `response.text()` call itself, which kept reading against a live (never-aborted) socket
|
|
356
|
+
// indefinitely. Because every CLI success path exits via a soft `process.exitCode`
|
|
357
|
+
// (#2544) rather than `process.exit()`, that orphaned read then kept the event loop open
|
|
358
|
+
// forever: strictly WORSE than the pre-#2587 baseline (which at least exited once its own
|
|
359
|
+
// unbounded read finally settled or errored). `drainBestEffort` now takes this attempt's
|
|
360
|
+
// own `budget.abort` and calls it when the cap fires, so the abandoned read is actually
|
|
361
|
+
// torn down — not merely ignored — and the socket (and the event loop) is released.
|
|
362
|
+
await drainBestEffort(response.text().catch(() => { }), budget.abort);
|
|
363
|
+
}
|
|
364
|
+
else {
|
|
365
|
+
// `toApiError` reads the error body, so it too runs inside the budget.
|
|
366
|
+
throw await toApiError(response);
|
|
223
367
|
}
|
|
224
|
-
throw await toApiError(response);
|
|
225
368
|
}
|
|
369
|
+
finally {
|
|
370
|
+
// Clear the timer on every exit path — success (body read included), HTTP error, or
|
|
371
|
+
// abort — so a healthy request leaves no dangling 120s timer.
|
|
372
|
+
budget.cleanup();
|
|
373
|
+
}
|
|
374
|
+
// OUTSIDE the budget's `finally`: the backoff is not spent against any attempt's budget, and
|
|
375
|
+
// the next attempt arms a fresh one. The `??` is unreachable in practice (only the branch
|
|
376
|
+
// that assigns `retryAfterMs` falls through to here) and exists solely to keep the delay a
|
|
377
|
+
// `number` without an assertion; it uses the same value `parseRetryAfterMs(null)` returns.
|
|
378
|
+
await sleep(retryAfterMs ?? DEFAULT_RETRY_AFTER_MS);
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
/**
|
|
382
|
+
* Build the CLI's JSON transport: the SDK's `request` contract, plus the single bounded 429 retry.
|
|
383
|
+
*
|
|
384
|
+
* Every SDK resource dereferences `this.client.request` at call time, so installing this as an own
|
|
385
|
+
* property of the client (see {@link buildClient}) routes all JSON commands through it from one
|
|
386
|
+
* seam. `/v1/fetch` does NOT ride this transport — it rides `requestRaw` (see
|
|
387
|
+
* {@link createRetryingRequestRaw}); `/v1/fetch/bulk` is a JSON job envelope and does.
|
|
388
|
+
*
|
|
389
|
+
* DECISION (#2586 dod-4): the 4th `RequestOptions` parameter of the SDK's contract is accepted
|
|
390
|
+
* here — so the signature matches `ByteKit['request']` exactly, closing the gap where a caller-typo
|
|
391
|
+
* of a 5th positional argument would silently no-op instead of a compile error — but deliberately
|
|
392
|
+
* NOT forwarded into `sendWithRetry`. Two reasons: (1) no CLI command constructs a `RequestOptions`
|
|
393
|
+
* today (no `--timeout` flag exists yet, and `signal` has no CLI-level source), so the parameter is
|
|
394
|
+
* always `undefined` in practice; (2) the CLI already owns a per-attempt abort budget with its own
|
|
395
|
+
* semantics (`createAttemptBudget`, `timeoutMs` above) that would need explicit composition with a
|
|
396
|
+
* caller `signal` to honor `RequestOptions` correctly (the SDK's `composeSignals`) — inventing that
|
|
397
|
+
* unexercised composition here, with no caller able to reach it, would be undertested surface for no
|
|
398
|
+
* present behavior change. If a future CLI flag needs per-call overrides, wire it through
|
|
399
|
+
* `sendWithRetry`'s `timeoutMs` (already per-attempt) at that point, with its own tests.
|
|
400
|
+
*/
|
|
401
|
+
export function createRetryingRequest(apiKey, baseUrl, timeoutMs = DEFAULT_TIMEOUT_MS) {
|
|
402
|
+
return async function request(method, path, body, _options) {
|
|
403
|
+
// The reader consumes the stream and nothing else, so the abort budget covers the read
|
|
404
|
+
// (#2546) while the 204/empty guard and `JSON.parse` stay OUTSIDE the guarded region —
|
|
405
|
+
// a malformed 200 body must still surface its own SyntaxError, never a connection error.
|
|
406
|
+
// Read-first ordering is preserved verbatim from before #2546: `.text()` is called
|
|
407
|
+
// unconditionally, and only then is the status inspected.
|
|
408
|
+
const { status, text } = await sendWithRetry(apiKey, baseUrl, timeoutMs, method, path, body, async (response) => ({ status: response.status, text: await response.text() }));
|
|
409
|
+
if (status === 204 || text.trim().length === 0) {
|
|
410
|
+
return undefined;
|
|
411
|
+
}
|
|
412
|
+
return JSON.parse(text);
|
|
226
413
|
};
|
|
227
414
|
}
|
|
415
|
+
/**
|
|
416
|
+
* Build the CLI's RAW transport: the SDK's `requestRaw` contract, plus the same bounded 429 retry.
|
|
417
|
+
*
|
|
418
|
+
* `/v1/fetch` returns the fetched content AS the body (HTML, markdown, or true binary) with its
|
|
419
|
+
* metadata in `X-Fetch-*` headers, so the SDK routes it through `requestRaw` rather than the JSON
|
|
420
|
+
* `request`. The CLI must override BOTH seams or fetch commands silently bypass the retry and the
|
|
421
|
+
* CLI's error taxonomy — the #2535 defect, where only `request` was replaced.
|
|
422
|
+
*
|
|
423
|
+
* The returned shape is the SDK's `requestRaw` contract verbatim (`{body, bytes, headers, status}`)
|
|
424
|
+
* so `FetchResource` builds an ordinary `FetchResult` from it and the CLI never forks fetch
|
|
425
|
+
* handling away from the SDK. `bytes` is byte-exact (binary-safe); `body` is the best-effort UTF-8
|
|
426
|
+
* decode, matching the SDK's own semantics.
|
|
427
|
+
*
|
|
428
|
+
* The 4th `RequestOptions` parameter is accepted but not forwarded — see the DECISION note on
|
|
429
|
+
* {@link createRetryingRequest}, which applies here verbatim. `FetchResource.create`/`get` DO thread
|
|
430
|
+
* a `requestOptions` argument through to this seam (unlike every other resource), but no CLI `fetch`
|
|
431
|
+
* command constructs one today (no `--timeout` flag), so it is still always `undefined` in practice.
|
|
432
|
+
*/
|
|
433
|
+
export function createRetryingRequestRaw(apiKey, baseUrl, timeoutMs = DEFAULT_TIMEOUT_MS) {
|
|
434
|
+
return async function requestRaw(method, path, body, _options) {
|
|
435
|
+
// As in `createRetryingRequest`: the reader only drains the stream (so the abort budget
|
|
436
|
+
// covers it, #2546); the UTF-8 decode happens outside the guarded region.
|
|
437
|
+
const { bytes, headers, status } = await sendWithRetry(apiKey, baseUrl, timeoutMs, method, path, body, async (response) => ({
|
|
438
|
+
bytes: new Uint8Array(await response.arrayBuffer()),
|
|
439
|
+
headers: response.headers,
|
|
440
|
+
status: response.status,
|
|
441
|
+
}));
|
|
442
|
+
return {
|
|
443
|
+
body: new TextDecoder('utf-8').decode(bytes),
|
|
444
|
+
bytes,
|
|
445
|
+
headers,
|
|
446
|
+
status,
|
|
447
|
+
};
|
|
448
|
+
};
|
|
449
|
+
}
|
|
450
|
+
/**
|
|
451
|
+
* Build the CLI's header-carrying JSON transport: the SDK's `requestWithResponse` contract
|
|
452
|
+
* (`{data, headers, status}`, #2547), plus the same bounded 429 retry every other CLI transport
|
|
453
|
+
* gets.
|
|
454
|
+
*
|
|
455
|
+
* Unused by any CLI command today — `screenshots.createWithResponse` / `getWithResponse` and
|
|
456
|
+
* `sitemap.getWithResponse` all exist on the SDK surface but no command builds on them yet. It is
|
|
457
|
+
* still installed (see {@link buildClient}) rather than left un-overridden: the transport-coverage
|
|
458
|
+
* contract test (#2586 dod-3) derives its method list from the real SDK prototype, and leaving a
|
|
459
|
+
* transport method un-overridden is exactly how #2535 shipped broken — the first command that DOES
|
|
460
|
+
* call a `*WithResponse` resource method must not silently ride the bare SDK transport and skip the
|
|
461
|
+
* CLI's retry/timeout/error taxonomy.
|
|
462
|
+
*
|
|
463
|
+
* Same body-read shape as {@link createRetryingRequest} (204/empty → `undefined`, `JSON.parse`
|
|
464
|
+
* otherwise), with `headers` and `status` threaded through from the successful response alongside
|
|
465
|
+
* the parsed `data`. The 4th `RequestOptions` parameter is accepted but not forwarded — see the
|
|
466
|
+
* DECISION note on {@link createRetryingRequest}.
|
|
467
|
+
*/
|
|
468
|
+
export function createRetryingRequestWithResponse(apiKey, baseUrl, timeoutMs = DEFAULT_TIMEOUT_MS) {
|
|
469
|
+
return async function requestWithResponse(method, path, body, _options) {
|
|
470
|
+
const { status, text, headers } = await sendWithRetry(apiKey, baseUrl, timeoutMs, method, path, body, async (response) => ({
|
|
471
|
+
status: response.status,
|
|
472
|
+
text: await response.text(),
|
|
473
|
+
headers: response.headers,
|
|
474
|
+
}));
|
|
475
|
+
const data = status === 204 || text.trim().length === 0 ? undefined : JSON.parse(text);
|
|
476
|
+
return { data, headers, status };
|
|
477
|
+
};
|
|
478
|
+
}
|
|
479
|
+
/**
|
|
480
|
+
* Strip a trailing slash (or run of them) from a base URL, mirroring the SDK's own constructor
|
|
481
|
+
* normalization verbatim (`packages/sdk-typescript/src/client.ts`, `/\/+$/`, #2481) — so
|
|
482
|
+
* `baseUrl + path` (every CLI-owned path already starts with `/`) never yields a double-slash URL.
|
|
483
|
+
* Only the trailing slash is removed; a path-carrying baseUrl (e.g. a reverse-proxy prefix
|
|
484
|
+
* `…/bytekit`) is preserved.
|
|
485
|
+
*/
|
|
486
|
+
function normalizeBaseUrl(url) {
|
|
487
|
+
return url.replace(/\/+$/, '');
|
|
488
|
+
}
|
|
228
489
|
/**
|
|
229
490
|
* Build an authenticated SDK client from global CLI options.
|
|
230
491
|
*
|
|
@@ -233,8 +494,19 @@ export function createRetryingRequest(apiKey, baseUrl, timeoutMs = DEFAULT_TIMEO
|
|
|
233
494
|
* baseUrl: --base-url flag → BYTEKIT_BASE_URL → DEFAULT_BASE_URL.
|
|
234
495
|
* Exits with a descriptive error when no key is available.
|
|
235
496
|
*
|
|
236
|
-
* The returned client's `request`
|
|
237
|
-
* the 429 retry
|
|
497
|
+
* The returned client's `request`, `requestRaw`, AND `requestWithResponse` are all replaced by the
|
|
498
|
+
* retrying transports, so every command inherits the 429 retry — JSON commands via `request`, the
|
|
499
|
+
* `/v1/fetch` content commands via `requestRaw`, and the header-carrying `*WithResponse` surface via
|
|
500
|
+
* `requestWithResponse`. Overriding only `request` is the #2535 defect: the SDK's fetch resource
|
|
501
|
+
* rides `requestRaw`, so fetch bypassed the retry and the CLI's error taxonomy entirely; leaving
|
|
502
|
+
* `requestWithResponse` un-overridden is the same class of defect (#2586) — it is unused by any CLI
|
|
503
|
+
* command TODAY, but a future command built on `screenshots.createWithResponse` /
|
|
504
|
+
* `sitemap.getWithResponse` would otherwise silently ride the bare SDK transport.
|
|
505
|
+
*
|
|
506
|
+
* `baseUrl` is normalized ONCE, here, before it reaches either the SDK constructor or the CLI's own
|
|
507
|
+
* transport builders below — the CLI's transports build their request URL from this local variable
|
|
508
|
+
* directly (they do not call the SDK's own normalized `send()`), so a trailing-slash flag would
|
|
509
|
+
* otherwise double-slash every request the CLI makes (#2586).
|
|
238
510
|
*/
|
|
239
511
|
export function buildClient(opts) {
|
|
240
512
|
const apiKey = opts.key ?? process.env['BYTEKIT_API_KEY'];
|
|
@@ -242,8 +514,10 @@ export function buildClient(opts) {
|
|
|
242
514
|
console.error('Error: API key required. Pass --key or set BYTEKIT_API_KEY.');
|
|
243
515
|
process.exit(1);
|
|
244
516
|
}
|
|
245
|
-
const baseUrl = opts.baseUrl ?? process.env['BYTEKIT_BASE_URL'] ?? DEFAULT_BASE_URL;
|
|
517
|
+
const baseUrl = normalizeBaseUrl(opts.baseUrl ?? process.env['BYTEKIT_BASE_URL'] ?? DEFAULT_BASE_URL);
|
|
246
518
|
const client = new ByteKit({ apiKey, baseUrl });
|
|
247
519
|
client.request = createRetryingRequest(apiKey, baseUrl);
|
|
520
|
+
client.requestRaw = createRetryingRequestRaw(apiKey, baseUrl);
|
|
521
|
+
client.requestWithResponse = createRetryingRequestWithResponse(apiKey, baseUrl);
|
|
248
522
|
return client;
|
|
249
523
|
}
|
|
@@ -1,5 +1,16 @@
|
|
|
1
1
|
import { Command } from 'commander';
|
|
2
2
|
import { type GlobalOptions } from '../client.js';
|
|
3
|
+
/**
|
|
4
|
+
* The `account` command group.
|
|
5
|
+
*
|
|
6
|
+
* Key management is DELIBERATELY absent (#2536). Those operations are mounted behind
|
|
7
|
+
* `clerkSession(db)` in the gateway and are annotated `x-internal: true` / `x-sdk-scope: internal`
|
|
8
|
+
* in the OpenAPI spec, so a bearer `sk_` key — the CLI's only credential — can never satisfy them.
|
|
9
|
+
* Keys are managed from the dashboard instead. Widening the gateway to accept bearer auth was
|
|
10
|
+
* rejected on security grounds: the reveal operation returns the live plaintext of any key on the
|
|
11
|
+
* account and there is no per-key scope model, so any leaked key would become a full
|
|
12
|
+
* account-credential factory. `internal-scope-guard.test.ts` mechanically enforces this boundary.
|
|
13
|
+
*/
|
|
3
14
|
export declare function makeAccountCommand(getGlobal: () => GlobalOptions & {
|
|
4
15
|
json: boolean;
|
|
5
16
|
}): Command;
|