@boy-offi9-inc/reqkit 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +146 -1
- package/RECIPES.md +243 -0
- package/dist/reqkit.mjs +2304 -0
- package/package.json +25 -5
- package/src/HTTPError.js +14 -0
- package/src/_cache.js +55 -0
- package/src/_http.js +44 -13
- package/src/_sleep.js +20 -1
- package/src/createCircuitBreaker.js +124 -0
- package/src/createFetch.js +225 -45
- package/src/createMemoryCache.js +40 -0
- package/src/createMockFetch.js +184 -0
- package/src/createRateLimiter.js +111 -0
- package/src/downloadResumable.js +193 -0
- package/src/index.js +15 -0
- package/src/index.mjs +9 -0
- package/src/streams.js +149 -0
- package/types/index.d.ts +175 -1
package/src/createFetch.js
CHANGED
|
@@ -4,24 +4,17 @@ const { withDedupe } = require("./withDedupe");
|
|
|
4
4
|
const { buildQuery } = require("./buildQuery");
|
|
5
5
|
const { parseJsonSafe } = require("./parseJsonSafe");
|
|
6
6
|
const { normalizeError } = require("./normalizeError");
|
|
7
|
+
const { HTTPError } = require("./HTTPError");
|
|
8
|
+
const { createRateLimiter } = require("./createRateLimiter");
|
|
9
|
+
const { createCircuitBreaker } = require("./createCircuitBreaker");
|
|
10
|
+
const { createMemoryCache } = require("./createMemoryCache");
|
|
11
|
+
const { cacheKey, snapshotResponse, entryToResponse } = require("./_cache");
|
|
12
|
+
const { discardBody } = require("./_http");
|
|
7
13
|
|
|
8
14
|
const DEFAULT_RETRY_STATUS_CODES = [408, 425, 429, 500, 502, 503, 504];
|
|
9
15
|
// Safe to repeat without side effects. POST/PATCH are deliberately absent.
|
|
10
16
|
const DEFAULT_RETRY_METHODS = ["GET", "HEAD", "OPTIONS", "PUT", "DELETE"];
|
|
11
17
|
|
|
12
|
-
class HTTPError extends Error {
|
|
13
|
-
constructor(response, request) {
|
|
14
|
-
const statusText = response.statusText ? ` ${response.statusText}` : "";
|
|
15
|
-
super(`Request failed with status ${response.status}${statusText}: ${request.method} ${request.url}`);
|
|
16
|
-
this.name = "HTTPError";
|
|
17
|
-
this.status = response.status;
|
|
18
|
-
this.statusText = response.statusText || "";
|
|
19
|
-
this.response = response;
|
|
20
|
-
this.url = request.url;
|
|
21
|
-
this.method = request.method;
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
|
|
25
18
|
function isAbsoluteUrl(url) {
|
|
26
19
|
return /^[a-z][a-z0-9+.-]*:/i.test(url);
|
|
27
20
|
}
|
|
@@ -53,14 +46,21 @@ function isStreamBody(body) {
|
|
|
53
46
|
return Boolean(body) && typeof body === "object" && typeof body.getReader === "function";
|
|
54
47
|
}
|
|
55
48
|
|
|
49
|
+
function originOf(url) {
|
|
50
|
+
try {
|
|
51
|
+
return new URL(url).origin;
|
|
52
|
+
} catch {
|
|
53
|
+
return url;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
56
57
|
function mergeDefaults(base, extra) {
|
|
57
|
-
const headersOf = (d) => d.headers;
|
|
58
58
|
const merged = { ...base, ...extra };
|
|
59
59
|
|
|
60
60
|
if (base.headers || extra.headers) {
|
|
61
61
|
merged.headers = async () => {
|
|
62
|
-
const a = await resolveHeaders(
|
|
63
|
-
const b = await resolveHeaders(
|
|
62
|
+
const a = await resolveHeaders(base.headers);
|
|
63
|
+
const b = await resolveHeaders(extra.headers);
|
|
64
64
|
return mergeHeaders(a, b);
|
|
65
65
|
};
|
|
66
66
|
}
|
|
@@ -81,6 +81,11 @@ function mergeDefaults(base, extra) {
|
|
|
81
81
|
* - per-attempt `timeout` (withTimeout) and external `signal`
|
|
82
82
|
* - retries (withRetry) on network errors/timeouts and retryable statuses,
|
|
83
83
|
* honoring Retry-After; only for idempotent methods unless you opt in
|
|
84
|
+
* - `rateLimit`: stay under a budget and obey X-RateLimit-* / Retry-After
|
|
85
|
+
* *before* the server has to say 429
|
|
86
|
+
* - `circuitBreaker`: fail fast against a service that keeps failing
|
|
87
|
+
* - `cache`: in-memory GET cache with TTL, stale-while-revalidate and
|
|
88
|
+
* ETag / Last-Modified revalidation (304s)
|
|
84
89
|
* - concurrent GET de-duplication (withDedupe)
|
|
85
90
|
* - `HTTPError` on non-2xx (turn off with `throwHttpErrors: false`)
|
|
86
91
|
* - `onRequest` / `onResponse` hooks
|
|
@@ -92,6 +97,13 @@ function mergeDefaults(base, extra) {
|
|
|
92
97
|
* @param {false | object} [defaults.retry] - false disables. Options:
|
|
93
98
|
* `retries`(2), `statusCodes`, `methods`, `baseDelayMs`, `maxDelayMs`,
|
|
94
99
|
* `factor`, `jitter`, `maxRetryTime`, `onRetry`, `calculateDelay`
|
|
100
|
+
* @param {boolean | object} [defaults.rateLimit] - `true` just learns from
|
|
101
|
+
* response headers; `{ requests, perMs }` also enforces a budget; or pass a
|
|
102
|
+
* createRateLimiter() instance to share one budget
|
|
103
|
+
* @param {boolean | object | Function} [defaults.circuitBreaker] - options for
|
|
104
|
+
* a per-origin breaker, a createCircuitBreaker() instance, or `(url) => breaker`
|
|
105
|
+
* @param {boolean | object} [defaults.cache] - `{ ttl, staleWhileRevalidate,
|
|
106
|
+
* maxEntries, maxBodyBytes, store, onEvent }`; `true` = 60s TTL
|
|
95
107
|
* @param {boolean} [defaults.dedupe=false] - merge identical concurrent GET/HEAD calls
|
|
96
108
|
* @param {boolean} [defaults.throwHttpErrors=true]
|
|
97
109
|
* @param {typeof fetch} [defaults.fetch=globalThis.fetch]
|
|
@@ -105,6 +117,9 @@ function createFetch(defaults = {}) {
|
|
|
105
117
|
headers: defaultHeaders,
|
|
106
118
|
timeout: defaultTimeout = 30000,
|
|
107
119
|
retry: defaultRetry = {},
|
|
120
|
+
rateLimit,
|
|
121
|
+
circuitBreaker,
|
|
122
|
+
cache,
|
|
108
123
|
dedupe = false,
|
|
109
124
|
throwHttpErrors: defaultThrow = true,
|
|
110
125
|
fetch: customFetch,
|
|
@@ -114,10 +129,67 @@ function createFetch(defaults = {}) {
|
|
|
114
129
|
|
|
115
130
|
const doFetch = (...args) => (customFetch || globalThis.fetch)(...args);
|
|
116
131
|
|
|
132
|
+
// --- stateful pieces, built once so `extend()` children share them -------
|
|
133
|
+
const limiter = !rateLimit
|
|
134
|
+
? null
|
|
135
|
+
: typeof rateLimit.acquire === "function"
|
|
136
|
+
? rateLimit
|
|
137
|
+
: createRateLimiter(rateLimit === true ? {} : rateLimit);
|
|
138
|
+
|
|
139
|
+
const breakerRegistry = new Map();
|
|
140
|
+
const breakerFor = !circuitBreaker
|
|
141
|
+
? null
|
|
142
|
+
: typeof circuitBreaker === "function"
|
|
143
|
+
? circuitBreaker
|
|
144
|
+
: typeof circuitBreaker.run === "function"
|
|
145
|
+
? () => circuitBreaker
|
|
146
|
+
: (url) => {
|
|
147
|
+
const origin = originOf(url);
|
|
148
|
+
if (!breakerRegistry.has(origin)) {
|
|
149
|
+
breakerRegistry.set(origin, createCircuitBreaker(circuitBreaker === true ? {} : circuitBreaker));
|
|
150
|
+
}
|
|
151
|
+
return breakerRegistry.get(origin);
|
|
152
|
+
};
|
|
153
|
+
|
|
154
|
+
const cacheCfg = !cache
|
|
155
|
+
? null
|
|
156
|
+
: {
|
|
157
|
+
ttl: 60000,
|
|
158
|
+
staleWhileRevalidate: 0,
|
|
159
|
+
maxEntries: 100,
|
|
160
|
+
maxBodyBytes: undefined,
|
|
161
|
+
...(cache === true ? {} : cache),
|
|
162
|
+
};
|
|
163
|
+
if (cacheCfg && !cacheCfg.store) cacheCfg.store = createMemoryCache({ maxEntries: cacheCfg.maxEntries });
|
|
164
|
+
// In-flight bookkeeping lives on the (shared) cache config so `extend()`
|
|
165
|
+
// children see the parent's pending writes and revalidations.
|
|
166
|
+
if (cacheCfg) {
|
|
167
|
+
cacheCfg.pending = cacheCfg.pending || new Map();
|
|
168
|
+
cacheCfg.revalidating = cacheCfg.revalidating || new Set();
|
|
169
|
+
}
|
|
170
|
+
const pendingStores = cacheCfg ? cacheCfg.pending : null;
|
|
171
|
+
const revalidating = cacheCfg ? cacheCfg.revalidating : null;
|
|
172
|
+
const emit = (type, url) => cacheCfg && cacheCfg.onEvent && cacheCfg.onEvent({ type, url });
|
|
173
|
+
|
|
117
174
|
// Shared in-flight GETs. Every caller gets its own clone of the response so
|
|
118
175
|
// body reads don't fight each other.
|
|
119
176
|
const sharedFetch = withDedupe((key, url, init) => doFetch(url, init), { keyFn: (key) => key });
|
|
120
177
|
|
|
178
|
+
function scheduleStore(key, response, url) {
|
|
179
|
+
const p = snapshotResponse(response, cacheCfg.maxBodyBytes)
|
|
180
|
+
.then(async (entry) => {
|
|
181
|
+
if (entry) {
|
|
182
|
+
await cacheCfg.store.set(key, entry);
|
|
183
|
+
emit("store", url);
|
|
184
|
+
}
|
|
185
|
+
})
|
|
186
|
+
.catch(() => {})
|
|
187
|
+
.finally(() => {
|
|
188
|
+
if (pendingStores.get(key) === p) pendingStores.delete(key);
|
|
189
|
+
});
|
|
190
|
+
pendingStores.set(key, p);
|
|
191
|
+
}
|
|
192
|
+
|
|
121
193
|
async function request(input, init = {}) {
|
|
122
194
|
const {
|
|
123
195
|
query,
|
|
@@ -127,6 +199,7 @@ function createFetch(defaults = {}) {
|
|
|
127
199
|
throwHttpErrors = defaultThrow,
|
|
128
200
|
signal,
|
|
129
201
|
headers: initHeaders,
|
|
202
|
+
cache: cacheMode,
|
|
130
203
|
...fetchInit
|
|
131
204
|
} = init;
|
|
132
205
|
|
|
@@ -152,30 +225,43 @@ function createFetch(defaults = {}) {
|
|
|
152
225
|
retryCfg !== false &&
|
|
153
226
|
!isStreamBody(ctx.init.body) && // a consumed stream can't be replayed
|
|
154
227
|
(methods.map((m) => m.toUpperCase()).includes(finalMethod) || hasIdempotencyKey);
|
|
155
|
-
|
|
156
|
-
const attemptOnce = () =>
|
|
157
|
-
withTimeout(
|
|
158
|
-
(attemptSignal) => {
|
|
159
|
-
const attemptInit = { ...ctx.init, method: finalMethod, headers: finalHeaders, signal: attemptSignal };
|
|
160
|
-
const useDedupe = dedupe && !signal && !ctx.init.body && (finalMethod === "GET" || finalMethod === "HEAD");
|
|
161
|
-
if (!useDedupe) return doFetch(ctx.url, attemptInit);
|
|
162
|
-
// No per-attempt signal on a shared request: aborting one caller
|
|
163
|
-
// must never cancel everyone else's.
|
|
164
|
-
// It still gets its own timeout so a hung server can't leak the request.
|
|
165
|
-
const sharedInit = { ...attemptInit, signal: timeout === false ? undefined : AbortSignal.timeout(timeout) };
|
|
166
|
-
const key = `${finalMethod} ${ctx.url} ${JSON.stringify([...finalHeaders.entries()].sort())}`;
|
|
167
|
-
return sharedFetch(key, ctx.url, sharedInit).then((res) => res.clone());
|
|
168
|
-
},
|
|
169
|
-
timeout === false ? Infinity : timeout,
|
|
170
|
-
{ signal }
|
|
171
|
-
);
|
|
172
|
-
|
|
173
228
|
const { statusCodes, methods: _methods, retries = 2, shouldRetry: customShouldRetry, ...retryRest } = retryCfg || {};
|
|
174
229
|
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
230
|
+
// One trip to the network: rate limit → circuit breaker → per-attempt
|
|
231
|
+
// timeout → fetch, retried per the policy above. `netHeaders` and
|
|
232
|
+
// `netSignal` are parameters so background revalidation can reuse this.
|
|
233
|
+
function network(netHeaders, netSignal) {
|
|
234
|
+
const attemptOnce = () => {
|
|
235
|
+
const attempt = async () => {
|
|
236
|
+
if (limiter) await limiter.acquire({ signal: netSignal });
|
|
237
|
+
const res = await withTimeout(
|
|
238
|
+
(attemptSignal) => {
|
|
239
|
+
const attemptInit = { ...ctx.init, method: finalMethod, headers: netHeaders, signal: attemptSignal };
|
|
240
|
+
const useDedupe =
|
|
241
|
+
dedupe && !netSignal && !ctx.init.body && (finalMethod === "GET" || finalMethod === "HEAD");
|
|
242
|
+
if (!useDedupe) return doFetch(ctx.url, attemptInit);
|
|
243
|
+
// No per-attempt signal on a shared request: aborting one caller
|
|
244
|
+
// must never cancel everyone else's. It still gets its own
|
|
245
|
+
// timeout so a hung server can't leak the request.
|
|
246
|
+
const sharedInit = {
|
|
247
|
+
...attemptInit,
|
|
248
|
+
signal: timeout === false ? undefined : AbortSignal.timeout(timeout),
|
|
249
|
+
};
|
|
250
|
+
const key = `${finalMethod} ${ctx.url} ${JSON.stringify([...netHeaders.entries()].sort())}`;
|
|
251
|
+
return sharedFetch(key, ctx.url, sharedInit).then((res) => res.clone());
|
|
252
|
+
},
|
|
253
|
+
timeout === false ? Infinity : timeout,
|
|
254
|
+
{ signal: netSignal }
|
|
255
|
+
);
|
|
256
|
+
if (limiter) limiter.update(res);
|
|
257
|
+
return res;
|
|
258
|
+
};
|
|
259
|
+
const breaker = breakerFor && breakerFor(ctx.url);
|
|
260
|
+
return breaker ? breaker.run(attempt) : attempt();
|
|
261
|
+
};
|
|
262
|
+
|
|
263
|
+
return canRetry
|
|
264
|
+
? withRetry(attemptOnce, {
|
|
179
265
|
...retryRest,
|
|
180
266
|
retries,
|
|
181
267
|
retryStatusCodes: statusCodes || DEFAULT_RETRY_STATUS_CODES,
|
|
@@ -185,20 +271,102 @@ function createFetch(defaults = {}) {
|
|
|
185
271
|
const n = normalizeError(err);
|
|
186
272
|
return n.isNetworkError || n.isTimeout;
|
|
187
273
|
}),
|
|
188
|
-
signal,
|
|
274
|
+
signal: netSignal,
|
|
189
275
|
})
|
|
190
|
-
:
|
|
276
|
+
: attemptOnce();
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
function conditionalHeaders(entry) {
|
|
280
|
+
const h = new Headers(finalHeaders);
|
|
281
|
+
if (entry) {
|
|
282
|
+
if (entry.etag && !h.has("if-none-match")) h.set("if-none-match", entry.etag);
|
|
283
|
+
if (entry.lastModified && !h.has("if-modified-since")) h.set("if-modified-since", entry.lastModified);
|
|
284
|
+
}
|
|
285
|
+
return h;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
// --- cache lookup --------------------------------------------------------
|
|
289
|
+
const useCache =
|
|
290
|
+
Boolean(cacheCfg) &&
|
|
291
|
+
finalMethod === "GET" &&
|
|
292
|
+
!ctx.init.body &&
|
|
293
|
+
cacheMode !== false &&
|
|
294
|
+
cacheMode !== "no-store";
|
|
295
|
+
const key = useCache ? cacheKey(finalMethod, ctx.url, finalHeaders) : null;
|
|
296
|
+
let entry = null;
|
|
297
|
+
|
|
298
|
+
const finish = async (response) => {
|
|
299
|
+
if (onResponse) await onResponse({ url: ctx.url, init: ctx.init, response });
|
|
300
|
+
if (throwHttpErrors && !response.ok) {
|
|
301
|
+
throw new HTTPError(response, { method: finalMethod, url: ctx.url });
|
|
302
|
+
}
|
|
303
|
+
return response;
|
|
304
|
+
};
|
|
305
|
+
|
|
306
|
+
if (useCache && cacheMode !== "reload") {
|
|
307
|
+
if (pendingStores.has(key)) await pendingStores.get(key);
|
|
308
|
+
entry = (await cacheCfg.store.get(key)) || null;
|
|
309
|
+
if (entry) {
|
|
310
|
+
const age = Date.now() - entry.storedAt;
|
|
311
|
+
if (age < cacheCfg.ttl) {
|
|
312
|
+
emit("hit", ctx.url);
|
|
313
|
+
return finish(entryToResponse(entry));
|
|
314
|
+
}
|
|
315
|
+
if (age < cacheCfg.ttl + cacheCfg.staleWhileRevalidate) {
|
|
316
|
+
emit("stale", ctx.url);
|
|
317
|
+
if (!revalidating.has(key)) {
|
|
318
|
+
revalidating.add(key);
|
|
319
|
+
const staleEntry = entry;
|
|
320
|
+
(async () => {
|
|
321
|
+
try {
|
|
322
|
+
const res = await network(conditionalHeaders(staleEntry), undefined);
|
|
323
|
+
if (res.status === 304) {
|
|
324
|
+
staleEntry.storedAt = Date.now();
|
|
325
|
+
await cacheCfg.store.set(key, staleEntry);
|
|
326
|
+
emit("revalidated", ctx.url);
|
|
327
|
+
await discardBody(res);
|
|
328
|
+
} else if (res.status === 200) {
|
|
329
|
+
scheduleStore(key, res, ctx.url);
|
|
330
|
+
await pendingStores.get(key);
|
|
331
|
+
} else {
|
|
332
|
+
await discardBody(res);
|
|
333
|
+
}
|
|
334
|
+
} catch {
|
|
335
|
+
// keep serving the stale copy; try again next time
|
|
336
|
+
} finally {
|
|
337
|
+
revalidating.delete(key);
|
|
338
|
+
}
|
|
339
|
+
})();
|
|
340
|
+
}
|
|
341
|
+
return finish(entryToResponse(entry));
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
// --- network -------------------------------------------------------------
|
|
347
|
+
let response;
|
|
348
|
+
try {
|
|
349
|
+
response = await network(useCache ? conditionalHeaders(entry) : finalHeaders, signal);
|
|
191
350
|
} catch (err) {
|
|
192
351
|
// Present a user abort as the plain AbortError they caused, not as retry internals.
|
|
193
352
|
if (err instanceof RetryAbortedError && signal && signal.aborted) throw signal.reason ?? err;
|
|
194
353
|
throw err;
|
|
195
354
|
}
|
|
196
355
|
|
|
197
|
-
if (
|
|
198
|
-
|
|
199
|
-
|
|
356
|
+
if (useCache) {
|
|
357
|
+
if (response.status === 304 && entry) {
|
|
358
|
+
entry.storedAt = Date.now();
|
|
359
|
+
await cacheCfg.store.set(key, entry);
|
|
360
|
+
emit("revalidated", ctx.url);
|
|
361
|
+
await discardBody(response);
|
|
362
|
+
response = entryToResponse(entry);
|
|
363
|
+
} else {
|
|
364
|
+
emit("miss", ctx.url);
|
|
365
|
+
if (response.status === 200) scheduleStore(key, response, ctx.url);
|
|
366
|
+
}
|
|
200
367
|
}
|
|
201
|
-
|
|
368
|
+
|
|
369
|
+
return finish(response);
|
|
202
370
|
}
|
|
203
371
|
|
|
204
372
|
/** Like `request`, but resolves to the parsed JSON body (null for an empty body). */
|
|
@@ -210,7 +378,19 @@ function createFetch(defaults = {}) {
|
|
|
210
378
|
};
|
|
211
379
|
|
|
212
380
|
/** A new fetch with these defaults merged over the current ones (headers and retry are merged, the rest replaced). */
|
|
213
|
-
request.extend = (overrides = {}) =>
|
|
381
|
+
request.extend = (overrides = {}) =>
|
|
382
|
+
createFetch(
|
|
383
|
+
mergeDefaults(
|
|
384
|
+
{
|
|
385
|
+
...defaults,
|
|
386
|
+
// children share the parent's budget, breakers and cache unless they bring their own
|
|
387
|
+
rateLimit: limiter || undefined,
|
|
388
|
+
circuitBreaker: breakerFor || undefined,
|
|
389
|
+
cache: cacheCfg || undefined,
|
|
390
|
+
},
|
|
391
|
+
overrides
|
|
392
|
+
)
|
|
393
|
+
);
|
|
214
394
|
|
|
215
395
|
return request;
|
|
216
396
|
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A small in-memory LRU store for createFetch's `cache` option. Anything with
|
|
3
|
+
* async-or-sync `get(key)`, `set(key, value)` and `delete(key)` can be used
|
|
4
|
+
* instead (Redis, IndexedDB, a Map...).
|
|
5
|
+
*
|
|
6
|
+
* @param {{ maxEntries?: number }} [opts]
|
|
7
|
+
*/
|
|
8
|
+
function createMemoryCache(opts = {}) {
|
|
9
|
+
const { maxEntries = 100 } = opts;
|
|
10
|
+
if (!(Number.isInteger(maxEntries) && maxEntries >= 1)) {
|
|
11
|
+
throw new TypeError("createMemoryCache: `maxEntries` must be an integer >= 1");
|
|
12
|
+
}
|
|
13
|
+
const map = new Map();
|
|
14
|
+
|
|
15
|
+
return {
|
|
16
|
+
get(key) {
|
|
17
|
+
if (!map.has(key)) return undefined;
|
|
18
|
+
const value = map.get(key);
|
|
19
|
+
map.delete(key); // re-insert so it counts as most recently used
|
|
20
|
+
map.set(key, value);
|
|
21
|
+
return value;
|
|
22
|
+
},
|
|
23
|
+
set(key, value) {
|
|
24
|
+
map.delete(key);
|
|
25
|
+
map.set(key, value);
|
|
26
|
+
while (map.size > maxEntries) map.delete(map.keys().next().value);
|
|
27
|
+
},
|
|
28
|
+
delete(key) {
|
|
29
|
+
return map.delete(key);
|
|
30
|
+
},
|
|
31
|
+
clear() {
|
|
32
|
+
map.clear();
|
|
33
|
+
},
|
|
34
|
+
get size() {
|
|
35
|
+
return map.size;
|
|
36
|
+
},
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
module.exports = { createMemoryCache };
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
const { abortableSleep } = require("./_sleep");
|
|
2
|
+
|
|
3
|
+
const BODYLESS = new Set([101, 204, 205, 304]);
|
|
4
|
+
|
|
5
|
+
function toBody(body, headers) {
|
|
6
|
+
if (body === null || body === undefined) return null;
|
|
7
|
+
if (typeof body === "string") return body;
|
|
8
|
+
if (
|
|
9
|
+
body instanceof ArrayBuffer ||
|
|
10
|
+
ArrayBuffer.isView(body) ||
|
|
11
|
+
(typeof Blob !== "undefined" && body instanceof Blob) ||
|
|
12
|
+
(typeof ReadableStream !== "undefined" && body instanceof ReadableStream) ||
|
|
13
|
+
body instanceof URLSearchParams
|
|
14
|
+
) {
|
|
15
|
+
return body;
|
|
16
|
+
}
|
|
17
|
+
if (!headers.has("content-type")) headers.set("content-type", "application/json");
|
|
18
|
+
return JSON.stringify(body);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function buildResponse(status, body, extraHeaders) {
|
|
22
|
+
const headers = new Headers(extraHeaders || undefined);
|
|
23
|
+
const payload = BODYLESS.has(status) ? null : toBody(body, headers);
|
|
24
|
+
return new Response(payload, { status, headers });
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function matchesUrl(matcher, url) {
|
|
28
|
+
if (typeof matcher === "function") return Boolean(matcher(url));
|
|
29
|
+
if (matcher instanceof RegExp) return matcher.test(url);
|
|
30
|
+
if (matcher === "*" || matcher === undefined) return true;
|
|
31
|
+
if (url === matcher) return true;
|
|
32
|
+
try {
|
|
33
|
+
const u = new URL(url);
|
|
34
|
+
if (matcher === u.pathname + u.search) return true;
|
|
35
|
+
if (!String(matcher).includes("?") && matcher === u.pathname) return true;
|
|
36
|
+
} catch {
|
|
37
|
+
// relative/odd URL — exact match only
|
|
38
|
+
}
|
|
39
|
+
return false;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* A scriptable, fetch-compatible test double — for testing code that uses
|
|
44
|
+
* retries, timeouts, rate limits and so on without a network.
|
|
45
|
+
*
|
|
46
|
+
* ```js
|
|
47
|
+
* const mock = createMockFetch();
|
|
48
|
+
* mock.on("GET", "/users/1")
|
|
49
|
+
* .replyOnce(503, "busy", { headers: { "retry-after": "0" } })
|
|
50
|
+
* .reply(200, { id: 1 });
|
|
51
|
+
* mock.on("POST", /\/orders/).fail(mock.networkError());
|
|
52
|
+
*
|
|
53
|
+
* const api = createFetch({ fetch: mock.fetch, baseUrl: "https://api.test" });
|
|
54
|
+
* await api.json("/users/1"); // → { id: 1 } after one retry
|
|
55
|
+
* mock.calls.length; // 2
|
|
56
|
+
* ```
|
|
57
|
+
*
|
|
58
|
+
* String matchers match the full URL, `pathname + search`, or the bare
|
|
59
|
+
* `pathname`; you can also pass a RegExp or a `(url) => boolean`. `"*"` as the
|
|
60
|
+
* method matches any method. Unmatched requests reject with a clear error.
|
|
61
|
+
*/
|
|
62
|
+
function createMockFetch() {
|
|
63
|
+
let routes = [];
|
|
64
|
+
const calls = [];
|
|
65
|
+
|
|
66
|
+
function on(method, matcher) {
|
|
67
|
+
const route = { method: String(method).toUpperCase(), matcher, once: [], fallback: null };
|
|
68
|
+
routes.push(route);
|
|
69
|
+
|
|
70
|
+
const chain = {
|
|
71
|
+
/** Always answer like this (once any `…Once` replies are used up). */
|
|
72
|
+
reply(status = 200, body, opts) {
|
|
73
|
+
route.fallback = replyHandler(status, body, opts);
|
|
74
|
+
return chain;
|
|
75
|
+
},
|
|
76
|
+
replyOnce(status = 200, body, opts) {
|
|
77
|
+
route.once.push(replyHandler(status, body, opts));
|
|
78
|
+
return chain;
|
|
79
|
+
},
|
|
80
|
+
/** `fn(call)` returns a Response, `{ status, body, headers }`, or throws/rejects. */
|
|
81
|
+
replyWith(fn) {
|
|
82
|
+
route.fallback = customHandler(fn);
|
|
83
|
+
return chain;
|
|
84
|
+
},
|
|
85
|
+
replyWithOnce(fn) {
|
|
86
|
+
route.once.push(customHandler(fn));
|
|
87
|
+
return chain;
|
|
88
|
+
},
|
|
89
|
+
/** Reject like fetch does on a network failure. */
|
|
90
|
+
fail(err = networkError()) {
|
|
91
|
+
route.fallback = async () => {
|
|
92
|
+
throw err;
|
|
93
|
+
};
|
|
94
|
+
return chain;
|
|
95
|
+
},
|
|
96
|
+
failOnce(err = networkError()) {
|
|
97
|
+
route.once.push(async () => {
|
|
98
|
+
throw err;
|
|
99
|
+
});
|
|
100
|
+
return chain;
|
|
101
|
+
},
|
|
102
|
+
};
|
|
103
|
+
return chain;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function replyHandler(status, body, opts = {}) {
|
|
107
|
+
return async (call) => {
|
|
108
|
+
if (opts.delayMs) await abortableSleep(opts.delayMs, call.signal);
|
|
109
|
+
return buildResponse(status, body, opts.headers);
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function customHandler(fn) {
|
|
114
|
+
return async (call) => {
|
|
115
|
+
const out = await fn(call);
|
|
116
|
+
if (out instanceof Response) return out;
|
|
117
|
+
return buildResponse(out.status ?? 200, out.body, out.headers);
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function networkError(code = "ECONNREFUSED") {
|
|
122
|
+
const err = new TypeError("fetch failed");
|
|
123
|
+
err.cause = Object.assign(new Error(`connect ${code}`), { code });
|
|
124
|
+
return err;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
async function mockFetch(input, init = {}) {
|
|
128
|
+
const url = input instanceof URL ? input.href : typeof input === "string" ? input : input.url;
|
|
129
|
+
const method = String(init.method || (typeof input === "object" && input.method) || "GET").toUpperCase();
|
|
130
|
+
const signal = init.signal || (typeof input === "object" ? input.signal : undefined);
|
|
131
|
+
signal?.throwIfAborted?.();
|
|
132
|
+
|
|
133
|
+
const headers = {};
|
|
134
|
+
new Headers(init.headers || (typeof input === "object" ? input.headers : undefined)).forEach((v, k) => {
|
|
135
|
+
headers[k] = v;
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
const call = {
|
|
139
|
+
url,
|
|
140
|
+
method,
|
|
141
|
+
headers,
|
|
142
|
+
body: init.body,
|
|
143
|
+
signal,
|
|
144
|
+
get json() {
|
|
145
|
+
try {
|
|
146
|
+
return typeof this.body === "string" ? JSON.parse(this.body) : undefined;
|
|
147
|
+
} catch {
|
|
148
|
+
return undefined;
|
|
149
|
+
}
|
|
150
|
+
},
|
|
151
|
+
};
|
|
152
|
+
calls.push(call);
|
|
153
|
+
|
|
154
|
+
const route = routes.find(
|
|
155
|
+
(r) => (r.method === "*" || r.method === method) && matchesUrl(r.matcher, url) && (r.once.length > 0 || r.fallback)
|
|
156
|
+
);
|
|
157
|
+
if (!route) {
|
|
158
|
+
throw new Error(`createMockFetch: no mock registered for ${method} ${url}`);
|
|
159
|
+
}
|
|
160
|
+
const handler = route.once.length > 0 ? route.once.shift() : route.fallback;
|
|
161
|
+
return handler(call);
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
return {
|
|
165
|
+
fetch: mockFetch,
|
|
166
|
+
on,
|
|
167
|
+
get: (m) => on("GET", m),
|
|
168
|
+
post: (m) => on("POST", m),
|
|
169
|
+
put: (m) => on("PUT", m),
|
|
170
|
+
patch: (m) => on("PATCH", m),
|
|
171
|
+
delete: (m) => on("DELETE", m),
|
|
172
|
+
/** A TypeError("fetch failed") with `cause.code`, like Node's fetch throws. */
|
|
173
|
+
networkError,
|
|
174
|
+
/** Every request made so far, in order. */
|
|
175
|
+
calls,
|
|
176
|
+
/** Remove all routes and forget recorded calls. */
|
|
177
|
+
reset() {
|
|
178
|
+
routes = [];
|
|
179
|
+
calls.length = 0;
|
|
180
|
+
},
|
|
181
|
+
};
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
module.exports = { createMockFetch };
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
const { abortableSleep } = require("./_sleep");
|
|
2
|
+
const { getRateLimitInfo } = require("./_http");
|
|
3
|
+
|
|
4
|
+
function abortable(promise, signal) {
|
|
5
|
+
if (!signal) return promise;
|
|
6
|
+
return new Promise((resolve, reject) => {
|
|
7
|
+
if (signal.aborted) {
|
|
8
|
+
reject(signal.reason);
|
|
9
|
+
return;
|
|
10
|
+
}
|
|
11
|
+
const onAbort = () => reject(signal.reason);
|
|
12
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
13
|
+
promise.then(
|
|
14
|
+
(v) => (signal.removeEventListener("abort", onAbort), resolve(v)),
|
|
15
|
+
(e) => (signal.removeEventListener("abort", onAbort), reject(e))
|
|
16
|
+
);
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* A *proactive* rate limiter: slows you down before the server has to say
|
|
22
|
+
* 429. Two things feed it:
|
|
23
|
+
* 1. your own budget — `requests` per `perMs` (sliding window), and
|
|
24
|
+
* 2. what the server tells you — `update(response)` reads X-RateLimit-* /
|
|
25
|
+
* RateLimit-* / Retry-After and pauses everything once `remaining` hits 0
|
|
26
|
+
* (or the server sent Retry-After), until the reset time.
|
|
27
|
+
*
|
|
28
|
+
* Callers are served FIFO.
|
|
29
|
+
*
|
|
30
|
+
* ```js
|
|
31
|
+
* const rl = createRateLimiter({ requests: 10, perMs: 1000 });
|
|
32
|
+
* const res = await rl.run(() => fetch(url)); // acquire → call → learn from headers
|
|
33
|
+
* ```
|
|
34
|
+
*
|
|
35
|
+
* @param {object} [opts]
|
|
36
|
+
* @param {number} [opts.requests=Infinity] - max requests per window
|
|
37
|
+
* @param {number} [opts.perMs=1000] - window length
|
|
38
|
+
*/
|
|
39
|
+
function createRateLimiter(opts = {}) {
|
|
40
|
+
const { requests = Infinity, perMs = 1000 } = opts;
|
|
41
|
+
if (requests !== Infinity && !(Number.isInteger(requests) && requests >= 1)) {
|
|
42
|
+
throw new TypeError("createRateLimiter: `requests` must be an integer >= 1 (or Infinity)");
|
|
43
|
+
}
|
|
44
|
+
if (!(typeof perMs === "number" && perMs > 0 && Number.isFinite(perMs))) {
|
|
45
|
+
throw new TypeError("createRateLimiter: `perMs` must be a positive number");
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const stamps = [];
|
|
49
|
+
let blockedUntil = 0;
|
|
50
|
+
let tail = Promise.resolve();
|
|
51
|
+
|
|
52
|
+
async function take(signal) {
|
|
53
|
+
for (;;) {
|
|
54
|
+
signal?.throwIfAborted?.();
|
|
55
|
+
const now = Date.now();
|
|
56
|
+
while (stamps.length > 0 && stamps[0] <= now - perMs) stamps.shift();
|
|
57
|
+
|
|
58
|
+
let waitMs = 0;
|
|
59
|
+
if (blockedUntil > now) waitMs = blockedUntil - now;
|
|
60
|
+
else if (stamps.length >= requests) waitMs = stamps[0] + perMs - now;
|
|
61
|
+
|
|
62
|
+
if (waitMs <= 0) {
|
|
63
|
+
stamps.push(now);
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
await abortableSleep(waitMs, signal);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Resolves when it's your turn. Rejects with the signal's reason if aborted while waiting. */
|
|
71
|
+
function acquire({ signal } = {}) {
|
|
72
|
+
const turn = tail.then(() => take(signal));
|
|
73
|
+
tail = turn.catch(() => {});
|
|
74
|
+
return abortable(turn, signal);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Feed a response (anything with `.headers`/`.status`) so the limiter learns the server's limits. */
|
|
78
|
+
function update(response) {
|
|
79
|
+
if (!response || !response.headers) return;
|
|
80
|
+
const { remaining, resetMs, retryAfterMs } = getRateLimitInfo(response.headers);
|
|
81
|
+
let blockMs = null;
|
|
82
|
+
if (retryAfterMs !== null && (response.status === 429 || response.status === 503)) blockMs = retryAfterMs;
|
|
83
|
+
else if (remaining === 0 && resetMs !== null) blockMs = resetMs;
|
|
84
|
+
if (blockMs !== null) blockedUntil = Math.max(blockedUntil, Date.now() + blockMs);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** acquire → fn() → update(result). Errors thrown by fn propagate (the slot stays spent). */
|
|
88
|
+
async function run(fn, { signal } = {}) {
|
|
89
|
+
await acquire({ signal });
|
|
90
|
+
const result = await fn();
|
|
91
|
+
update(result);
|
|
92
|
+
return result;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
return {
|
|
96
|
+
acquire,
|
|
97
|
+
update,
|
|
98
|
+
run,
|
|
99
|
+
/** Ms until the server-imposed pause ends (0 if none). */
|
|
100
|
+
get blockedForMs() {
|
|
101
|
+
return Math.max(0, blockedUntil - Date.now());
|
|
102
|
+
},
|
|
103
|
+
/** Forget all history and server hints. */
|
|
104
|
+
reset() {
|
|
105
|
+
stamps.length = 0;
|
|
106
|
+
blockedUntil = 0;
|
|
107
|
+
},
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
module.exports = { createRateLimiter };
|