@boy-offi9-inc/reqkit 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +92 -2
- package/dist/reqkit.mjs +452 -3
- package/package.json +6 -2
- package/src/createFetch.js +69 -1
- package/src/index.js +7 -0
- package/src/index.mjs +5 -0
- package/src/normalizeError.js +1 -1
- package/src/paginate.js +166 -0
- package/src/streamSSE.js +197 -0
- package/types/index.d.ts +111 -0
package/src/createFetch.js
CHANGED
|
@@ -54,6 +54,18 @@ function originOf(url) {
|
|
|
54
54
|
}
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
+
// One in-flight token refresh per `auth` config object, so concurrent 401s —
|
|
58
|
+
// even across extend() children — trigger exactly one refresh.
|
|
59
|
+
const refreshStates = new WeakMap();
|
|
60
|
+
function refreshStateFor(auth) {
|
|
61
|
+
let state = refreshStates.get(auth);
|
|
62
|
+
if (!state) {
|
|
63
|
+
state = { inflight: null };
|
|
64
|
+
refreshStates.set(auth, state);
|
|
65
|
+
}
|
|
66
|
+
return state;
|
|
67
|
+
}
|
|
68
|
+
|
|
57
69
|
function mergeDefaults(base, extra) {
|
|
58
70
|
const merged = { ...base, ...extra };
|
|
59
71
|
|
|
@@ -87,6 +99,8 @@ function mergeDefaults(base, extra) {
|
|
|
87
99
|
* - `cache`: in-memory GET cache with TTL, stale-while-revalidate and
|
|
88
100
|
* ETag / Last-Modified revalidation (304s)
|
|
89
101
|
* - concurrent GET de-duplication (withDedupe)
|
|
102
|
+
* - `auth`: attach a token, and on a 401 refresh it ONCE (shared by all
|
|
103
|
+
* concurrent requests) and replay the request
|
|
90
104
|
* - `HTTPError` on non-2xx (turn off with `throwHttpErrors: false`)
|
|
91
105
|
* - `onRequest` / `onResponse` hooks
|
|
92
106
|
*
|
|
@@ -104,6 +118,11 @@ function mergeDefaults(base, extra) {
|
|
|
104
118
|
* a per-origin breaker, a createCircuitBreaker() instance, or `(url) => breaker`
|
|
105
119
|
* @param {boolean | object} [defaults.cache] - `{ ttl, staleWhileRevalidate,
|
|
106
120
|
* maxEntries, maxBodyBytes, store, onEvent }`; `true` = 60s TTL
|
|
121
|
+
* @param {object} [defaults.auth] - `{ getToken, refresh, header?, scheme?, shouldRefresh? }`.
|
|
122
|
+
* `getToken()` supplies the current token (sync or async); on a 401 (or
|
|
123
|
+
* whatever `shouldRefresh(response)` says) `refresh({ token, response })` is
|
|
124
|
+
* called once for all concurrent callers and the request is replayed with the
|
|
125
|
+
* new token. `header` defaults to "authorization", `scheme` to "Bearer" ("" for none).
|
|
107
126
|
* @param {boolean} [defaults.dedupe=false] - merge identical concurrent GET/HEAD calls
|
|
108
127
|
* @param {boolean} [defaults.throwHttpErrors=true]
|
|
109
128
|
* @param {typeof fetch} [defaults.fetch=globalThis.fetch]
|
|
@@ -120,6 +139,7 @@ function createFetch(defaults = {}) {
|
|
|
120
139
|
rateLimit,
|
|
121
140
|
circuitBreaker,
|
|
122
141
|
cache,
|
|
142
|
+
auth,
|
|
123
143
|
dedupe = false,
|
|
124
144
|
throwHttpErrors: defaultThrow = true,
|
|
125
145
|
fetch: customFetch,
|
|
@@ -190,6 +210,27 @@ function createFetch(defaults = {}) {
|
|
|
190
210
|
pendingStores.set(key, p);
|
|
191
211
|
}
|
|
192
212
|
|
|
213
|
+
// Returns a token newer than `usedToken`, refreshing at most once no matter
|
|
214
|
+
// how many requests were rejected at the same time. If someone else already
|
|
215
|
+
// refreshed (the current token isn't the one we used), just use that.
|
|
216
|
+
async function obtainFreshToken(authCfg, tokenUsed, response) {
|
|
217
|
+
const current = await authCfg.getToken();
|
|
218
|
+
if (current && current !== tokenUsed) return current;
|
|
219
|
+
|
|
220
|
+
const state = refreshStateFor(authCfg);
|
|
221
|
+
if (!state.inflight) {
|
|
222
|
+
state.inflight = (async () => {
|
|
223
|
+
try {
|
|
224
|
+
const out = await authCfg.refresh({ token: tokenUsed, response });
|
|
225
|
+
return typeof out === "string" && out ? out : await authCfg.getToken();
|
|
226
|
+
} finally {
|
|
227
|
+
state.inflight = null;
|
|
228
|
+
}
|
|
229
|
+
})();
|
|
230
|
+
}
|
|
231
|
+
return state.inflight;
|
|
232
|
+
}
|
|
233
|
+
|
|
193
234
|
async function request(input, init = {}) {
|
|
194
235
|
const {
|
|
195
236
|
query,
|
|
@@ -217,6 +258,15 @@ function createFetch(defaults = {}) {
|
|
|
217
258
|
const finalHeaders = ctx.init.headers instanceof Headers ? ctx.init.headers : new Headers(ctx.init.headers);
|
|
218
259
|
const finalMethod = String(ctx.init.method || method).toUpperCase();
|
|
219
260
|
|
|
261
|
+
// --- auth: attach the current token (unless the caller set the header) ----
|
|
262
|
+
const authHeader = auth ? String(auth.header || "authorization").toLowerCase() : null;
|
|
263
|
+
const formatToken = (token) => (auth.scheme === "" ? String(token) : `${auth.scheme === undefined ? "Bearer" : auth.scheme} ${token}`);
|
|
264
|
+
let usedToken = null;
|
|
265
|
+
if (auth && !finalHeaders.has(authHeader)) {
|
|
266
|
+
usedToken = (await auth.getToken()) || null;
|
|
267
|
+
if (usedToken) finalHeaders.set(authHeader, formatToken(usedToken));
|
|
268
|
+
}
|
|
269
|
+
|
|
220
270
|
// --- retry policy ------------------------------------------------------
|
|
221
271
|
const retryCfg = retryOpt === false || defaultRetry === false ? false : { ...defaultRetry, ...retryOpt };
|
|
222
272
|
const methods = (retryCfg && retryCfg.methods) || DEFAULT_RETRY_METHODS;
|
|
@@ -292,7 +342,7 @@ function createFetch(defaults = {}) {
|
|
|
292
342
|
!ctx.init.body &&
|
|
293
343
|
cacheMode !== false &&
|
|
294
344
|
cacheMode !== "no-store";
|
|
295
|
-
|
|
345
|
+
let key = useCache ? cacheKey(finalMethod, ctx.url, finalHeaders) : null;
|
|
296
346
|
let entry = null;
|
|
297
347
|
|
|
298
348
|
const finish = async (response) => {
|
|
@@ -353,6 +403,24 @@ function createFetch(defaults = {}) {
|
|
|
353
403
|
throw err;
|
|
354
404
|
}
|
|
355
405
|
|
|
406
|
+
// --- auth: a rejected token is refreshed once, then the request replays ----
|
|
407
|
+
const needsRefresh = auth && (auth.shouldRefresh ? auth.shouldRefresh(response) : response.status === 401);
|
|
408
|
+
if (usedToken && needsRefresh && !isStreamBody(ctx.init.body)) {
|
|
409
|
+
const fresh = await obtainFreshToken(auth, usedToken, response);
|
|
410
|
+
if (fresh) {
|
|
411
|
+
await discardBody(response);
|
|
412
|
+
finalHeaders.set(authHeader, formatToken(fresh));
|
|
413
|
+
entry = null; // belongs to the old token's cache key
|
|
414
|
+
if (useCache) key = cacheKey(finalMethod, ctx.url, finalHeaders);
|
|
415
|
+
try {
|
|
416
|
+
response = await network(finalHeaders, signal);
|
|
417
|
+
} catch (err) {
|
|
418
|
+
if (err instanceof RetryAbortedError && signal && signal.aborted) throw signal.reason ?? err;
|
|
419
|
+
throw err;
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
|
|
356
424
|
if (useCache) {
|
|
357
425
|
if (response.status === 304 && entry) {
|
|
358
426
|
entry.storedAt = Date.now();
|
package/src/index.js
CHANGED
|
@@ -14,6 +14,8 @@ const { createMemoryCache } = require("./createMemoryCache");
|
|
|
14
14
|
const { createMockFetch } = require("./createMockFetch");
|
|
15
15
|
const { downloadResumable } = require("./downloadResumable");
|
|
16
16
|
const { readLines, readNDJSON, readSSE } = require("./streams");
|
|
17
|
+
const { streamSSE } = require("./streamSSE");
|
|
18
|
+
const { paginate, paginatePages, paginateLinks, parseLinkHeader } = require("./paginate");
|
|
17
19
|
|
|
18
20
|
module.exports = {
|
|
19
21
|
withRetry,
|
|
@@ -42,4 +44,9 @@ module.exports = {
|
|
|
42
44
|
readLines,
|
|
43
45
|
readNDJSON,
|
|
44
46
|
readSSE,
|
|
47
|
+
streamSSE,
|
|
48
|
+
paginate,
|
|
49
|
+
paginatePages,
|
|
50
|
+
paginateLinks,
|
|
51
|
+
parseLinkHeader,
|
|
45
52
|
};
|
package/src/index.mjs
CHANGED
package/src/normalizeError.js
CHANGED
|
@@ -54,7 +54,7 @@ function isNetworkCode(code) {
|
|
|
54
54
|
|
|
55
55
|
// Browsers and Node throw a bare TypeError for network-level fetch failures.
|
|
56
56
|
const FETCH_NETWORK_MESSAGE =
|
|
57
|
-
/fetch failed|failed to fetch|network ?request failed|load failed|networkerror when attempting to fetch
|
|
57
|
+
/fetch failed|failed to fetch|network ?request failed|load failed|networkerror when attempting to fetch|^terminated$|^network error$/i;
|
|
58
58
|
|
|
59
59
|
function isFetchNetworkTypeError(err) {
|
|
60
60
|
return err.name === "TypeError" && FETCH_NETWORK_MESSAGE.test(err.message || "");
|
package/src/paginate.js
ADDED
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
const { HTTPError } = require("./HTTPError");
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Parses an RFC 8288 `Link` header (GitHub, GitLab, WordPress... use it for
|
|
5
|
+
* pagination) into `{ rel: url }`. Pass `base` to resolve relative URLs.
|
|
6
|
+
*
|
|
7
|
+
* ```js
|
|
8
|
+
* parseLinkHeader('<https://api.x/items?page=2>; rel="next", <https://api.x/items?page=9>; rel="last"');
|
|
9
|
+
* // { next: "https://api.x/items?page=2", last: "https://api.x/items?page=9" }
|
|
10
|
+
* ```
|
|
11
|
+
*
|
|
12
|
+
* A link with several rels (`rel="next last"`) is listed under each; the first
|
|
13
|
+
* link wins when a rel appears twice. Commas inside `<...>` are handled.
|
|
14
|
+
*
|
|
15
|
+
* @param {string | null | undefined} header
|
|
16
|
+
* @param {string} [base]
|
|
17
|
+
* @returns {Record<string, string>}
|
|
18
|
+
*/
|
|
19
|
+
function parseLinkHeader(header, base) {
|
|
20
|
+
const links = {};
|
|
21
|
+
if (!header) return links;
|
|
22
|
+
|
|
23
|
+
// <url> followed by any number of `; name` / `; name=token` / `; name="quoted, value"` params
|
|
24
|
+
const linkRe = /<([^>]*)>((?:\s*;\s*[^=;,<\s]+(?:\s*=\s*(?:"[^"]*"|[^;,<\s]*))?)*)/g;
|
|
25
|
+
let match;
|
|
26
|
+
while ((match = linkRe.exec(header)) !== null) {
|
|
27
|
+
const rel = /;\s*rel\s*=\s*(?:"([^"]*)"|([^\s;,]+))/i.exec(match[2]);
|
|
28
|
+
if (!rel) continue;
|
|
29
|
+
|
|
30
|
+
let url = match[1].trim();
|
|
31
|
+
if (base) {
|
|
32
|
+
try {
|
|
33
|
+
url = new URL(url, base).href;
|
|
34
|
+
} catch {
|
|
35
|
+
// leave it as written
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
for (const name of (rel[1] ?? rel[2]).toLowerCase().split(/\s+/).filter(Boolean)) {
|
|
39
|
+
if (!(name in links)) links[name] = url;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
return links;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function stuckError(message) {
|
|
46
|
+
return Object.assign(new Error(message), { code: "PAGINATE_STUCK" });
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Walks a paginated API page by page. You write `fetchPage(cursor, ctx)` —
|
|
51
|
+
* it returns `{ items, next }`, where `next` is whatever identifies the next
|
|
52
|
+
* page (a cursor token, an offset, a URL...) or `undefined`/`null` on the
|
|
53
|
+
* last page — and you get an async iterator of pages.
|
|
54
|
+
*
|
|
55
|
+
* ```js
|
|
56
|
+
* for await (const { items, index } of paginatePages(async (cursor) => {
|
|
57
|
+
* const r = await api.json("/users", { query: { cursor, limit: 100 } });
|
|
58
|
+
* return { items: r.data, next: r.next_cursor };
|
|
59
|
+
* })) { ... }
|
|
60
|
+
* ```
|
|
61
|
+
*
|
|
62
|
+
* A `next` that equals the cursor just used throws `code: "PAGINATE_STUCK"`
|
|
63
|
+
* instead of looping forever.
|
|
64
|
+
*
|
|
65
|
+
* @template T
|
|
66
|
+
* @param {(cursor: any, ctx: { index: number, signal?: AbortSignal }) => Promise<{ items: T[], next?: any }>} fetchPage
|
|
67
|
+
* @param {object} [opts]
|
|
68
|
+
* @param {any} [opts.start] - the cursor for the first page (default undefined)
|
|
69
|
+
* @param {number} [opts.maxPages=Infinity]
|
|
70
|
+
* @param {AbortSignal} [opts.signal]
|
|
71
|
+
* @returns {AsyncGenerator<{ items: T[], index: number, cursor: any }>}
|
|
72
|
+
*/
|
|
73
|
+
async function* paginatePages(fetchPage, opts = {}) {
|
|
74
|
+
const { start, maxPages = Infinity, signal } = opts;
|
|
75
|
+
let cursor = start;
|
|
76
|
+
let index = 0;
|
|
77
|
+
|
|
78
|
+
for (;;) {
|
|
79
|
+
if (index >= maxPages) return;
|
|
80
|
+
signal?.throwIfAborted?.();
|
|
81
|
+
|
|
82
|
+
const result = await fetchPage(cursor, { index, signal });
|
|
83
|
+
if (!result || !Array.isArray(result.items)) {
|
|
84
|
+
throw new TypeError("paginate: fetchPage must return { items: array, next?: cursor }");
|
|
85
|
+
}
|
|
86
|
+
yield { items: result.items, index, cursor };
|
|
87
|
+
|
|
88
|
+
const next = result.next;
|
|
89
|
+
if (next === undefined || next === null) return;
|
|
90
|
+
if (next === cursor) {
|
|
91
|
+
throw stuckError("paginate: `next` equals the cursor just used, so the page didn't advance — stopping to avoid an infinite loop");
|
|
92
|
+
}
|
|
93
|
+
cursor = next;
|
|
94
|
+
index++;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Like `paginatePages`, but yields the items one by one. `maxItems` stops
|
|
100
|
+
* early (mid-page) — and since nothing is fetched until you ask for it,
|
|
101
|
+
* `break`-ing out of the loop never fetches a page you didn't need.
|
|
102
|
+
*
|
|
103
|
+
* ```js
|
|
104
|
+
* for await (const user of paginate(fetchUsersPage, { maxItems: 250 })) { ... }
|
|
105
|
+
* ```
|
|
106
|
+
*
|
|
107
|
+
* @template T
|
|
108
|
+
* @param {(cursor: any, ctx: { index: number, signal?: AbortSignal }) => Promise<{ items: T[], next?: any }>} fetchPage
|
|
109
|
+
* @param {object} [opts] - same as paginatePages, plus `maxItems`
|
|
110
|
+
* @param {number} [opts.maxItems=Infinity]
|
|
111
|
+
* @returns {AsyncGenerator<T>}
|
|
112
|
+
*/
|
|
113
|
+
async function* paginate(fetchPage, opts = {}) {
|
|
114
|
+
const { maxItems = Infinity } = opts;
|
|
115
|
+
if (maxItems <= 0) return;
|
|
116
|
+
|
|
117
|
+
let count = 0;
|
|
118
|
+
for await (const page of paginatePages(fetchPage, opts)) {
|
|
119
|
+
for (const item of page.items) {
|
|
120
|
+
yield item;
|
|
121
|
+
if (++count >= maxItems) return;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function defaultGetItems(data) {
|
|
127
|
+
if (Array.isArray(data)) return data;
|
|
128
|
+
for (const key of ["items", "data", "results", "records"]) {
|
|
129
|
+
if (data && Array.isArray(data[key])) return data[key];
|
|
130
|
+
}
|
|
131
|
+
throw new TypeError("paginateLinks: couldn't find the items array in the response — pass `getItems(data)`");
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Follows `Link: <...>; rel="next"` headers (GitHub-style) and yields the
|
|
136
|
+
* items of every page. `request` is anything fetch-like — a `createFetch`
|
|
137
|
+
* instance or plain `fetch` (a non-2xx response throws `HTTPError`).
|
|
138
|
+
*
|
|
139
|
+
* ```js
|
|
140
|
+
* for await (const repo of paginateLinks(api, "/orgs/acme/repos?per_page=100")) { ... }
|
|
141
|
+
* ```
|
|
142
|
+
*
|
|
143
|
+
* @param {(url: string, init?: object) => Promise<Response>} request
|
|
144
|
+
* @param {string} url - the first page
|
|
145
|
+
* @param {object} [opts]
|
|
146
|
+
* @param {(data: any, response: Response) => any[]} [opts.getItems] - default: the body if it's an array, else its `items` / `data` / `results` / `records`
|
|
147
|
+
* @param {number} [opts.maxPages=Infinity]
|
|
148
|
+
* @param {number} [opts.maxItems=Infinity]
|
|
149
|
+
* @param {AbortSignal} [opts.signal]
|
|
150
|
+
* @param {object} [opts.init] - extra options passed to every `request`
|
|
151
|
+
*/
|
|
152
|
+
async function* paginateLinks(request, url, opts = {}) {
|
|
153
|
+
const { getItems = defaultGetItems, init = {}, signal } = opts;
|
|
154
|
+
|
|
155
|
+
async function fetchPage(pageUrl) {
|
|
156
|
+
const res = await request(pageUrl, { ...init, signal });
|
|
157
|
+
if (!res.ok) throw new HTTPError(res, { method: init.method || "GET", url: pageUrl });
|
|
158
|
+
const items = getItems(await res.json(), res);
|
|
159
|
+
const next = parseLinkHeader(res.headers.get("link"), pageUrl).next;
|
|
160
|
+
return { items, next };
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
yield* paginate(fetchPage, { ...opts, start: url });
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
module.exports = { paginate, paginatePages, paginateLinks, parseLinkHeader };
|
package/src/streamSSE.js
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
const { readSSE } = require("./streams");
|
|
2
|
+
const { HTTPError } = require("./HTTPError");
|
|
3
|
+
const { normalizeError } = require("./normalizeError");
|
|
4
|
+
const { getWaitMsFromHeaders, discardBody } = require("./_http");
|
|
5
|
+
const { abortableSleep } = require("./_sleep");
|
|
6
|
+
|
|
7
|
+
function isRetryableStatus(status) {
|
|
8
|
+
return status === 408 || status === 425 || status === 429 || status >= 500;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
function isRetryableError(err) {
|
|
12
|
+
const n = normalizeError(err);
|
|
13
|
+
return !n.isAborted && (n.isNetworkError || n.isTimeout);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
async function resolveHeaders(source) {
|
|
17
|
+
const value = typeof source === "function" ? await source() : source;
|
|
18
|
+
return new Headers(value || undefined);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Connects to a Server-Sent Events endpoint and yields its events — and when
|
|
23
|
+
* the connection drops (or the server closes it), reconnects by itself,
|
|
24
|
+
* sending `Last-Event-ID` so the server can resume where you left off. This is
|
|
25
|
+
* what the browser's `EventSource` does, but with `fetch`: custom headers,
|
|
26
|
+
* POST bodies, AbortSignal, your own retry/backoff rules.
|
|
27
|
+
*
|
|
28
|
+
* ```js
|
|
29
|
+
* for await (const { data } of streamSSE("https://api.example.com/events", { signal })) {
|
|
30
|
+
* console.log(data);
|
|
31
|
+
* }
|
|
32
|
+
* ```
|
|
33
|
+
*
|
|
34
|
+
* Reconnect behavior:
|
|
35
|
+
* - `Last-Event-ID` is sent on every reconnect (the last `id:` seen).
|
|
36
|
+
* - A server `retry: <ms>` field sets the base reconnect delay, like EventSource.
|
|
37
|
+
* - Waits back off exponentially (capped at `maxDelayMs`) while connecting keeps
|
|
38
|
+
* failing; 429/503 `Retry-After` is honored. A connection that delivered at
|
|
39
|
+
* least one event resets the failure count (only no-progress attempts count
|
|
40
|
+
* toward `retries`).
|
|
41
|
+
* - HTTP 204 means "stop reconnecting" (iteration ends). A non-retryable
|
|
42
|
+
* status (e.g. 401/404) or a wrong content-type throws immediately.
|
|
43
|
+
* - Only GET reconnects by default. For other methods (e.g. a POST that starts
|
|
44
|
+
* an LLM stream) re-sending the request could repeat side effects, so
|
|
45
|
+
* nothing is retried unless you pass `reconnect: true`.
|
|
46
|
+
* - Breaking out of the loop, or aborting `signal`, closes the connection.
|
|
47
|
+
*
|
|
48
|
+
* @param {string} url
|
|
49
|
+
* @param {object} [opts]
|
|
50
|
+
* @param {typeof fetch} [opts.fetch=globalThis.fetch] - e.g. a createFetch() instance
|
|
51
|
+
* @param {string} [opts.method="GET"]
|
|
52
|
+
* @param {HeadersInit | (() => HeadersInit | Promise<HeadersInit>)} [opts.headers] -
|
|
53
|
+
* a function is called before every (re)connect — handy for refreshing a token
|
|
54
|
+
* @param {BodyInit} [opts.body]
|
|
55
|
+
* @param {AbortSignal} [opts.signal]
|
|
56
|
+
* @param {string} [opts.lastEventId] - resume from here on the very first connect
|
|
57
|
+
* @param {boolean} [opts.reconnect] - default: true for GET/HEAD, false otherwise
|
|
58
|
+
* @param {number} [opts.retries=10] - consecutive failed connections before giving up (Infinity = never)
|
|
59
|
+
* @param {number} [opts.reconnectDelayMs=1000] - base delay (the server's `retry:` overrides it)
|
|
60
|
+
* @param {number} [opts.maxDelayMs=30000]
|
|
61
|
+
* @param {(event: object) => boolean} [opts.stopWhen] - end the stream (no reconnect) after this event, e.g. `e => e.data === "[DONE]"`
|
|
62
|
+
* @param {(info: {connections: number, lastEventId: string}) => void} [opts.onOpen]
|
|
63
|
+
* @param {(info: {reason: "error"|"end"|"status", error?: any, status?: number, attempt: number, delayMs: number, lastEventId: string}) => void} [opts.onReconnect]
|
|
64
|
+
* @returns {AsyncGenerator<{event: string, data: string, id: string, retry: number|undefined}>}
|
|
65
|
+
*/
|
|
66
|
+
async function* streamSSE(url, opts = {}) {
|
|
67
|
+
const {
|
|
68
|
+
fetch: fetchFn = globalThis.fetch,
|
|
69
|
+
method = "GET",
|
|
70
|
+
headers,
|
|
71
|
+
body,
|
|
72
|
+
signal,
|
|
73
|
+
reconnect = method === "GET" || method === "HEAD",
|
|
74
|
+
retries = 10,
|
|
75
|
+
reconnectDelayMs = 1000,
|
|
76
|
+
maxDelayMs = 30000,
|
|
77
|
+
stopWhen,
|
|
78
|
+
onOpen,
|
|
79
|
+
onReconnect,
|
|
80
|
+
} = opts;
|
|
81
|
+
|
|
82
|
+
let lastId = opts.lastEventId || "";
|
|
83
|
+
let baseDelay = reconnectDelayMs;
|
|
84
|
+
let failures = 0; // consecutive connections that delivered nothing
|
|
85
|
+
let connections = 0;
|
|
86
|
+
|
|
87
|
+
for (;;) {
|
|
88
|
+
signal?.throwIfAborted?.();
|
|
89
|
+
let failure = null; // { reason, error?, status?, response? }
|
|
90
|
+
let res = null;
|
|
91
|
+
|
|
92
|
+
// ---- connect -----------------------------------------------------------
|
|
93
|
+
try {
|
|
94
|
+
const h = await resolveHeaders(headers);
|
|
95
|
+
if (!h.has("accept")) h.set("accept", "text/event-stream");
|
|
96
|
+
if (!h.has("cache-control")) h.set("cache-control", "no-cache");
|
|
97
|
+
if (lastId) h.set("last-event-id", lastId);
|
|
98
|
+
res = await fetchFn(url, { method, headers: h, body, signal });
|
|
99
|
+
} catch (err) {
|
|
100
|
+
if (signal?.aborted) throw signal.reason ?? err;
|
|
101
|
+
if (err instanceof HTTPError) {
|
|
102
|
+
if (!isRetryableStatus(err.status)) throw err;
|
|
103
|
+
failure = { reason: "status", status: err.status, response: err.response, error: err };
|
|
104
|
+
} else if (isRetryableError(err)) {
|
|
105
|
+
failure = { reason: "error", error: err };
|
|
106
|
+
} else {
|
|
107
|
+
throw err;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// ---- inspect the response ---------------------------------------------
|
|
112
|
+
if (!failure) {
|
|
113
|
+
if (res.status === 204) {
|
|
114
|
+
await discardBody(res);
|
|
115
|
+
return; // the server's way of saying "don't reconnect"
|
|
116
|
+
}
|
|
117
|
+
if (!res.ok) {
|
|
118
|
+
if (!isRetryableStatus(res.status)) {
|
|
119
|
+
await discardBody(res);
|
|
120
|
+
throw new HTTPError(res, { method, url });
|
|
121
|
+
}
|
|
122
|
+
failure = { reason: "status", status: res.status, response: res };
|
|
123
|
+
} else {
|
|
124
|
+
const type = res.headers.get("content-type") || "";
|
|
125
|
+
if (!/text\/event-stream/i.test(type)) {
|
|
126
|
+
await discardBody(res);
|
|
127
|
+
throw Object.assign(new Error(`streamSSE: expected "text/event-stream" but the server sent "${type || "no content-type"}"`), {
|
|
128
|
+
code: "SSE_BAD_CONTENT_TYPE",
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// ---- stream ------------------------------------------------------------
|
|
135
|
+
let gotEvent = false;
|
|
136
|
+
if (!failure) {
|
|
137
|
+
connections++;
|
|
138
|
+
if (onOpen) onOpen({ connections, lastEventId: lastId });
|
|
139
|
+
try {
|
|
140
|
+
for await (const event of readSSE(res, { signal })) {
|
|
141
|
+
gotEvent = true;
|
|
142
|
+
lastId = event.id;
|
|
143
|
+
if (event.retry !== undefined) baseDelay = event.retry;
|
|
144
|
+
yield event;
|
|
145
|
+
if (stopWhen && stopWhen(event)) return;
|
|
146
|
+
}
|
|
147
|
+
failure = { reason: "end" };
|
|
148
|
+
} catch (err) {
|
|
149
|
+
if (signal?.aborted) throw signal.reason ?? err;
|
|
150
|
+
if (!isRetryableError(err)) throw err;
|
|
151
|
+
failure = { reason: "error", error: err };
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// ---- decide whether and when to reconnect ------------------------------
|
|
156
|
+
if (!reconnect) {
|
|
157
|
+
if (failure.reason === "end") return;
|
|
158
|
+
if (failure.reason === "status") {
|
|
159
|
+
throw failure.error || new HTTPError(failure.response, { method, url });
|
|
160
|
+
}
|
|
161
|
+
throw failure.error;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// A connection that delivered events made progress: reconnect promptly and
|
|
165
|
+
// start the failure count over. Only no-progress attempts count toward `retries`.
|
|
166
|
+
failures = gotEvent ? 0 : failures + 1;
|
|
167
|
+
if (failures > retries) {
|
|
168
|
+
if (failure.reason === "status") throw failure.error || new HTTPError(failure.response, { method, url });
|
|
169
|
+
if (failure.reason === "error") throw failure.error;
|
|
170
|
+
throw Object.assign(new Error(`streamSSE: the server keeps closing the stream without sending events — gave up after ${retries} retries`), {
|
|
171
|
+
code: "SSE_GAVE_UP",
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
let delayMs = Math.min(baseDelay * 2 ** Math.max(failures - 1, 0), maxDelayMs);
|
|
176
|
+
if (failure.reason === "status" && failure.response) {
|
|
177
|
+
const headerMs = getWaitMsFromHeaders(failure.response.headers);
|
|
178
|
+
if (headerMs !== null) delayMs = Math.min(headerMs, 60000);
|
|
179
|
+
await discardBody(failure.response);
|
|
180
|
+
}
|
|
181
|
+
delayMs += Math.random() * delayMs * 0.1; // a little jitter so clients don't reconnect in lockstep
|
|
182
|
+
|
|
183
|
+
if (onReconnect) {
|
|
184
|
+
onReconnect({
|
|
185
|
+
reason: failure.reason,
|
|
186
|
+
error: failure.error,
|
|
187
|
+
status: failure.status,
|
|
188
|
+
attempt: Math.max(failures, 1),
|
|
189
|
+
delayMs,
|
|
190
|
+
lastEventId: lastId,
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
await abortableSleep(delayMs, signal);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
module.exports = { streamSSE };
|
package/types/index.d.ts
CHANGED
|
@@ -210,6 +210,8 @@ export interface CreateFetchDefaults {
|
|
|
210
210
|
rateLimit?: boolean | RateLimiterOptions | RateLimiter;
|
|
211
211
|
/** Options for a per-origin breaker, a shared breaker instance, or `(url) => breaker`. */
|
|
212
212
|
circuitBreaker?: boolean | CircuitBreakerOptions | CircuitBreaker | ((url: string) => CircuitBreaker);
|
|
213
|
+
/** Attach a token and, on a 401, refresh it once (shared by all concurrent requests) and replay. */
|
|
214
|
+
auth?: AuthOptions;
|
|
213
215
|
/** In-memory GET cache with TTL, stale-while-revalidate and ETag/Last-Modified revalidation. `true` = 60s TTL. */
|
|
214
216
|
cache?: boolean | CacheOptions;
|
|
215
217
|
/** Merge identical concurrent GET/HEAD calls. Default false. */
|
|
@@ -405,3 +407,112 @@ export function readNDJSON<T = unknown>(
|
|
|
405
407
|
opts?: StreamOptions & { skipInvalid?: boolean }
|
|
406
408
|
): AsyncGenerator<T, void, undefined>;
|
|
407
409
|
export function readSSE(source: StreamSource, opts?: StreamOptions): AsyncGenerator<SSEEvent, void, undefined>;
|
|
410
|
+
|
|
411
|
+
export interface StreamSSEOptions {
|
|
412
|
+
/** e.g. a createFetch() instance. Default: globalThis.fetch. */
|
|
413
|
+
fetch?: typeof fetch;
|
|
414
|
+
/** Default "GET". Only GET/HEAD reconnect unless `reconnect: true`. */
|
|
415
|
+
method?: string;
|
|
416
|
+
/** A function is re-evaluated before every (re)connect — handy for refreshing a token. */
|
|
417
|
+
headers?: HeadersInit | (() => HeadersInit | Promise<HeadersInit>);
|
|
418
|
+
body?: BodyInit | null;
|
|
419
|
+
signal?: AbortSignal;
|
|
420
|
+
/** Resume from this id on the very first connect. */
|
|
421
|
+
lastEventId?: string;
|
|
422
|
+
/** Default: true for GET/HEAD, false otherwise. */
|
|
423
|
+
reconnect?: boolean;
|
|
424
|
+
/** Consecutive no-progress connection failures before giving up. Default 10 (Infinity = never). */
|
|
425
|
+
retries?: number;
|
|
426
|
+
/** Base reconnect delay; the server's `retry:` field overrides it. Default 1000. */
|
|
427
|
+
reconnectDelayMs?: number;
|
|
428
|
+
/** Cap on the exponential backoff. Default 30000. */
|
|
429
|
+
maxDelayMs?: number;
|
|
430
|
+
/** End the stream (no reconnect) after this event, e.g. `e => e.data === "[DONE]"`. */
|
|
431
|
+
stopWhen?: (event: SSEEvent) => boolean;
|
|
432
|
+
onOpen?: (info: { connections: number; lastEventId: string }) => void;
|
|
433
|
+
onReconnect?: (info: {
|
|
434
|
+
reason: "error" | "end" | "status";
|
|
435
|
+
error?: unknown;
|
|
436
|
+
status?: number;
|
|
437
|
+
/** Consecutive reconnects without progress (at least 1). */
|
|
438
|
+
attempt: number;
|
|
439
|
+
delayMs: number;
|
|
440
|
+
lastEventId: string;
|
|
441
|
+
}) => void;
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
/** Auto-reconnecting SSE: resumes with Last-Event-ID, honors `retry:` and Retry-After. */
|
|
445
|
+
export function streamSSE(url: string, opts?: StreamSSEOptions): AsyncGenerator<SSEEvent, void, undefined>;
|
|
446
|
+
|
|
447
|
+
export interface AuthOptions {
|
|
448
|
+
/** The current token (sync or async). Falsy = send no auth header. */
|
|
449
|
+
getToken(): string | null | undefined | Promise<string | null | undefined>;
|
|
450
|
+
/**
|
|
451
|
+
* Called ONCE when requests are rejected (401 by default), no matter how many
|
|
452
|
+
* are waiting. Return the new token, or update your store and return nothing
|
|
453
|
+
* (reqkit then calls getToken() again). Throwing rejects every waiting request.
|
|
454
|
+
*/
|
|
455
|
+
refresh(ctx: { token: string; response: Response }): string | void | undefined | Promise<string | void | undefined>;
|
|
456
|
+
/** Default "authorization". */
|
|
457
|
+
header?: string;
|
|
458
|
+
/** Default "Bearer". Use "" to send the bare token (e.g. for an X-API-Key header). */
|
|
459
|
+
scheme?: string;
|
|
460
|
+
/** Default: `res.status === 401`. */
|
|
461
|
+
shouldRefresh?: (response: Response) => boolean;
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
export interface PageContext {
|
|
465
|
+
index: number;
|
|
466
|
+
signal?: AbortSignal;
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
export interface PageResult<T, C = any> {
|
|
470
|
+
items: T[];
|
|
471
|
+
/** Whatever identifies the next page (token, offset, URL...). undefined/null = last page. */
|
|
472
|
+
next?: C | null;
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
export interface Page<T, C = any> {
|
|
476
|
+
items: T[];
|
|
477
|
+
index: number;
|
|
478
|
+
/** The cursor this page was fetched with (undefined for the first, unless `start` was given). */
|
|
479
|
+
cursor: C | undefined;
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
export interface PaginateOptions<C = any> {
|
|
483
|
+
/** The cursor for the first page. */
|
|
484
|
+
start?: C;
|
|
485
|
+
maxPages?: number;
|
|
486
|
+
maxItems?: number;
|
|
487
|
+
signal?: AbortSignal;
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
export type FetchPage<T, C = any> = (cursor: C | undefined, ctx: PageContext) => Promise<PageResult<T, C>>;
|
|
491
|
+
|
|
492
|
+
/** Yields every item, fetching pages lazily. */
|
|
493
|
+
export function paginate<T, C = any>(fetchPage: FetchPage<T, C>, opts?: PaginateOptions<C>): AsyncGenerator<T, void, undefined>;
|
|
494
|
+
/** Yields whole pages. */
|
|
495
|
+
export function paginatePages<T, C = any>(
|
|
496
|
+
fetchPage: FetchPage<T, C>,
|
|
497
|
+
opts?: Omit<PaginateOptions<C>, "maxItems">
|
|
498
|
+
): AsyncGenerator<Page<T, C>, void, undefined>;
|
|
499
|
+
|
|
500
|
+
export interface PaginateLinksOptions<T = any> {
|
|
501
|
+
/** Default: the body if it's an array, else its `items` / `data` / `results` / `records`. */
|
|
502
|
+
getItems?: (data: any, response: Response) => T[];
|
|
503
|
+
maxPages?: number;
|
|
504
|
+
maxItems?: number;
|
|
505
|
+
signal?: AbortSignal;
|
|
506
|
+
/** Extra options passed to every request. */
|
|
507
|
+
init?: Record<string, any>;
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
/** Follows `Link: <...>; rel="next"` headers and yields the items of every page. */
|
|
511
|
+
export function paginateLinks<T = any>(
|
|
512
|
+
request: (url: string, init?: any) => Promise<Response>,
|
|
513
|
+
url: string,
|
|
514
|
+
opts?: PaginateLinksOptions<T>
|
|
515
|
+
): AsyncGenerator<T, void, undefined>;
|
|
516
|
+
|
|
517
|
+
/** Parses an RFC 8288 Link header into `{ rel: url }`. Pass `base` to resolve relative URLs. */
|
|
518
|
+
export function parseLinkHeader(header: string | null | undefined, base?: string): Record<string, string>;
|