@boy-offi9-inc/reqkit 0.1.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +1 -1
- package/README.md +22 -2
- package/package.json +23 -6
- package/src/_sleep.js +32 -0
- package/src/buildQuery.js +22 -8
- package/src/index.js +2 -1
- package/src/index.mjs +1 -0
- package/src/normalizeError.js +83 -38
- package/src/retryAfterAware.js +58 -11
- package/src/withDedupe.js +43 -5
- package/src/withProgress.js +76 -17
- package/src/withRetry.js +4 -29
- package/types/index.d.mts +3 -0
- package/types/index.d.ts +25 -1
package/LICENSE
CHANGED
package/README.md
CHANGED
|
@@ -76,6 +76,8 @@ const promise = withRetry(() => fetch(url), { retries: 5, signal: controller.sig
|
|
|
76
76
|
controller.abort(); // rejects with RetryAbortedError, even mid-backoff
|
|
77
77
|
```
|
|
78
78
|
|
|
79
|
+
> **Heads up:** `fetch` doesn't throw on 4xx/5xx, so `withRetry` alone won't retry a `500`. Throw inside `fn` for the statuses you care about (`if (!r.ok) throw ...`), or use `retryAfterAware` below, which retries on status codes.
|
|
80
|
+
|
|
79
81
|
### `withTimeout(fn, ms)`
|
|
80
82
|
|
|
81
83
|
Races an async function against a timeout, passing it an `AbortSignal` so the underlying request can actually be cancelled.
|
|
@@ -93,9 +95,11 @@ const data = await withTimeout(
|
|
|
93
95
|
One consistent error shape regardless of source (fetch, axios, node http, generic):
|
|
94
96
|
|
|
95
97
|
```js
|
|
96
|
-
const { message, status, code, isNetworkError, isTimeout } = normalizeError(err);
|
|
98
|
+
const { message, status, code, isNetworkError, isTimeout, isAborted } = normalizeError(err);
|
|
97
99
|
```
|
|
98
100
|
|
|
101
|
+
Understands Node's `TypeError: fetch failed` (real code read from `err.cause`), browser `Failed to fetch`, undici timeouts, and axios. `isTimeout` is only for real timeouts; a deliberate `AbortController.abort()` sets `isAborted` instead.
|
|
102
|
+
|
|
99
103
|
Every field is always present (`null`/`false` when not applicable) — no existence checks needed.
|
|
100
104
|
|
|
101
105
|
### `parseJsonSafe(input)`
|
|
@@ -116,6 +120,8 @@ buildQuery({ q: "hello world", tag: ["a", "b"] });
|
|
|
116
120
|
// "q=hello%20world&tag=a&tag=b"
|
|
117
121
|
```
|
|
118
122
|
|
|
123
|
+
`Date` values are serialized as ISO 8601. Nested objects throw a `TypeError` (flatten or `JSON.stringify` them first) instead of silently becoming `[object Object]`.
|
|
124
|
+
|
|
119
125
|
For full parsing/nested-object support, use [`query-string`](https://www.npmjs.com/package/query-string) instead — this only covers the common flat-object case with zero dependencies.
|
|
120
126
|
|
|
121
127
|
### `retryAfterAware(fn, opts?)`
|
|
@@ -136,6 +142,11 @@ const response = await retryAfterAware(
|
|
|
136
142
|
| `fallbackDelayMs` | `1000` | used when no usable header is present |
|
|
137
143
|
| `maxDelayMs` | `60000` | |
|
|
138
144
|
| `onRetry` | — | `(status, waitMs, attempt) => void` |
|
|
145
|
+
| `shouldRetryError` | network errors + timeouts | `(err, attempt) => boolean` — decides whether a *thrown* error is retried |
|
|
146
|
+
| `onRetryError` | — | `(err, waitMs, attempt) => void` |
|
|
147
|
+
| `signal` | — | `AbortSignal` — cancels the loop (even mid-wait), throwing `RetryAbortedError` |
|
|
148
|
+
|
|
149
|
+
The body of every response that gets retried is cancelled so the connection is freed.
|
|
139
150
|
|
|
140
151
|
### `withProgress(response, onProgress?)`
|
|
141
152
|
|
|
@@ -147,7 +158,14 @@ const bytes = await withProgress(response, ({ loaded, total, percent }) => {
|
|
|
147
158
|
});
|
|
148
159
|
```
|
|
149
160
|
|
|
150
|
-
Works without `Content-Length` too — `total`/`percent` are just `null` in that case, `loaded` is still accurate. Falls back to a single-shot read (still calling `onProgress` once) if the response body is not a stream.
|
|
161
|
+
Works without `Content-Length` too — `total`/`percent` are just `null` in that case, `loaded` is still accurate. They're also `null` when a `Content-Encoding` is applied (the header is the compressed size, but you receive decompressed bytes), and `percent` never exceeds 100. Falls back to a single-shot read (still calling `onProgress` once) if the response body is not a stream.
|
|
162
|
+
|
|
163
|
+
```js
|
|
164
|
+
const bytes = await withProgress(response, onProgress, {
|
|
165
|
+
maxBytes: 50 * 1024 * 1024, // throws BodyTooLargeError (code "EBODYTOOLARGE") past this
|
|
166
|
+
signal: controller.signal, // abort the read
|
|
167
|
+
});
|
|
168
|
+
```
|
|
151
169
|
|
|
152
170
|
### `withDedupe(fn, opts?)`
|
|
153
171
|
|
|
@@ -164,6 +182,8 @@ const [a, b] = await Promise.all([getUser(1), getUser(1)]);
|
|
|
164
182
|
| --- | --- | --- |
|
|
165
183
|
| `keyFn` | `(...args) => JSON.stringify(args)` | derives the dedupe key from call args |
|
|
166
184
|
|
|
185
|
+
Plain arguments are compared by value (object key order doesn't matter); anything else — `AbortSignal`s, `Response`s, class instances, functions — is compared by identity, so calls with different `AbortSignal`s are never merged. Note that merged callers share one promise, so aborting the request affects everyone waiting on it.
|
|
186
|
+
|
|
167
187
|
This is deduplication of *concurrent* calls, not a cache — once a call settles (success or failure), the next call with the same key starts fresh.
|
|
168
188
|
|
|
169
189
|
```js
|
package/package.json
CHANGED
|
@@ -1,20 +1,26 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@boy-offi9-inc/reqkit",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "Small, composable HTTP helper functions — retry with backoff, timeouts, normalized errors, safe JSON parsing, rate-limit-aware retry, and download progress. Not a client, works alongside fetch/axios/anything.",
|
|
5
5
|
"main": "./src/index.js",
|
|
6
6
|
"types": "./types/index.d.ts",
|
|
7
7
|
"exports": {
|
|
8
8
|
".": {
|
|
9
|
-
"
|
|
10
|
-
|
|
11
|
-
|
|
9
|
+
"import": {
|
|
10
|
+
"types": "./types/index.d.mts",
|
|
11
|
+
"default": "./src/index.mjs"
|
|
12
|
+
},
|
|
13
|
+
"require": {
|
|
14
|
+
"types": "./types/index.d.ts",
|
|
15
|
+
"default": "./src/index.js"
|
|
16
|
+
}
|
|
12
17
|
},
|
|
13
18
|
"./package.json": "./package.json"
|
|
14
19
|
},
|
|
15
20
|
"sideEffects": false,
|
|
16
21
|
"scripts": {
|
|
17
|
-
"test": "node --test"
|
|
22
|
+
"test": "node --test",
|
|
23
|
+
"typecheck": "tsc -p tsconfig.json"
|
|
18
24
|
},
|
|
19
25
|
"keywords": [
|
|
20
26
|
"http",
|
|
@@ -30,6 +36,14 @@
|
|
|
30
36
|
],
|
|
31
37
|
"author": "Boy Offi9",
|
|
32
38
|
"license": "MIT",
|
|
39
|
+
"homepage": "https://github.com/boy-offi9-inc/reqkit#readme",
|
|
40
|
+
"repository": {
|
|
41
|
+
"type": "git",
|
|
42
|
+
"url": "git+https://github.com/boy-offi9-inc/reqkit.git"
|
|
43
|
+
},
|
|
44
|
+
"bugs": {
|
|
45
|
+
"url": "https://github.com/boy-offi9-inc/reqkit/issues"
|
|
46
|
+
},
|
|
33
47
|
"publishConfig": {
|
|
34
48
|
"access": "public"
|
|
35
49
|
},
|
|
@@ -41,5 +55,8 @@
|
|
|
41
55
|
"types",
|
|
42
56
|
"README.md",
|
|
43
57
|
"LICENSE"
|
|
44
|
-
]
|
|
58
|
+
],
|
|
59
|
+
"devDependencies": {
|
|
60
|
+
"typescript": "^5.6.0"
|
|
61
|
+
}
|
|
45
62
|
}
|
package/src/_sleep.js
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
class RetryAbortedError extends Error {
|
|
2
|
+
constructor(cause) {
|
|
3
|
+
super("Retry loop was aborted");
|
|
4
|
+
this.name = "RetryAbortedError";
|
|
5
|
+
if (cause !== undefined) this.cause = cause;
|
|
6
|
+
}
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Abortable sleep shared by withRetry and retryAfterAware. Rejects with
|
|
11
|
+
* RetryAbortedError (carrying `cause`, usually the last failure) if the
|
|
12
|
+
* signal aborts before or during the wait.
|
|
13
|
+
*/
|
|
14
|
+
function sleep(ms, signal, cause) {
|
|
15
|
+
return new Promise((resolve, reject) => {
|
|
16
|
+
if (signal?.aborted) {
|
|
17
|
+
reject(new RetryAbortedError(cause));
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
20
|
+
const onAbort = () => {
|
|
21
|
+
clearTimeout(timer);
|
|
22
|
+
reject(new RetryAbortedError(cause));
|
|
23
|
+
};
|
|
24
|
+
const timer = setTimeout(() => {
|
|
25
|
+
signal?.removeEventListener("abort", onAbort);
|
|
26
|
+
resolve();
|
|
27
|
+
}, ms);
|
|
28
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
module.exports = { RetryAbortedError, sleep };
|
package/src/buildQuery.js
CHANGED
|
@@ -1,6 +1,23 @@
|
|
|
1
|
+
function toParam(key, value) {
|
|
2
|
+
if (value instanceof Date) {
|
|
3
|
+
if (Number.isNaN(value.getTime())) {
|
|
4
|
+
throw new TypeError(`buildQuery: value for "${key}" is an invalid Date`);
|
|
5
|
+
}
|
|
6
|
+
return value.toISOString();
|
|
7
|
+
}
|
|
8
|
+
if (typeof value === "object" || typeof value === "function" || typeof value === "symbol") {
|
|
9
|
+
throw new TypeError(
|
|
10
|
+
`buildQuery: value for "${key}" is not a primitive, Date or array of those — flatten it or JSON.stringify it first`
|
|
11
|
+
);
|
|
12
|
+
}
|
|
13
|
+
return String(value);
|
|
14
|
+
}
|
|
15
|
+
|
|
1
16
|
/**
|
|
2
17
|
* Builds a query string from a params object. Skips null/undefined values,
|
|
3
|
-
* repeats the key for arrays (`?tag=a&tag=b`),
|
|
18
|
+
* repeats the key for arrays (`?tag=a&tag=b`), serializes Dates as ISO 8601,
|
|
19
|
+
* and encodes everything. Nested objects throw a TypeError instead of
|
|
20
|
+
* silently becoming "[object Object]".
|
|
4
21
|
*
|
|
5
22
|
* Deliberately minimal — for full query-string parsing/nested-object support,
|
|
6
23
|
* reach for `query-string` (sindresorhus) instead. This just covers the
|
|
@@ -15,13 +32,10 @@ function buildQuery(params) {
|
|
|
15
32
|
for (const [key, value] of Object.entries(params)) {
|
|
16
33
|
if (value === null || value === undefined) continue;
|
|
17
34
|
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
}
|
|
23
|
-
} else {
|
|
24
|
-
pairs.push(`${encodeURIComponent(key)}=${encodeURIComponent(String(value))}`);
|
|
35
|
+
const values = Array.isArray(value) ? value : [value];
|
|
36
|
+
for (const v of values) {
|
|
37
|
+
if (v === null || v === undefined) continue;
|
|
38
|
+
pairs.push(`${encodeURIComponent(key)}=${encodeURIComponent(toParam(key, v))}`);
|
|
25
39
|
}
|
|
26
40
|
}
|
|
27
41
|
|
package/src/index.js
CHANGED
|
@@ -4,7 +4,7 @@ const { normalizeError } = require("./normalizeError");
|
|
|
4
4
|
const { parseJsonSafe } = require("./parseJsonSafe");
|
|
5
5
|
const { buildQuery } = require("./buildQuery");
|
|
6
6
|
const { retryAfterAware, parseRetryAfter, getWaitMsFromHeaders } = require("./retryAfterAware");
|
|
7
|
-
const { withProgress } = require("./withProgress");
|
|
7
|
+
const { withProgress, BodyTooLargeError } = require("./withProgress");
|
|
8
8
|
const { withDedupe } = require("./withDedupe");
|
|
9
9
|
|
|
10
10
|
module.exports = {
|
|
@@ -19,5 +19,6 @@ module.exports = {
|
|
|
19
19
|
parseRetryAfter,
|
|
20
20
|
getWaitMsFromHeaders,
|
|
21
21
|
withProgress,
|
|
22
|
+
BodyTooLargeError,
|
|
22
23
|
withDedupe,
|
|
23
24
|
};
|
package/src/index.mjs
CHANGED
package/src/normalizeError.js
CHANGED
|
@@ -1,79 +1,124 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Normalizes errors from fetch, axios, node's http/https, and generic
|
|
2
|
+
* Normalizes errors from fetch, axios, node's http/https, undici and generic
|
|
3
3
|
* Error objects into one consistent shape:
|
|
4
|
-
* { message, status, code, isNetworkError, isTimeout, cause }
|
|
4
|
+
* { message, status, code, isNetworkError, isTimeout, isAborted, cause }
|
|
5
5
|
*
|
|
6
6
|
* Every field is always present (null/false when not applicable) so callers
|
|
7
7
|
* can destructure without existence checks.
|
|
8
|
+
*
|
|
9
|
+
* - `isTimeout` is true only for real timeouts (reqkit's TimeoutError,
|
|
10
|
+
* AbortSignal.timeout(), ETIMEDOUT, undici timeouts) — not for a plain
|
|
11
|
+
* user-initiated abort.
|
|
12
|
+
* - `isAborted` is true when the request was cancelled on purpose
|
|
13
|
+
* (AbortController.abort(), axios cancel).
|
|
8
14
|
*/
|
|
9
|
-
|
|
10
|
-
|
|
15
|
+
|
|
16
|
+
function shape(err, overrides) {
|
|
17
|
+
return {
|
|
11
18
|
message: "Unknown error",
|
|
12
19
|
status: null,
|
|
13
20
|
code: null,
|
|
14
21
|
isNetworkError: false,
|
|
15
22
|
isTimeout: false,
|
|
23
|
+
isAborted: false,
|
|
16
24
|
cause: err,
|
|
25
|
+
...overrides,
|
|
17
26
|
};
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// Node's fetch (undici) throws `TypeError: fetch failed` and puts the real
|
|
30
|
+
// reason (ECONNREFUSED, ENOTFOUND, UND_ERR_*...) on `err.cause`, sometimes
|
|
31
|
+
// inside an AggregateError. Walk the chain to find the first string code.
|
|
32
|
+
function findCode(err) {
|
|
33
|
+
const seen = new Set();
|
|
34
|
+
const queue = [err];
|
|
35
|
+
while (queue.length && seen.size < 10) {
|
|
36
|
+
const cur = queue.shift();
|
|
37
|
+
if (!cur || typeof cur !== "object" || seen.has(cur)) continue;
|
|
38
|
+
seen.add(cur);
|
|
39
|
+
if (typeof cur.code === "string") return cur.code;
|
|
40
|
+
queue.push(cur.cause);
|
|
41
|
+
if (Array.isArray(cur.errors)) queue.push(...cur.errors);
|
|
42
|
+
}
|
|
43
|
+
return null;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function isTimeoutCode(code) {
|
|
47
|
+
return code === "ETIMEDOUT" || code === "ESOCKETTIMEDOUT" || /^UND_ERR_.*TIMEOUT$/.test(code || "");
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function isNetworkCode(code) {
|
|
51
|
+
if (!code) return false;
|
|
52
|
+
return /^E[A-Z0-9]+$/.test(code) || code === "EAI_AGAIN" || code === "EAI_FAIL" || code.startsWith("UND_ERR_");
|
|
53
|
+
}
|
|
18
54
|
|
|
19
|
-
|
|
55
|
+
// Browsers and Node throw a bare TypeError for network-level fetch failures.
|
|
56
|
+
const FETCH_NETWORK_MESSAGE =
|
|
57
|
+
/fetch failed|failed to fetch|network ?request failed|load failed|networkerror when attempting to fetch/i;
|
|
58
|
+
|
|
59
|
+
function isFetchNetworkTypeError(err) {
|
|
60
|
+
return err.name === "TypeError" && FETCH_NETWORK_MESSAGE.test(err.message || "");
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function normalizeError(err) {
|
|
64
|
+
if (!err) return shape(err);
|
|
20
65
|
|
|
21
66
|
// axios error shape: err.response.{status,data}, err.request, err.code
|
|
22
67
|
if (err.isAxiosError || (err.response && typeof err.response.status === "number" && err.config)) {
|
|
23
|
-
return {
|
|
68
|
+
return shape(err, {
|
|
24
69
|
message: err.response?.data?.message || err.message || "Request failed",
|
|
25
70
|
status: err.response?.status ?? null,
|
|
26
71
|
code: err.code ?? null,
|
|
27
72
|
isNetworkError: !err.response && Boolean(err.request),
|
|
28
|
-
isTimeout: err.code === "ECONNABORTED" || /timeout/i.test(err.message || ""),
|
|
29
|
-
|
|
30
|
-
};
|
|
73
|
+
isTimeout: err.code === "ECONNABORTED" || err.code === "ETIMEDOUT" || /timeout/i.test(err.message || ""),
|
|
74
|
+
isAborted: err.code === "ERR_CANCELED" || err.name === "CanceledError",
|
|
75
|
+
});
|
|
31
76
|
}
|
|
32
77
|
|
|
33
78
|
// fetch Response passed in directly (not thrown, but common enough to handle)
|
|
34
79
|
if (typeof err.status === "number" && typeof err.ok === "boolean") {
|
|
35
|
-
return {
|
|
80
|
+
return shape(err, {
|
|
36
81
|
message: err.statusText || `Request failed with status ${err.status}`,
|
|
37
82
|
status: err.status,
|
|
38
|
-
|
|
39
|
-
isNetworkError: false,
|
|
40
|
-
isTimeout: false,
|
|
41
|
-
cause: err,
|
|
42
|
-
};
|
|
83
|
+
});
|
|
43
84
|
}
|
|
44
85
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
86
|
+
const chainCode = findCode(err);
|
|
87
|
+
|
|
88
|
+
// reqkit's TimeoutError, AbortSignal.timeout() (DOMException TimeoutError),
|
|
89
|
+
// node ETIMEDOUT, undici connect/headers/body timeouts
|
|
90
|
+
if (err.name === "TimeoutError" || isTimeoutCode(chainCode)) {
|
|
91
|
+
return shape(err, {
|
|
48
92
|
message: err.message || "Request timed out",
|
|
49
|
-
|
|
50
|
-
code: "ETIMEDOUT",
|
|
51
|
-
isNetworkError: false,
|
|
93
|
+
code: chainCode || "ETIMEDOUT",
|
|
52
94
|
isTimeout: true,
|
|
53
|
-
|
|
54
|
-
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// Deliberate cancellation via AbortController — not a timeout.
|
|
99
|
+
if (err.name === "AbortError") {
|
|
100
|
+
return shape(err, {
|
|
101
|
+
message: err.message || "Request aborted",
|
|
102
|
+
code: "ABORT_ERR",
|
|
103
|
+
isAborted: true,
|
|
104
|
+
});
|
|
55
105
|
}
|
|
56
106
|
|
|
57
|
-
// node http/https / undici network-level failures
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
code:
|
|
107
|
+
// node http/https / undici network-level failures, including
|
|
108
|
+
// `TypeError: fetch failed` with the real code on err.cause
|
|
109
|
+
if (isNetworkCode(chainCode) || isFetchNetworkTypeError(err)) {
|
|
110
|
+
return shape(err, {
|
|
111
|
+
message: err.message || `Network error (${chainCode})`,
|
|
112
|
+
code: chainCode,
|
|
63
113
|
isNetworkError: true,
|
|
64
|
-
|
|
65
|
-
cause: err,
|
|
66
|
-
};
|
|
114
|
+
});
|
|
67
115
|
}
|
|
68
116
|
|
|
69
|
-
return {
|
|
117
|
+
return shape(err, {
|
|
70
118
|
message: err.message || String(err),
|
|
71
|
-
status: err.status ?? err.statusCode ?? null,
|
|
119
|
+
status: err.status ?? err.statusCode ?? err.response?.status ?? err.response?.statusCode ?? null,
|
|
72
120
|
code: err.code ?? null,
|
|
73
|
-
|
|
74
|
-
isTimeout: false,
|
|
75
|
-
cause: err,
|
|
76
|
-
};
|
|
121
|
+
});
|
|
77
122
|
}
|
|
78
123
|
|
|
79
124
|
module.exports = { normalizeError };
|
package/src/retryAfterAware.js
CHANGED
|
@@ -1,20 +1,21 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
}
|
|
1
|
+
const { RetryAbortedError, sleep } = require("./_sleep");
|
|
2
|
+
const { normalizeError } = require("./normalizeError");
|
|
4
3
|
|
|
5
4
|
/**
|
|
6
5
|
* Parses a Retry-After header value into milliseconds. Per HTTP spec it's
|
|
7
6
|
* either an integer number of seconds, or an HTTP-date.
|
|
8
7
|
*/
|
|
9
8
|
function parseRetryAfter(headerValue) {
|
|
10
|
-
if (
|
|
9
|
+
if (headerValue === null || headerValue === undefined) return null;
|
|
10
|
+
const value = String(headerValue).trim();
|
|
11
|
+
if (!value) return null;
|
|
11
12
|
|
|
12
|
-
|
|
13
|
-
if (
|
|
14
|
-
return
|
|
13
|
+
// delta-seconds (spec says integer; tolerate "1.5" from sloppy servers)
|
|
14
|
+
if (/^\d+(\.\d+)?$/.test(value)) {
|
|
15
|
+
return Number(value) * 1000;
|
|
15
16
|
}
|
|
16
17
|
|
|
17
|
-
const dateMs = Date.parse(
|
|
18
|
+
const dateMs = Date.parse(value);
|
|
18
19
|
if (!Number.isNaN(dateMs)) {
|
|
19
20
|
return Math.max(0, dateMs - Date.now());
|
|
20
21
|
}
|
|
@@ -46,22 +47,50 @@ function getWaitMsFromHeaders(headers) {
|
|
|
46
47
|
return null;
|
|
47
48
|
}
|
|
48
49
|
|
|
50
|
+
function defaultShouldRetryError(err) {
|
|
51
|
+
const normalized = normalizeError(err);
|
|
52
|
+
return normalized.isNetworkError || normalized.isTimeout;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
async function discardBody(response) {
|
|
56
|
+
// An unread body keeps the underlying connection busy; cancel it before
|
|
57
|
+
// we sit through a backoff and fire the next request.
|
|
58
|
+
try {
|
|
59
|
+
if (response.body && typeof response.body.cancel === "function" && !response.bodyUsed) {
|
|
60
|
+
await response.body.cancel();
|
|
61
|
+
}
|
|
62
|
+
} catch {
|
|
63
|
+
// locked / already consumed / not a web stream — safe to ignore
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
49
67
|
/**
|
|
50
68
|
* Retries a fetch-like call, honoring Retry-After / X-RateLimit-Reset
|
|
51
69
|
* headers on 429/503 responses instead of blind exponential backoff — most
|
|
52
70
|
* generic retry libraries treat rate-limit responses like any other failure
|
|
53
71
|
* and ignore the header the server is explicitly giving you.
|
|
54
72
|
*
|
|
73
|
+
* Also retries thrown network errors / timeouts (see `shouldRetryError`),
|
|
74
|
+
* and the whole loop — including any backoff wait — can be cancelled with
|
|
75
|
+
* `signal`. The body of each response that gets retried is cancelled so the
|
|
76
|
+
* connection is released.
|
|
77
|
+
*
|
|
55
78
|
* `fn` should return a fetch Response (or anything with `.status` and
|
|
56
79
|
* `.headers.get()`).
|
|
57
80
|
*
|
|
58
|
-
* @param {() => Promise<Response>} fn
|
|
81
|
+
* @param {(attempt: number) => Promise<Response>} fn
|
|
59
82
|
* @param {object} [opts]
|
|
60
83
|
* @param {number} [opts.retries=3]
|
|
61
84
|
* @param {number[]} [opts.retryStatusCodes=[429, 503]]
|
|
62
85
|
* @param {number} [opts.fallbackDelayMs=1000] - used when no usable header is present
|
|
63
86
|
* @param {number} [opts.maxDelayMs=60000]
|
|
64
87
|
* @param {(status: number, waitMs: number, attempt: number) => void} [opts.onRetry]
|
|
88
|
+
* @param {(err: any, attempt: number) => boolean} [opts.shouldRetryError] -
|
|
89
|
+
* decides whether a thrown error is retried. Default: network errors and
|
|
90
|
+
* timeouts only.
|
|
91
|
+
* @param {(err: any, waitMs: number, attempt: number) => void} [opts.onRetryError]
|
|
92
|
+
* @param {AbortSignal} [opts.signal] - cancels the retry loop, throwing
|
|
93
|
+
* RetryAbortedError
|
|
65
94
|
*/
|
|
66
95
|
async function retryAfterAware(fn, opts = {}) {
|
|
67
96
|
const {
|
|
@@ -70,10 +99,27 @@ async function retryAfterAware(fn, opts = {}) {
|
|
|
70
99
|
fallbackDelayMs = 1000,
|
|
71
100
|
maxDelayMs = 60000,
|
|
72
101
|
onRetry,
|
|
102
|
+
shouldRetryError = defaultShouldRetryError,
|
|
103
|
+
onRetryError,
|
|
104
|
+
signal,
|
|
73
105
|
} = opts;
|
|
74
106
|
|
|
107
|
+
let lastErr;
|
|
75
108
|
for (let attempt = 0; attempt <= retries; attempt++) {
|
|
76
|
-
|
|
109
|
+
if (signal?.aborted) throw new RetryAbortedError(lastErr);
|
|
110
|
+
|
|
111
|
+
let response;
|
|
112
|
+
try {
|
|
113
|
+
response = await fn(attempt);
|
|
114
|
+
} catch (err) {
|
|
115
|
+
lastErr = err;
|
|
116
|
+
if (attempt === retries || !shouldRetryError(err, attempt)) throw err;
|
|
117
|
+
|
|
118
|
+
const waitMs = Math.min(fallbackDelayMs * 2 ** attempt, maxDelayMs);
|
|
119
|
+
if (onRetryError) onRetryError(err, waitMs, attempt);
|
|
120
|
+
await sleep(waitMs, signal, err);
|
|
121
|
+
continue;
|
|
122
|
+
}
|
|
77
123
|
|
|
78
124
|
if (!retryStatusCodes.includes(response.status) || attempt === retries) {
|
|
79
125
|
return response;
|
|
@@ -82,8 +128,9 @@ async function retryAfterAware(fn, opts = {}) {
|
|
|
82
128
|
const headerWaitMs = getWaitMsFromHeaders(response.headers);
|
|
83
129
|
const waitMs = Math.min(headerWaitMs ?? fallbackDelayMs * 2 ** attempt, maxDelayMs);
|
|
84
130
|
|
|
131
|
+
await discardBody(response);
|
|
85
132
|
if (onRetry) onRetry(response.status, waitMs, attempt);
|
|
86
|
-
await
|
|
133
|
+
await sleep(waitMs, signal);
|
|
87
134
|
}
|
|
88
135
|
}
|
|
89
136
|
|
package/src/withDedupe.js
CHANGED
|
@@ -1,10 +1,45 @@
|
|
|
1
|
+
// Non-plain objects (AbortSignal, Response, class instances, functions...)
|
|
2
|
+
// can't be meaningfully JSON-serialized — JSON.stringify turns every
|
|
3
|
+
// AbortSignal into "{}", which would make unrelated calls share one request.
|
|
4
|
+
// Give them a stable identity-based id instead.
|
|
5
|
+
const identityIds = new WeakMap();
|
|
6
|
+
let nextIdentityId = 1;
|
|
7
|
+
|
|
8
|
+
function identityOf(value) {
|
|
9
|
+
let id = identityIds.get(value);
|
|
10
|
+
if (id === undefined) {
|
|
11
|
+
id = nextIdentityId++;
|
|
12
|
+
identityIds.set(value, id);
|
|
13
|
+
}
|
|
14
|
+
return id;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function isPlainObject(value) {
|
|
18
|
+
const proto = Object.getPrototypeOf(value);
|
|
19
|
+
return proto === Object.prototype || proto === null;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function keyReplacer(_key, value) {
|
|
23
|
+
if (typeof value === "bigint") return { $bigint: String(value) };
|
|
24
|
+
if (typeof value === "function") return { $ref: identityOf(value) };
|
|
25
|
+
if (value === null || typeof value !== "object") return value;
|
|
26
|
+
if (Array.isArray(value)) return value;
|
|
27
|
+
if (!isPlainObject(value)) return { $ref: identityOf(value) };
|
|
28
|
+
|
|
29
|
+
// Sort plain-object keys so {a:1,b:2} and {b:2,a:1} dedupe together.
|
|
30
|
+
const sorted = {};
|
|
31
|
+
for (const k of Object.keys(value).sort()) sorted[k] = value[k];
|
|
32
|
+
return sorted;
|
|
33
|
+
}
|
|
34
|
+
|
|
1
35
|
function defaultKeyFn(...args) {
|
|
2
36
|
try {
|
|
3
|
-
return JSON.stringify(args);
|
|
37
|
+
return JSON.stringify(args, keyReplacer);
|
|
4
38
|
} catch {
|
|
5
|
-
// Fall back to
|
|
6
|
-
|
|
7
|
-
|
|
39
|
+
// Circular structures etc. Fall back to identity for objects.
|
|
40
|
+
return args
|
|
41
|
+
.map((a) => ((a && typeof a === "object") || typeof a === "function" ? `#${identityOf(a)}` : String(a)))
|
|
42
|
+
.join("|");
|
|
8
43
|
}
|
|
9
44
|
}
|
|
10
45
|
|
|
@@ -22,7 +57,10 @@ function defaultKeyFn(...args) {
|
|
|
22
57
|
* @param {(...args: any[]) => Promise<any>} fn
|
|
23
58
|
* @param {object} [opts]
|
|
24
59
|
* @param {(...args: any[]) => string} [opts.keyFn] - derives a dedupe key
|
|
25
|
-
* from the call arguments.
|
|
60
|
+
* from the call arguments. By default plain data is compared by value
|
|
61
|
+
* (object key order doesn't matter) and anything else — AbortSignals,
|
|
62
|
+
* Responses, class instances, functions — by identity, so two calls with
|
|
63
|
+
* different AbortSignals are never merged.
|
|
26
64
|
* @returns {(...args: any[]) => Promise<any>}
|
|
27
65
|
*/
|
|
28
66
|
function withDedupe(fn, opts = {}) {
|
package/src/withProgress.js
CHANGED
|
@@ -1,48 +1,107 @@
|
|
|
1
|
+
class BodyTooLargeError extends Error {
|
|
2
|
+
constructor(maxBytes) {
|
|
3
|
+
super(`Response body exceeded the ${maxBytes}-byte limit`);
|
|
4
|
+
this.name = "BodyTooLargeError";
|
|
5
|
+
this.code = "EBODYTOOLARGE";
|
|
6
|
+
this.maxBytes = maxBytes;
|
|
7
|
+
}
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
function cancelQuietly(reader) {
|
|
11
|
+
try {
|
|
12
|
+
const p = reader.cancel();
|
|
13
|
+
if (p && typeof p.catch === "function") p.catch(() => {});
|
|
14
|
+
} catch {
|
|
15
|
+
// already closed / released — nothing to do
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
|
|
1
19
|
/**
|
|
2
20
|
* Reads a fetch Response body while reporting progress, and returns the
|
|
3
21
|
* fully collected bytes as a Uint8Array. Works for any streamed response,
|
|
4
22
|
* not just file downloads — same shape whether it's a media file, a large
|
|
5
23
|
* JSON payload, or anything else worth showing progress for.
|
|
6
24
|
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
25
|
+
* `total` is null (and `percent` with it) when the size isn't trustworthy:
|
|
26
|
+
* no Content-Length, or a Content-Encoding is applied (Content-Length is then
|
|
27
|
+
* the *compressed* size while fetch hands you *decompressed* bytes, so
|
|
28
|
+
* loaded could exceed total). `percent` is always clamped to 0–100.
|
|
9
29
|
*
|
|
10
30
|
* @param {Response} response
|
|
11
|
-
* @param {(progress: { loaded: number, total: number | null, percent: number | null }) => void} onProgress
|
|
31
|
+
* @param {(progress: { loaded: number, total: number | null, percent: number | null }) => void} [onProgress]
|
|
32
|
+
* @param {object} [opts]
|
|
33
|
+
* @param {number} [opts.maxBytes] - abort with BodyTooLargeError once more
|
|
34
|
+
* than this many bytes have been received
|
|
35
|
+
* @param {AbortSignal} [opts.signal] - cancels the read; rejects with the
|
|
36
|
+
* signal's reason (an AbortError by default)
|
|
12
37
|
* @returns {Promise<Uint8Array>}
|
|
13
38
|
*/
|
|
14
|
-
async function withProgress(response, onProgress) {
|
|
39
|
+
async function withProgress(response, onProgress, opts = {}) {
|
|
40
|
+
const { maxBytes, signal } = opts;
|
|
41
|
+
signal?.throwIfAborted?.();
|
|
42
|
+
|
|
15
43
|
if (!response.body || typeof response.body.getReader !== "function") {
|
|
16
44
|
// No streaming support available — fall back to a single-shot read,
|
|
17
45
|
// still reporting one final progress event so callers don't need a branch.
|
|
18
46
|
const buf = new Uint8Array(await response.arrayBuffer());
|
|
47
|
+
signal?.throwIfAborted?.();
|
|
48
|
+
if (maxBytes !== undefined && buf.byteLength > maxBytes) {
|
|
49
|
+
throw new BodyTooLargeError(maxBytes);
|
|
50
|
+
}
|
|
19
51
|
if (onProgress) {
|
|
20
52
|
onProgress({ loaded: buf.byteLength, total: buf.byteLength, percent: 100 });
|
|
21
53
|
}
|
|
22
54
|
return buf;
|
|
23
55
|
}
|
|
24
56
|
|
|
57
|
+
const encoding = response.headers?.get?.("content-encoding");
|
|
58
|
+
const encoded = Boolean(encoding) && encoding.trim().toLowerCase() !== "identity";
|
|
25
59
|
const contentLength = response.headers?.get?.("content-length");
|
|
26
|
-
const
|
|
60
|
+
const parsed = contentLength ? Number(contentLength) : NaN;
|
|
61
|
+
const total = !encoded && Number.isFinite(parsed) && parsed >= 0 ? parsed : null;
|
|
27
62
|
|
|
28
63
|
const reader = response.body.getReader();
|
|
64
|
+
|
|
65
|
+
// Fail fast when the declared size already exceeds the cap.
|
|
66
|
+
if (maxBytes !== undefined && total !== null && total > maxBytes) {
|
|
67
|
+
cancelQuietly(reader);
|
|
68
|
+
throw new BodyTooLargeError(maxBytes);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const onAbort = () => cancelQuietly(reader);
|
|
72
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
73
|
+
|
|
29
74
|
const chunks = [];
|
|
30
75
|
let loaded = 0;
|
|
76
|
+
let finished = false;
|
|
31
77
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
78
|
+
try {
|
|
79
|
+
for (;;) {
|
|
80
|
+
const { done, value } = await reader.read();
|
|
81
|
+
// reader.cancel() on abort makes read() resolve as done — check the
|
|
82
|
+
// signal first so an aborted read never looks like a complete one.
|
|
83
|
+
signal?.throwIfAborted?.();
|
|
84
|
+
if (done) break;
|
|
35
85
|
|
|
36
|
-
|
|
37
|
-
|
|
86
|
+
chunks.push(value);
|
|
87
|
+
loaded += value.byteLength;
|
|
38
88
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
89
|
+
if (maxBytes !== undefined && loaded > maxBytes) {
|
|
90
|
+
throw new BodyTooLargeError(maxBytes);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
if (onProgress) {
|
|
94
|
+
onProgress({
|
|
95
|
+
loaded,
|
|
96
|
+
total,
|
|
97
|
+
percent: total ? Math.min(100, Math.round((loaded / total) * 100)) : null,
|
|
98
|
+
});
|
|
99
|
+
}
|
|
45
100
|
}
|
|
101
|
+
finished = true;
|
|
102
|
+
} finally {
|
|
103
|
+
signal?.removeEventListener("abort", onAbort);
|
|
104
|
+
if (!finished) cancelQuietly(reader);
|
|
46
105
|
}
|
|
47
106
|
|
|
48
107
|
const result = new Uint8Array(loaded);
|
|
@@ -54,4 +113,4 @@ async function withProgress(response, onProgress) {
|
|
|
54
113
|
return result;
|
|
55
114
|
}
|
|
56
115
|
|
|
57
|
-
module.exports = { withProgress };
|
|
116
|
+
module.exports = { withProgress, BodyTooLargeError };
|
package/src/withRetry.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
const { RetryAbortedError, sleep } = require("./_sleep");
|
|
2
|
+
|
|
1
3
|
function defaultShouldRetry(err) {
|
|
2
4
|
// Retry network-level failures by default. HTTP status handling is the
|
|
3
5
|
// caller's job (they know what "failure" means for their response shape) —
|
|
@@ -5,33 +7,6 @@ function defaultShouldRetry(err) {
|
|
|
5
7
|
return Boolean(err) && err.name !== "AbortError";
|
|
6
8
|
}
|
|
7
9
|
|
|
8
|
-
class RetryAbortedError extends Error {
|
|
9
|
-
constructor() {
|
|
10
|
-
super("Retry loop was aborted");
|
|
11
|
-
this.name = "RetryAbortedError";
|
|
12
|
-
}
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
function delay(ms, signal) {
|
|
16
|
-
return new Promise((resolve, reject) => {
|
|
17
|
-
if (signal?.aborted) {
|
|
18
|
-
reject(new RetryAbortedError());
|
|
19
|
-
return;
|
|
20
|
-
}
|
|
21
|
-
const timer = setTimeout(resolve, ms);
|
|
22
|
-
if (signal) {
|
|
23
|
-
signal.addEventListener(
|
|
24
|
-
"abort",
|
|
25
|
-
() => {
|
|
26
|
-
clearTimeout(timer);
|
|
27
|
-
reject(new RetryAbortedError());
|
|
28
|
-
},
|
|
29
|
-
{ once: true }
|
|
30
|
-
);
|
|
31
|
-
}
|
|
32
|
-
});
|
|
33
|
-
}
|
|
34
|
-
|
|
35
10
|
/**
|
|
36
11
|
* Retries an async function with exponential backoff + jitter.
|
|
37
12
|
*
|
|
@@ -60,7 +35,7 @@ async function withRetry(fn, opts = {}) {
|
|
|
60
35
|
let lastErr;
|
|
61
36
|
for (let attempt = 0; attempt <= retries; attempt++) {
|
|
62
37
|
if (signal?.aborted) {
|
|
63
|
-
throw new RetryAbortedError();
|
|
38
|
+
throw new RetryAbortedError(lastErr);
|
|
64
39
|
}
|
|
65
40
|
try {
|
|
66
41
|
return await fn(attempt);
|
|
@@ -74,7 +49,7 @@ async function withRetry(fn, opts = {}) {
|
|
|
74
49
|
const jitter = Math.random() * exponential * 0.2;
|
|
75
50
|
const waitMs = Math.round(exponential + jitter);
|
|
76
51
|
if (onRetry) onRetry(err, attempt, waitMs);
|
|
77
|
-
await
|
|
52
|
+
await sleep(waitMs, signal, err);
|
|
78
53
|
}
|
|
79
54
|
}
|
|
80
55
|
throw lastErr;
|
package/types/index.d.ts
CHANGED
|
@@ -9,6 +9,8 @@ export interface RetryOptions {
|
|
|
9
9
|
|
|
10
10
|
export class RetryAbortedError extends Error {
|
|
11
11
|
name: "RetryAbortedError";
|
|
12
|
+
/** The last failure before the loop was aborted, if there was one. */
|
|
13
|
+
cause?: unknown;
|
|
12
14
|
}
|
|
13
15
|
|
|
14
16
|
export function withRetry<T>(fn: (attempt: number) => Promise<T>, opts?: RetryOptions): Promise<T>;
|
|
@@ -26,6 +28,8 @@ export interface NormalizedError {
|
|
|
26
28
|
code: string | null;
|
|
27
29
|
isNetworkError: boolean;
|
|
28
30
|
isTimeout: boolean;
|
|
31
|
+
/** True when the request was cancelled on purpose (AbortController, axios cancel). */
|
|
32
|
+
isAborted: boolean;
|
|
29
33
|
cause: unknown;
|
|
30
34
|
}
|
|
31
35
|
|
|
@@ -46,6 +50,11 @@ export interface RetryAfterAwareOptions {
|
|
|
46
50
|
fallbackDelayMs?: number;
|
|
47
51
|
maxDelayMs?: number;
|
|
48
52
|
onRetry?: (status: number, waitMs: number, attempt: number) => void;
|
|
53
|
+
/** Decide whether a thrown error is retried. Default: network errors and timeouts only. */
|
|
54
|
+
shouldRetryError?: (err: unknown, attempt: number) => boolean;
|
|
55
|
+
onRetryError?: (err: unknown, waitMs: number, attempt: number) => void;
|
|
56
|
+
/** Cancels the retry loop, including any pending wait. Throws RetryAbortedError. */
|
|
57
|
+
signal?: AbortSignal;
|
|
49
58
|
}
|
|
50
59
|
|
|
51
60
|
export function retryAfterAware(
|
|
@@ -62,9 +71,24 @@ export interface DownloadProgress {
|
|
|
62
71
|
percent: number | null;
|
|
63
72
|
}
|
|
64
73
|
|
|
74
|
+
export interface WithProgressOptions {
|
|
75
|
+
/** Reject with BodyTooLargeError once more than this many bytes arrive. */
|
|
76
|
+
maxBytes?: number;
|
|
77
|
+
/** Cancels the read; rejects with the signal's reason. */
|
|
78
|
+
signal?: AbortSignal;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export class BodyTooLargeError extends Error {
|
|
82
|
+
name: "BodyTooLargeError";
|
|
83
|
+
code: "EBODYTOOLARGE";
|
|
84
|
+
maxBytes: number;
|
|
85
|
+
constructor(maxBytes: number);
|
|
86
|
+
}
|
|
87
|
+
|
|
65
88
|
export function withProgress(
|
|
66
89
|
response: Response,
|
|
67
|
-
onProgress?: (progress: DownloadProgress) => void
|
|
90
|
+
onProgress?: (progress: DownloadProgress) => void,
|
|
91
|
+
opts?: WithProgressOptions
|
|
68
92
|
): Promise<Uint8Array>;
|
|
69
93
|
|
|
70
94
|
export interface DedupeOptions {
|