@naturalcycles/js-lib 14.147.0 → 14.149.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/http/fetcher.d.ts +20 -19
- package/dist/http/fetcher.js +33 -10
- package/dist/http/fetcher.model.d.ts +3 -18
- package/dist-esm/http/fetcher.js +33 -10
- package/package.json +1 -1
- package/src/http/fetcher.model.ts +4 -22
- package/src/http/fetcher.ts +63 -44
package/dist/http/fetcher.d.ts
CHANGED
|
@@ -14,36 +14,36 @@ export declare class Fetcher {
|
|
|
14
14
|
onBeforeRetry(hook: FetcherBeforeRetryHook): this;
|
|
15
15
|
cfg: FetcherNormalizedCfg;
|
|
16
16
|
static create(cfg?: FetcherCfg & FetcherOptions): Fetcher;
|
|
17
|
-
get: <T = unknown>(url: string, opt?: FetcherOptions
|
|
18
|
-
post: <T = unknown>(url: string, opt?: FetcherOptions
|
|
19
|
-
put: <T = unknown>(url: string, opt?: FetcherOptions
|
|
20
|
-
patch: <T = unknown>(url: string, opt?: FetcherOptions
|
|
21
|
-
delete: <T = unknown>(url: string, opt?: FetcherOptions
|
|
22
|
-
getText: (url: string, opt?: FetcherOptions
|
|
23
|
-
postText: (url: string, opt?: FetcherOptions
|
|
24
|
-
putText: (url: string, opt?: FetcherOptions
|
|
25
|
-
patchText: (url: string, opt?: FetcherOptions
|
|
26
|
-
deleteText: (url: string, opt?: FetcherOptions
|
|
27
|
-
getVoid: (url: string, opt?: FetcherOptions
|
|
28
|
-
postVoid: (url: string, opt?: FetcherOptions
|
|
29
|
-
putVoid: (url: string, opt?: FetcherOptions
|
|
30
|
-
patchVoid: (url: string, opt?: FetcherOptions
|
|
31
|
-
deleteVoid: (url: string, opt?: FetcherOptions
|
|
32
|
-
headVoid: (url: string, opt?: FetcherOptions
|
|
17
|
+
get: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>;
|
|
18
|
+
post: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>;
|
|
19
|
+
put: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>;
|
|
20
|
+
patch: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>;
|
|
21
|
+
delete: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>;
|
|
22
|
+
getText: (url: string, opt?: FetcherOptions) => Promise<string>;
|
|
23
|
+
postText: (url: string, opt?: FetcherOptions) => Promise<string>;
|
|
24
|
+
putText: (url: string, opt?: FetcherOptions) => Promise<string>;
|
|
25
|
+
patchText: (url: string, opt?: FetcherOptions) => Promise<string>;
|
|
26
|
+
deleteText: (url: string, opt?: FetcherOptions) => Promise<string>;
|
|
27
|
+
getVoid: (url: string, opt?: FetcherOptions) => Promise<void>;
|
|
28
|
+
postVoid: (url: string, opt?: FetcherOptions) => Promise<void>;
|
|
29
|
+
putVoid: (url: string, opt?: FetcherOptions) => Promise<void>;
|
|
30
|
+
patchVoid: (url: string, opt?: FetcherOptions) => Promise<void>;
|
|
31
|
+
deleteVoid: (url: string, opt?: FetcherOptions) => Promise<void>;
|
|
32
|
+
headVoid: (url: string, opt?: FetcherOptions) => Promise<void>;
|
|
33
33
|
/**
|
|
34
34
|
* Returns raw fetchResponse.body, which is a ReadableStream<Uint8Array>
|
|
35
35
|
*
|
|
36
36
|
* More on streams and Node interop:
|
|
37
37
|
* https://css-tricks.com/web-streams-everywhere-and-fetch-for-node-js/
|
|
38
38
|
*/
|
|
39
|
-
getReadableStream(url: string, opt?: FetcherOptions
|
|
40
|
-
fetch<T = unknown>(opt: FetcherOptions
|
|
39
|
+
getReadableStream(url: string, opt?: FetcherOptions): Promise<ReadableStream<Uint8Array>>;
|
|
40
|
+
fetch<T = unknown>(opt: FetcherOptions): Promise<T>;
|
|
41
41
|
/**
|
|
42
42
|
* Returns FetcherResponse.
|
|
43
43
|
* Never throws, returns `err` property in the response instead.
|
|
44
44
|
* Use this method instead of `throwHttpErrors: false` or try-catching.
|
|
45
45
|
*/
|
|
46
|
-
doFetch<T = unknown>(opt: FetcherOptions
|
|
46
|
+
doFetch<T = unknown>(opt: FetcherOptions): Promise<FetcherResponse<T>>;
|
|
47
47
|
private onOkResponse;
|
|
48
48
|
/**
|
|
49
49
|
* This method exists to be able to easily mock it.
|
|
@@ -51,6 +51,7 @@ export declare class Fetcher {
|
|
|
51
51
|
callNativeFetch(url: string, init: RequestInit): Promise<Response>;
|
|
52
52
|
private onNotOkResponse;
|
|
53
53
|
private processRetry;
|
|
54
|
+
private getRetryTimeout;
|
|
54
55
|
/**
|
|
55
56
|
* Default is yes,
|
|
56
57
|
* unless there's reason not to (e.g method is POST).
|
package/dist/http/fetcher.js
CHANGED
|
@@ -151,6 +151,8 @@ class Fetcher {
|
|
|
151
151
|
try {
|
|
152
152
|
res.fetchResponse = await this.callNativeFetch(req.fullUrl, req.init);
|
|
153
153
|
res.ok = res.fetchResponse.ok;
|
|
154
|
+
// important to set it to undefined, otherwise it can keep the previous value (from previous try)
|
|
155
|
+
res.err = undefined;
|
|
154
156
|
}
|
|
155
157
|
catch (err) {
|
|
156
158
|
// For example, CORS error would result in "TypeError: failed to fetch" here
|
|
@@ -171,12 +173,6 @@ class Fetcher {
|
|
|
171
173
|
for (const hook of this.cfg.hooks.afterResponse || []) {
|
|
172
174
|
await hook(res);
|
|
173
175
|
}
|
|
174
|
-
if (req.paginate && res.ok) {
|
|
175
|
-
const proceeed = await req.paginate(res, opt);
|
|
176
|
-
if (proceeed) {
|
|
177
|
-
return await this.doFetch(opt);
|
|
178
|
-
}
|
|
179
|
-
}
|
|
180
176
|
return res;
|
|
181
177
|
}
|
|
182
178
|
async onOkResponse(res, timeout) {
|
|
@@ -308,12 +304,11 @@ class Fetcher {
|
|
|
308
304
|
// but we should log all previous errors, otherwise they are lost.
|
|
309
305
|
// Here is the right place where we know it's not the "last error"
|
|
310
306
|
if (res.err) {
|
|
311
|
-
const { retryAttempt } = retryStatus;
|
|
312
307
|
this.cfg.logger.error([
|
|
313
308
|
' <<',
|
|
314
309
|
res.fetchResponse?.status || 0,
|
|
315
310
|
res.signature,
|
|
316
|
-
`try#${retryAttempt + 1}/${count + 1}`,
|
|
311
|
+
`try#${retryStatus.retryAttempt + 1}/${count + 1}`,
|
|
317
312
|
(0, time_util_1._since)(res.req.started),
|
|
318
313
|
]
|
|
319
314
|
.filter(Boolean)
|
|
@@ -321,8 +316,36 @@ class Fetcher {
|
|
|
321
316
|
}
|
|
322
317
|
retryStatus.retryAttempt++;
|
|
323
318
|
retryStatus.retryTimeout = (0, number_util_1._clamp)(retryStatus.retryTimeout * timeoutMultiplier, 0, timeoutMax);
|
|
324
|
-
|
|
325
|
-
|
|
319
|
+
await (0, pDelay_1.pDelay)(this.getRetryTimeout(res));
|
|
320
|
+
}
|
|
321
|
+
getRetryTimeout(res) {
|
|
322
|
+
let timeout = 0;
|
|
323
|
+
// Handling http 429 with specific retry headers
|
|
324
|
+
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
|
|
325
|
+
if (res.fetchResponse && [429, 503].includes(res.fetchResponse.status)) {
|
|
326
|
+
const retryAfterStr = res.fetchResponse.headers.get('retry-after') ??
|
|
327
|
+
res.fetchResponse.headers.get('x-ratelimit-reset');
|
|
328
|
+
if (retryAfterStr) {
|
|
329
|
+
if (Number(retryAfterStr)) {
|
|
330
|
+
timeout = Number(retryAfterStr) * 1000;
|
|
331
|
+
}
|
|
332
|
+
else {
|
|
333
|
+
const date = new Date(retryAfterStr);
|
|
334
|
+
if (!isNaN(date)) {
|
|
335
|
+
timeout = Number(date) - Date.now();
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
this.cfg.logger.log(`retry-after: ${retryAfterStr}`);
|
|
339
|
+
if (!timeout) {
|
|
340
|
+
this.cfg.logger.warn(`retry-after could not be parsed`);
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
if (!timeout) {
|
|
345
|
+
const noise = Math.random() * 500;
|
|
346
|
+
timeout = res.retryStatus.retryTimeout + noise;
|
|
347
|
+
}
|
|
348
|
+
return timeout;
|
|
326
349
|
}
|
|
327
350
|
/**
|
|
328
351
|
* Default is yes,
|
|
@@ -6,7 +6,7 @@ export interface FetcherNormalizedCfg extends Required<FetcherCfg>, Omit<Fetcher
|
|
|
6
6
|
logger: CommonLogger;
|
|
7
7
|
searchParams: Record<string, any>;
|
|
8
8
|
}
|
|
9
|
-
export type FetcherBeforeRequestHook =
|
|
9
|
+
export type FetcherBeforeRequestHook = (req: FetcherRequest) => Promisable<void>;
|
|
10
10
|
export type FetcherAfterResponseHook = <BODY = unknown>(res: FetcherResponse<BODY>) => Promisable<void>;
|
|
11
11
|
export type FetcherBeforeRetryHook = <BODY = unknown>(res: FetcherResponse<BODY>) => Promisable<void>;
|
|
12
12
|
export interface FetcherCfg {
|
|
@@ -77,7 +77,7 @@ export interface FetcherRetryOptions {
|
|
|
77
77
|
timeoutMax: number;
|
|
78
78
|
timeoutMultiplier: number;
|
|
79
79
|
}
|
|
80
|
-
export interface FetcherRequest
|
|
80
|
+
export interface FetcherRequest extends Omit<FetcherOptions, 'method' | 'headers' | 'baseUrl' | 'url'> {
|
|
81
81
|
/**
|
|
82
82
|
* inputUrl is only the part that was passed in the request,
|
|
83
83
|
* without baseUrl or searchParams.
|
|
@@ -97,7 +97,7 @@ export interface FetcherRequest<BODY = unknown> extends Omit<FetcherOptions<BODY
|
|
|
97
97
|
retry5xx: boolean;
|
|
98
98
|
started: UnixTimestampMillisNumber;
|
|
99
99
|
}
|
|
100
|
-
export interface FetcherOptions
|
|
100
|
+
export interface FetcherOptions {
|
|
101
101
|
method?: HttpMethod;
|
|
102
102
|
/**
|
|
103
103
|
* If defined - this `url` will override the original given `url`.
|
|
@@ -148,21 +148,6 @@ export interface FetcherOptions<BODY = unknown> {
|
|
|
148
148
|
*/
|
|
149
149
|
retry5xx?: boolean;
|
|
150
150
|
jsonReviver?: Reviver;
|
|
151
|
-
/**
|
|
152
|
-
* Allows to walk over multiple pages of results.
|
|
153
|
-
* Paginate take a function.
|
|
154
|
-
* Function has access to FetcherResponse and FetcherOptions
|
|
155
|
-
* and has to make a decision to continue pagination or not.
|
|
156
|
-
*
|
|
157
|
-
* Return false to stop pagination.
|
|
158
|
-
* Return true to continue pagination.
|
|
159
|
-
* Feel free to mutate/modify opt (FetcherOptions), for example:
|
|
160
|
-
*
|
|
161
|
-
* opt.searchParams!['page']++
|
|
162
|
-
*
|
|
163
|
-
* @experimental
|
|
164
|
-
*/
|
|
165
|
-
paginate?: (res: FetcherSuccessResponse<BODY>, opt: FetcherOptions<BODY>) => Promisable<boolean>;
|
|
166
151
|
}
|
|
167
152
|
export type RequestInitNormalized = Omit<RequestInit, 'method' | 'headers'> & {
|
|
168
153
|
method: HttpMethod;
|
package/dist-esm/http/fetcher.js
CHANGED
|
@@ -136,6 +136,8 @@ export class Fetcher {
|
|
|
136
136
|
try {
|
|
137
137
|
res.fetchResponse = await this.callNativeFetch(req.fullUrl, req.init);
|
|
138
138
|
res.ok = res.fetchResponse.ok;
|
|
139
|
+
// important to set it to undefined, otherwise it can keep the previous value (from previous try)
|
|
140
|
+
res.err = undefined;
|
|
139
141
|
}
|
|
140
142
|
catch (err) {
|
|
141
143
|
// For example, CORS error would result in "TypeError: failed to fetch" here
|
|
@@ -156,12 +158,6 @@ export class Fetcher {
|
|
|
156
158
|
for (const hook of this.cfg.hooks.afterResponse || []) {
|
|
157
159
|
await hook(res);
|
|
158
160
|
}
|
|
159
|
-
if (req.paginate && res.ok) {
|
|
160
|
-
const proceeed = await req.paginate(res, opt);
|
|
161
|
-
if (proceeed) {
|
|
162
|
-
return await this.doFetch(opt);
|
|
163
|
-
}
|
|
164
|
-
}
|
|
165
161
|
return res;
|
|
166
162
|
}
|
|
167
163
|
async onOkResponse(res, timeout) {
|
|
@@ -295,12 +291,11 @@ export class Fetcher {
|
|
|
295
291
|
// but we should log all previous errors, otherwise they are lost.
|
|
296
292
|
// Here is the right place where we know it's not the "last error"
|
|
297
293
|
if (res.err) {
|
|
298
|
-
const { retryAttempt } = retryStatus;
|
|
299
294
|
this.cfg.logger.error([
|
|
300
295
|
' <<',
|
|
301
296
|
((_a = res.fetchResponse) === null || _a === void 0 ? void 0 : _a.status) || 0,
|
|
302
297
|
res.signature,
|
|
303
|
-
`try#${retryAttempt + 1}/${count + 1}`,
|
|
298
|
+
`try#${retryStatus.retryAttempt + 1}/${count + 1}`,
|
|
304
299
|
_since(res.req.started),
|
|
305
300
|
]
|
|
306
301
|
.filter(Boolean)
|
|
@@ -308,8 +303,36 @@ export class Fetcher {
|
|
|
308
303
|
}
|
|
309
304
|
retryStatus.retryAttempt++;
|
|
310
305
|
retryStatus.retryTimeout = _clamp(retryStatus.retryTimeout * timeoutMultiplier, 0, timeoutMax);
|
|
311
|
-
|
|
312
|
-
|
|
306
|
+
await pDelay(this.getRetryTimeout(res));
|
|
307
|
+
}
|
|
308
|
+
getRetryTimeout(res) {
|
|
309
|
+
var _a;
|
|
310
|
+
let timeout = 0;
|
|
311
|
+
// Handling http 429 with specific retry headers
|
|
312
|
+
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
|
|
313
|
+
if (res.fetchResponse && [429, 503].includes(res.fetchResponse.status)) {
|
|
314
|
+
const retryAfterStr = (_a = res.fetchResponse.headers.get('retry-after')) !== null && _a !== void 0 ? _a : res.fetchResponse.headers.get('x-ratelimit-reset');
|
|
315
|
+
if (retryAfterStr) {
|
|
316
|
+
if (Number(retryAfterStr)) {
|
|
317
|
+
timeout = Number(retryAfterStr) * 1000;
|
|
318
|
+
}
|
|
319
|
+
else {
|
|
320
|
+
const date = new Date(retryAfterStr);
|
|
321
|
+
if (!isNaN(date)) {
|
|
322
|
+
timeout = Number(date) - Date.now();
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
this.cfg.logger.log(`retry-after: ${retryAfterStr}`);
|
|
326
|
+
if (!timeout) {
|
|
327
|
+
this.cfg.logger.warn(`retry-after could not be parsed`);
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
if (!timeout) {
|
|
332
|
+
const noise = Math.random() * 500;
|
|
333
|
+
timeout = res.retryStatus.retryTimeout + noise;
|
|
334
|
+
}
|
|
335
|
+
return timeout;
|
|
313
336
|
}
|
|
314
337
|
/**
|
|
315
338
|
* Default is yes,
|
package/package.json
CHANGED
|
@@ -10,9 +10,7 @@ export interface FetcherNormalizedCfg
|
|
|
10
10
|
searchParams: Record<string, any>
|
|
11
11
|
}
|
|
12
12
|
|
|
13
|
-
export type FetcherBeforeRequestHook =
|
|
14
|
-
req: FetcherRequest<BODY>,
|
|
15
|
-
) => Promisable<void>
|
|
13
|
+
export type FetcherBeforeRequestHook = (req: FetcherRequest) => Promisable<void>
|
|
16
14
|
export type FetcherAfterResponseHook = <BODY = unknown>(
|
|
17
15
|
res: FetcherResponse<BODY>,
|
|
18
16
|
) => Promisable<void>
|
|
@@ -96,8 +94,8 @@ export interface FetcherRetryOptions {
|
|
|
96
94
|
timeoutMultiplier: number
|
|
97
95
|
}
|
|
98
96
|
|
|
99
|
-
export interface FetcherRequest
|
|
100
|
-
extends Omit<FetcherOptions
|
|
97
|
+
export interface FetcherRequest
|
|
98
|
+
extends Omit<FetcherOptions, 'method' | 'headers' | 'baseUrl' | 'url'> {
|
|
101
99
|
/**
|
|
102
100
|
* inputUrl is only the part that was passed in the request,
|
|
103
101
|
* without baseUrl or searchParams.
|
|
@@ -118,7 +116,7 @@ export interface FetcherRequest<BODY = unknown>
|
|
|
118
116
|
started: UnixTimestampMillisNumber
|
|
119
117
|
}
|
|
120
118
|
|
|
121
|
-
export interface FetcherOptions
|
|
119
|
+
export interface FetcherOptions {
|
|
122
120
|
method?: HttpMethod
|
|
123
121
|
|
|
124
122
|
/**
|
|
@@ -183,22 +181,6 @@ export interface FetcherOptions<BODY = unknown> {
|
|
|
183
181
|
retry5xx?: boolean
|
|
184
182
|
|
|
185
183
|
jsonReviver?: Reviver
|
|
186
|
-
|
|
187
|
-
/**
|
|
188
|
-
* Allows to walk over multiple pages of results.
|
|
189
|
-
* Paginate take a function.
|
|
190
|
-
* Function has access to FetcherResponse and FetcherOptions
|
|
191
|
-
* and has to make a decision to continue pagination or not.
|
|
192
|
-
*
|
|
193
|
-
* Return false to stop pagination.
|
|
194
|
-
* Return true to continue pagination.
|
|
195
|
-
* Feel free to mutate/modify opt (FetcherOptions), for example:
|
|
196
|
-
*
|
|
197
|
-
* opt.searchParams!['page']++
|
|
198
|
-
*
|
|
199
|
-
* @experimental
|
|
200
|
-
*/
|
|
201
|
-
paginate?: (res: FetcherSuccessResponse<BODY>, opt: FetcherOptions<BODY>) => Promisable<boolean>
|
|
202
184
|
}
|
|
203
185
|
|
|
204
186
|
export type RequestInitNormalized = Omit<RequestInit, 'method' | 'headers'> & {
|
package/src/http/fetcher.ts
CHANGED
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
import { pDelay } from '../promise/pDelay'
|
|
16
16
|
import { _jsonParse, _jsonParseIfPossible } from '../string/json.util'
|
|
17
17
|
import { _since } from '../time/time.util'
|
|
18
|
+
import { NumberOfMilliseconds } from '../types'
|
|
18
19
|
import type {
|
|
19
20
|
FetcherAfterResponseHook,
|
|
20
21
|
FetcherBeforeRequestHook,
|
|
@@ -53,10 +54,7 @@ export class Fetcher {
|
|
|
53
54
|
const m = method.toLowerCase()
|
|
54
55
|
|
|
55
56
|
// mode=void
|
|
56
|
-
;(this as any)[`${m}Void`] = async (
|
|
57
|
-
url: string,
|
|
58
|
-
opt?: FetcherOptions<void>,
|
|
59
|
-
): Promise<void> => {
|
|
57
|
+
;(this as any)[`${m}Void`] = async (url: string, opt?: FetcherOptions): Promise<void> => {
|
|
60
58
|
return await this.fetch<void>({
|
|
61
59
|
url,
|
|
62
60
|
method,
|
|
@@ -66,10 +64,7 @@ export class Fetcher {
|
|
|
66
64
|
}
|
|
67
65
|
|
|
68
66
|
if (method === 'HEAD') return // mode=text
|
|
69
|
-
;(this as any)[`${m}Text`] = async (
|
|
70
|
-
url: string,
|
|
71
|
-
opt?: FetcherOptions<string>,
|
|
72
|
-
): Promise<string> => {
|
|
67
|
+
;(this as any)[`${m}Text`] = async (url: string, opt?: FetcherOptions): Promise<string> => {
|
|
73
68
|
return await this.fetch<string>({
|
|
74
69
|
url,
|
|
75
70
|
method,
|
|
@@ -79,7 +74,7 @@ export class Fetcher {
|
|
|
79
74
|
}
|
|
80
75
|
|
|
81
76
|
// Default mode=json, but overridable
|
|
82
|
-
;(this as any)[m] = async <T = unknown>(url: string, opt?: FetcherOptions
|
|
77
|
+
;(this as any)[m] = async <T = unknown>(url: string, opt?: FetcherOptions): Promise<T> => {
|
|
83
78
|
return await this.fetch<T>({
|
|
84
79
|
url,
|
|
85
80
|
method,
|
|
@@ -116,26 +111,26 @@ export class Fetcher {
|
|
|
116
111
|
|
|
117
112
|
// These methods are generated dynamically in the constructor
|
|
118
113
|
// These default methods use mode=json
|
|
119
|
-
get!: <T = unknown>(url: string, opt?: FetcherOptions
|
|
120
|
-
post!: <T = unknown>(url: string, opt?: FetcherOptions
|
|
121
|
-
put!: <T = unknown>(url: string, opt?: FetcherOptions
|
|
122
|
-
patch!: <T = unknown>(url: string, opt?: FetcherOptions
|
|
123
|
-
delete!: <T = unknown>(url: string, opt?: FetcherOptions
|
|
114
|
+
get!: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>
|
|
115
|
+
post!: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>
|
|
116
|
+
put!: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>
|
|
117
|
+
patch!: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>
|
|
118
|
+
delete!: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>
|
|
124
119
|
|
|
125
120
|
// mode=text
|
|
126
|
-
getText!: (url: string, opt?: FetcherOptions
|
|
127
|
-
postText!: (url: string, opt?: FetcherOptions
|
|
128
|
-
putText!: (url: string, opt?: FetcherOptions
|
|
129
|
-
patchText!: (url: string, opt?: FetcherOptions
|
|
130
|
-
deleteText!: (url: string, opt?: FetcherOptions
|
|
121
|
+
getText!: (url: string, opt?: FetcherOptions) => Promise<string>
|
|
122
|
+
postText!: (url: string, opt?: FetcherOptions) => Promise<string>
|
|
123
|
+
putText!: (url: string, opt?: FetcherOptions) => Promise<string>
|
|
124
|
+
patchText!: (url: string, opt?: FetcherOptions) => Promise<string>
|
|
125
|
+
deleteText!: (url: string, opt?: FetcherOptions) => Promise<string>
|
|
131
126
|
|
|
132
127
|
// mode=void (no body fetching/parsing)
|
|
133
|
-
getVoid!: (url: string, opt?: FetcherOptions
|
|
134
|
-
postVoid!: (url: string, opt?: FetcherOptions
|
|
135
|
-
putVoid!: (url: string, opt?: FetcherOptions
|
|
136
|
-
patchVoid!: (url: string, opt?: FetcherOptions
|
|
137
|
-
deleteVoid!: (url: string, opt?: FetcherOptions
|
|
138
|
-
headVoid!: (url: string, opt?: FetcherOptions
|
|
128
|
+
getVoid!: (url: string, opt?: FetcherOptions) => Promise<void>
|
|
129
|
+
postVoid!: (url: string, opt?: FetcherOptions) => Promise<void>
|
|
130
|
+
putVoid!: (url: string, opt?: FetcherOptions) => Promise<void>
|
|
131
|
+
patchVoid!: (url: string, opt?: FetcherOptions) => Promise<void>
|
|
132
|
+
deleteVoid!: (url: string, opt?: FetcherOptions) => Promise<void>
|
|
133
|
+
headVoid!: (url: string, opt?: FetcherOptions) => Promise<void>
|
|
139
134
|
|
|
140
135
|
// mode=readableStream
|
|
141
136
|
/**
|
|
@@ -144,10 +139,7 @@ export class Fetcher {
|
|
|
144
139
|
* More on streams and Node interop:
|
|
145
140
|
* https://css-tricks.com/web-streams-everywhere-and-fetch-for-node-js/
|
|
146
141
|
*/
|
|
147
|
-
async getReadableStream(
|
|
148
|
-
url: string,
|
|
149
|
-
opt?: FetcherOptions<ReadableStream<Uint8Array>>,
|
|
150
|
-
): Promise<ReadableStream<Uint8Array>> {
|
|
142
|
+
async getReadableStream(url: string, opt?: FetcherOptions): Promise<ReadableStream<Uint8Array>> {
|
|
151
143
|
return await this.fetch({
|
|
152
144
|
url,
|
|
153
145
|
mode: 'readableStream',
|
|
@@ -155,7 +147,7 @@ export class Fetcher {
|
|
|
155
147
|
})
|
|
156
148
|
}
|
|
157
149
|
|
|
158
|
-
async fetch<T = unknown>(opt: FetcherOptions
|
|
150
|
+
async fetch<T = unknown>(opt: FetcherOptions): Promise<T> {
|
|
159
151
|
const res = await this.doFetch<T>(opt)
|
|
160
152
|
if (res.err) {
|
|
161
153
|
if (res.req.throwHttpErrors) throw res.err
|
|
@@ -169,7 +161,7 @@ export class Fetcher {
|
|
|
169
161
|
* Never throws, returns `err` property in the response instead.
|
|
170
162
|
* Use this method instead of `throwHttpErrors: false` or try-catching.
|
|
171
163
|
*/
|
|
172
|
-
async doFetch<T = unknown>(opt: FetcherOptions
|
|
164
|
+
async doFetch<T = unknown>(opt: FetcherOptions): Promise<FetcherResponse<T>> {
|
|
173
165
|
const req = this.normalizeOptions(opt)
|
|
174
166
|
const { logger } = this.cfg
|
|
175
167
|
const {
|
|
@@ -224,6 +216,8 @@ export class Fetcher {
|
|
|
224
216
|
try {
|
|
225
217
|
res.fetchResponse = await this.callNativeFetch(req.fullUrl, req.init)
|
|
226
218
|
res.ok = res.fetchResponse.ok
|
|
219
|
+
// important to set it to undefined, otherwise it can keep the previous value (from previous try)
|
|
220
|
+
res.err = undefined
|
|
227
221
|
} catch (err) {
|
|
228
222
|
// For example, CORS error would result in "TypeError: failed to fetch" here
|
|
229
223
|
res.err = err as Error
|
|
@@ -245,13 +239,6 @@ export class Fetcher {
|
|
|
245
239
|
await hook(res)
|
|
246
240
|
}
|
|
247
241
|
|
|
248
|
-
if (req.paginate && res.ok) {
|
|
249
|
-
const proceeed = await req.paginate(res, opt)
|
|
250
|
-
if (proceeed) {
|
|
251
|
-
return await this.doFetch(opt)
|
|
252
|
-
}
|
|
253
|
-
}
|
|
254
|
-
|
|
255
242
|
return res
|
|
256
243
|
}
|
|
257
244
|
|
|
@@ -405,13 +392,12 @@ export class Fetcher {
|
|
|
405
392
|
// but we should log all previous errors, otherwise they are lost.
|
|
406
393
|
// Here is the right place where we know it's not the "last error"
|
|
407
394
|
if (res.err) {
|
|
408
|
-
const { retryAttempt } = retryStatus
|
|
409
395
|
this.cfg.logger.error(
|
|
410
396
|
[
|
|
411
397
|
' <<',
|
|
412
398
|
res.fetchResponse?.status || 0,
|
|
413
399
|
res.signature,
|
|
414
|
-
`try#${retryAttempt + 1}/${count + 1}`,
|
|
400
|
+
`try#${retryStatus.retryAttempt + 1}/${count + 1}`,
|
|
415
401
|
_since(res.req.started),
|
|
416
402
|
]
|
|
417
403
|
.filter(Boolean)
|
|
@@ -423,8 +409,41 @@ export class Fetcher {
|
|
|
423
409
|
retryStatus.retryAttempt++
|
|
424
410
|
retryStatus.retryTimeout = _clamp(retryStatus.retryTimeout * timeoutMultiplier, 0, timeoutMax)
|
|
425
411
|
|
|
426
|
-
|
|
427
|
-
|
|
412
|
+
await pDelay(this.getRetryTimeout(res))
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
private getRetryTimeout(res: FetcherResponse): NumberOfMilliseconds {
|
|
416
|
+
let timeout: NumberOfMilliseconds = 0
|
|
417
|
+
|
|
418
|
+
// Handling http 429 with specific retry headers
|
|
419
|
+
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
|
|
420
|
+
if (res.fetchResponse && [429, 503].includes(res.fetchResponse.status)) {
|
|
421
|
+
const retryAfterStr =
|
|
422
|
+
res.fetchResponse.headers.get('retry-after') ??
|
|
423
|
+
res.fetchResponse.headers.get('x-ratelimit-reset')
|
|
424
|
+
if (retryAfterStr) {
|
|
425
|
+
if (Number(retryAfterStr)) {
|
|
426
|
+
timeout = Number(retryAfterStr) * 1000
|
|
427
|
+
} else {
|
|
428
|
+
const date = new Date(retryAfterStr)
|
|
429
|
+
if (!isNaN(date as any)) {
|
|
430
|
+
timeout = Number(date) - Date.now()
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
this.cfg.logger.log(`retry-after: ${retryAfterStr}`)
|
|
435
|
+
if (!timeout) {
|
|
436
|
+
this.cfg.logger.warn(`retry-after could not be parsed`)
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
if (!timeout) {
|
|
442
|
+
const noise = Math.random() * 500
|
|
443
|
+
timeout = res.retryStatus.retryTimeout + noise
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
return timeout
|
|
428
447
|
}
|
|
429
448
|
|
|
430
449
|
/**
|
|
@@ -525,7 +544,7 @@ export class Fetcher {
|
|
|
525
544
|
return norm
|
|
526
545
|
}
|
|
527
546
|
|
|
528
|
-
private normalizeOptions
|
|
547
|
+
private normalizeOptions(opt: FetcherOptions): FetcherRequest {
|
|
529
548
|
const {
|
|
530
549
|
timeoutSeconds,
|
|
531
550
|
throwHttpErrors,
|
|
@@ -537,7 +556,7 @@ export class Fetcher {
|
|
|
537
556
|
jsonReviver,
|
|
538
557
|
} = this.cfg
|
|
539
558
|
|
|
540
|
-
const req: FetcherRequest
|
|
559
|
+
const req: FetcherRequest = {
|
|
541
560
|
started: Date.now(),
|
|
542
561
|
mode,
|
|
543
562
|
timeoutSeconds,
|