@naturalcycles/js-lib 14.145.0 → 14.147.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/http/fetcher.d.ts +19 -19
- package/dist/http/fetcher.js +60 -32
- package/dist/http/fetcher.model.d.ts +37 -8
- package/dist/promise/pMap.js +1 -1
- package/dist-esm/http/fetcher.js +62 -91
- package/dist-esm/promise/pMap.js +15 -34
- package/package.json +1 -1
- package/src/http/fetcher.model.ts +48 -8
- package/src/http/fetcher.ts +94 -66
- package/src/promise/pMap.ts +1 -1
package/dist/http/fetcher.d.ts
CHANGED
|
@@ -14,36 +14,36 @@ export declare class Fetcher {
|
|
|
14
14
|
onBeforeRetry(hook: FetcherBeforeRetryHook): this;
|
|
15
15
|
cfg: FetcherNormalizedCfg;
|
|
16
16
|
static create(cfg?: FetcherCfg & FetcherOptions): Fetcher;
|
|
17
|
-
get: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>;
|
|
18
|
-
post: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>;
|
|
19
|
-
put: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>;
|
|
20
|
-
patch: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>;
|
|
21
|
-
delete: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>;
|
|
22
|
-
getText: (url: string, opt?: FetcherOptions) => Promise<string>;
|
|
23
|
-
postText: (url: string, opt?: FetcherOptions) => Promise<string>;
|
|
24
|
-
putText: (url: string, opt?: FetcherOptions) => Promise<string>;
|
|
25
|
-
patchText: (url: string, opt?: FetcherOptions) => Promise<string>;
|
|
26
|
-
deleteText: (url: string, opt?: FetcherOptions) => Promise<string>;
|
|
27
|
-
getVoid: (url: string, opt?: FetcherOptions) => Promise<void>;
|
|
28
|
-
postVoid: (url: string, opt?: FetcherOptions) => Promise<void>;
|
|
29
|
-
putVoid: (url: string, opt?: FetcherOptions) => Promise<void>;
|
|
30
|
-
patchVoid: (url: string, opt?: FetcherOptions) => Promise<void>;
|
|
31
|
-
deleteVoid: (url: string, opt?: FetcherOptions) => Promise<void>;
|
|
32
|
-
headVoid: (url: string, opt?: FetcherOptions) => Promise<void>;
|
|
17
|
+
get: <T = unknown>(url: string, opt?: FetcherOptions<T>) => Promise<T>;
|
|
18
|
+
post: <T = unknown>(url: string, opt?: FetcherOptions<T>) => Promise<T>;
|
|
19
|
+
put: <T = unknown>(url: string, opt?: FetcherOptions<T>) => Promise<T>;
|
|
20
|
+
patch: <T = unknown>(url: string, opt?: FetcherOptions<T>) => Promise<T>;
|
|
21
|
+
delete: <T = unknown>(url: string, opt?: FetcherOptions<T>) => Promise<T>;
|
|
22
|
+
getText: (url: string, opt?: FetcherOptions<string>) => Promise<string>;
|
|
23
|
+
postText: (url: string, opt?: FetcherOptions<string>) => Promise<string>;
|
|
24
|
+
putText: (url: string, opt?: FetcherOptions<string>) => Promise<string>;
|
|
25
|
+
patchText: (url: string, opt?: FetcherOptions<string>) => Promise<string>;
|
|
26
|
+
deleteText: (url: string, opt?: FetcherOptions<string>) => Promise<string>;
|
|
27
|
+
getVoid: (url: string, opt?: FetcherOptions<void>) => Promise<void>;
|
|
28
|
+
postVoid: (url: string, opt?: FetcherOptions<void>) => Promise<void>;
|
|
29
|
+
putVoid: (url: string, opt?: FetcherOptions<void>) => Promise<void>;
|
|
30
|
+
patchVoid: (url: string, opt?: FetcherOptions<void>) => Promise<void>;
|
|
31
|
+
deleteVoid: (url: string, opt?: FetcherOptions<void>) => Promise<void>;
|
|
32
|
+
headVoid: (url: string, opt?: FetcherOptions<void>) => Promise<void>;
|
|
33
33
|
/**
|
|
34
34
|
* Returns raw fetchResponse.body, which is a ReadableStream<Uint8Array>
|
|
35
35
|
*
|
|
36
36
|
* More on streams and Node interop:
|
|
37
37
|
* https://css-tricks.com/web-streams-everywhere-and-fetch-for-node-js/
|
|
38
38
|
*/
|
|
39
|
-
getReadableStream(url: string, opt?: FetcherOptions): Promise<ReadableStream<Uint8Array>>;
|
|
40
|
-
fetch<T = unknown>(
|
|
39
|
+
getReadableStream(url: string, opt?: FetcherOptions<ReadableStream<Uint8Array>>): Promise<ReadableStream<Uint8Array>>;
|
|
40
|
+
fetch<T = unknown>(opt: FetcherOptions<T>): Promise<T>;
|
|
41
41
|
/**
|
|
42
42
|
* Returns FetcherResponse.
|
|
43
43
|
* Never throws, returns `err` property in the response instead.
|
|
44
44
|
* Use this method instead of `throwHttpErrors: false` or try-catching.
|
|
45
45
|
*/
|
|
46
|
-
doFetch<T = unknown>(
|
|
46
|
+
doFetch<T = unknown>(opt: FetcherOptions<T>): Promise<FetcherResponse<T>>;
|
|
47
47
|
private onOkResponse;
|
|
48
48
|
/**
|
|
49
49
|
* This method exists to be able to easily mock it.
|
package/dist/http/fetcher.js
CHANGED
|
@@ -31,7 +31,8 @@ class Fetcher {
|
|
|
31
31
|
http_model_1.HTTP_METHODS.forEach(method => {
|
|
32
32
|
const m = method.toLowerCase();
|
|
33
33
|
this[`${m}Void`] = async (url, opt) => {
|
|
34
|
-
return await this.fetch(
|
|
34
|
+
return await this.fetch({
|
|
35
|
+
url,
|
|
35
36
|
method,
|
|
36
37
|
mode: 'void',
|
|
37
38
|
...opt,
|
|
@@ -41,14 +42,16 @@ class Fetcher {
|
|
|
41
42
|
return // mode=text
|
|
42
43
|
;
|
|
43
44
|
this[`${m}Text`] = async (url, opt) => {
|
|
44
|
-
return await this.fetch(
|
|
45
|
+
return await this.fetch({
|
|
46
|
+
url,
|
|
45
47
|
method,
|
|
46
48
|
mode: 'text',
|
|
47
49
|
...opt,
|
|
48
50
|
});
|
|
49
51
|
};
|
|
50
52
|
this[m] = async (url, opt) => {
|
|
51
|
-
return await this.fetch(
|
|
53
|
+
return await this.fetch({
|
|
54
|
+
url,
|
|
52
55
|
method,
|
|
53
56
|
mode: 'json',
|
|
54
57
|
...opt,
|
|
@@ -85,13 +88,14 @@ class Fetcher {
|
|
|
85
88
|
* https://css-tricks.com/web-streams-everywhere-and-fetch-for-node-js/
|
|
86
89
|
*/
|
|
87
90
|
async getReadableStream(url, opt) {
|
|
88
|
-
return await this.fetch(
|
|
91
|
+
return await this.fetch({
|
|
92
|
+
url,
|
|
89
93
|
mode: 'readableStream',
|
|
90
94
|
...opt,
|
|
91
95
|
});
|
|
92
96
|
}
|
|
93
|
-
async fetch(
|
|
94
|
-
const res = await this.doFetch(
|
|
97
|
+
async fetch(opt) {
|
|
98
|
+
const res = await this.doFetch(opt);
|
|
95
99
|
if (res.err) {
|
|
96
100
|
if (res.req.throwHttpErrors)
|
|
97
101
|
throw res.err;
|
|
@@ -104,9 +108,9 @@ class Fetcher {
|
|
|
104
108
|
* Never throws, returns `err` property in the response instead.
|
|
105
109
|
* Use this method instead of `throwHttpErrors: false` or try-catching.
|
|
106
110
|
*/
|
|
107
|
-
async doFetch(
|
|
111
|
+
async doFetch(opt) {
|
|
112
|
+
const req = this.normalizeOptions(opt);
|
|
108
113
|
const { logger } = this.cfg;
|
|
109
|
-
const req = this.normalizeOptions(url, rawOpt);
|
|
110
114
|
const { timeoutSeconds, init: { method }, } = req;
|
|
111
115
|
// setup timeout
|
|
112
116
|
let timeout;
|
|
@@ -117,12 +121,12 @@ class Fetcher {
|
|
|
117
121
|
abortController.abort(`timeout of ${timeoutSeconds} sec`);
|
|
118
122
|
}, timeoutSeconds * 1000);
|
|
119
123
|
}
|
|
120
|
-
for
|
|
124
|
+
for (const hook of this.cfg.hooks.beforeRequest || []) {
|
|
121
125
|
await hook(req);
|
|
122
126
|
}
|
|
123
|
-
const isFullUrl = req.
|
|
124
|
-
const fullUrl = isFullUrl ? new URL(req.
|
|
125
|
-
const shortUrl = fullUrl ? this.getShortUrl(fullUrl) : req.
|
|
127
|
+
const isFullUrl = req.fullUrl.includes('://');
|
|
128
|
+
const fullUrl = isFullUrl ? new URL(req.fullUrl) : undefined;
|
|
129
|
+
const shortUrl = fullUrl ? this.getShortUrl(fullUrl) : req.fullUrl;
|
|
126
130
|
const signature = [method, shortUrl].join(' ');
|
|
127
131
|
const res = {
|
|
128
132
|
req,
|
|
@@ -134,7 +138,7 @@ class Fetcher {
|
|
|
134
138
|
signature,
|
|
135
139
|
};
|
|
136
140
|
while (!res.retryStatus.retryStopped) {
|
|
137
|
-
|
|
141
|
+
req.started = Date.now();
|
|
138
142
|
if (this.cfg.logRequest) {
|
|
139
143
|
const { retryAttempt } = res.retryStatus;
|
|
140
144
|
logger.log([' >>', signature, retryAttempt && `try#${retryAttempt + 1}/${req.retry.count + 1}`]
|
|
@@ -145,7 +149,7 @@ class Fetcher {
|
|
|
145
149
|
}
|
|
146
150
|
}
|
|
147
151
|
try {
|
|
148
|
-
res.fetchResponse = await this.callNativeFetch(req.
|
|
152
|
+
res.fetchResponse = await this.callNativeFetch(req.fullUrl, req.init);
|
|
149
153
|
res.ok = res.fetchResponse.ok;
|
|
150
154
|
}
|
|
151
155
|
catch (err) {
|
|
@@ -157,19 +161,25 @@ class Fetcher {
|
|
|
157
161
|
}
|
|
158
162
|
res.statusFamily = this.getStatusFamily(res);
|
|
159
163
|
if (res.fetchResponse?.ok) {
|
|
160
|
-
await this.onOkResponse(res,
|
|
164
|
+
await this.onOkResponse(res, timeout);
|
|
161
165
|
}
|
|
162
166
|
else {
|
|
163
167
|
// !res.ok
|
|
164
|
-
await this.onNotOkResponse(res,
|
|
168
|
+
await this.onNotOkResponse(res, timeout);
|
|
165
169
|
}
|
|
166
170
|
}
|
|
167
|
-
for
|
|
171
|
+
for (const hook of this.cfg.hooks.afterResponse || []) {
|
|
168
172
|
await hook(res);
|
|
169
173
|
}
|
|
174
|
+
if (req.paginate && res.ok) {
|
|
175
|
+
const proceeed = await req.paginate(res, opt);
|
|
176
|
+
if (proceeed) {
|
|
177
|
+
return await this.doFetch(opt);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
170
180
|
return res;
|
|
171
181
|
}
|
|
172
|
-
async onOkResponse(res,
|
|
182
|
+
async onOkResponse(res, timeout) {
|
|
173
183
|
const { req } = res;
|
|
174
184
|
const { mode } = res.req;
|
|
175
185
|
if (mode === 'json') {
|
|
@@ -192,7 +202,7 @@ class Fetcher {
|
|
|
192
202
|
// } satisfies HttpRequestErrorData)
|
|
193
203
|
res.err = (0, error_util_1._anyToError)(err);
|
|
194
204
|
res.ok = false;
|
|
195
|
-
return await this.onNotOkResponse(res,
|
|
205
|
+
return await this.onNotOkResponse(res, timeout);
|
|
196
206
|
}
|
|
197
207
|
}
|
|
198
208
|
else {
|
|
@@ -220,7 +230,7 @@ class Fetcher {
|
|
|
220
230
|
if (res.body === null) {
|
|
221
231
|
res.err = new Error(`fetchResponse.body is null`);
|
|
222
232
|
res.ok = false;
|
|
223
|
-
return await this.onNotOkResponse(res,
|
|
233
|
+
return await this.onNotOkResponse(res, timeout);
|
|
224
234
|
}
|
|
225
235
|
}
|
|
226
236
|
clearTimeout(timeout);
|
|
@@ -234,7 +244,7 @@ class Fetcher {
|
|
|
234
244
|
res.fetchResponse.status,
|
|
235
245
|
res.signature,
|
|
236
246
|
retryAttempt && `try#${retryAttempt + 1}/${req.retry.count + 1}`,
|
|
237
|
-
(0, time_util_1._since)(started),
|
|
247
|
+
(0, time_util_1._since)(res.req.started),
|
|
238
248
|
]
|
|
239
249
|
.filter(Boolean)
|
|
240
250
|
.join(' '));
|
|
@@ -249,7 +259,7 @@ class Fetcher {
|
|
|
249
259
|
async callNativeFetch(url, init) {
|
|
250
260
|
return await globalThis.fetch(url, init);
|
|
251
261
|
}
|
|
252
|
-
async onNotOkResponse(res,
|
|
262
|
+
async onNotOkResponse(res, timeout) {
|
|
253
263
|
clearTimeout(timeout);
|
|
254
264
|
let cause;
|
|
255
265
|
if (res.err) {
|
|
@@ -271,11 +281,11 @@ class Fetcher {
|
|
|
271
281
|
// Enabled, cause `data` is not printed by default when error is HttpError
|
|
272
282
|
// method: req.method,
|
|
273
283
|
// tryCount: req.tryCount,
|
|
274
|
-
requestUrl: res.req.
|
|
284
|
+
requestUrl: res.req.fullUrl,
|
|
275
285
|
requestBaseUrl: this.cfg.baseUrl || null,
|
|
276
286
|
requestMethod: res.req.init.method,
|
|
277
287
|
requestSignature: res.signature,
|
|
278
|
-
requestDuration: Date.now() - started,
|
|
288
|
+
requestDuration: Date.now() - res.req.started,
|
|
279
289
|
}), cause);
|
|
280
290
|
await this.processRetry(res);
|
|
281
291
|
}
|
|
@@ -284,7 +294,7 @@ class Fetcher {
|
|
|
284
294
|
if (!this.shouldRetry(res)) {
|
|
285
295
|
retryStatus.retryStopped = true;
|
|
286
296
|
}
|
|
287
|
-
for
|
|
297
|
+
for (const hook of this.cfg.hooks.beforeRetry || []) {
|
|
288
298
|
await hook(res);
|
|
289
299
|
}
|
|
290
300
|
const { count, timeoutMultiplier, timeoutMax } = res.req.retry;
|
|
@@ -293,6 +303,22 @@ class Fetcher {
|
|
|
293
303
|
}
|
|
294
304
|
if (retryStatus.retryStopped)
|
|
295
305
|
return;
|
|
306
|
+
// Here we know that more retries will be attempted
|
|
307
|
+
// We don't log "last error", because it will be thrown and logged by consumer,
|
|
308
|
+
// but we should log all previous errors, otherwise they are lost.
|
|
309
|
+
// Here is the right place where we know it's not the "last error"
|
|
310
|
+
if (res.err) {
|
|
311
|
+
const { retryAttempt } = retryStatus;
|
|
312
|
+
this.cfg.logger.error([
|
|
313
|
+
' <<',
|
|
314
|
+
res.fetchResponse?.status || 0,
|
|
315
|
+
res.signature,
|
|
316
|
+
`try#${retryAttempt + 1}/${count + 1}`,
|
|
317
|
+
(0, time_util_1._since)(res.req.started),
|
|
318
|
+
]
|
|
319
|
+
.filter(Boolean)
|
|
320
|
+
.join(' '), res.err.cause || res.err);
|
|
321
|
+
}
|
|
296
322
|
retryStatus.retryAttempt++;
|
|
297
323
|
retryStatus.retryTimeout = (0, number_util_1._clamp)(retryStatus.retryTimeout * timeoutMultiplier, 0, timeoutMax);
|
|
298
324
|
const noise = Math.random() * 500;
|
|
@@ -362,7 +388,7 @@ class Fetcher {
|
|
|
362
388
|
const { debug = false } = cfg;
|
|
363
389
|
const norm = (0, object_util_1._merge)({
|
|
364
390
|
baseUrl: '',
|
|
365
|
-
|
|
391
|
+
inputUrl: '',
|
|
366
392
|
mode: 'void',
|
|
367
393
|
searchParams: {},
|
|
368
394
|
timeoutSeconds: 30,
|
|
@@ -390,11 +416,11 @@ class Fetcher {
|
|
|
390
416
|
norm.init.headers = (0, object_util_1._mapKeys)(norm.init.headers, k => k.toLowerCase());
|
|
391
417
|
return norm;
|
|
392
418
|
}
|
|
393
|
-
normalizeOptions(
|
|
419
|
+
normalizeOptions(opt) {
|
|
394
420
|
const { timeoutSeconds, throwHttpErrors, retryPost, retry4xx, retry5xx, retry, mode, jsonReviver, } = this.cfg;
|
|
395
421
|
const req = {
|
|
422
|
+
started: Date.now(),
|
|
396
423
|
mode,
|
|
397
|
-
url,
|
|
398
424
|
timeoutSeconds,
|
|
399
425
|
throwHttpErrors,
|
|
400
426
|
retryPost,
|
|
@@ -402,6 +428,8 @@ class Fetcher {
|
|
|
402
428
|
retry5xx,
|
|
403
429
|
jsonReviver,
|
|
404
430
|
...(0, object_util_1._omit)(opt, ['method', 'headers', 'credentials']),
|
|
431
|
+
inputUrl: opt.url || '',
|
|
432
|
+
fullUrl: opt.url || '',
|
|
405
433
|
retry: {
|
|
406
434
|
...retry,
|
|
407
435
|
...(0, object_util_1._filterUndefinedValues)(opt.retry || {}),
|
|
@@ -418,11 +446,11 @@ class Fetcher {
|
|
|
418
446
|
// setup url
|
|
419
447
|
const baseUrl = opt.baseUrl || this.cfg.baseUrl;
|
|
420
448
|
if (baseUrl) {
|
|
421
|
-
if (
|
|
449
|
+
if (req.fullUrl.startsWith('/')) {
|
|
422
450
|
console.warn(`Fetcher: url should not start with / when baseUrl is specified`);
|
|
423
|
-
|
|
451
|
+
req.fullUrl = req.fullUrl.slice(1);
|
|
424
452
|
}
|
|
425
|
-
req.
|
|
453
|
+
req.fullUrl = `${baseUrl}/${req.inputUrl}`;
|
|
426
454
|
}
|
|
427
455
|
const searchParams = (0, object_util_1._filterUndefinedValues)({
|
|
428
456
|
...this.cfg.searchParams,
|
|
@@ -430,7 +458,7 @@ class Fetcher {
|
|
|
430
458
|
});
|
|
431
459
|
if (Object.keys(searchParams).length) {
|
|
432
460
|
const qs = new URLSearchParams(searchParams).toString();
|
|
433
|
-
req.
|
|
461
|
+
req.fullUrl += req.fullUrl.includes('?') ? '&' : '?' + qs;
|
|
434
462
|
}
|
|
435
463
|
// setup request body
|
|
436
464
|
if (opt.json !== undefined) {
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import type { CommonLogger } from '../log/commonLogger';
|
|
2
2
|
import type { Promisable } from '../typeFest';
|
|
3
|
-
import type { Reviver } from '../types';
|
|
3
|
+
import type { Reviver, UnixTimestampMillisNumber } from '../types';
|
|
4
4
|
import type { HttpMethod, HttpStatusFamily } from './http.model';
|
|
5
|
-
export interface FetcherNormalizedCfg extends Required<FetcherCfg>, FetcherRequest {
|
|
5
|
+
export interface FetcherNormalizedCfg extends Required<FetcherCfg>, Omit<FetcherRequest, 'started' | 'fullUrl'> {
|
|
6
6
|
logger: CommonLogger;
|
|
7
7
|
searchParams: Record<string, any>;
|
|
8
8
|
}
|
|
9
|
-
export type FetcherBeforeRequestHook = (req: FetcherRequest) => Promisable<void>;
|
|
10
|
-
export type FetcherAfterResponseHook = (res: FetcherResponse) => Promisable<void>;
|
|
11
|
-
export type FetcherBeforeRetryHook = (res: FetcherResponse) => Promisable<void>;
|
|
9
|
+
export type FetcherBeforeRequestHook = <BODY = unknown>(req: FetcherRequest<BODY>) => Promisable<void>;
|
|
10
|
+
export type FetcherAfterResponseHook = <BODY = unknown>(res: FetcherResponse<BODY>) => Promisable<void>;
|
|
11
|
+
export type FetcherBeforeRetryHook = <BODY = unknown>(res: FetcherResponse<BODY>) => Promisable<void>;
|
|
12
12
|
export interface FetcherCfg {
|
|
13
13
|
/**
|
|
14
14
|
* Should **not** contain trailing slash.
|
|
@@ -77,8 +77,16 @@ export interface FetcherRetryOptions {
|
|
|
77
77
|
timeoutMax: number;
|
|
78
78
|
timeoutMultiplier: number;
|
|
79
79
|
}
|
|
80
|
-
export interface FetcherRequest extends Omit<FetcherOptions
|
|
81
|
-
|
|
80
|
+
export interface FetcherRequest<BODY = unknown> extends Omit<FetcherOptions<BODY>, 'method' | 'headers' | 'baseUrl' | 'url'> {
|
|
81
|
+
/**
|
|
82
|
+
* inputUrl is only the part that was passed in the request,
|
|
83
|
+
* without baseUrl or searchParams.
|
|
84
|
+
*/
|
|
85
|
+
inputUrl: string;
|
|
86
|
+
/**
|
|
87
|
+
* fullUrl includes baseUrl and searchParams.
|
|
88
|
+
*/
|
|
89
|
+
fullUrl: string;
|
|
82
90
|
init: RequestInitNormalized;
|
|
83
91
|
mode: FetcherMode;
|
|
84
92
|
throwHttpErrors: boolean;
|
|
@@ -87,9 +95,15 @@ export interface FetcherRequest extends Omit<FetcherOptions, 'method' | 'headers
|
|
|
87
95
|
retryPost: boolean;
|
|
88
96
|
retry4xx: boolean;
|
|
89
97
|
retry5xx: boolean;
|
|
98
|
+
started: UnixTimestampMillisNumber;
|
|
90
99
|
}
|
|
91
|
-
export interface FetcherOptions {
|
|
100
|
+
export interface FetcherOptions<BODY = unknown> {
|
|
92
101
|
method?: HttpMethod;
|
|
102
|
+
/**
|
|
103
|
+
* If defined - this `url` will override the original given `url`.
|
|
104
|
+
* baseUrl (and searchParams) will still modify it.
|
|
105
|
+
*/
|
|
106
|
+
url?: string;
|
|
93
107
|
baseUrl?: string;
|
|
94
108
|
throwHttpErrors?: boolean;
|
|
95
109
|
/**
|
|
@@ -134,6 +148,21 @@ export interface FetcherOptions {
|
|
|
134
148
|
*/
|
|
135
149
|
retry5xx?: boolean;
|
|
136
150
|
jsonReviver?: Reviver;
|
|
151
|
+
/**
|
|
152
|
+
* Allows to walk over multiple pages of results.
|
|
153
|
+
* Paginate take a function.
|
|
154
|
+
* Function has access to FetcherResponse and FetcherOptions
|
|
155
|
+
* and has to make a decision to continue pagination or not.
|
|
156
|
+
*
|
|
157
|
+
* Return false to stop pagination.
|
|
158
|
+
* Return true to continue pagination.
|
|
159
|
+
* Feel free to mutate/modify opt (FetcherOptions), for example:
|
|
160
|
+
*
|
|
161
|
+
* opt.searchParams!['page']++
|
|
162
|
+
*
|
|
163
|
+
* @experimental
|
|
164
|
+
*/
|
|
165
|
+
paginate?: (res: FetcherSuccessResponse<BODY>, opt: FetcherOptions<BODY>) => Promisable<boolean>;
|
|
137
166
|
}
|
|
138
167
|
export type RequestInitNormalized = Omit<RequestInit, 'method' | 'headers'> & {
|
|
139
168
|
method: HttpMethod;
|
package/dist/promise/pMap.js
CHANGED
|
@@ -51,7 +51,7 @@ async function pMap(iterable, mapper, opt = {}) {
|
|
|
51
51
|
// Special cases that are able to preserve async stack traces
|
|
52
52
|
if (concurrency === 1) {
|
|
53
53
|
// Special case for concurrency == 1
|
|
54
|
-
for
|
|
54
|
+
for (const item of items) {
|
|
55
55
|
try {
|
|
56
56
|
const r = await mapper(item, currentIndex++);
|
|
57
57
|
if (r === __1.END)
|
package/dist-esm/http/fetcher.js
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
/// <reference lib="dom"/>
|
|
2
|
-
import { __asyncValues } from "tslib";
|
|
3
2
|
import { isServerSide } from '../env';
|
|
4
3
|
import { _anyToError, _anyToErrorObject, _errorToErrorObject } from '../error/error.util';
|
|
5
4
|
import { HttpRequestError } from '../error/httpRequestError';
|
|
@@ -29,16 +28,19 @@ export class Fetcher {
|
|
|
29
28
|
HTTP_METHODS.forEach(method => {
|
|
30
29
|
const m = method.toLowerCase();
|
|
31
30
|
this[`${m}Void`] = async (url, opt) => {
|
|
32
|
-
return await this.fetch(
|
|
31
|
+
return await this.fetch(Object.assign({ url,
|
|
32
|
+
method, mode: 'void' }, opt));
|
|
33
33
|
};
|
|
34
34
|
if (method === 'HEAD')
|
|
35
35
|
return // mode=text
|
|
36
36
|
;
|
|
37
37
|
this[`${m}Text`] = async (url, opt) => {
|
|
38
|
-
return await this.fetch(
|
|
38
|
+
return await this.fetch(Object.assign({ url,
|
|
39
|
+
method, mode: 'text' }, opt));
|
|
39
40
|
};
|
|
40
41
|
this[m] = async (url, opt) => {
|
|
41
|
-
return await this.fetch(
|
|
42
|
+
return await this.fetch(Object.assign({ url,
|
|
43
|
+
method, mode: 'json' }, opt));
|
|
42
44
|
};
|
|
43
45
|
});
|
|
44
46
|
}
|
|
@@ -74,10 +76,10 @@ export class Fetcher {
|
|
|
74
76
|
* https://css-tricks.com/web-streams-everywhere-and-fetch-for-node-js/
|
|
75
77
|
*/
|
|
76
78
|
async getReadableStream(url, opt) {
|
|
77
|
-
return await this.fetch(
|
|
79
|
+
return await this.fetch(Object.assign({ url, mode: 'readableStream' }, opt));
|
|
78
80
|
}
|
|
79
|
-
async fetch(
|
|
80
|
-
const res = await this.doFetch(
|
|
81
|
+
async fetch(opt) {
|
|
82
|
+
const res = await this.doFetch(opt);
|
|
81
83
|
if (res.err) {
|
|
82
84
|
if (res.req.throwHttpErrors)
|
|
83
85
|
throw res.err;
|
|
@@ -90,11 +92,10 @@ export class Fetcher {
|
|
|
90
92
|
* Never throws, returns `err` property in the response instead.
|
|
91
93
|
* Use this method instead of `throwHttpErrors: false` or try-catching.
|
|
92
94
|
*/
|
|
93
|
-
async doFetch(
|
|
94
|
-
var _a
|
|
95
|
-
|
|
95
|
+
async doFetch(opt) {
|
|
96
|
+
var _a;
|
|
97
|
+
const req = this.normalizeOptions(opt);
|
|
96
98
|
const { logger } = this.cfg;
|
|
97
|
-
const req = this.normalizeOptions(url, rawOpt);
|
|
98
99
|
const { timeoutSeconds, init: { method }, } = req;
|
|
99
100
|
// setup timeout
|
|
100
101
|
let timeout;
|
|
@@ -105,29 +106,12 @@ export class Fetcher {
|
|
|
105
106
|
abortController.abort(`timeout of ${timeoutSeconds} sec`);
|
|
106
107
|
}, timeoutSeconds * 1000);
|
|
107
108
|
}
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
_c = _k.value;
|
|
111
|
-
_h = false;
|
|
112
|
-
try {
|
|
113
|
-
const hook = _c;
|
|
114
|
-
await hook(req);
|
|
115
|
-
}
|
|
116
|
-
finally {
|
|
117
|
-
_h = true;
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
122
|
-
finally {
|
|
123
|
-
try {
|
|
124
|
-
if (!_h && !_a && (_b = _j.return)) await _b.call(_j);
|
|
125
|
-
}
|
|
126
|
-
finally { if (e_1) throw e_1.error; }
|
|
109
|
+
for (const hook of this.cfg.hooks.beforeRequest || []) {
|
|
110
|
+
await hook(req);
|
|
127
111
|
}
|
|
128
|
-
const isFullUrl = req.
|
|
129
|
-
const fullUrl = isFullUrl ? new URL(req.
|
|
130
|
-
const shortUrl = fullUrl ? this.getShortUrl(fullUrl) : req.
|
|
112
|
+
const isFullUrl = req.fullUrl.includes('://');
|
|
113
|
+
const fullUrl = isFullUrl ? new URL(req.fullUrl) : undefined;
|
|
114
|
+
const shortUrl = fullUrl ? this.getShortUrl(fullUrl) : req.fullUrl;
|
|
131
115
|
const signature = [method, shortUrl].join(' ');
|
|
132
116
|
const res = {
|
|
133
117
|
req,
|
|
@@ -139,7 +123,7 @@ export class Fetcher {
|
|
|
139
123
|
signature,
|
|
140
124
|
};
|
|
141
125
|
while (!res.retryStatus.retryStopped) {
|
|
142
|
-
|
|
126
|
+
req.started = Date.now();
|
|
143
127
|
if (this.cfg.logRequest) {
|
|
144
128
|
const { retryAttempt } = res.retryStatus;
|
|
145
129
|
logger.log([' >>', signature, retryAttempt && `try#${retryAttempt + 1}/${req.retry.count + 1}`]
|
|
@@ -150,7 +134,7 @@ export class Fetcher {
|
|
|
150
134
|
}
|
|
151
135
|
}
|
|
152
136
|
try {
|
|
153
|
-
res.fetchResponse = await this.callNativeFetch(req.
|
|
137
|
+
res.fetchResponse = await this.callNativeFetch(req.fullUrl, req.init);
|
|
154
138
|
res.ok = res.fetchResponse.ok;
|
|
155
139
|
}
|
|
156
140
|
catch (err) {
|
|
@@ -161,37 +145,26 @@ export class Fetcher {
|
|
|
161
145
|
res.fetchResponse = undefined;
|
|
162
146
|
}
|
|
163
147
|
res.statusFamily = this.getStatusFamily(res);
|
|
164
|
-
if ((
|
|
165
|
-
await this.onOkResponse(res,
|
|
148
|
+
if ((_a = res.fetchResponse) === null || _a === void 0 ? void 0 : _a.ok) {
|
|
149
|
+
await this.onOkResponse(res, timeout);
|
|
166
150
|
}
|
|
167
151
|
else {
|
|
168
152
|
// !res.ok
|
|
169
|
-
await this.onNotOkResponse(res,
|
|
153
|
+
await this.onNotOkResponse(res, timeout);
|
|
170
154
|
}
|
|
171
155
|
}
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
_f = _o.value;
|
|
175
|
-
_l = false;
|
|
176
|
-
try {
|
|
177
|
-
const hook = _f;
|
|
178
|
-
await hook(res);
|
|
179
|
-
}
|
|
180
|
-
finally {
|
|
181
|
-
_l = true;
|
|
182
|
-
}
|
|
183
|
-
}
|
|
156
|
+
for (const hook of this.cfg.hooks.afterResponse || []) {
|
|
157
|
+
await hook(res);
|
|
184
158
|
}
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
159
|
+
if (req.paginate && res.ok) {
|
|
160
|
+
const proceeed = await req.paginate(res, opt);
|
|
161
|
+
if (proceeed) {
|
|
162
|
+
return await this.doFetch(opt);
|
|
189
163
|
}
|
|
190
|
-
finally { if (e_2) throw e_2.error; }
|
|
191
164
|
}
|
|
192
165
|
return res;
|
|
193
166
|
}
|
|
194
|
-
async onOkResponse(res,
|
|
167
|
+
async onOkResponse(res, timeout) {
|
|
195
168
|
const { req } = res;
|
|
196
169
|
const { mode } = res.req;
|
|
197
170
|
if (mode === 'json') {
|
|
@@ -214,7 +187,7 @@ export class Fetcher {
|
|
|
214
187
|
// } satisfies HttpRequestErrorData)
|
|
215
188
|
res.err = _anyToError(err);
|
|
216
189
|
res.ok = false;
|
|
217
|
-
return await this.onNotOkResponse(res,
|
|
190
|
+
return await this.onNotOkResponse(res, timeout);
|
|
218
191
|
}
|
|
219
192
|
}
|
|
220
193
|
else {
|
|
@@ -242,7 +215,7 @@ export class Fetcher {
|
|
|
242
215
|
if (res.body === null) {
|
|
243
216
|
res.err = new Error(`fetchResponse.body is null`);
|
|
244
217
|
res.ok = false;
|
|
245
|
-
return await this.onNotOkResponse(res,
|
|
218
|
+
return await this.onNotOkResponse(res, timeout);
|
|
246
219
|
}
|
|
247
220
|
}
|
|
248
221
|
clearTimeout(timeout);
|
|
@@ -256,7 +229,7 @@ export class Fetcher {
|
|
|
256
229
|
res.fetchResponse.status,
|
|
257
230
|
res.signature,
|
|
258
231
|
retryAttempt && `try#${retryAttempt + 1}/${req.retry.count + 1}`,
|
|
259
|
-
_since(started),
|
|
232
|
+
_since(res.req.started),
|
|
260
233
|
]
|
|
261
234
|
.filter(Boolean)
|
|
262
235
|
.join(' '));
|
|
@@ -271,7 +244,7 @@ export class Fetcher {
|
|
|
271
244
|
async callNativeFetch(url, init) {
|
|
272
245
|
return await globalThis.fetch(url, init);
|
|
273
246
|
}
|
|
274
|
-
async onNotOkResponse(res,
|
|
247
|
+
async onNotOkResponse(res, timeout) {
|
|
275
248
|
var _a, _b;
|
|
276
249
|
clearTimeout(timeout);
|
|
277
250
|
let cause;
|
|
@@ -294,39 +267,22 @@ export class Fetcher {
|
|
|
294
267
|
// Enabled, cause `data` is not printed by default when error is HttpError
|
|
295
268
|
// method: req.method,
|
|
296
269
|
// tryCount: req.tryCount,
|
|
297
|
-
requestUrl: res.req.
|
|
270
|
+
requestUrl: res.req.fullUrl,
|
|
298
271
|
requestBaseUrl: this.cfg.baseUrl || null,
|
|
299
272
|
requestMethod: res.req.init.method,
|
|
300
273
|
requestSignature: res.signature,
|
|
301
|
-
requestDuration: Date.now() - started,
|
|
274
|
+
requestDuration: Date.now() - res.req.started,
|
|
302
275
|
}), cause);
|
|
303
276
|
await this.processRetry(res);
|
|
304
277
|
}
|
|
305
278
|
async processRetry(res) {
|
|
306
|
-
var _a
|
|
279
|
+
var _a;
|
|
307
280
|
const { retryStatus } = res;
|
|
308
281
|
if (!this.shouldRetry(res)) {
|
|
309
282
|
retryStatus.retryStopped = true;
|
|
310
283
|
}
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
_c = _f.value;
|
|
314
|
-
_d = false;
|
|
315
|
-
try {
|
|
316
|
-
const hook = _c;
|
|
317
|
-
await hook(res);
|
|
318
|
-
}
|
|
319
|
-
finally {
|
|
320
|
-
_d = true;
|
|
321
|
-
}
|
|
322
|
-
}
|
|
323
|
-
}
|
|
324
|
-
catch (e_3_1) { e_3 = { error: e_3_1 }; }
|
|
325
|
-
finally {
|
|
326
|
-
try {
|
|
327
|
-
if (!_d && !_a && (_b = _e.return)) await _b.call(_e);
|
|
328
|
-
}
|
|
329
|
-
finally { if (e_3) throw e_3.error; }
|
|
284
|
+
for (const hook of this.cfg.hooks.beforeRetry || []) {
|
|
285
|
+
await hook(res);
|
|
330
286
|
}
|
|
331
287
|
const { count, timeoutMultiplier, timeoutMax } = res.req.retry;
|
|
332
288
|
if (retryStatus.retryAttempt >= count) {
|
|
@@ -334,6 +290,22 @@ export class Fetcher {
|
|
|
334
290
|
}
|
|
335
291
|
if (retryStatus.retryStopped)
|
|
336
292
|
return;
|
|
293
|
+
// Here we know that more retries will be attempted
|
|
294
|
+
// We don't log "last error", because it will be thrown and logged by consumer,
|
|
295
|
+
// but we should log all previous errors, otherwise they are lost.
|
|
296
|
+
// Here is the right place where we know it's not the "last error"
|
|
297
|
+
if (res.err) {
|
|
298
|
+
const { retryAttempt } = retryStatus;
|
|
299
|
+
this.cfg.logger.error([
|
|
300
|
+
' <<',
|
|
301
|
+
((_a = res.fetchResponse) === null || _a === void 0 ? void 0 : _a.status) || 0,
|
|
302
|
+
res.signature,
|
|
303
|
+
`try#${retryAttempt + 1}/${count + 1}`,
|
|
304
|
+
_since(res.req.started),
|
|
305
|
+
]
|
|
306
|
+
.filter(Boolean)
|
|
307
|
+
.join(' '), res.err.cause || res.err);
|
|
308
|
+
}
|
|
337
309
|
retryStatus.retryAttempt++;
|
|
338
310
|
retryStatus.retryTimeout = _clamp(retryStatus.retryTimeout * timeoutMultiplier, 0, timeoutMax);
|
|
339
311
|
const noise = Math.random() * 500;
|
|
@@ -406,7 +378,7 @@ export class Fetcher {
|
|
|
406
378
|
const { debug = false } = cfg;
|
|
407
379
|
const norm = _merge({
|
|
408
380
|
baseUrl: '',
|
|
409
|
-
|
|
381
|
+
inputUrl: '',
|
|
410
382
|
mode: 'void',
|
|
411
383
|
searchParams: {},
|
|
412
384
|
timeoutSeconds: 30,
|
|
@@ -434,32 +406,31 @@ export class Fetcher {
|
|
|
434
406
|
norm.init.headers = _mapKeys(norm.init.headers, k => k.toLowerCase());
|
|
435
407
|
return norm;
|
|
436
408
|
}
|
|
437
|
-
normalizeOptions(
|
|
409
|
+
normalizeOptions(opt) {
|
|
438
410
|
var _a, _b;
|
|
439
411
|
const { timeoutSeconds, throwHttpErrors, retryPost, retry4xx, retry5xx, retry, mode, jsonReviver, } = this.cfg;
|
|
440
|
-
const req = Object.assign(Object.assign({ mode,
|
|
441
|
-
url,
|
|
412
|
+
const req = Object.assign(Object.assign({ started: Date.now(), mode,
|
|
442
413
|
timeoutSeconds,
|
|
443
414
|
throwHttpErrors,
|
|
444
415
|
retryPost,
|
|
445
416
|
retry4xx,
|
|
446
417
|
retry5xx,
|
|
447
|
-
jsonReviver }, _omit(opt, ['method', 'headers', 'credentials'])), { retry: Object.assign(Object.assign({}, retry), _filterUndefinedValues(opt.retry || {})), init: _merge(Object.assign(Object.assign({}, this.cfg.init), { method: opt.method || this.cfg.init.method, credentials: opt.credentials || this.cfg.init.credentials, redirect: ((_b = (_a = opt.followRedirects) !== null && _a !== void 0 ? _a : this.cfg.followRedirects) !== null && _b !== void 0 ? _b : true) ? 'follow' : 'error' }), {
|
|
418
|
+
jsonReviver }, _omit(opt, ['method', 'headers', 'credentials'])), { inputUrl: opt.url || '', fullUrl: opt.url || '', retry: Object.assign(Object.assign({}, retry), _filterUndefinedValues(opt.retry || {})), init: _merge(Object.assign(Object.assign({}, this.cfg.init), { method: opt.method || this.cfg.init.method, credentials: opt.credentials || this.cfg.init.credentials, redirect: ((_b = (_a = opt.followRedirects) !== null && _a !== void 0 ? _a : this.cfg.followRedirects) !== null && _b !== void 0 ? _b : true) ? 'follow' : 'error' }), {
|
|
448
419
|
headers: _mapKeys(opt.headers || {}, k => k.toLowerCase()),
|
|
449
420
|
}) });
|
|
450
421
|
// setup url
|
|
451
422
|
const baseUrl = opt.baseUrl || this.cfg.baseUrl;
|
|
452
423
|
if (baseUrl) {
|
|
453
|
-
if (
|
|
424
|
+
if (req.fullUrl.startsWith('/')) {
|
|
454
425
|
console.warn(`Fetcher: url should not start with / when baseUrl is specified`);
|
|
455
|
-
|
|
426
|
+
req.fullUrl = req.fullUrl.slice(1);
|
|
456
427
|
}
|
|
457
|
-
req.
|
|
428
|
+
req.fullUrl = `${baseUrl}/${req.inputUrl}`;
|
|
458
429
|
}
|
|
459
430
|
const searchParams = _filterUndefinedValues(Object.assign(Object.assign({}, this.cfg.searchParams), opt.searchParams));
|
|
460
431
|
if (Object.keys(searchParams).length) {
|
|
461
432
|
const qs = new URLSearchParams(searchParams).toString();
|
|
462
|
-
req.
|
|
433
|
+
req.fullUrl += req.fullUrl.includes('?') ? '&' : '?' + qs;
|
|
463
434
|
}
|
|
464
435
|
// setup request body
|
|
465
436
|
if (opt.json !== undefined) {
|
package/dist-esm/promise/pMap.js
CHANGED
|
@@ -6,7 +6,6 @@ Improvements:
|
|
|
6
6
|
- Included Typescript typings (no need for @types/p-map)
|
|
7
7
|
- Compatible with pProps (that had typings issues)
|
|
8
8
|
*/
|
|
9
|
-
import { __asyncValues } from "tslib";
|
|
10
9
|
import { END, ErrorMode, SKIP } from '..';
|
|
11
10
|
/**
|
|
12
11
|
* Returns a `Promise` that is fulfilled when all promises in `input` and ones returned from `mapper` are fulfilled,
|
|
@@ -35,7 +34,6 @@ import { END, ErrorMode, SKIP } from '..';
|
|
|
35
34
|
* })();
|
|
36
35
|
*/
|
|
37
36
|
export async function pMap(iterable, mapper, opt = {}) {
|
|
38
|
-
var _a, e_1, _b, _c;
|
|
39
37
|
const ret = [];
|
|
40
38
|
// const iterator = iterable[Symbol.iterator]()
|
|
41
39
|
const items = [...iterable];
|
|
@@ -49,40 +47,23 @@ export async function pMap(iterable, mapper, opt = {}) {
|
|
|
49
47
|
let currentIndex = 0;
|
|
50
48
|
// Special cases that are able to preserve async stack traces
|
|
51
49
|
if (concurrency === 1) {
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
for (var _d = true, items_1 = __asyncValues(items), items_1_1; items_1_1 = await items_1.next(), _a = items_1_1.done, !_a;) {
|
|
55
|
-
_c = items_1_1.value;
|
|
56
|
-
_d = false;
|
|
57
|
-
try {
|
|
58
|
-
const item = _c;
|
|
59
|
-
try {
|
|
60
|
-
const r = await mapper(item, currentIndex++);
|
|
61
|
-
if (r === END)
|
|
62
|
-
break;
|
|
63
|
-
if (r !== SKIP)
|
|
64
|
-
ret.push(r);
|
|
65
|
-
}
|
|
66
|
-
catch (err) {
|
|
67
|
-
if (errorMode === ErrorMode.THROW_IMMEDIATELY)
|
|
68
|
-
throw err;
|
|
69
|
-
if (errorMode === ErrorMode.THROW_AGGREGATED) {
|
|
70
|
-
errors.push(err);
|
|
71
|
-
}
|
|
72
|
-
// otherwise, suppress completely
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
finally {
|
|
76
|
-
_d = true;
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
}
|
|
80
|
-
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
81
|
-
finally {
|
|
50
|
+
// Special case for concurrency == 1
|
|
51
|
+
for (const item of items) {
|
|
82
52
|
try {
|
|
83
|
-
|
|
53
|
+
const r = await mapper(item, currentIndex++);
|
|
54
|
+
if (r === END)
|
|
55
|
+
break;
|
|
56
|
+
if (r !== SKIP)
|
|
57
|
+
ret.push(r);
|
|
58
|
+
}
|
|
59
|
+
catch (err) {
|
|
60
|
+
if (errorMode === ErrorMode.THROW_IMMEDIATELY)
|
|
61
|
+
throw err;
|
|
62
|
+
if (errorMode === ErrorMode.THROW_AGGREGATED) {
|
|
63
|
+
errors.push(err);
|
|
64
|
+
}
|
|
65
|
+
// otherwise, suppress completely
|
|
84
66
|
}
|
|
85
|
-
finally { if (e_1) throw e_1.error; }
|
|
86
67
|
}
|
|
87
68
|
if (errors.length) {
|
|
88
69
|
throw new AggregateError(errors, `pMap resulted in ${errors.length} error(s)`);
|
package/package.json
CHANGED
|
@@ -1,16 +1,24 @@
|
|
|
1
1
|
import type { CommonLogger } from '../log/commonLogger'
|
|
2
2
|
import type { Promisable } from '../typeFest'
|
|
3
|
-
import type { Reviver } from '../types'
|
|
3
|
+
import type { Reviver, UnixTimestampMillisNumber } from '../types'
|
|
4
4
|
import type { HttpMethod, HttpStatusFamily } from './http.model'
|
|
5
5
|
|
|
6
|
-
export interface FetcherNormalizedCfg
|
|
6
|
+
export interface FetcherNormalizedCfg
|
|
7
|
+
extends Required<FetcherCfg>,
|
|
8
|
+
Omit<FetcherRequest, 'started' | 'fullUrl'> {
|
|
7
9
|
logger: CommonLogger
|
|
8
10
|
searchParams: Record<string, any>
|
|
9
11
|
}
|
|
10
12
|
|
|
11
|
-
export type FetcherBeforeRequestHook =
|
|
12
|
-
|
|
13
|
-
|
|
13
|
+
export type FetcherBeforeRequestHook = <BODY = unknown>(
|
|
14
|
+
req: FetcherRequest<BODY>,
|
|
15
|
+
) => Promisable<void>
|
|
16
|
+
export type FetcherAfterResponseHook = <BODY = unknown>(
|
|
17
|
+
res: FetcherResponse<BODY>,
|
|
18
|
+
) => Promisable<void>
|
|
19
|
+
export type FetcherBeforeRetryHook = <BODY = unknown>(
|
|
20
|
+
res: FetcherResponse<BODY>,
|
|
21
|
+
) => Promisable<void>
|
|
14
22
|
|
|
15
23
|
export interface FetcherCfg {
|
|
16
24
|
/**
|
|
@@ -88,8 +96,17 @@ export interface FetcherRetryOptions {
|
|
|
88
96
|
timeoutMultiplier: number
|
|
89
97
|
}
|
|
90
98
|
|
|
91
|
-
export interface FetcherRequest
|
|
92
|
-
url
|
|
99
|
+
export interface FetcherRequest<BODY = unknown>
|
|
100
|
+
extends Omit<FetcherOptions<BODY>, 'method' | 'headers' | 'baseUrl' | 'url'> {
|
|
101
|
+
/**
|
|
102
|
+
* inputUrl is only the part that was passed in the request,
|
|
103
|
+
* without baseUrl or searchParams.
|
|
104
|
+
*/
|
|
105
|
+
inputUrl: string
|
|
106
|
+
/**
|
|
107
|
+
* fullUrl includes baseUrl and searchParams.
|
|
108
|
+
*/
|
|
109
|
+
fullUrl: string
|
|
93
110
|
init: RequestInitNormalized
|
|
94
111
|
mode: FetcherMode
|
|
95
112
|
throwHttpErrors: boolean
|
|
@@ -98,11 +115,18 @@ export interface FetcherRequest extends Omit<FetcherOptions, 'method' | 'headers
|
|
|
98
115
|
retryPost: boolean
|
|
99
116
|
retry4xx: boolean
|
|
100
117
|
retry5xx: boolean
|
|
118
|
+
started: UnixTimestampMillisNumber
|
|
101
119
|
}
|
|
102
120
|
|
|
103
|
-
export interface FetcherOptions {
|
|
121
|
+
export interface FetcherOptions<BODY = unknown> {
|
|
104
122
|
method?: HttpMethod
|
|
105
123
|
|
|
124
|
+
/**
|
|
125
|
+
* If defined - this `url` will override the original given `url`.
|
|
126
|
+
* baseUrl (and searchParams) will still modify it.
|
|
127
|
+
*/
|
|
128
|
+
url?: string
|
|
129
|
+
|
|
106
130
|
baseUrl?: string
|
|
107
131
|
|
|
108
132
|
throwHttpErrors?: boolean
|
|
@@ -159,6 +183,22 @@ export interface FetcherOptions {
|
|
|
159
183
|
retry5xx?: boolean
|
|
160
184
|
|
|
161
185
|
jsonReviver?: Reviver
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* Allows to walk over multiple pages of results.
|
|
189
|
+
* Paginate take a function.
|
|
190
|
+
* Function has access to FetcherResponse and FetcherOptions
|
|
191
|
+
* and has to make a decision to continue pagination or not.
|
|
192
|
+
*
|
|
193
|
+
* Return false to stop pagination.
|
|
194
|
+
* Return true to continue pagination.
|
|
195
|
+
* Feel free to mutate/modify opt (FetcherOptions), for example:
|
|
196
|
+
*
|
|
197
|
+
* opt.searchParams!['page']++
|
|
198
|
+
*
|
|
199
|
+
* @experimental
|
|
200
|
+
*/
|
|
201
|
+
paginate?: (res: FetcherSuccessResponse<BODY>, opt: FetcherOptions<BODY>) => Promisable<boolean>
|
|
162
202
|
}
|
|
163
203
|
|
|
164
204
|
export type RequestInitNormalized = Omit<RequestInit, 'method' | 'headers'> & {
|
package/src/http/fetcher.ts
CHANGED
|
@@ -15,7 +15,6 @@ import {
|
|
|
15
15
|
import { pDelay } from '../promise/pDelay'
|
|
16
16
|
import { _jsonParse, _jsonParseIfPossible } from '../string/json.util'
|
|
17
17
|
import { _since } from '../time/time.util'
|
|
18
|
-
import { UnixTimestampNumber } from '../types'
|
|
19
18
|
import type {
|
|
20
19
|
FetcherAfterResponseHook,
|
|
21
20
|
FetcherBeforeRequestHook,
|
|
@@ -54,8 +53,12 @@ export class Fetcher {
|
|
|
54
53
|
const m = method.toLowerCase()
|
|
55
54
|
|
|
56
55
|
// mode=void
|
|
57
|
-
;(this as any)[`${m}Void`] = async (
|
|
58
|
-
|
|
56
|
+
;(this as any)[`${m}Void`] = async (
|
|
57
|
+
url: string,
|
|
58
|
+
opt?: FetcherOptions<void>,
|
|
59
|
+
): Promise<void> => {
|
|
60
|
+
return await this.fetch<void>({
|
|
61
|
+
url,
|
|
59
62
|
method,
|
|
60
63
|
mode: 'void',
|
|
61
64
|
...opt,
|
|
@@ -63,8 +66,12 @@ export class Fetcher {
|
|
|
63
66
|
}
|
|
64
67
|
|
|
65
68
|
if (method === 'HEAD') return // mode=text
|
|
66
|
-
;(this as any)[`${m}Text`] = async (
|
|
67
|
-
|
|
69
|
+
;(this as any)[`${m}Text`] = async (
|
|
70
|
+
url: string,
|
|
71
|
+
opt?: FetcherOptions<string>,
|
|
72
|
+
): Promise<string> => {
|
|
73
|
+
return await this.fetch<string>({
|
|
74
|
+
url,
|
|
68
75
|
method,
|
|
69
76
|
mode: 'text',
|
|
70
77
|
...opt,
|
|
@@ -72,8 +79,9 @@ export class Fetcher {
|
|
|
72
79
|
}
|
|
73
80
|
|
|
74
81
|
// Default mode=json, but overridable
|
|
75
|
-
;(this as any)[m] = async <T = unknown>(url: string, opt?: FetcherOptions): Promise<T> => {
|
|
76
|
-
return await this.fetch<T>(
|
|
82
|
+
;(this as any)[m] = async <T = unknown>(url: string, opt?: FetcherOptions<T>): Promise<T> => {
|
|
83
|
+
return await this.fetch<T>({
|
|
84
|
+
url,
|
|
77
85
|
method,
|
|
78
86
|
mode: 'json',
|
|
79
87
|
...opt,
|
|
@@ -108,26 +116,26 @@ export class Fetcher {
|
|
|
108
116
|
|
|
109
117
|
// These methods are generated dynamically in the constructor
|
|
110
118
|
// These default methods use mode=json
|
|
111
|
-
get!: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>
|
|
112
|
-
post!: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>
|
|
113
|
-
put!: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>
|
|
114
|
-
patch!: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>
|
|
115
|
-
delete!: <T = unknown>(url: string, opt?: FetcherOptions) => Promise<T>
|
|
119
|
+
get!: <T = unknown>(url: string, opt?: FetcherOptions<T>) => Promise<T>
|
|
120
|
+
post!: <T = unknown>(url: string, opt?: FetcherOptions<T>) => Promise<T>
|
|
121
|
+
put!: <T = unknown>(url: string, opt?: FetcherOptions<T>) => Promise<T>
|
|
122
|
+
patch!: <T = unknown>(url: string, opt?: FetcherOptions<T>) => Promise<T>
|
|
123
|
+
delete!: <T = unknown>(url: string, opt?: FetcherOptions<T>) => Promise<T>
|
|
116
124
|
|
|
117
125
|
// mode=text
|
|
118
|
-
getText!: (url: string, opt?: FetcherOptions) => Promise<string>
|
|
119
|
-
postText!: (url: string, opt?: FetcherOptions) => Promise<string>
|
|
120
|
-
putText!: (url: string, opt?: FetcherOptions) => Promise<string>
|
|
121
|
-
patchText!: (url: string, opt?: FetcherOptions) => Promise<string>
|
|
122
|
-
deleteText!: (url: string, opt?: FetcherOptions) => Promise<string>
|
|
126
|
+
getText!: (url: string, opt?: FetcherOptions<string>) => Promise<string>
|
|
127
|
+
postText!: (url: string, opt?: FetcherOptions<string>) => Promise<string>
|
|
128
|
+
putText!: (url: string, opt?: FetcherOptions<string>) => Promise<string>
|
|
129
|
+
patchText!: (url: string, opt?: FetcherOptions<string>) => Promise<string>
|
|
130
|
+
deleteText!: (url: string, opt?: FetcherOptions<string>) => Promise<string>
|
|
123
131
|
|
|
124
132
|
// mode=void (no body fetching/parsing)
|
|
125
|
-
getVoid!: (url: string, opt?: FetcherOptions) => Promise<void>
|
|
126
|
-
postVoid!: (url: string, opt?: FetcherOptions) => Promise<void>
|
|
127
|
-
putVoid!: (url: string, opt?: FetcherOptions) => Promise<void>
|
|
128
|
-
patchVoid!: (url: string, opt?: FetcherOptions) => Promise<void>
|
|
129
|
-
deleteVoid!: (url: string, opt?: FetcherOptions) => Promise<void>
|
|
130
|
-
headVoid!: (url: string, opt?: FetcherOptions) => Promise<void>
|
|
133
|
+
getVoid!: (url: string, opt?: FetcherOptions<void>) => Promise<void>
|
|
134
|
+
postVoid!: (url: string, opt?: FetcherOptions<void>) => Promise<void>
|
|
135
|
+
putVoid!: (url: string, opt?: FetcherOptions<void>) => Promise<void>
|
|
136
|
+
patchVoid!: (url: string, opt?: FetcherOptions<void>) => Promise<void>
|
|
137
|
+
deleteVoid!: (url: string, opt?: FetcherOptions<void>) => Promise<void>
|
|
138
|
+
headVoid!: (url: string, opt?: FetcherOptions<void>) => Promise<void>
|
|
131
139
|
|
|
132
140
|
// mode=readableStream
|
|
133
141
|
/**
|
|
@@ -136,15 +144,19 @@ export class Fetcher {
|
|
|
136
144
|
* More on streams and Node interop:
|
|
137
145
|
* https://css-tricks.com/web-streams-everywhere-and-fetch-for-node-js/
|
|
138
146
|
*/
|
|
139
|
-
async getReadableStream(
|
|
140
|
-
|
|
147
|
+
async getReadableStream(
|
|
148
|
+
url: string,
|
|
149
|
+
opt?: FetcherOptions<ReadableStream<Uint8Array>>,
|
|
150
|
+
): Promise<ReadableStream<Uint8Array>> {
|
|
151
|
+
return await this.fetch({
|
|
152
|
+
url,
|
|
141
153
|
mode: 'readableStream',
|
|
142
154
|
...opt,
|
|
143
155
|
})
|
|
144
156
|
}
|
|
145
157
|
|
|
146
|
-
async fetch<T = unknown>(
|
|
147
|
-
const res = await this.doFetch<T>(
|
|
158
|
+
async fetch<T = unknown>(opt: FetcherOptions<T>): Promise<T> {
|
|
159
|
+
const res = await this.doFetch<T>(opt)
|
|
148
160
|
if (res.err) {
|
|
149
161
|
if (res.req.throwHttpErrors) throw res.err
|
|
150
162
|
return res as any
|
|
@@ -157,13 +169,9 @@ export class Fetcher {
|
|
|
157
169
|
* Never throws, returns `err` property in the response instead.
|
|
158
170
|
* Use this method instead of `throwHttpErrors: false` or try-catching.
|
|
159
171
|
*/
|
|
160
|
-
async doFetch<T = unknown>(
|
|
161
|
-
|
|
162
|
-
rawOpt: FetcherOptions = {},
|
|
163
|
-
): Promise<FetcherResponse<T>> {
|
|
172
|
+
async doFetch<T = unknown>(opt: FetcherOptions<T>): Promise<FetcherResponse<T>> {
|
|
173
|
+
const req = this.normalizeOptions(opt)
|
|
164
174
|
const { logger } = this.cfg
|
|
165
|
-
|
|
166
|
-
const req = this.normalizeOptions(url, rawOpt)
|
|
167
175
|
const {
|
|
168
176
|
timeoutSeconds,
|
|
169
177
|
init: { method },
|
|
@@ -179,13 +187,13 @@ export class Fetcher {
|
|
|
179
187
|
}, timeoutSeconds * 1000) as any as number
|
|
180
188
|
}
|
|
181
189
|
|
|
182
|
-
for
|
|
190
|
+
for (const hook of this.cfg.hooks.beforeRequest || []) {
|
|
183
191
|
await hook(req)
|
|
184
192
|
}
|
|
185
193
|
|
|
186
|
-
const isFullUrl = req.
|
|
187
|
-
const fullUrl = isFullUrl ? new URL(req.
|
|
188
|
-
const shortUrl = fullUrl ? this.getShortUrl(fullUrl) : req.
|
|
194
|
+
const isFullUrl = req.fullUrl.includes('://')
|
|
195
|
+
const fullUrl = isFullUrl ? new URL(req.fullUrl) : undefined
|
|
196
|
+
const shortUrl = fullUrl ? this.getShortUrl(fullUrl) : req.fullUrl
|
|
189
197
|
const signature = [method, shortUrl].join(' ')
|
|
190
198
|
|
|
191
199
|
const res = {
|
|
@@ -199,7 +207,7 @@ export class Fetcher {
|
|
|
199
207
|
} as FetcherResponse<any>
|
|
200
208
|
|
|
201
209
|
while (!res.retryStatus.retryStopped) {
|
|
202
|
-
|
|
210
|
+
req.started = Date.now()
|
|
203
211
|
|
|
204
212
|
if (this.cfg.logRequest) {
|
|
205
213
|
const { retryAttempt } = res.retryStatus
|
|
@@ -214,7 +222,7 @@ export class Fetcher {
|
|
|
214
222
|
}
|
|
215
223
|
|
|
216
224
|
try {
|
|
217
|
-
res.fetchResponse = await this.callNativeFetch(req.
|
|
225
|
+
res.fetchResponse = await this.callNativeFetch(req.fullUrl, req.init)
|
|
218
226
|
res.ok = res.fetchResponse.ok
|
|
219
227
|
} catch (err) {
|
|
220
228
|
// For example, CORS error would result in "TypeError: failed to fetch" here
|
|
@@ -226,27 +234,29 @@ export class Fetcher {
|
|
|
226
234
|
res.statusFamily = this.getStatusFamily(res)
|
|
227
235
|
|
|
228
236
|
if (res.fetchResponse?.ok) {
|
|
229
|
-
await this.onOkResponse(
|
|
230
|
-
res as FetcherResponse<T> & { fetchResponse: Response },
|
|
231
|
-
started,
|
|
232
|
-
timeout,
|
|
233
|
-
)
|
|
237
|
+
await this.onOkResponse(res as FetcherResponse<T> & { fetchResponse: Response }, timeout)
|
|
234
238
|
} else {
|
|
235
239
|
// !res.ok
|
|
236
|
-
await this.onNotOkResponse(res,
|
|
240
|
+
await this.onNotOkResponse(res, timeout)
|
|
237
241
|
}
|
|
238
242
|
}
|
|
239
243
|
|
|
240
|
-
for
|
|
244
|
+
for (const hook of this.cfg.hooks.afterResponse || []) {
|
|
241
245
|
await hook(res)
|
|
242
246
|
}
|
|
243
247
|
|
|
248
|
+
if (req.paginate && res.ok) {
|
|
249
|
+
const proceeed = await req.paginate(res, opt)
|
|
250
|
+
if (proceeed) {
|
|
251
|
+
return await this.doFetch(opt)
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
|
|
244
255
|
return res
|
|
245
256
|
}
|
|
246
257
|
|
|
247
258
|
private async onOkResponse(
|
|
248
259
|
res: FetcherResponse<any> & { fetchResponse: Response },
|
|
249
|
-
started: UnixTimestampNumber,
|
|
250
260
|
timeout?: number,
|
|
251
261
|
): Promise<void> {
|
|
252
262
|
const { req } = res
|
|
@@ -273,7 +283,7 @@ export class Fetcher {
|
|
|
273
283
|
res.err = _anyToError(err)
|
|
274
284
|
res.ok = false
|
|
275
285
|
|
|
276
|
-
return await this.onNotOkResponse(res,
|
|
286
|
+
return await this.onNotOkResponse(res, timeout)
|
|
277
287
|
}
|
|
278
288
|
} else {
|
|
279
289
|
// Body had a '' (empty string)
|
|
@@ -296,7 +306,7 @@ export class Fetcher {
|
|
|
296
306
|
if (res.body === null) {
|
|
297
307
|
res.err = new Error(`fetchResponse.body is null`)
|
|
298
308
|
res.ok = false
|
|
299
|
-
return await this.onNotOkResponse(res,
|
|
309
|
+
return await this.onNotOkResponse(res, timeout)
|
|
300
310
|
}
|
|
301
311
|
}
|
|
302
312
|
|
|
@@ -313,7 +323,7 @@ export class Fetcher {
|
|
|
313
323
|
res.fetchResponse.status,
|
|
314
324
|
res.signature,
|
|
315
325
|
retryAttempt && `try#${retryAttempt + 1}/${req.retry.count + 1}`,
|
|
316
|
-
_since(started),
|
|
326
|
+
_since(res.req.started),
|
|
317
327
|
]
|
|
318
328
|
.filter(Boolean)
|
|
319
329
|
.join(' '),
|
|
@@ -332,11 +342,7 @@ export class Fetcher {
|
|
|
332
342
|
return await globalThis.fetch(url, init)
|
|
333
343
|
}
|
|
334
344
|
|
|
335
|
-
private async onNotOkResponse(
|
|
336
|
-
res: FetcherResponse,
|
|
337
|
-
started: UnixTimestampNumber,
|
|
338
|
-
timeout?: number,
|
|
339
|
-
): Promise<void> {
|
|
345
|
+
private async onNotOkResponse(res: FetcherResponse, timeout?: number): Promise<void> {
|
|
340
346
|
clearTimeout(timeout)
|
|
341
347
|
|
|
342
348
|
let cause: ErrorObject | undefined
|
|
@@ -363,11 +369,11 @@ export class Fetcher {
|
|
|
363
369
|
// Enabled, cause `data` is not printed by default when error is HttpError
|
|
364
370
|
// method: req.method,
|
|
365
371
|
// tryCount: req.tryCount,
|
|
366
|
-
requestUrl: res.req.
|
|
372
|
+
requestUrl: res.req.fullUrl,
|
|
367
373
|
requestBaseUrl: this.cfg.baseUrl || (null as any),
|
|
368
374
|
requestMethod: res.req.init.method,
|
|
369
375
|
requestSignature: res.signature,
|
|
370
|
-
requestDuration: Date.now() - started,
|
|
376
|
+
requestDuration: Date.now() - res.req.started,
|
|
371
377
|
}),
|
|
372
378
|
cause,
|
|
373
379
|
)
|
|
@@ -382,7 +388,7 @@ export class Fetcher {
|
|
|
382
388
|
retryStatus.retryStopped = true
|
|
383
389
|
}
|
|
384
390
|
|
|
385
|
-
for
|
|
391
|
+
for (const hook of this.cfg.hooks.beforeRetry || []) {
|
|
386
392
|
await hook(res)
|
|
387
393
|
}
|
|
388
394
|
|
|
@@ -394,6 +400,26 @@ export class Fetcher {
|
|
|
394
400
|
|
|
395
401
|
if (retryStatus.retryStopped) return
|
|
396
402
|
|
|
403
|
+
// Here we know that more retries will be attempted
|
|
404
|
+
// We don't log "last error", because it will be thrown and logged by consumer,
|
|
405
|
+
// but we should log all previous errors, otherwise they are lost.
|
|
406
|
+
// Here is the right place where we know it's not the "last error"
|
|
407
|
+
if (res.err) {
|
|
408
|
+
const { retryAttempt } = retryStatus
|
|
409
|
+
this.cfg.logger.error(
|
|
410
|
+
[
|
|
411
|
+
' <<',
|
|
412
|
+
res.fetchResponse?.status || 0,
|
|
413
|
+
res.signature,
|
|
414
|
+
`try#${retryAttempt + 1}/${count + 1}`,
|
|
415
|
+
_since(res.req.started),
|
|
416
|
+
]
|
|
417
|
+
.filter(Boolean)
|
|
418
|
+
.join(' '),
|
|
419
|
+
res.err.cause || res.err,
|
|
420
|
+
)
|
|
421
|
+
}
|
|
422
|
+
|
|
397
423
|
retryStatus.retryAttempt++
|
|
398
424
|
retryStatus.retryTimeout = _clamp(retryStatus.retryTimeout * timeoutMultiplier, 0, timeoutMax)
|
|
399
425
|
|
|
@@ -466,7 +492,7 @@ export class Fetcher {
|
|
|
466
492
|
const norm: FetcherNormalizedCfg = _merge(
|
|
467
493
|
{
|
|
468
494
|
baseUrl: '',
|
|
469
|
-
|
|
495
|
+
inputUrl: '',
|
|
470
496
|
mode: 'void',
|
|
471
497
|
searchParams: {},
|
|
472
498
|
timeoutSeconds: 30,
|
|
@@ -499,7 +525,7 @@ export class Fetcher {
|
|
|
499
525
|
return norm
|
|
500
526
|
}
|
|
501
527
|
|
|
502
|
-
private normalizeOptions(
|
|
528
|
+
private normalizeOptions<BODY>(opt: FetcherOptions<BODY>): FetcherRequest<BODY> {
|
|
503
529
|
const {
|
|
504
530
|
timeoutSeconds,
|
|
505
531
|
throwHttpErrors,
|
|
@@ -511,9 +537,9 @@ export class Fetcher {
|
|
|
511
537
|
jsonReviver,
|
|
512
538
|
} = this.cfg
|
|
513
539
|
|
|
514
|
-
const req: FetcherRequest = {
|
|
540
|
+
const req: FetcherRequest<BODY> = {
|
|
541
|
+
started: Date.now(),
|
|
515
542
|
mode,
|
|
516
|
-
url,
|
|
517
543
|
timeoutSeconds,
|
|
518
544
|
throwHttpErrors,
|
|
519
545
|
retryPost,
|
|
@@ -521,6 +547,8 @@ export class Fetcher {
|
|
|
521
547
|
retry5xx,
|
|
522
548
|
jsonReviver,
|
|
523
549
|
..._omit(opt, ['method', 'headers', 'credentials']),
|
|
550
|
+
inputUrl: opt.url || '',
|
|
551
|
+
fullUrl: opt.url || '',
|
|
524
552
|
retry: {
|
|
525
553
|
...retry,
|
|
526
554
|
..._filterUndefinedValues(opt.retry || {}),
|
|
@@ -541,11 +569,11 @@ export class Fetcher {
|
|
|
541
569
|
// setup url
|
|
542
570
|
const baseUrl = opt.baseUrl || this.cfg.baseUrl
|
|
543
571
|
if (baseUrl) {
|
|
544
|
-
if (
|
|
572
|
+
if (req.fullUrl.startsWith('/')) {
|
|
545
573
|
console.warn(`Fetcher: url should not start with / when baseUrl is specified`)
|
|
546
|
-
|
|
574
|
+
req.fullUrl = req.fullUrl.slice(1)
|
|
547
575
|
}
|
|
548
|
-
req.
|
|
576
|
+
req.fullUrl = `${baseUrl}/${req.inputUrl}`
|
|
549
577
|
}
|
|
550
578
|
|
|
551
579
|
const searchParams = _filterUndefinedValues({
|
|
@@ -555,7 +583,7 @@ export class Fetcher {
|
|
|
555
583
|
|
|
556
584
|
if (Object.keys(searchParams).length) {
|
|
557
585
|
const qs = new URLSearchParams(searchParams).toString()
|
|
558
|
-
req.
|
|
586
|
+
req.fullUrl += req.fullUrl.includes('?') ? '&' : '?' + qs
|
|
559
587
|
}
|
|
560
588
|
|
|
561
589
|
// setup request body
|
package/src/promise/pMap.ts
CHANGED