@naturalcycles/js-lib 14.157.0 → 14.157.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/http/fetcher.js +47 -43
- package/dist/http/fetcher.model.d.ts +9 -5
- package/dist/promise/pTimeout.d.ts +2 -2
- package/dist-esm/http/fetcher.js +49 -45
- package/package.json +1 -1
- package/src/http/fetcher.model.ts +9 -4
- package/src/http/fetcher.ts +51 -47
- package/src/promise/pTimeout.ts +2 -2
package/dist/http/fetcher.js
CHANGED
|
@@ -121,19 +121,6 @@ class Fetcher {
|
|
|
121
121
|
const req = this.normalizeOptions(opt);
|
|
122
122
|
const { logger } = this.cfg;
|
|
123
123
|
const { timeoutSeconds, init: { method }, } = req;
|
|
124
|
-
// setup timeout
|
|
125
|
-
let timeout;
|
|
126
|
-
if (timeoutSeconds) {
|
|
127
|
-
const abortController = new AbortController();
|
|
128
|
-
req.init.signal = abortController.signal;
|
|
129
|
-
timeout = setTimeout(() => {
|
|
130
|
-
// Apparently, providing a `string` reason to abort() causes Undici to throw `invalid_argument` error,
|
|
131
|
-
// so, we're wrapping it in a TimeoutError instance
|
|
132
|
-
abortController.abort(new pTimeout_1.TimeoutError(`request timed out after ${timeoutSeconds} sec`));
|
|
133
|
-
// abortController.abort(`timeout of ${timeoutSeconds} sec`)
|
|
134
|
-
// abortController.abort()
|
|
135
|
-
}, timeoutSeconds * 1000);
|
|
136
|
-
}
|
|
137
124
|
for (const hook of this.cfg.hooks.beforeRequest || []) {
|
|
138
125
|
await hook(req);
|
|
139
126
|
}
|
|
@@ -152,6 +139,18 @@ class Fetcher {
|
|
|
152
139
|
};
|
|
153
140
|
while (!res.retryStatus.retryStopped) {
|
|
154
141
|
req.started = Date.now();
|
|
142
|
+
// setup timeout
|
|
143
|
+
let timeoutId;
|
|
144
|
+
if (timeoutSeconds) {
|
|
145
|
+
const abortController = new AbortController();
|
|
146
|
+
req.init.signal = abortController.signal;
|
|
147
|
+
timeoutId = setTimeout(() => {
|
|
148
|
+
// console.log(`actual request timed out in ${_since(req.started)}`)
|
|
149
|
+
// Apparently, providing a `string` reason to abort() causes Undici to throw `invalid_argument` error,
|
|
150
|
+
// so, we're wrapping it in a TimeoutError instance
|
|
151
|
+
abortController.abort(new pTimeout_1.TimeoutError(`request timed out after ${timeoutSeconds} sec`));
|
|
152
|
+
}, timeoutSeconds * 1000);
|
|
153
|
+
}
|
|
155
154
|
if (this.cfg.logRequest) {
|
|
156
155
|
const { retryAttempt } = res.retryStatus;
|
|
157
156
|
logger.log([' >>', signature, retryAttempt && `try#${retryAttempt + 1}/${req.retry.count + 1}`]
|
|
@@ -175,14 +174,30 @@ class Fetcher {
|
|
|
175
174
|
// important to set it to undefined, otherwise it can keep the previous value (from previous try)
|
|
176
175
|
res.fetchResponse = undefined;
|
|
177
176
|
}
|
|
177
|
+
finally {
|
|
178
|
+
clearTimeout(timeoutId);
|
|
179
|
+
// Separate Timeout will be introduced to "download and parse the body"
|
|
180
|
+
}
|
|
178
181
|
res.statusFamily = this.getStatusFamily(res);
|
|
179
182
|
res.statusCode = res.fetchResponse?.status;
|
|
180
183
|
if (res.fetchResponse?.ok) {
|
|
181
|
-
|
|
184
|
+
try {
|
|
185
|
+
// We are applying a separate Timeout (as long as original Timeout for now) to "download and parse the body"
|
|
186
|
+
await (0, pTimeout_1.pTimeout)(async () => await this.onOkResponse(res), {
|
|
187
|
+
timeout: timeoutSeconds * 1000 || Number.POSITIVE_INFINITY,
|
|
188
|
+
name: 'Fetcher.onOkResponse',
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
catch (err) {
|
|
192
|
+
// onOkResponse can still fail, e.g when loading/parsing json, text or doing other response manipulation
|
|
193
|
+
res.err = (0, error_util_1._anyToError)(err);
|
|
194
|
+
res.ok = false;
|
|
195
|
+
await this.onNotOkResponse(res);
|
|
196
|
+
}
|
|
182
197
|
}
|
|
183
198
|
else {
|
|
184
199
|
// !res.ok
|
|
185
|
-
await this.onNotOkResponse(res
|
|
200
|
+
await this.onNotOkResponse(res);
|
|
186
201
|
}
|
|
187
202
|
}
|
|
188
203
|
for (const hook of this.cfg.hooks.afterResponse || []) {
|
|
@@ -190,31 +205,17 @@ class Fetcher {
|
|
|
190
205
|
}
|
|
191
206
|
return res;
|
|
192
207
|
}
|
|
193
|
-
async onOkResponse(res
|
|
208
|
+
async onOkResponse(res) {
|
|
194
209
|
const { req } = res;
|
|
195
210
|
const { responseType } = res.req;
|
|
211
|
+
// This function is subject to a separate timeout to "download and parse the data"
|
|
196
212
|
if (responseType === 'json') {
|
|
197
213
|
if (res.fetchResponse.body) {
|
|
198
214
|
const text = await res.fetchResponse.text();
|
|
199
215
|
if (text) {
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
}
|
|
204
|
-
catch (err) {
|
|
205
|
-
// Error while parsing json
|
|
206
|
-
// res.err = _anyToError(err, HttpRequestError, {
|
|
207
|
-
// requestUrl: res.req.url,
|
|
208
|
-
// requestBaseUrl: this.cfg.baseUrl,
|
|
209
|
-
// requestMethod: res.req.init.method,
|
|
210
|
-
// requestSignature: res.signature,
|
|
211
|
-
// requestDuration: Date.now() - started,
|
|
212
|
-
// responseStatusCode: res.fetchResponse.status,
|
|
213
|
-
// } satisfies HttpRequestErrorData)
|
|
214
|
-
res.err = (0, error_util_1._anyToError)(err);
|
|
215
|
-
res.ok = false;
|
|
216
|
-
return await this.onNotOkResponse(res, timeout);
|
|
217
|
-
}
|
|
216
|
+
res.body = text;
|
|
217
|
+
res.body = (0, json_util_1._jsonParse)(text, req.jsonReviver);
|
|
218
|
+
// Error while parsing json can happen - it'll be handled upstream
|
|
218
219
|
}
|
|
219
220
|
else {
|
|
220
221
|
// Body had a '' (empty string)
|
|
@@ -239,12 +240,10 @@ class Fetcher {
|
|
|
239
240
|
else if (responseType === 'readableStream') {
|
|
240
241
|
res.body = res.fetchResponse.body;
|
|
241
242
|
if (res.body === null) {
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
return await this.onNotOkResponse(res, timeout);
|
|
243
|
+
// Error is to be handled upstream
|
|
244
|
+
throw new Error(`fetchResponse.body is null`);
|
|
245
245
|
}
|
|
246
246
|
}
|
|
247
|
-
clearTimeout(timeout);
|
|
248
247
|
res.retryStatus.retryStopped = true;
|
|
249
248
|
// res.err can happen on `failed to fetch` type of error, e.g JSON.parse, CORS, unexpected redirect
|
|
250
249
|
if (!res.err && this.cfg.logResponse) {
|
|
@@ -270,8 +269,7 @@ class Fetcher {
|
|
|
270
269
|
async callNativeFetch(url, init) {
|
|
271
270
|
return await globalThis.fetch(url, init);
|
|
272
271
|
}
|
|
273
|
-
async onNotOkResponse(res
|
|
274
|
-
clearTimeout(timeout);
|
|
272
|
+
async onNotOkResponse(res) {
|
|
275
273
|
let cause;
|
|
276
274
|
if (res.err) {
|
|
277
275
|
// This is only possible on JSON.parse error, or CORS error,
|
|
@@ -337,7 +335,11 @@ class Fetcher {
|
|
|
337
335
|
return;
|
|
338
336
|
retryStatus.retryAttempt++;
|
|
339
337
|
retryStatus.retryTimeout = (0, number_util_1._clamp)(retryStatus.retryTimeout * timeoutMultiplier, 0, timeoutMax);
|
|
340
|
-
|
|
338
|
+
const timeout = this.getRetryTimeout(res);
|
|
339
|
+
if (res.req.debug) {
|
|
340
|
+
this.cfg.logger.log(` .. ${res.signature} waiting ${(0, time_util_1._ms)(timeout)}`);
|
|
341
|
+
}
|
|
342
|
+
await (0, pDelay_1.pDelay)(timeout);
|
|
341
343
|
}
|
|
342
344
|
getRetryTimeout(res) {
|
|
343
345
|
let timeout = 0;
|
|
@@ -392,8 +394,9 @@ class Fetcher {
|
|
|
392
394
|
if (statusFamily === 3 && !retry3xx)
|
|
393
395
|
return false;
|
|
394
396
|
// should not retry on `unexpected redirect` in error.cause.cause
|
|
395
|
-
if (res.err?.cause?.cause?.message?.includes('unexpected redirect'))
|
|
397
|
+
if (res.err?.cause?.cause?.message?.includes('unexpected redirect')) {
|
|
396
398
|
return false;
|
|
399
|
+
}
|
|
397
400
|
return true; // default is true
|
|
398
401
|
}
|
|
399
402
|
getStatusFamily(res) {
|
|
@@ -431,7 +434,7 @@ class Fetcher {
|
|
|
431
434
|
}
|
|
432
435
|
normalizeCfg(cfg) {
|
|
433
436
|
if (cfg.baseUrl?.endsWith('/')) {
|
|
434
|
-
console.warn(`Fetcher: baseUrl should not end with
|
|
437
|
+
console.warn(`Fetcher: baseUrl should not end with slash: ${cfg.baseUrl}`);
|
|
435
438
|
cfg.baseUrl = cfg.baseUrl.slice(0, cfg.baseUrl.length - 1);
|
|
436
439
|
}
|
|
437
440
|
const { debug = false } = cfg;
|
|
@@ -482,6 +485,7 @@ class Fetcher {
|
|
|
482
485
|
'logRequestBody',
|
|
483
486
|
'logResponse',
|
|
484
487
|
'logResponseBody',
|
|
488
|
+
'debug',
|
|
485
489
|
]),
|
|
486
490
|
started: Date.now(),
|
|
487
491
|
...(0, object_util_1._omit)(opt, ['method', 'headers', 'credentials']),
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import type { CommonLogger } from '../log/commonLogger';
|
|
2
2
|
import type { Promisable } from '../typeFest';
|
|
3
|
-
import type { AnyObject, Reviver, UnixTimestampMillisNumber } from '../types';
|
|
3
|
+
import type { AnyObject, NumberOfMilliseconds, Reviver, UnixTimestampMillisNumber } from '../types';
|
|
4
4
|
import type { HttpMethod, HttpStatusFamily } from './http.model';
|
|
5
|
-
export interface FetcherNormalizedCfg extends Required<FetcherCfg>, Omit<FetcherRequest, 'started' | 'fullUrl' | 'logRequest' | 'logRequestBody' | 'logResponse' | 'logResponseBody' | 'redirect' | 'credentials'> {
|
|
5
|
+
export interface FetcherNormalizedCfg extends Required<FetcherCfg>, Omit<FetcherRequest, 'started' | 'fullUrl' | 'logRequest' | 'logRequestBody' | 'logResponse' | 'logResponseBody' | 'debug' | 'redirect' | 'credentials'> {
|
|
6
6
|
logger: CommonLogger;
|
|
7
7
|
searchParams: Record<string, any>;
|
|
8
8
|
}
|
|
@@ -68,13 +68,13 @@ export interface FetcherCfg {
|
|
|
68
68
|
}
|
|
69
69
|
export interface FetcherRetryStatus {
|
|
70
70
|
retryAttempt: number;
|
|
71
|
-
retryTimeout:
|
|
71
|
+
retryTimeout: NumberOfMilliseconds;
|
|
72
72
|
retryStopped: boolean;
|
|
73
73
|
}
|
|
74
74
|
export interface FetcherRetryOptions {
|
|
75
75
|
count: number;
|
|
76
|
-
timeout:
|
|
77
|
-
timeoutMax:
|
|
76
|
+
timeout: NumberOfMilliseconds;
|
|
77
|
+
timeoutMax: NumberOfMilliseconds;
|
|
78
78
|
timeoutMultiplier: number;
|
|
79
79
|
}
|
|
80
80
|
export interface FetcherRequest extends Omit<FetcherOptions, 'method' | 'headers' | 'baseUrl' | 'url'> {
|
|
@@ -172,6 +172,10 @@ export interface FetcherOptions {
|
|
|
172
172
|
logRequestBody?: boolean;
|
|
173
173
|
logResponse?: boolean;
|
|
174
174
|
logResponseBody?: boolean;
|
|
175
|
+
/**
|
|
176
|
+
* If true - enables all possible logging.
|
|
177
|
+
*/
|
|
178
|
+
debug?: boolean;
|
|
175
179
|
}
|
|
176
180
|
export type RequestInitNormalized = Omit<RequestInit, 'method' | 'headers'> & {
|
|
177
181
|
method: HttpMethod;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { AppError } from '../error/app.error';
|
|
2
2
|
import type { ErrorData, ErrorObject } from '../error/error.model';
|
|
3
|
-
import type { AnyAsyncFunction } from '../types';
|
|
3
|
+
import type { AnyAsyncFunction, NumberOfMilliseconds } from '../types';
|
|
4
4
|
export declare class TimeoutError extends AppError {
|
|
5
5
|
constructor(message: string, data?: {}, cause?: ErrorObject);
|
|
6
6
|
}
|
|
@@ -8,7 +8,7 @@ export interface PTimeoutOptions {
|
|
|
8
8
|
/**
|
|
9
9
|
* Timeout in milliseconds.
|
|
10
10
|
*/
|
|
11
|
-
timeout:
|
|
11
|
+
timeout: NumberOfMilliseconds;
|
|
12
12
|
/**
|
|
13
13
|
* If set - will be included in the error message.
|
|
14
14
|
* Can be used to identify the place in the code that failed.
|
package/dist-esm/http/fetcher.js
CHANGED
|
@@ -6,10 +6,10 @@ import { HttpRequestError } from '../error/httpRequestError';
|
|
|
6
6
|
import { _clamp } from '../number/number.util';
|
|
7
7
|
import { _filterNullishValues, _filterUndefinedValues, _mapKeys, _merge, _omit, _pick, } from '../object/object.util';
|
|
8
8
|
import { pDelay } from '../promise/pDelay';
|
|
9
|
-
import { TimeoutError } from '../promise/pTimeout';
|
|
9
|
+
import { pTimeout, TimeoutError } from '../promise/pTimeout';
|
|
10
10
|
import { _jsonParse, _jsonParseIfPossible } from '../string/json.util';
|
|
11
11
|
import { _stringifyAny } from '../string/stringifyAny';
|
|
12
|
-
import { _since } from '../time/time.util';
|
|
12
|
+
import { _ms, _since } from '../time/time.util';
|
|
13
13
|
import { HTTP_METHODS } from './http.model';
|
|
14
14
|
const acceptByResponseType = {
|
|
15
15
|
text: 'text/plain',
|
|
@@ -106,19 +106,6 @@ export class Fetcher {
|
|
|
106
106
|
const req = this.normalizeOptions(opt);
|
|
107
107
|
const { logger } = this.cfg;
|
|
108
108
|
const { timeoutSeconds, init: { method }, } = req;
|
|
109
|
-
// setup timeout
|
|
110
|
-
let timeout;
|
|
111
|
-
if (timeoutSeconds) {
|
|
112
|
-
const abortController = new AbortController();
|
|
113
|
-
req.init.signal = abortController.signal;
|
|
114
|
-
timeout = setTimeout(() => {
|
|
115
|
-
// Apparently, providing a `string` reason to abort() causes Undici to throw `invalid_argument` error,
|
|
116
|
-
// so, we're wrapping it in a TimeoutError instance
|
|
117
|
-
abortController.abort(new TimeoutError(`request timed out after ${timeoutSeconds} sec`));
|
|
118
|
-
// abortController.abort(`timeout of ${timeoutSeconds} sec`)
|
|
119
|
-
// abortController.abort()
|
|
120
|
-
}, timeoutSeconds * 1000);
|
|
121
|
-
}
|
|
122
109
|
for (const hook of this.cfg.hooks.beforeRequest || []) {
|
|
123
110
|
await hook(req);
|
|
124
111
|
}
|
|
@@ -137,6 +124,18 @@ export class Fetcher {
|
|
|
137
124
|
};
|
|
138
125
|
while (!res.retryStatus.retryStopped) {
|
|
139
126
|
req.started = Date.now();
|
|
127
|
+
// setup timeout
|
|
128
|
+
let timeoutId;
|
|
129
|
+
if (timeoutSeconds) {
|
|
130
|
+
const abortController = new AbortController();
|
|
131
|
+
req.init.signal = abortController.signal;
|
|
132
|
+
timeoutId = setTimeout(() => {
|
|
133
|
+
// console.log(`actual request timed out in ${_since(req.started)}`)
|
|
134
|
+
// Apparently, providing a `string` reason to abort() causes Undici to throw `invalid_argument` error,
|
|
135
|
+
// so, we're wrapping it in a TimeoutError instance
|
|
136
|
+
abortController.abort(new TimeoutError(`request timed out after ${timeoutSeconds} sec`));
|
|
137
|
+
}, timeoutSeconds * 1000);
|
|
138
|
+
}
|
|
140
139
|
if (this.cfg.logRequest) {
|
|
141
140
|
const { retryAttempt } = res.retryStatus;
|
|
142
141
|
logger.log([' >>', signature, retryAttempt && `try#${retryAttempt + 1}/${req.retry.count + 1}`]
|
|
@@ -160,14 +159,30 @@ export class Fetcher {
|
|
|
160
159
|
// important to set it to undefined, otherwise it can keep the previous value (from previous try)
|
|
161
160
|
res.fetchResponse = undefined;
|
|
162
161
|
}
|
|
162
|
+
finally {
|
|
163
|
+
clearTimeout(timeoutId);
|
|
164
|
+
// Separate Timeout will be introduced to "download and parse the body"
|
|
165
|
+
}
|
|
163
166
|
res.statusFamily = this.getStatusFamily(res);
|
|
164
167
|
res.statusCode = (_a = res.fetchResponse) === null || _a === void 0 ? void 0 : _a.status;
|
|
165
168
|
if ((_b = res.fetchResponse) === null || _b === void 0 ? void 0 : _b.ok) {
|
|
166
|
-
|
|
169
|
+
try {
|
|
170
|
+
// We are applying a separate Timeout (as long as original Timeout for now) to "download and parse the body"
|
|
171
|
+
await pTimeout(async () => await this.onOkResponse(res), {
|
|
172
|
+
timeout: timeoutSeconds * 1000 || Number.POSITIVE_INFINITY,
|
|
173
|
+
name: 'Fetcher.onOkResponse',
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
catch (err) {
|
|
177
|
+
// onOkResponse can still fail, e.g when loading/parsing json, text or doing other response manipulation
|
|
178
|
+
res.err = _anyToError(err);
|
|
179
|
+
res.ok = false;
|
|
180
|
+
await this.onNotOkResponse(res);
|
|
181
|
+
}
|
|
167
182
|
}
|
|
168
183
|
else {
|
|
169
184
|
// !res.ok
|
|
170
|
-
await this.onNotOkResponse(res
|
|
185
|
+
await this.onNotOkResponse(res);
|
|
171
186
|
}
|
|
172
187
|
}
|
|
173
188
|
for (const hook of this.cfg.hooks.afterResponse || []) {
|
|
@@ -175,31 +190,17 @@ export class Fetcher {
|
|
|
175
190
|
}
|
|
176
191
|
return res;
|
|
177
192
|
}
|
|
178
|
-
async onOkResponse(res
|
|
193
|
+
async onOkResponse(res) {
|
|
179
194
|
const { req } = res;
|
|
180
195
|
const { responseType } = res.req;
|
|
196
|
+
// This function is subject to a separate timeout to "download and parse the data"
|
|
181
197
|
if (responseType === 'json') {
|
|
182
198
|
if (res.fetchResponse.body) {
|
|
183
199
|
const text = await res.fetchResponse.text();
|
|
184
200
|
if (text) {
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
}
|
|
189
|
-
catch (err) {
|
|
190
|
-
// Error while parsing json
|
|
191
|
-
// res.err = _anyToError(err, HttpRequestError, {
|
|
192
|
-
// requestUrl: res.req.url,
|
|
193
|
-
// requestBaseUrl: this.cfg.baseUrl,
|
|
194
|
-
// requestMethod: res.req.init.method,
|
|
195
|
-
// requestSignature: res.signature,
|
|
196
|
-
// requestDuration: Date.now() - started,
|
|
197
|
-
// responseStatusCode: res.fetchResponse.status,
|
|
198
|
-
// } satisfies HttpRequestErrorData)
|
|
199
|
-
res.err = _anyToError(err);
|
|
200
|
-
res.ok = false;
|
|
201
|
-
return await this.onNotOkResponse(res, timeout);
|
|
202
|
-
}
|
|
201
|
+
res.body = text;
|
|
202
|
+
res.body = _jsonParse(text, req.jsonReviver);
|
|
203
|
+
// Error while parsing json can happen - it'll be handled upstream
|
|
203
204
|
}
|
|
204
205
|
else {
|
|
205
206
|
// Body had a '' (empty string)
|
|
@@ -224,12 +225,10 @@ export class Fetcher {
|
|
|
224
225
|
else if (responseType === 'readableStream') {
|
|
225
226
|
res.body = res.fetchResponse.body;
|
|
226
227
|
if (res.body === null) {
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
return await this.onNotOkResponse(res, timeout);
|
|
228
|
+
// Error is to be handled upstream
|
|
229
|
+
throw new Error(`fetchResponse.body is null`);
|
|
230
230
|
}
|
|
231
231
|
}
|
|
232
|
-
clearTimeout(timeout);
|
|
233
232
|
res.retryStatus.retryStopped = true;
|
|
234
233
|
// res.err can happen on `failed to fetch` type of error, e.g JSON.parse, CORS, unexpected redirect
|
|
235
234
|
if (!res.err && this.cfg.logResponse) {
|
|
@@ -255,9 +254,8 @@ export class Fetcher {
|
|
|
255
254
|
async callNativeFetch(url, init) {
|
|
256
255
|
return await globalThis.fetch(url, init);
|
|
257
256
|
}
|
|
258
|
-
async onNotOkResponse(res
|
|
257
|
+
async onNotOkResponse(res) {
|
|
259
258
|
var _a, _b;
|
|
260
|
-
clearTimeout(timeout);
|
|
261
259
|
let cause;
|
|
262
260
|
if (res.err) {
|
|
263
261
|
// This is only possible on JSON.parse error, or CORS error,
|
|
@@ -324,7 +322,11 @@ export class Fetcher {
|
|
|
324
322
|
return;
|
|
325
323
|
retryStatus.retryAttempt++;
|
|
326
324
|
retryStatus.retryTimeout = _clamp(retryStatus.retryTimeout * timeoutMultiplier, 0, timeoutMax);
|
|
327
|
-
|
|
325
|
+
const timeout = this.getRetryTimeout(res);
|
|
326
|
+
if (res.req.debug) {
|
|
327
|
+
this.cfg.logger.log(` .. ${res.signature} waiting ${_ms(timeout)}`);
|
|
328
|
+
}
|
|
329
|
+
await pDelay(timeout);
|
|
328
330
|
}
|
|
329
331
|
getRetryTimeout(res) {
|
|
330
332
|
var _a;
|
|
@@ -380,8 +382,9 @@ export class Fetcher {
|
|
|
380
382
|
if (statusFamily === 3 && !retry3xx)
|
|
381
383
|
return false;
|
|
382
384
|
// should not retry on `unexpected redirect` in error.cause.cause
|
|
383
|
-
if ((_e = (_d = (_c = (_b = res.err) === null || _b === void 0 ? void 0 : _b.cause) === null || _c === void 0 ? void 0 : _c.cause) === null || _d === void 0 ? void 0 : _d.message) === null || _e === void 0 ? void 0 : _e.includes('unexpected redirect'))
|
|
385
|
+
if ((_e = (_d = (_c = (_b = res.err) === null || _b === void 0 ? void 0 : _b.cause) === null || _c === void 0 ? void 0 : _c.cause) === null || _d === void 0 ? void 0 : _d.message) === null || _e === void 0 ? void 0 : _e.includes('unexpected redirect')) {
|
|
384
386
|
return false;
|
|
387
|
+
}
|
|
385
388
|
return true; // default is true
|
|
386
389
|
}
|
|
387
390
|
getStatusFamily(res) {
|
|
@@ -421,7 +424,7 @@ export class Fetcher {
|
|
|
421
424
|
normalizeCfg(cfg) {
|
|
422
425
|
var _a;
|
|
423
426
|
if ((_a = cfg.baseUrl) === null || _a === void 0 ? void 0 : _a.endsWith('/')) {
|
|
424
|
-
console.warn(`Fetcher: baseUrl should not end with
|
|
427
|
+
console.warn(`Fetcher: baseUrl should not end with slash: ${cfg.baseUrl}`);
|
|
425
428
|
cfg.baseUrl = cfg.baseUrl.slice(0, cfg.baseUrl.length - 1);
|
|
426
429
|
}
|
|
427
430
|
const { debug = false } = cfg;
|
|
@@ -469,6 +472,7 @@ export class Fetcher {
|
|
|
469
472
|
'logRequestBody',
|
|
470
473
|
'logResponse',
|
|
471
474
|
'logResponseBody',
|
|
475
|
+
'debug',
|
|
472
476
|
])), { started: Date.now() }), _omit(opt, ['method', 'headers', 'credentials'])), { inputUrl: opt.url || '', fullUrl: opt.url || '', retry: Object.assign(Object.assign({}, this.cfg.retry), _filterUndefinedValues(opt.retry || {})), init: _merge(Object.assign(Object.assign({}, this.cfg.init), { method: opt.method || this.cfg.init.method, credentials: opt.credentials || this.cfg.init.credentials, redirect: opt.redirect || this.cfg.init.redirect || 'follow' }), {
|
|
473
477
|
headers: _mapKeys(opt.headers || {}, k => k.toLowerCase()),
|
|
474
478
|
}) });
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { CommonLogger } from '../log/commonLogger'
|
|
2
2
|
import type { Promisable } from '../typeFest'
|
|
3
|
-
import type { AnyObject, Reviver, UnixTimestampMillisNumber } from '../types'
|
|
3
|
+
import type { AnyObject, NumberOfMilliseconds, Reviver, UnixTimestampMillisNumber } from '../types'
|
|
4
4
|
import type { HttpMethod, HttpStatusFamily } from './http.model'
|
|
5
5
|
|
|
6
6
|
export interface FetcherNormalizedCfg
|
|
@@ -13,6 +13,7 @@ export interface FetcherNormalizedCfg
|
|
|
13
13
|
| 'logRequestBody'
|
|
14
14
|
| 'logResponse'
|
|
15
15
|
| 'logResponseBody'
|
|
16
|
+
| 'debug'
|
|
16
17
|
| 'redirect'
|
|
17
18
|
| 'credentials'
|
|
18
19
|
> {
|
|
@@ -93,14 +94,14 @@ export interface FetcherCfg {
|
|
|
93
94
|
|
|
94
95
|
export interface FetcherRetryStatus {
|
|
95
96
|
retryAttempt: number
|
|
96
|
-
retryTimeout:
|
|
97
|
+
retryTimeout: NumberOfMilliseconds
|
|
97
98
|
retryStopped: boolean
|
|
98
99
|
}
|
|
99
100
|
|
|
100
101
|
export interface FetcherRetryOptions {
|
|
101
102
|
count: number
|
|
102
|
-
timeout:
|
|
103
|
-
timeoutMax:
|
|
103
|
+
timeout: NumberOfMilliseconds
|
|
104
|
+
timeoutMax: NumberOfMilliseconds
|
|
104
105
|
timeoutMultiplier: number
|
|
105
106
|
}
|
|
106
107
|
|
|
@@ -219,6 +220,10 @@ export interface FetcherOptions {
|
|
|
219
220
|
logRequestBody?: boolean
|
|
220
221
|
logResponse?: boolean
|
|
221
222
|
logResponseBody?: boolean
|
|
223
|
+
/**
|
|
224
|
+
* If true - enables all possible logging.
|
|
225
|
+
*/
|
|
226
|
+
debug?: boolean
|
|
222
227
|
}
|
|
223
228
|
|
|
224
229
|
export type RequestInitNormalized = Omit<RequestInit, 'method' | 'headers'> & {
|
package/src/http/fetcher.ts
CHANGED
|
@@ -15,10 +15,10 @@ import {
|
|
|
15
15
|
_pick,
|
|
16
16
|
} from '../object/object.util'
|
|
17
17
|
import { pDelay } from '../promise/pDelay'
|
|
18
|
-
import { TimeoutError } from '../promise/pTimeout'
|
|
18
|
+
import { pTimeout, TimeoutError } from '../promise/pTimeout'
|
|
19
19
|
import { _jsonParse, _jsonParseIfPossible } from '../string/json.util'
|
|
20
20
|
import { _stringifyAny } from '../string/stringifyAny'
|
|
21
|
-
import { _since } from '../time/time.util'
|
|
21
|
+
import { _ms, _since } from '../time/time.util'
|
|
22
22
|
import { NumberOfMilliseconds } from '../types'
|
|
23
23
|
import type {
|
|
24
24
|
FetcherAfterResponseHook,
|
|
@@ -182,20 +182,6 @@ export class Fetcher {
|
|
|
182
182
|
init: { method },
|
|
183
183
|
} = req
|
|
184
184
|
|
|
185
|
-
// setup timeout
|
|
186
|
-
let timeout: number | undefined
|
|
187
|
-
if (timeoutSeconds) {
|
|
188
|
-
const abortController = new AbortController()
|
|
189
|
-
req.init.signal = abortController.signal
|
|
190
|
-
timeout = setTimeout(() => {
|
|
191
|
-
// Apparently, providing a `string` reason to abort() causes Undici to throw `invalid_argument` error,
|
|
192
|
-
// so, we're wrapping it in a TimeoutError instance
|
|
193
|
-
abortController.abort(new TimeoutError(`request timed out after ${timeoutSeconds} sec`))
|
|
194
|
-
// abortController.abort(`timeout of ${timeoutSeconds} sec`)
|
|
195
|
-
// abortController.abort()
|
|
196
|
-
}, timeoutSeconds * 1000) as any as number
|
|
197
|
-
}
|
|
198
|
-
|
|
199
185
|
for (const hook of this.cfg.hooks.beforeRequest || []) {
|
|
200
186
|
await hook(req)
|
|
201
187
|
}
|
|
@@ -218,6 +204,19 @@ export class Fetcher {
|
|
|
218
204
|
while (!res.retryStatus.retryStopped) {
|
|
219
205
|
req.started = Date.now()
|
|
220
206
|
|
|
207
|
+
// setup timeout
|
|
208
|
+
let timeoutId: number | undefined
|
|
209
|
+
if (timeoutSeconds) {
|
|
210
|
+
const abortController = new AbortController()
|
|
211
|
+
req.init.signal = abortController.signal
|
|
212
|
+
timeoutId = setTimeout(() => {
|
|
213
|
+
// console.log(`actual request timed out in ${_since(req.started)}`)
|
|
214
|
+
// Apparently, providing a `string` reason to abort() causes Undici to throw `invalid_argument` error,
|
|
215
|
+
// so, we're wrapping it in a TimeoutError instance
|
|
216
|
+
abortController.abort(new TimeoutError(`request timed out after ${timeoutSeconds} sec`))
|
|
217
|
+
}, timeoutSeconds * 1000) as any as number
|
|
218
|
+
}
|
|
219
|
+
|
|
221
220
|
if (this.cfg.logRequest) {
|
|
222
221
|
const { retryAttempt } = res.retryStatus
|
|
223
222
|
logger.log(
|
|
@@ -242,15 +241,33 @@ export class Fetcher {
|
|
|
242
241
|
res.ok = false
|
|
243
242
|
// important to set it to undefined, otherwise it can keep the previous value (from previous try)
|
|
244
243
|
res.fetchResponse = undefined
|
|
244
|
+
} finally {
|
|
245
|
+
clearTimeout(timeoutId)
|
|
246
|
+
// Separate Timeout will be introduced to "download and parse the body"
|
|
245
247
|
}
|
|
246
248
|
res.statusFamily = this.getStatusFamily(res)
|
|
247
249
|
res.statusCode = res.fetchResponse?.status
|
|
248
250
|
|
|
249
251
|
if (res.fetchResponse?.ok) {
|
|
250
|
-
|
|
252
|
+
try {
|
|
253
|
+
// We are applying a separate Timeout (as long as original Timeout for now) to "download and parse the body"
|
|
254
|
+
await pTimeout(
|
|
255
|
+
async () =>
|
|
256
|
+
await this.onOkResponse(res as FetcherResponse<T> & { fetchResponse: Response }),
|
|
257
|
+
{
|
|
258
|
+
timeout: timeoutSeconds * 1000 || Number.POSITIVE_INFINITY,
|
|
259
|
+
name: 'Fetcher.onOkResponse',
|
|
260
|
+
},
|
|
261
|
+
)
|
|
262
|
+
} catch (err) {
|
|
263
|
+
// onOkResponse can still fail, e.g when loading/parsing json, text or doing other response manipulation
|
|
264
|
+
res.err = _anyToError(err)
|
|
265
|
+
res.ok = false
|
|
266
|
+
await this.onNotOkResponse(res)
|
|
267
|
+
}
|
|
251
268
|
} else {
|
|
252
269
|
// !res.ok
|
|
253
|
-
await this.onNotOkResponse(res
|
|
270
|
+
await this.onNotOkResponse(res)
|
|
254
271
|
}
|
|
255
272
|
}
|
|
256
273
|
|
|
@@ -263,34 +280,19 @@ export class Fetcher {
|
|
|
263
280
|
|
|
264
281
|
private async onOkResponse(
|
|
265
282
|
res: FetcherResponse<any> & { fetchResponse: Response },
|
|
266
|
-
timeout?: number,
|
|
267
283
|
): Promise<void> {
|
|
268
284
|
const { req } = res
|
|
269
285
|
const { responseType } = res.req
|
|
270
286
|
|
|
287
|
+
// This function is subject to a separate timeout to "download and parse the data"
|
|
271
288
|
if (responseType === 'json') {
|
|
272
289
|
if (res.fetchResponse.body) {
|
|
273
290
|
const text = await res.fetchResponse.text()
|
|
274
291
|
|
|
275
292
|
if (text) {
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
} catch (err) {
|
|
280
|
-
// Error while parsing json
|
|
281
|
-
// res.err = _anyToError(err, HttpRequestError, {
|
|
282
|
-
// requestUrl: res.req.url,
|
|
283
|
-
// requestBaseUrl: this.cfg.baseUrl,
|
|
284
|
-
// requestMethod: res.req.init.method,
|
|
285
|
-
// requestSignature: res.signature,
|
|
286
|
-
// requestDuration: Date.now() - started,
|
|
287
|
-
// responseStatusCode: res.fetchResponse.status,
|
|
288
|
-
// } satisfies HttpRequestErrorData)
|
|
289
|
-
res.err = _anyToError(err)
|
|
290
|
-
res.ok = false
|
|
291
|
-
|
|
292
|
-
return await this.onNotOkResponse(res, timeout)
|
|
293
|
-
}
|
|
293
|
+
res.body = text
|
|
294
|
+
res.body = _jsonParse(text, req.jsonReviver)
|
|
295
|
+
// Error while parsing json can happen - it'll be handled upstream
|
|
294
296
|
} else {
|
|
295
297
|
// Body had a '' (empty string)
|
|
296
298
|
res.body = {}
|
|
@@ -310,13 +312,11 @@ export class Fetcher {
|
|
|
310
312
|
res.body = res.fetchResponse.body
|
|
311
313
|
|
|
312
314
|
if (res.body === null) {
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
return await this.onNotOkResponse(res, timeout)
|
|
315
|
+
// Error is to be handled upstream
|
|
316
|
+
throw new Error(`fetchResponse.body is null`)
|
|
316
317
|
}
|
|
317
318
|
}
|
|
318
319
|
|
|
319
|
-
clearTimeout(timeout)
|
|
320
320
|
res.retryStatus.retryStopped = true
|
|
321
321
|
|
|
322
322
|
// res.err can happen on `failed to fetch` type of error, e.g JSON.parse, CORS, unexpected redirect
|
|
@@ -348,9 +348,7 @@ export class Fetcher {
|
|
|
348
348
|
return await globalThis.fetch(url, init)
|
|
349
349
|
}
|
|
350
350
|
|
|
351
|
-
private async onNotOkResponse(res: FetcherResponse
|
|
352
|
-
clearTimeout(timeout)
|
|
353
|
-
|
|
351
|
+
private async onNotOkResponse(res: FetcherResponse): Promise<void> {
|
|
354
352
|
let cause: ErrorObject | undefined
|
|
355
353
|
|
|
356
354
|
if (res.err) {
|
|
@@ -433,7 +431,11 @@ export class Fetcher {
|
|
|
433
431
|
retryStatus.retryAttempt++
|
|
434
432
|
retryStatus.retryTimeout = _clamp(retryStatus.retryTimeout * timeoutMultiplier, 0, timeoutMax)
|
|
435
433
|
|
|
436
|
-
|
|
434
|
+
const timeout = this.getRetryTimeout(res)
|
|
435
|
+
if (res.req.debug) {
|
|
436
|
+
this.cfg.logger.log(` .. ${res.signature} waiting ${_ms(timeout)}`)
|
|
437
|
+
}
|
|
438
|
+
await pDelay(timeout)
|
|
437
439
|
}
|
|
438
440
|
|
|
439
441
|
private getRetryTimeout(res: FetcherResponse): NumberOfMilliseconds {
|
|
@@ -491,8 +493,9 @@ export class Fetcher {
|
|
|
491
493
|
if (statusFamily === 3 && !retry3xx) return false
|
|
492
494
|
|
|
493
495
|
// should not retry on `unexpected redirect` in error.cause.cause
|
|
494
|
-
if ((res.err?.cause as ErrorLike | void)?.cause?.message?.includes('unexpected redirect'))
|
|
496
|
+
if ((res.err?.cause as ErrorLike | void)?.cause?.message?.includes('unexpected redirect')) {
|
|
495
497
|
return false
|
|
498
|
+
}
|
|
496
499
|
|
|
497
500
|
return true // default is true
|
|
498
501
|
}
|
|
@@ -533,7 +536,7 @@ export class Fetcher {
|
|
|
533
536
|
|
|
534
537
|
private normalizeCfg(cfg: FetcherCfg & FetcherOptions): FetcherNormalizedCfg {
|
|
535
538
|
if (cfg.baseUrl?.endsWith('/')) {
|
|
536
|
-
console.warn(`Fetcher: baseUrl should not end with
|
|
539
|
+
console.warn(`Fetcher: baseUrl should not end with slash: ${cfg.baseUrl}`)
|
|
537
540
|
cfg.baseUrl = cfg.baseUrl.slice(0, cfg.baseUrl.length - 1)
|
|
538
541
|
}
|
|
539
542
|
const { debug = false } = cfg
|
|
@@ -591,6 +594,7 @@ export class Fetcher {
|
|
|
591
594
|
'logRequestBody',
|
|
592
595
|
'logResponse',
|
|
593
596
|
'logResponseBody',
|
|
597
|
+
'debug',
|
|
594
598
|
]),
|
|
595
599
|
started: Date.now(),
|
|
596
600
|
..._omit(opt, ['method', 'headers', 'credentials']),
|
package/src/promise/pTimeout.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { AppError } from '../error/app.error'
|
|
2
2
|
import type { ErrorData, ErrorObject } from '../error/error.model'
|
|
3
3
|
import { _errorDataAppend } from '../error/error.util'
|
|
4
|
-
import type { AnyAsyncFunction } from '../types'
|
|
4
|
+
import type { AnyAsyncFunction, NumberOfMilliseconds } from '../types'
|
|
5
5
|
|
|
6
6
|
export class TimeoutError extends AppError {
|
|
7
7
|
constructor(message: string, data = {}, cause?: ErrorObject) {
|
|
@@ -13,7 +13,7 @@ export interface PTimeoutOptions {
|
|
|
13
13
|
/**
|
|
14
14
|
* Timeout in milliseconds.
|
|
15
15
|
*/
|
|
16
|
-
timeout:
|
|
16
|
+
timeout: NumberOfMilliseconds
|
|
17
17
|
|
|
18
18
|
/**
|
|
19
19
|
* If set - will be included in the error message.
|