got 11.5.1 → 11.6.2

package/dist/source/core/index.js

@@ -25,7 +25,11 @@ const options_to_url_1 = require("./utils/options-to-url");
 const weakable_map_1 = require("./utils/weakable-map");
 const get_buffer_1 = require("./utils/get-buffer");
 const dns_ip_version_1 = require("./utils/dns-ip-version");
+const is_response_ok_1 = require("./utils/is-response-ok");
 const deprecation_warning_1 = require("../utils/deprecation-warning");
+const normalize_arguments_1 = require("../as-promise/normalize-arguments");
+const calculate_retry_delay_1 = require("./calculate-retry-delay");
+const globalDnsCache = new cacheable_lookup_1.default();
 const kRequest = Symbol('request');
 const kResponse = Symbol('response');
 const kResponseSize = Symbol('responseSize');
@@ -42,10 +46,19 @@ const kTriggerRead = Symbol('triggerRead');
 const kBody = Symbol('body');
 const kJobs = Symbol('jobs');
 const kOriginalResponse = Symbol('originalResponse');
+const kRetryTimeout = Symbol('retryTimeout');
 exports.kIsNormalizedAlready = Symbol('isNormalizedAlready');
 const supportsBrotli = is_1.default.string(process.versions.brotli);
 exports.withoutBody = new Set(['GET', 'HEAD']);
-exports.knownHookEvents = ['init', 'beforeRequest', 'beforeRedirect', 'beforeError'];
+exports.knownHookEvents = [
+    'init',
+    'beforeRequest',
+    'beforeRedirect',
+    'beforeError',
+    'beforeRetry',
+    // Promise-Only
+    'afterResponse'
+];
 function validateSearchParameters(searchParameters) {
     // eslint-disable-next-line guard-for-in
     for (const key in searchParameters) {
@@ -102,6 +115,10 @@ exports.setNonEnumerableProperties = (sources, to) => {
     }
     Object.defineProperties(to, properties);
 };
+/**
+An error to be thrown when a request fails.
+Contains a `code` property with error class code, like `ECONNREFUSED`.
+*/
 class RequestError extends Error {
     constructor(message, error, self) {
         var _a;
@@ -148,6 +165,10 @@ class RequestError extends Error {
     }
 }
 exports.RequestError = RequestError;
+/**
+An error to be thrown when the server redirects you more than ten times.
+Includes a `response` property.
+*/
 class MaxRedirectsError extends RequestError {
     constructor(request) {
         super(`Redirected ${request.options.maxRedirects} times. Aborting.`, {}, request);
@@ -155,6 +176,10 @@ class MaxRedirectsError extends RequestError {
     }
 }
 exports.MaxRedirectsError = MaxRedirectsError;
+/**
+An error to be thrown when the server response code is not 2xx nor 3xx if `options.followRedirect` is `true`, but always except for 304.
+Includes a `response` property.
+*/
 class HTTPError extends RequestError {
     constructor(response) {
         super(`Response code ${response.statusCode} (${response.statusMessage})`, {}, response.request);
@@ -162,6 +187,10 @@ class HTTPError extends RequestError {
     }
 }
 exports.HTTPError = HTTPError;
+/**
+An error to be thrown when a cache method fails.
+For example, if the database goes down or there's a filesystem error.
+*/
 class CacheError extends RequestError {
     constructor(error, request) {
         super(error.message, error, request);
@@ -169,6 +198,9 @@ class CacheError extends RequestError {
     }
 }
 exports.CacheError = CacheError;
+/**
+An error to be thrown when the request body is a stream and an error occurs while reading from that stream.
+*/
 class UploadError extends RequestError {
     constructor(error, request) {
         super(error.message, error, request);
@@ -176,6 +208,10 @@ class UploadError extends RequestError {
     }
 }
 exports.UploadError = UploadError;
+/**
+An error to be thrown when the request is aborted due to a timeout.
+Includes an `event` and `timings` property.
+*/
 class TimeoutError extends RequestError {
     constructor(error, timings, request) {
         super(error.message, error, request);
@@ -185,6 +221,9 @@ class TimeoutError extends RequestError {
     }
 }
 exports.TimeoutError = TimeoutError;
+/**
+An error to be thrown when reading from response stream fails.
+*/
 class ReadError extends RequestError {
     constructor(error, request) {
         super(error.message, error, request);
@@ -192,6 +231,9 @@ class ReadError extends RequestError {
     }
 }
 exports.ReadError = ReadError;
+/**
+An error to be thrown when given an unsupported protocol.
+*/
 class UnsupportedProtocolError extends RequestError {
     constructor(options) {
         super(`Unsupported protocol "${options.url.protocol}"`, {}, options);
@@ -210,6 +252,9 @@ const proxiedRequestEvents = [
 class Request extends stream_1.Duplex {
     constructor(url, options = {}, defaults) {
         super({
+            // This must be false, to enable throwing after destroy
+            // It is used for retry logic in Promise API
+            autoDestroy: false,
             // It needs to be zero because we're just proxying the data to another stream
             highWaterMark: 0
         });
@@ -221,6 +266,7 @@ class Request extends stream_1.Duplex {
         this[kStopReading] = false;
         this[kTriggerRead] = false;
         this[kJobs] = [];
+        this.retryCount = 0;
         // TODO: Remove this when targeting Node.js >= 12
         this._progressCallbacks = [];
         const unlockWrite = () => this._unlockWrite();
@@ -278,6 +324,8 @@ class Request extends stream_1.Duplex {
                 for (const job of this[kJobs]) {
                     job();
                 }
+                // Prevent memory leak
+                this[kJobs].length = 0;
                 this.requestInitialized = true;
             }
             catch (error) {
@@ -293,7 +341,7 @@ class Request extends stream_1.Duplex {
         })(options);
     }
     static normalizeArguments(url, options, defaults) {
-        var _a, _b, _c, _d;
+        var _a, _b, _c, _d, _e;
         const rawOptions = options;
         if (is_1.default.object(url) && !is_1.default.urlInstance(url)) {
             options = { ...defaults, ...url, ...options };
@@ -347,6 +395,7 @@ class Request extends stream_1.Duplex {
             is_1.assert.any([is_1.default.string, is_1.default.object, is_1.default.array, is_1.default.undefined], options.https.certificate);
             is_1.assert.any([is_1.default.string, is_1.default.undefined], options.https.passphrase);
         }
+        is_1.assert.any([is_1.default.object, is_1.default.undefined], options.cacheOptions);
         // `options.method`
         if (is_1.default.string(options.method)) {
             options.method = options.method.toUpperCase();
@@ -404,15 +453,15 @@ class Request extends stream_1.Duplex {
         options.username = (_b = options.username) !== null && _b !== void 0 ? _b : '';
         options.password = (_c = options.password) !== null && _c !== void 0 ? _c : '';
         // `options.prefixUrl` & `options.url`
-        if (options.prefixUrl) {
+        if (is_1.default.undefined(options.prefixUrl)) {
+            options.prefixUrl = (_d = defaults === null || defaults === void 0 ? void 0 : defaults.prefixUrl) !== null && _d !== void 0 ? _d : '';
+        }
+        else {
             options.prefixUrl = options.prefixUrl.toString();
             if (options.prefixUrl !== '' && !options.prefixUrl.endsWith('/')) {
                 options.prefixUrl += '/';
             }
         }
-        else {
-            options.prefixUrl = '';
-        }
         if (is_1.default.string(options.url)) {
             if (options.url.startsWith('/')) {
                 throw new Error('`input` must not start with a slash when using `prefixUrl`');
@@ -478,9 +527,7 @@ class Request extends stream_1.Duplex {
                 getCookieString = util_1.promisify(getCookieString.bind(options.cookieJar));
                 options.cookieJar = {
                     setCookie,
-                    // TODO: Fix this when upgrading to TypeScript 4.
-                    // @ts-expect-error TypeScript thinks that promisifying callback(error, string) will result in Promise<void>
-                    getCookieString
+                    getCookieString: getCookieString
                 };
             }
         }
@@ -521,9 +568,11 @@ class Request extends stream_1.Duplex {
                 }), cache));
             }
         }
+        // `options.cacheOptions`
+        options.cacheOptions = { ...options.cacheOptions };
         // `options.dnsCache`
         if (options.dnsCache === true) {
-            options.dnsCache = new cacheable_lookup_1.default();
+            options.dnsCache = globalDnsCache;
         }
         else if (!is_1.default.undefined(options.dnsCache) && !options.dnsCache.lookup) {
             throw new TypeError(`Parameter \`dnsCache\` must be a CacheableLookup instance or a boolean, got ${is_1.default(options.dnsCache)}`);
@@ -611,10 +660,10 @@ class Request extends stream_1.Duplex {
                 }
             }
         }
-        options.maxRedirects = (_d = options.maxRedirects) !== null && _d !== void 0 ? _d : 0;
+        options.maxRedirects = (_e = options.maxRedirects) !== null && _e !== void 0 ? _e : 0;
         // Set non-enumerable properties
         exports.setNonEnumerableProperties([defaults, rawOptions], options);
-        return options;
+        return normalize_arguments_1.default(options, defaults);
     }
     _lockWrite() {
         const onLockedWrite = () => {
@@ -716,6 +765,7 @@ class Request extends stream_1.Duplex {
         typedResponse.request = this;
         typedResponse.isFromCache = response.fromCache || false;
         typedResponse.ip = this.ip;
+        typedResponse.retryCount = this.retryCount;
         this[kIsFromCache] = typedResponse.isFromCache;
         this[kResponseSize] = Number(response.headers['content-length']) || undefined;
         this[kResponse] = response;
@@ -753,7 +803,7 @@ class Request extends stream_1.Duplex {
         }
         if (options.followRedirect && response.headers.location && redirectCodes.has(statusCode)) {
             // We're being redirected, we don't care about the response.
-            // It'd be besto to abort the request, but we can't because
+            // It'd be best to abort the request, but we can't because
             // we would have to sacrifice the TCP connection. We don't want that.
             response.resume();
             if (this[kRequest]) {
@@ -789,7 +839,7 @@ class Request extends stream_1.Duplex {
                 const redirectString = redirectUrl.toString();
                 decodeURI(redirectString);
                 // Redirecting to a different site, clear sensitive data.
-                if (redirectUrl.hostname !== url.hostname) {
+                if (redirectUrl.hostname !== url.hostname || redirectUrl.port !== url.port) {
                     if ('host' in options.headers) {
                         delete options.headers.host;
                     }
@@ -800,9 +850,15 @@ class Request extends stream_1.Duplex {
                         delete options.headers.authorization;
                     }
                     if (options.username || options.password) {
+                        // TODO: Fix this ignore.
+                        // @ts-expect-error
                         delete options.username;
+                        // @ts-expect-error
                         delete options.password;
                     }
+                    if ('port' in options) {
+                        delete options.port;
+                    }
                 }
                 this.redirects.push(redirectString);
                 options.url = redirectUrl;
@@ -819,16 +875,9 @@ class Request extends stream_1.Duplex {
             }
             return;
         }
-        const limitStatusCode = options.followRedirect ? 299 : 399;
-        const isOk = (statusCode >= 200 && statusCode <= limitStatusCode) || statusCode === 304;
-        if (options.throwHttpErrors && !isOk) {
-            // Normally we would have to use `void [await] this._beforeError(error)` everywhere,
-            // but since there's `void (async () => { ... })()` inside of it, we don't have to.
+        if (options.isStream && options.throwHttpErrors && !is_response_ok_1.isResponseOk(typedResponse)) {
             this._beforeError(new HTTPError(typedResponse));
-            // This is equivalent to this.destroyed
-            if (this[kStopReading]) {
-                return;
-            }
+            return;
         }
         response.on('readable', () => {
             if (this[kTriggerRead]) {
@@ -865,6 +914,7 @@ class Request extends stream_1.Duplex {
             await this._onResponseBase(response);
         }
         catch (error) {
+            /* istanbul ignore next: better safe than sorry */
             this._beforeError(error);
         }
     }
@@ -902,9 +952,6 @@ class Request extends stream_1.Duplex {
             body.once('error', (error) => {
                 this._beforeError(new UploadError(error, this));
             });
-            body.once('end', () => {
-                delete options.body;
-            });
         }
         else {
             this._unlockWrite();
@@ -925,6 +972,8 @@ class Request extends stream_1.Duplex {
             // TODO: Remove `utils/url-to-options.ts` when `cacheable-request` is fixed
             Object.assign(options, url_to_options_1.default(url));
             // `http-cache-semantics` checks this
+            // TODO: Fix this ignore.
+            // @ts-expect-error
             delete options.url;
             let request;
             // This is ugly
@@ -946,7 +995,7 @@ class Request extends stream_1.Duplex {
         });
     }
     async _makeRequest() {
-        var _a;
+        var _a, _b, _c, _d, _e;
         const { options } = this;
         const { headers } = options;
         for (const key in headers) {
@@ -1012,14 +1061,20 @@ class Request extends stream_1.Duplex {
         // Prepare plain HTTP request options
         options[kRequest] = realFn;
         delete options.request;
+        // TODO: Fix this ignore.
+        // @ts-expect-error
         delete options.timeout;
         const requestOptions = options;
+        requestOptions.shared = (_b = options.cacheOptions) === null || _b === void 0 ? void 0 : _b.shared;
+        requestOptions.cacheHeuristic = (_c = options.cacheOptions) === null || _c === void 0 ? void 0 : _c.cacheHeuristic;
+        requestOptions.immutableMinTimeToLive = (_d = options.cacheOptions) === null || _d === void 0 ? void 0 : _d.immutableMinTimeToLive;
+        requestOptions.ignoreCargoCult = (_e = options.cacheOptions) === null || _e === void 0 ? void 0 : _e.ignoreCargoCult;
         // If `dnsLookupIpVersion` is not present do not override `family`
         if (options.dnsLookupIpVersion !== undefined) {
             try {
                 requestOptions.family = dns_ip_version_1.dnsLookupIpVersionToFamily(options.dnsLookupIpVersion);
             }
-            catch (_b) {
+            catch (_f) {
                 throw new Error('Invalid `dnsLookupIpVersion` option value');
             }
         }
@@ -1053,6 +1108,28 @@ class Request extends stream_1.Duplex {
             options.request = request;
             options.timeout = timeout;
             options.agent = agent;
+            // HTTPS options restore
+            if (options.https) {
+                if ('rejectUnauthorized' in options.https) {
+                    delete requestOptions.rejectUnauthorized;
+                }
+                if (options.https.checkServerIdentity) {
+                    // @ts-expect-error - This one will be removed when we remove the alias.
+                    delete requestOptions.checkServerIdentity;
+                }
+                if (options.https.certificateAuthority) {
+                    delete requestOptions.ca;
+                }
+                if (options.https.certificate) {
+                    delete requestOptions.cert;
+                }
+                if (options.https.key) {
+                    delete requestOptions.key;
+                }
+                if (options.https.passphrase) {
+                    delete requestOptions.passphrase;
+                }
+            }
             if (isClientRequest(requestOrResponse)) {
                 this._onRequest(requestOrResponse);
                 // Emit the response after the stream has been ended
@@ -1076,34 +1153,97 @@ class Request extends stream_1.Duplex {
             throw new RequestError(error.message, error, this);
         }
     }
+    async _error(error) {
+        try {
+            for (const hook of this.options.hooks.beforeError) {
+                // eslint-disable-next-line no-await-in-loop
+                error = await hook(error);
+            }
+        }
+        catch (error_) {
+            error = new RequestError(error_.message, error_, this);
+        }
+        this.destroy(error);
+    }
     _beforeError(error) {
-        if (this.destroyed) {
+        if (this[kStopReading]) {
             return;
         }
+        const { options } = this;
+        const retryCount = this.retryCount + 1;
         this[kStopReading] = true;
         if (!(error instanceof RequestError)) {
             error = new RequestError(error.message, error, this);
         }
+        const typedError = error;
+        const { response } = typedError;
         void (async () => {
-            try {
-                const { response } = error;
-                if (response) {
-                    response.setEncoding(this._readableState.encoding);
+            if (response && !response.body) {
+                response.setEncoding(this._readableState.encoding);
+                try {
                     response.rawBody = await get_buffer_1.default(response);
                     response.body = response.rawBody.toString();
                 }
-            }
-            catch (_a) { }
-            try {
-                for (const hook of this.options.hooks.beforeError) {
-                    // eslint-disable-next-line no-await-in-loop
-                    error = await hook(error);
+                catch (_a) { }
+            }
+            if (this.listenerCount('retry') !== 0) {
+                let backoff;
+                try {
+                    let retryAfter;
+                    if (response && 'retry-after' in response.headers) {
+                        retryAfter = Number(response.headers['retry-after']);
+                        if (Number.isNaN(retryAfter)) {
+                            retryAfter = Date.parse(response.headers['retry-after']) - Date.now();
+                            if (retryAfter <= 0) {
+                                retryAfter = 1;
+                            }
+                        }
+                        else {
+                            retryAfter *= 1000;
+                        }
+                    }
+                    backoff = await options.retry.calculateDelay({
+                        attemptCount: retryCount,
+                        retryOptions: options.retry,
+                        error: typedError,
+                        retryAfter,
+                        computedValue: calculate_retry_delay_1.default({
+                            attemptCount: retryCount,
+                            retryOptions: options.retry,
+                            error: typedError,
+                            retryAfter,
+                            computedValue: 0
+                        })
+                    });
+                }
+                catch (error_) {
+                    void this._error(new RequestError(error_.message, error_, this));
+                    return;
+                }
+                if (backoff) {
+                    const retry = async () => {
+                        try {
+                            for (const hook of this.options.hooks.beforeRetry) {
+                                // eslint-disable-next-line no-await-in-loop
+                                await hook(this.options, typedError, retryCount);
+                            }
+                        }
+                        catch (error_) {
+                            void this._error(new RequestError(error_.message, error, this));
+                            return;
+                        }
+                        // Something forced us to abort the retry
+                        if (this.destroyed) {
+                            return;
+                        }
+                        this.destroy();
+                        this.emit('retry', retryCount, error);
+                    };
+                    this[kRetryTimeout] = setTimeout(retry, backoff);
+                    return;
                 }
             }
-            catch (error_) {
-                error = new RequestError(error_.message, error_, this);
-            }
-            this.destroy(error);
+            void this._error(typedError);
         })();
     }
     _read() {
@@ -1140,6 +1280,10 @@ class Request extends stream_1.Duplex {
         }
     }
     _writeRequest(chunk, encoding, callback) {
+        if (this[kRequest].destroyed) {
+            // Probably the `ClientRequest` instance will throw
+            return;
+        }
         this._progressCallbacks.push(() => {
             this[kUploadedSize] += Buffer.byteLength(chunk, encoding);
             const progress = this.uploadProgress;
@@ -1190,6 +1334,8 @@ class Request extends stream_1.Duplex {
     _destroy(error, callback) {
         var _a;
         this[kStopReading] = true;
+        // Prevent further retries
+        clearTimeout(this[kRetryTimeout]);
         if (kRequest in this) {
             this[kCancelTimeouts]();
             // TODO: Remove the next `if` when these get fixed:
@@ -1203,10 +1349,19 @@ class Request extends stream_1.Duplex {
         }
         callback(error);
     }
+    get _isAboutToError() {
+        return this[kStopReading];
+    }
+    /**
+    The remote IP address.
+    */
     get ip() {
         var _a;
         return (_a = this[kRequest]) === null || _a === void 0 ? void 0 : _a.socket.remoteAddress;
     }
+    /**
+    Indicates whether the request has been aborted or not.
+    */
     get aborted() {
         var _a, _b, _c;
         return ((_b = (_a = this[kRequest]) === null || _a === void 0 ? void 0 : _a.destroyed) !== null && _b !== void 0 ? _b : this.destroyed) && !((_c = this[kOriginalResponse]) === null || _c === void 0 ? void 0 : _c.complete);
@@ -1215,6 +1370,9 @@ class Request extends stream_1.Duplex {
         var _a;
         return (_a = this[kRequest]) === null || _a === void 0 ? void 0 : _a.socket;
     }
+    /**
+    Progress event for downloading (receiving a response).
+    */
     get downloadProgress() {
         let percent;
         if (this[kResponseSize]) {
@@ -1232,6 +1390,9 @@ class Request extends stream_1.Duplex {
             total: this[kResponseSize]
         };
     }
+    /**
+    Progress event for uploading (sending a request).
+    */
     get uploadProgress() {
         let percent;
         if (this[kBodySize]) {
@@ -1249,16 +1410,43 @@ class Request extends stream_1.Duplex {
             total: this[kBodySize]
         };
     }
+    /**
+    The object contains the following properties:
+
+    - `start` - Time when the request started.
+    - `socket` - Time when a socket was assigned to the request.
+    - `lookup` - Time when the DNS lookup finished.
+    - `connect` - Time when the socket successfully connected.
+    - `secureConnect` - Time when the socket securely connected.
+    - `upload` - Time when the request finished uploading.
+    - `response` - Time when the request fired `response` event.
+    - `end` - Time when the response fired `end` event.
+    - `error` - Time when the request fired `error` event.
+    - `abort` - Time when the request fired `abort` event.
+    - `phases`
+        - `wait` - `timings.socket - timings.start`
+        - `dns` - `timings.lookup - timings.socket`
+        - `tcp` - `timings.connect - timings.lookup`
+        - `tls` - `timings.secureConnect - timings.connect`
+        - `request` - `timings.upload - (timings.secureConnect || timings.connect)`
+        - `firstByte` - `timings.response - timings.upload`
+        - `download` - `timings.end - timings.response`
+        - `total` - `(timings.end || timings.error || timings.abort) - timings.start`
+
+    If something has not been measured yet, it will be `undefined`.
+
+    __Note__: The time is a `number` representing the milliseconds elapsed since the UNIX epoch.
+    */
     get timings() {
         var _a;
         return (_a = this[kRequest]) === null || _a === void 0 ? void 0 : _a.timings;
     }
+    /**
+    Whether the response was retrieved from the cache.
+    */
     get isFromCache() {
         return this[kIsFromCache];
     }
-    get _response() {
-        return this[kResponse];
-    }
     pipe(destination, options) {
         if (this[kStartedReading]) {
             throw new Error('Failed to pipe. The response has been emitted already.');

package/dist/source/core/utils/is-response-ok.d.ts

@@ -0,0 +1,2 @@
+import { Response } from '..';
+export declare const isResponseOk: (response: Response) => boolean;

package/dist/source/core/utils/is-response-ok.js

@@ -0,0 +1,8 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isResponseOk = void 0;
+exports.isResponseOk = (response) => {
+    const { statusCode } = response;
+    const limitStatusCode = response.request.options.followRedirect ? 299 : 399;
+    return (statusCode >= 200 && statusCode <= limitStatusCode) || statusCode === 304;
+};