got 11.1.3 → 11.1.4

package/dist/source/as-promise/calculate-retry-delay.d.ts

@@ -1,3 +1,4 @@
 import { RetryFunction } from './types';
-declare const calculateRetryDelay: RetryFunction;
+declare type Returns<T extends (...args: any) => unknown, V> = (...args: Parameters<T>) => V;
+declare const calculateRetryDelay: Returns<RetryFunction, number>;
 export default calculateRetryDelay;

package/dist/source/as-promise/core.js

@@ -39,6 +39,11 @@ class PromisableRequest extends core_1.default {
         is_1.assert.any([is_1.default.boolean, is_1.default.undefined], options.resolveBodyOnly);
         is_1.assert.any([is_1.default.boolean, is_1.default.undefined], options.methodRewriting);
         is_1.assert.any([is_1.default.boolean, is_1.default.undefined], options.isStream);
+        is_1.assert.any([is_1.default.string, is_1.default.undefined], options.responseType);
+        // `options.responseType`
+        if (options.responseType === undefined) {
+            options.responseType = 'text';
+        }
         // `options.retry`
         const { retry } = options;
         if (defaults) {

package/dist/source/as-promise/index.js

@@ -51,7 +51,7 @@ function asPromise(options) {
                 _reject(error);
             };
             globalRequest = request;
-            request.once('response', async (response) => {
+            const onResponse = async (response) => {
                 response.retryCount = retryCount;
                 if (response.request.aborted) {
                     // Canceled while downloading - will throw a `CancelError` or `TimeoutError` error
@@ -126,8 +126,9 @@ function asPromise(options) {
                 }
                 globalResponse = response;
                 resolve(options.resolveBodyOnly ? response.body : response);
-            });
-            request.once('error', (error) => {
+            };
+            request.once('response', onResponse);
+            request.once('error', async (error) => {
                 if (promise.isCanceled) {
                     return;
                 }
@@ -135,10 +136,11 @@ function asPromise(options) {
                     reject(error);
                     return;
                 }
+                request.off('response', onResponse);
                 let backoff;
                 retryCount++;
                 try {
-                    backoff = options.retry.calculateDelay({
+                    backoff = await options.retry.calculateDelay({
                         attemptCount: retryCount,
                         retryOptions: options.retry,
                         error,
@@ -181,7 +183,8 @@ function asPromise(options) {
                 // The retry has not been made
                 retryCount--;
                 if (error instanceof types_1.HTTPError) {
-                    // It will be handled by the `response` event
+                    // The error will be handled by the `response` event
+                    onResponse(request._response);
                     return;
                 }
                 // Don't emit the `response` event

package/dist/source/as-promise/types.d.ts

@@ -13,7 +13,7 @@ export interface RetryObject {
     error: TimeoutError | RequestError;
     computedValue: number;
 }
-export declare type RetryFunction = (retryObject: RetryObject) => number;
+export declare type RetryFunction = (retryObject: RetryObject) => number | Promise<number>;
 export interface RequiredRetryOptions {
     limit: number;
     methods: Method[];

package/dist/source/core/index.d.ts

@@ -4,7 +4,7 @@ import { URL, URLSearchParams } from 'url';
 import { Socket } from 'net';
 import { SecureContextOptions } from 'tls';
 import http = require('http');
-import { ClientRequest, RequestOptions, IncomingMessage, ServerResponse, request as httpRequest } from 'http';
+import { ClientRequest, RequestOptions, ServerResponse, request as httpRequest } from 'http';
 import https = require('https');
 import { Timings, IncomingMessageWithTimings } from '@szmarczak/http-timer';
 import CacheableLookup from 'cacheable-lookup';
@@ -65,7 +65,7 @@ export interface Hooks {
 }
 export declare type HookEvent = 'init' | 'beforeRequest' | 'beforeRedirect' | 'beforeError';
 export declare const knownHookEvents: HookEvent[];
-declare type AcceptableResponse = IncomingMessage | ResponseLike;
+declare type AcceptableResponse = IncomingMessageWithTimings | ResponseLike;
 declare type AcceptableRequestResult = AcceptableResponse | ClientRequest | Promise<AcceptableResponse | ClientRequest> | undefined;
 export declare type RequestFunction = (url: URL, options: RequestOptions, callback?: (response: AcceptableResponse) => void) => AcceptableRequestResult;
 export declare type Headers = Record<string, string | string[] | undefined>;
@@ -249,8 +249,8 @@ export default class Request extends Duplex implements RequestEvents<Request> {
     [kStartedReading]?: boolean;
     [kCancelTimeouts]?: () => void;
     [kResponseSize]?: number;
-    [kResponse]?: IncomingMessage;
-    [kOriginalResponse]?: IncomingMessage;
+    [kResponse]?: IncomingMessageWithTimings;
+    [kOriginalResponse]?: IncomingMessageWithTimings;
     [kRequest]?: ClientRequest;
     _noPipe?: boolean;
     _progressCallbacks: Array<() => void>;
@@ -263,7 +263,7 @@ export default class Request extends Duplex implements RequestEvents<Request> {
     _lockWrite(): void;
     _unlockWrite(): void;
     _finalizeBody(): Promise<void>;
-    _onResponse(response: IncomingMessage): Promise<void>;
+    _onResponse(response: IncomingMessageWithTimings): Promise<void>;
     _onRequest(request: ClientRequest): void;
     _createCacheableRequest(url: URL, options: RequestOptions): Promise<ClientRequest | ResponseLike>;
     _makeRequest(): Promise<void>;
@@ -280,6 +280,7 @@ export default class Request extends Duplex implements RequestEvents<Request> {
     get uploadProgress(): Progress;
     get timings(): Timings | undefined;
     get isFromCache(): boolean | undefined;
+    get _response(): Response | undefined;
     pipe<T extends NodeJS.WritableStream>(destination: T, options?: {
         end?: boolean;
     }): T;

package/dist/source/core/index.js

@@ -8,9 +8,9 @@ const http = require("http");
 const http_1 = require("http");
 const https = require("https");
 const http_timer_1 = require("@szmarczak/http-timer");
-const decompressResponse = require("decompress-response");
 const cacheable_lookup_1 = require("cacheable-lookup");
 const CacheableRequest = require("cacheable-request");
+const decompressResponse = require("decompress-response");
 // @ts-ignore Missing types
 const http2wrapper = require("http2-wrapper");
 const lowercaseKeys = require("lowercase-keys");
@@ -329,6 +329,7 @@ class Request extends stream_1.Duplex {
         is_1.assert.any([is_1.default.boolean, is_1.default.undefined], options.http2);
         is_1.assert.any([is_1.default.boolean, is_1.default.undefined], options.allowGetBody);
         is_1.assert.any([is_1.default.boolean, is_1.default.undefined], options.rejectUnauthorized);
+        is_1.assert.any([is_1.default.string, is_1.default.undefined], options.localAddress);
         // `options.method`
         if (is_1.default.string(options.method)) {
             options.method = options.method.toUpperCase();
@@ -414,12 +415,6 @@ class Request extends stream_1.Duplex {
             if (options.searchParams) {
                 options.url.search = options.searchParams.toString();
             }
-            // Trigger search params normalization
-            if (options.url.search) {
-                const triggerSearchParameters = '_GOT_INTERNAL_TRIGGER_NORMALIZATION';
-                options.url.searchParams.append(triggerSearchParameters, '');
-                options.url.searchParams.delete(triggerSearchParameters);
-            }
             // Protocol check
             if (protocol !== 'http:' && protocol !== 'https:') {
                 throw new UnsupportedProtocolError(options);
@@ -466,7 +461,7 @@ class Request extends stream_1.Duplex {
         if (options.dnsCache === true) {
             options.dnsCache = new cacheable_lookup_1.default();
         }
-        else if (!is_1.default.undefined(options.dnsCache) && !(options.dnsCache instanceof cacheable_lookup_1.default)) {
+        else if (!is_1.default.undefined(options.dnsCache) && !options.dnsCache.lookup) {
             throw new TypeError(`Parameter \`dnsCache\` must be a CacheableLookup instance or a boolean, got ${is_1.default(options.dnsCache)}`);
         }
         // `options.timeout`
@@ -699,8 +694,9 @@ class Request extends stream_1.Duplex {
                 return;
             }
             try {
-                // Handles invalid URLs. See https://github.com/sindresorhus/got/issues/604
+                // Do not remove. See https://github.com/sindresorhus/got/pull/214
                 const redirectBuffer = Buffer.from(response.headers.location, 'binary').toString();
+                // Handles invalid URLs. See https://github.com/sindresorhus/got/issues/604
                 const redirectUrl = new url_1.URL(redirectBuffer, url);
                 const redirectString = redirectUrl.toString();
                 decodeURI(redirectString);
@@ -832,6 +828,8 @@ class Request extends stream_1.Duplex {
             const cacheRequest = cacheableStore.get(options.cache)(options, response => {
                 const typedResponse = response;
                 const { req } = typedResponse;
+                // TODO: Fix `cacheable-response`
+                typedResponse._readableState.autoDestroy = false;
                 if (req) {
                     req.emit('cacheableResponse', typedResponse);
                 }
@@ -1030,6 +1028,10 @@ class Request extends stream_1.Duplex {
                 callback();
                 return;
             }
+            if (this[kRequest].destroyed) {
+                callback();
+                return;
+            }
             this[kRequest].end((error) => {
                 if (!error) {
                     this[kBodySize] = this[kUploadedSize];
@@ -1114,6 +1116,9 @@ class Request extends stream_1.Duplex {
     get isFromCache() {
         return this[kIsFromCache];
     }
+    get _response() {
+        return this[kResponse];
+    }
     pipe(destination, options) {
         if (this[kStartedReading]) {
             throw new Error('Failed to pipe. The response has been emitted already.');

package/dist/source/create.js

@@ -60,6 +60,7 @@ const create = (defaults) => {
         }
         return result;
     }));
+    // Got interface
     const got = ((url, options) => {
         var _a, _b;
         let iteration = 0;
@@ -128,7 +129,8 @@ const create = (defaults) => {
             mutableDefaults: Boolean(isMutableDefaults)
         });
     };
-    got.paginate = (async function* (url, options) {
+    // Pagination
+    const paginateEach = (async function* (url, options) {
         let normalizedOptions = normalizeArguments(url, options, defaults.options);
         normalizedOptions.resolveBodyOnly = false;
         const pagination = normalizedOptions.pagination;
@@ -173,6 +175,9 @@ const create = (defaults) => {
             numberOfRequests++;
         }
     });
+    got.paginate = ((url, options) => {
+        return paginateEach(url, options);
+    });
     got.paginate.all = (async (url, options) => {
         const results = [];
         for await (const item of got.paginate(url, options)) {
@@ -180,7 +185,11 @@ const create = (defaults) => {
         }
         return results;
     });
+    // For those who like very descriptive names
+    got.paginate.each = paginateEach;
+    // Stream API
     got.stream = ((url, options) => got(url, { ...options, isStream: true }));
+    // Shortcuts
     for (const method of aliases) {
         got[method] = ((url, options) => got(url, { ...options, method }));
         got.stream[method] = ((url, options) => {

package/dist/source/types.d.ts

@@ -42,8 +42,10 @@ declare type ResponseBodyOnly = {
 export declare type OptionsWithPagination<T = unknown, R = unknown> = Merge<Options, PaginationOptions<T, R>>;
 export interface GotPaginate {
     <T, R = unknown>(url: string | URL, options?: OptionsWithPagination<T, R>): AsyncIterableIterator<T>;
-    all<T, R = unknown>(url: string | URL, options?: OptionsWithPagination<T, R>): Promise<T[]>;
     <T, R = unknown>(options?: OptionsWithPagination<T, R>): AsyncIterableIterator<T>;
+    each<T, R = unknown>(url: string | URL, options?: OptionsWithPagination<T, R>): AsyncIterableIterator<T>;
+    each<T, R = unknown>(options?: OptionsWithPagination<T, R>): AsyncIterableIterator<T>;
+    all<T, R = unknown>(url: string | URL, options?: OptionsWithPagination<T, R>): Promise<T[]>;
     all<T, R = unknown>(options?: OptionsWithPagination<T, R>): Promise<T[]>;
 }
 export interface GotRequestFunction {

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "got",
-	"version": "11.1.3",
+	"version": "11.1.4",
 	"description": "Human-friendly and powerful HTTP request library for Node.js",
 	"license": "MIT",
 	"repository": "sindresorhus/got",
@@ -47,9 +47,9 @@
 		"@szmarczak/http-timer": "^4.0.5",
 		"@types/cacheable-request": "^6.0.1",
 		"@types/responselike": "^1.0.0",
-		"cacheable-lookup": "^4.3.0",
+		"cacheable-lookup": "^5.0.3",
 		"cacheable-request": "^7.0.1",
-		"decompress-response": "^5.0.0",
+		"decompress-response": "^6.0.0",
 		"get-stream": "^5.1.0",
 		"http2-wrapper": "^1.0.0-beta.4.5",
 		"lowercase-keys": "^2.0.0",
@@ -59,9 +59,9 @@
 	"devDependencies": {
 		"@ava/typescript": "^1.1.1",
 		"@sindresorhus/tsconfig": "^0.7.0",
+		"@sinonjs/fake-timers": "^6.0.1",
 		"@types/benchmark": "^1.0.31",
 		"@types/express": "^4.17.6",
-		"@types/lolex": "^5.1.0",
 		"@types/node": "^13.13.4",
 		"@types/node-fetch": "^2.5.5",
 		"@types/request": "^2.48.4",
@@ -76,7 +76,6 @@
 		"delay": "^4.3.0",
 		"express": "^4.17.1",
 		"form-data": "^3.0.0",
-		"lolex": "^6.0.0",
 		"nock": "^12.0.0",
 		"node-fetch": "^2.6.0",
 		"np": "^6.0.0",
@@ -125,5 +124,6 @@
 			"@typescript-eslint/method-signature-style": "off",
 			"unicorn/no-fn-reference-in-iterator": "off"
 		}
-	}
+	},
+	"runkitExampleFilename": "./documentation/examples/runkit-example.js"
 }

package/readme.md

@@ -139,6 +139,16 @@ Properties from `options` will override properties in the parsed `url`.
 
 If no protocol is specified, it will throw a `TypeError`.
 
+**Note:** The query string is **not** parsed as search params. Example:
+
+```
+got('https://example.com/?query=a b'); //=> https://example.com/?query=a%20b
+got('https://example.com/', {searchParams: {query: 'a b'}}); //=> https://example.com/?query=a+b
+
+// The query string is overridden by `searchParams`
+got('https://example.com/?query=a b', {searchParams: {query: 'a b'}}); //=> https://example.com/?query=a+b
+```
+
 ##### options
 
 Type: `object`
@@ -402,7 +412,7 @@ This also accepts an `object` with the following fields to constrain the duratio
 Type: `number | object`\
 Default:
 - limit: `2`
-- calculateDelay: `({attemptCount, retryOptions, error, computedValue}) => computedValue`
+- calculateDelay: `({attemptCount, retryOptions, error, computedValue}) => computedValue | Promise<computedValue>`
 - methods: `GET` `PUT` `HEAD` `DELETE` `OPTIONS` `TRACE`
 - statusCodes: [`408`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/408) [`413`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/413) [`429`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/429) [`500`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/500) [`502`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/502) [`503`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503) [`504`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/504) [`521`](https://support.cloudflare.com/hc/en-us/articles/115003011431#521error) [`522`](https://support.cloudflare.com/hc/en-us/articles/115003011431#522error) [`524`](https://support.cloudflare.com/hc/en-us/articles/115003011431#524error)
 - maxRetryAfter: `undefined`
@@ -417,7 +427,7 @@ If [`Retry-After`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Ret
 
 Delays between retries counts with function `1000 * Math.pow(2, retry) + Math.random() * 100`, where `retry` is attempt number (starts from 1).
 
-The `calculateDelay` property is a `function` that receives an object with `attemptCount`, `retryOptions`, `error` and `computedValue` properties for current retry count, the retry options, error and default computed value. The function must return a delay in milliseconds (`0` return value cancels retry).
+The `calculateDelay` property is a `function` that receives an object with `attemptCount`, `retryOptions`, `error` and `computedValue` properties for current retry count, the retry options, error and default computed value. The function must return a delay in milliseconds (or a Promise resolving with it) (`0` return value cancels retry).
 
 By default, it retries *only* on the specified methods, status codes, and on these network errors:
 - `ETIMEDOUT`: One of the [timeout](#timeout) limits were reached.
@@ -523,7 +533,7 @@ const got = require('got');
 Type: `boolean`\
 Default: `true`
 
-Determines if a `got.HTTPError` is thrown for error responses (non-2xx status codes).
+Determines if a [`got.HTTPError`](#gothttperror) is thrown for unsuccessful responses.
 
 If this is disabled, requests that encounter an error status code will be resolved with the `response` instead of throwing. This may be useful if you are checking for resource availability and are expecting error responses.
 
@@ -809,6 +819,31 @@ Type: `string`
 
 The IP address used to send the request from.
 
+##### rejectUnauthorized
+
+Type: `boolean`\
+Default: `true`
+
+If set to `false`, all invalid SSL certificates will be ignored and no error will be thrown.\
+If set to `true`, it will throw an error whenever an invalid SSL certificate is detected.
+
+We strongly recommend to have this set to `true` for security reasons.
+
+```js
+const got = require('got');
+
+(async () => {
+	// Correct:
+	await got('https://example.com', {rejectUnauthorized: true});
+
+	// You can disable it when developing an HTTPS app:
+	await got('https://localhost', {rejectUnauthorized: false});
+
+	// Never do this:
+	await got('https://example.com', {rejectUnauthorized: false});
+})();
+```
+
 #### Response
 
 The response object will typically be a [Node.js HTTP response stream](https://nodejs.org/api/http.html#http_class_http_incomingmessage), however, if returned from the cache it will be a [response-like object](https://github.com/lukechilds/responselike) which behaves in the same way.
@@ -1302,7 +1337,7 @@ When server response code is 2xx, and parsing body fails. Includes a `response`
 
 #### got.HTTPError
 
-When the server response code is not 2xx. Includes a `response` property.
+When the server response code is not 2xx nor 3xx if `options.followRedirect` is `true`, but always except for 304. Includes a `response` property.
 
 #### got.MaxRedirectsError