got 9.2.1 → 9.3.2

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "got",
-	"version": "9.2.1",
+	"version": "9.3.2",
 	"description": "Simplified HTTP requests",
 	"license": "MIT",
 	"repository": "sindresorhus/got",
@@ -9,7 +9,8 @@
 		"node": ">=8.6"
 	},
 	"scripts": {
-		"test": "xo && nyc ava"
+		"test": "xo && nyc ava",
+		"release": "np"
 	},
 	"files": [
 		"source"
@@ -33,28 +34,30 @@
 		"electron"
 	],
 	"dependencies": {
-		"@sindresorhus/is": "^0.11.0",
+		"@sindresorhus/is": "^0.12.0",
 		"@szmarczak/http-timer": "^1.1.0",
-		"cacheable-request": "^5.0.0",
+		"cacheable-request": "^5.1.0",
 		"decompress-response": "^3.3.0",
 		"duplexer3": "^0.1.4",
-		"get-stream": "^4.0.0",
+		"get-stream": "^4.1.0",
+		"lowercase-keys": "^1.0.1",
 		"mimic-response": "^1.0.1",
-		"p-cancelable": "^0.5.0",
+		"p-cancelable": "^1.0.0",
 		"to-readable-stream": "^1.0.0",
 		"url-parse-lax": "^3.0.0"
 	},
 	"devDependencies": {
-		"ava": "1.0.0-beta.8",
+		"ava": "1.0.0-rc.1",
 		"coveralls": "^3.0.0",
-		"delay": "^4.0.0",
-		"form-data": "^2.1.1",
+		"delay": "^4.1.0",
+		"form-data": "^2.3.3",
 		"get-port": "^4.0.0",
-		"nyc": "^13.0.1",
+		"np": "^3.0.4",
+		"nyc": "^13.1.0",
 		"p-event": "^2.1.0",
-		"pem": "^1.4.4",
+		"pem": "^1.13.2",
 		"proxyquire": "^2.0.1",
-		"sinon": "^6.1.0",
+		"sinon": "^7.1.0",
 		"slow-stream": "0.0.4",
 		"tempfile": "^2.0.0",
 		"tempy": "^0.2.1",

package/readme.md

@@ -35,9 +35,10 @@ Got is for Node.js. For browsers, we recommend [Ky](https://github.com/sindresor
 - [Errors with metadata](#errors)
 - [JSON mode](#json)
 - [WHATWG URL support](#url)
-- [Electron support](#useelectronnet)
+- [Hooks](https://github.com/sindresorhus/got#hooks)
 - [Instances with custom defaults](#instances)
 - [Composable](advanced-creation.md#merging-instances)
+- [Electron support](#useelectronnet)
 - [Used by ~2000 packages and ~500K repos](https://github.com/sindresorhus/got/network/dependents)
 - Actively maintained
 
@@ -120,7 +121,7 @@ Very useful when used with `got.extend()` to create niche-specific Got instances
 
 Can be a string or a [WHATWG `URL`](https://nodejs.org/api/url.html#url_class_url).
 
-Backslash at the end of `baseUrl` and at the beginning of the `url` argument is optional:
+Slash at the end of `baseUrl` and at the beginning of the `url` argument is optional:
 
 ```js
 await got('hello', {baseUrl: 'https://example.com/v1'});
@@ -202,9 +203,36 @@ Parse response body with `JSON.parse` and set `accept` header to `application/js
 
 ###### query
 
-Type: `string` `Object` [`URLSearchParams`](https://developer.mozilla.org/en-US/docs/Web/API/URLSearchParams)
+Type: `string` `Object<string, string|number>` [`URLSearchParams`](https://developer.mozilla.org/en-US/docs/Web/API/URLSearchParams)
+
+Query string that will be added to the request URL. This will override the query string in `url`.
+
+If you need to pass in an array, you can do it using a `URLSearchParams` instance:
+
+```js
+const got = require('got');
+
+const query = new URLSearchParams([['key', 'a'], ['key', 'b']]);
+
+got('https://example.com', {query});
+
+console.log(query.toString());
+//=> 'key=a&key=b'
+```
+
+And if you need a different array format, you could use the [`query-string`](https://github.com/sindresorhus/query-string) package:
+
+```js
+const got = require('got');
+const queryString = require('query-string');
+
+const query = queryString.stringify({key: ['a', 'b']}, {arrayFormat: 'bracket'});
+
+got('https://example.com', {query});
 
-Query string object that will be added to the request URL. This will override the query string in `url`.
+console.log(query);
+//=> 'key[]=a&key[]=b'
+```
 
 ###### timeout
 
@@ -236,7 +264,7 @@ An object representing `retries`, `methods`, `statusCodes` and `maxRetryAfter` f
 If `maxRetryAfter` is set to `undefined`, it will use `options.timeout`.<br>
 If [`Retry-After`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After) header is greater than `maxRetryAfter`, it will cancel the request.
 
-Delays between retries counts with function `1000 * Math.pow(2, retry) + Math.random() * 100`, where `retry` is attempt number (starts from 0).
+Delays between retries counts with function `1000 * Math.pow(2, retry) + Math.random() * 100`, where `retry` is attempt number (starts from 1).
 
 The `retries` property can be a `number` or a `function` with `retry` and `error` arguments. The function must return a delay in milliseconds (`0` return value cancels retry).
 
@@ -316,8 +344,7 @@ got('sindresorhus.com', {
 
 ###### hooks
 
-Type: `Object<string, Function[]>`<br>
-Default: `{beforeRequest: []}`
+Type: `Object<string, Function[]>`
 
 Hooks allow modifications during the request lifecycle. Hook functions may be async and are run serially.
 
@@ -326,11 +353,96 @@ Hooks allow modifications during the request lifecycle. Hook functions may be as
 Type: `Function[]`<br>
 Default: `[]`
 
-Called with the normalized request options. Got will make no further changes to the request before it is sent. This is especially useful in conjunction with [`got.extend()`](#instances) and [`got.create()`](advanced-creation.md) when you want to create an API client that, for example, uses HMAC-signing.
+Called with [normalized](source/normalize-arguments.js) [request options](#options). Got will make no further changes to the request before it is sent. This is especially useful in conjunction with [`got.extend()`](#instances) and [`got.create()`](advanced-creation.md) when you want to create an API client that, for example, uses HMAC-signing.
 
 See the [AWS section](#aws) for an example.
 
-**Note**: Modifying the `body` is not recommended because the `content-length` header has already been computed and assigned.
+**Note**: If you modify the `body` you will need to modify the `content-length` header too, because it has already been computed and assigned.
+
+###### hooks.beforeRedirect
+
+Type: `Function[]`<br>
+Default: `[]`
+
+Called with [normalized](source/normalize-arguments.js) [request options](#options). Got will make no further changes to the request. This is especially useful when you want to avoid dead sites. Example:
+
+```js
+const got = require('got');
+
+got('example.com', {
+	hooks: {
+		beforeRedirect: [
+			options => {
+				if (options.hostname === 'deadSite') {
+					options.hostname = 'fallbackSite';
+				}
+			}
+		]
+	}
+});
+```
+
+###### hooks.beforeRetry
+
+Type: `Function[]`<br>
+Default: `[]`
+
+Called with [normalized](source/normalize-arguments.js) [request options](#options), the error and the retry count. Got will make no further changes to the request. This is especially useful when some extra work is required before the next try. Example:
+
+```js
+const got = require('got');
+
+got('example.com', {
+	hooks: {
+		beforeRetry: [
+			(options, error, retryCount) => {
+				if (error.statusCode === 413) { // Payload too large
+					options.body = getNewBody();
+				}
+			}
+		]
+	}
+});
+```
+
+###### hooks.afterResponse
+
+Type: `Function[]`<br>
+Default: `[]`
+
+Called with [response object](#response) and a retry function.
+
+Each function should return the response. This is especially useful when you want to refresh an access token. Example:
+
+```js
+const got = require('got');
+
+const instance = got.extend({
+	hooks: {
+		afterResponse: [
+			(response, retryWithMergedOptions) => {
+				if (response.statusCode === 401) { // Unauthorized
+					const updatedOptions = {
+						headers: {
+							token: getNewToken() // Refresh the access token
+						}
+					};
+
+					// Save for further requests
+					instance.defaults.options = got.mergeOptions(instance.defaults.options, updatedOptions);
+
+					// Make a new retry
+					return retryWithMergedOptions(updatedOptions);
+				}
+
+				// No changes otherwise
+				return response;
+			}
+		]
+	},
+	mutableDefaults: true
+});
+```
 
 #### Response
 
@@ -472,7 +584,7 @@ Sets `options.method` to the method name and makes a request.
 
 #### got.extend([options])
 
-Configure a new `got` instance with default `options`. `options` are merged with the parent instance's `defaults.options` using [`got.mergeOptions`](#gotmergeoptionsparentoptions-newoptions).
+Configure a new `got` instance with default `options`. The `options` are merged with the parent instance's `defaults.options` using [`got.mergeOptions`](#gotmergeoptionsparentoptions-newoptions). You can access the resolved options with the `.defaults` property on the instance.
 
 ```js
 const client = got.extend({
@@ -538,9 +650,15 @@ Options are deeply merged to a new object. The value of each key is determined a
 - If the new property is an `Array`, it overwrites the old one with a deep clone of the new property.
 - Otherwise, the new value is assigned to the key.
 
+#### got.defaults
+
+Type: `Object`
+
+The default Got options.
+
 ## Errors
 
-Each error contains (if available) `statusCode`, `statusMessage`, `host`, `hostname`, `method`, `path`, `protocol` and `url` properties to make debugging easier.
+Each error contains (if available) `body`, `statusCode`, `statusMessage`, `host`, `hostname`, `method`, `path`, `protocol` and `url` properties to make debugging easier.
 
 In Promise mode, the `response` is attached to the error.
 
@@ -670,7 +788,7 @@ You can use the [`tunnel`](https://github.com/koichik/node-tunnel) package with
 
 ```js
 const got = require('got');
-const tunnel = require('tunnel-agent');
+const tunnel = require('tunnel');
 
 got('sindresorhus.com', {
 	agent: tunnel.httpOverHttp({
@@ -910,7 +1028,7 @@ const h2got = got.extend({request});
 | Browser support       |       ✖      |       ✖      |       ✔*     |       ✔      |
 | Electron support      |       ✔      |       ✖      |       ✖      |       ✖      |
 | Promise API           |       ✔      |       ✔      |       ✔      |       ✔      |
-| Stream API            |       ✔      |       ✔      |       ✖      |       ✖      |
+| Stream API            |       ✔      |       ✔      | Node.js only |       ✖      |
 | Request cancelation   |       ✔      |       ✖      |       ✖      |       ✔      |
 | RFC compliant caching |       ✔      |       ✖      |       ✖      |       ✖      |
 | Cookies (out-of-box)  |       ✔      |       ✔      |       ✖      |       ✖      |

package/source/as-promise.js

@@ -5,43 +5,16 @@ const is = require('@sindresorhus/is');
 const PCancelable = require('p-cancelable');
 const requestAsEventEmitter = require('./request-as-event-emitter');
 const {HTTPError, ParseError, ReadError} = require('./errors');
+const {options: mergeOptions} = require('./merge');
+const {reNormalize} = require('./normalize-arguments');
 
-module.exports = options => {
+const asPromise = options => {
 	const proxy = new EventEmitter();
 
 	const promise = new PCancelable((resolve, reject, onCancel) => {
 		const emitter = requestAsEventEmitter(options);
-		let cancelOnRequest = false;
 
-		onCancel(() => {
-			cancelOnRequest = true;
-		});
-
-		emitter.on('request', request => {
-			if (cancelOnRequest) {
-				request.abort();
-				return;
-			}
-
-			proxy.emit('request', request);
-
-			const uploadComplete = () => {
-				request.emit('upload-complete');
-			};
-
-			onCancel(() => {
-				request.abort();
-			});
-
-			if (is.nodeStream(options.body)) {
-				options.body.once('end', uploadComplete);
-				options.body.pipe(request);
-				options.body = undefined;
-				return;
-			}
-
-			request.end(options.body, uploadComplete);
-		});
+		onCancel(emitter.abort);
 
 		emitter.on('response', async response => {
 			proxy.emit('response', response);
@@ -56,11 +29,34 @@ module.exports = options => {
 				return;
 			}
 
-			const {statusCode} = response;
 			const limitStatusCode = options.followRedirect ? 299 : 399;
 
 			response.body = data;
 
+			try {
+				for (const [index, hook] of Object.entries(options.hooks.afterResponse)) {
+					// eslint-disable-next-line no-await-in-loop
+					response = await hook(response, updatedOptions => {
+						updatedOptions = reNormalize(mergeOptions(options, {
+							...updatedOptions,
+							retry: 0,
+							throwHttpErrors: false
+						}));
+
+						// Remove any further hooks for that request, because we we'll call them anyway.
+						// The loop continues. We don't want duplicates (asPromise recursion).
+						updatedOptions.hooks.afterResponse = options.hooks.afterResponse.slice(0, index);
+
+						return asPromise(updatedOptions);
+					});
+				}
+			} catch (error) {
+				reject(error);
+				return;
+			}
+
+			const {statusCode} = response;
+
 			if (options.json && response.body) {
 				try {
 					response.body = JSON.parse(response.body);
@@ -69,23 +65,22 @@ module.exports = options => {
 						const parseError = new ParseError(error, statusCode, options, data);
 						Object.defineProperty(parseError, 'response', {value: response});
 						reject(parseError);
+						return;
 					}
 				}
 			}
 
 			if (statusCode !== 304 && (statusCode < 200 || statusCode > limitStatusCode)) {
-				const error = new HTTPError(statusCode, response.statusMessage, response.headers, options);
+				const error = new HTTPError(response, options);
 				Object.defineProperty(error, 'response', {value: response});
-				emitter.emit('retry', error, retried => {
-					if (!retried) {
-						if (options.throwHttpErrors) {
-							reject(error);
-							return;
-						}
-
-						resolve(response);
+				if (emitter.retry(error) === false) {
+					if (options.throwHttpErrors) {
+						reject(error);
+						return;
 					}
-				});
+
+					resolve(response);
+				}
 				return;
 			}
 
@@ -94,6 +89,7 @@ module.exports = options => {
 
 		emitter.once('error', reject);
 		[
+			'request',
 			'redirect',
 			'uploadProgress',
 			'downloadProgress'
@@ -107,3 +103,5 @@ module.exports = options => {
 
 	return promise;
 };
+
+module.exports = asPromise;

package/source/as-stream.js

@@ -1,7 +1,6 @@
 'use strict';
 const {PassThrough} = require('stream');
 const duplexer3 = require('duplexer3');
-const is = require('@sindresorhus/is');
 const requestAsEventEmitter = require('./request-as-event-emitter');
 const {HTTPError, ReadError} = require('./errors');
 
@@ -12,7 +11,7 @@ module.exports = options => {
 	const piped = new Set();
 	let isFinished = false;
 
-	options.gotRetry.retries = () => 0;
+	options.retry.retries = () => 0;
 
 	if (options.body) {
 		proxy.write = () => {
@@ -20,33 +19,10 @@ module.exports = options => {
 		};
 	}
 
-	const emitter = requestAsEventEmitter(options);
+	const emitter = requestAsEventEmitter(options, input);
 
-	emitter.on('request', request => {
-		proxy.emit('request', request);
-		const uploadComplete = () => {
-			request.emit('upload-complete');
-		};
-
-		if (is.nodeStream(options.body)) {
-			options.body.once('end', uploadComplete);
-			options.body.pipe(request);
-			return;
-		}
-
-		if (options.body) {
-			request.end(options.body, uploadComplete);
-			return;
-		}
-
-		if (options.method === 'POST' || options.method === 'PUT' || options.method === 'PATCH') {
-			input.once('end', uploadComplete);
-			input.pipe(request);
-			return;
-		}
-
-		request.end(uploadComplete);
-	});
+	// Cancels the request
+	proxy._destroy = emitter.abort;
 
 	emitter.on('response', response => {
 		const {statusCode} = response;
@@ -56,7 +32,7 @@ module.exports = options => {
 		});
 
 		if (options.throwHttpErrors && statusCode !== 304 && (statusCode < 200 || statusCode > 299)) {
-			proxy.emit('error', new HTTPError(statusCode, response.statusMessage, response.headers, options), null, response);
+			proxy.emit('error', new HTTPError(response, options), null, response);
 			return;
 		}
 
@@ -70,8 +46,8 @@ module.exports = options => {
 			}
 
 			for (const [key, value] of Object.entries(response.headers)) {
-				// Got gives *uncompressed* data. Overriding `content-encoding` header would result in an error.
-				// It's not possible to decompress uncompressed data, is it?
+				// Got gives *decompressed* data. Overriding `content-encoding` header would result in an error.
+				// It's not possible to decompress already decompressed data, is it?
 				const allowed = options.decompress ? key !== 'content-encoding' : true;
 				if (allowed) {
 					destination.setHeader(key, value);
@@ -86,6 +62,7 @@ module.exports = options => {
 
 	[
 		'error',
+		'request',
 		'redirect',
 		'uploadProgress',
 		'downloadProgress'

package/source/create.js

@@ -4,11 +4,9 @@ const asStream = require('./as-stream');
 const asPromise = require('./as-promise');
 const normalizeArguments = require('./normalize-arguments');
 const merge = require('./merge');
-const deepFreeze = require('./deep-freeze');
-const mergeInstances = require('./merge-instances');
+const deepFreeze = require('./utils/deep-freeze');
 
 const getPromiseOrStream = options => options.stream ? asStream(options) : asPromise(options);
-const mergeOptions = (defaults, options = {}) => merge({}, defaults, options);
 
 const aliases = [
 	'get',
@@ -21,7 +19,8 @@ const aliases = [
 
 const create = defaults => {
 	defaults = merge({}, defaults);
-	defaults.options = normalizeArguments.preNormalize(defaults.options);
+	normalizeArguments.preNormalize(defaults.options);
+
 	if (!defaults.handler) {
 		// This can't be getPromiseOrStream, because when merging
 		// the chain would stop at this point and no further handlers would be called.
@@ -41,12 +40,23 @@ const create = defaults => {
 	}
 
 	got.create = create;
-	got.extend = options => create({
-		options: mergeOptions(defaults.options, options),
-		handler: defaults.handler
-	});
+	got.extend = options => {
+		let mutableDefaults;
+		if (options && Reflect.has(options, 'mutableDefaults')) {
+			mutableDefaults = options.mutableDefaults;
+			delete options.mutableDefaults;
+		} else {
+			mutableDefaults = defaults.mutableDefaults;
+		}
+
+		return create({
+			options: merge.options(defaults.options, options),
+			handler: defaults.handler,
+			mutableDefaults
+		});
+	};
 
-	got.mergeInstances = (...args) => create(mergeInstances(args));
+	got.mergeInstances = (...args) => create(merge.instances(args));
 
 	got.stream = (url, options) => got(url, {...options, stream: true});
 
@@ -55,12 +65,12 @@ const create = defaults => {
 		got.stream[method] = (url, options) => got.stream(url, {...options, method});
 	}
 
-	Object.assign(got, {...errors, mergeOptions});
+	Object.assign(got, {...errors, mergeOptions: merge.options});
 	Object.defineProperty(got, 'defaults', {
-		value: deepFreeze(defaults),
-		writable: false,
-		enumerable: true,
-		configurable: true
+		value: defaults.mutableDefaults ? defaults : deepFreeze(defaults),
+		writable: defaults.mutableDefaults,
+		configurable: defaults.mutableDefaults,
+		enumerable: true
 	});
 
 	return got;

package/source/errors.js

@@ -59,7 +59,10 @@ module.exports.ParseError = class extends GotError {
 };
 
 module.exports.HTTPError = class extends GotError {
-	constructor(statusCode, statusMessage, headers, opts) {
+	constructor(response, opts) {
+		const {statusCode} = response;
+		let {statusMessage} = response;
+
 		if (statusMessage) {
 			statusMessage = statusMessage.replace(/\r?\n/g, ' ').trim();
 		} else {
@@ -69,7 +72,8 @@ module.exports.HTTPError = class extends GotError {
 		this.name = 'HTTPError';
 		this.statusCode = statusCode;
 		this.statusMessage = statusMessage;
-		this.headers = headers;
+		this.headers = response.headers;
+		this.body = response.body;
 	}
 };
 
@@ -91,10 +95,10 @@ module.exports.UnsupportedProtocolError = class extends GotError {
 };
 
 module.exports.TimeoutError = class extends GotError {
-	constructor(threshold, event, opts) {
-		super(`Timeout awaiting '${event}' for ${threshold}ms`, {code: 'ETIMEDOUT'}, opts);
+	constructor(error, opts) {
+		super(error.message, {code: 'ETIMEDOUT'}, opts);
 		this.name = 'TimeoutError';
-		this.event = event;
+		this.event = error.event;
 	}
 };
 

package/source/get-response.js

@@ -1,44 +1,15 @@
 'use strict';
-const {Transform} = require('stream');
 const decompressResponse = require('decompress-response');
 const is = require('@sindresorhus/is');
 const mimicResponse = require('mimic-response');
+const progress = require('./progress');
 
-module.exports = (response, options, emitter, redirects) => {
+module.exports = (response, options, emitter) => {
 	const downloadBodySize = Number(response.headers['content-length']) || null;
-	let downloaded = 0;
-
-	const progressStream = new Transform({
-		transform(chunk, encoding, callback) {
-			downloaded += chunk.length;
-
-			const percent = downloadBodySize ? downloaded / downloadBodySize : 0;
-
-			// Let `flush()` be responsible for emitting the last event
-			if (percent < 1) {
-				emitter.emit('downloadProgress', {
-					percent,
-					transferred: downloaded,
-					total: downloadBodySize
-				});
-			}
-
-			callback(null, chunk);
-		},
-
-		flush(callback) {
-			emitter.emit('downloadProgress', {
-				percent: 1,
-				transferred: downloaded,
-				total: downloadBodySize
-			});
-
-			callback();
-		}
-	});
+
+	const progressStream = progress.download(response, emitter, downloadBodySize);
 
 	mimicResponse(response, progressStream);
-	progressStream.redirectUrls = redirects;
 
 	const newResponse = options.decompress === true &&
 		is.function(decompressResponse) &&

package/source/index.js

@@ -24,17 +24,25 @@ const defaults = {
 				504
 			]
 		},
-		cache: false,
-		decompress: true,
-		useElectronNet: false,
-		throwHttpErrors: true,
 		headers: {
 			'user-agent': `${pkg.name}/${pkg.version} (https://github.com/sindresorhus/got)`
 		},
 		hooks: {
-			beforeRequest: []
-		}
-	}
+			beforeRequest: [],
+			beforeRedirect: [],
+			beforeRetry: [],
+			afterResponse: []
+		},
+		decompress: true,
+		throwHttpErrors: true,
+		followRedirect: true,
+		stream: false,
+		form: false,
+		json: false,
+		cache: false,
+		useElectronNet: false
+	},
+	mutableDefaults: false
 };
 
 const got = create(defaults);

package/source/known-hook-events.js

@@ -1,3 +1,8 @@
 'use strict';
 
-module.exports = ['beforeRequest'];
+module.exports = [
+	'beforeRequest',
+	'beforeRedirect',
+	'beforeRetry',
+	'afterResponse'
+];