got 9.2.1 → 9.3.2

package/source/merge.js

@@ -1,6 +1,7 @@
 'use strict';
 const {URL} = require('url');
 const is = require('@sindresorhus/is');
+const knownHookEvents = require('./known-hook-events');
 
 const merge = (target, ...sources) => {
 	for (const source of sources) {
@@ -29,4 +30,44 @@ const merge = (target, ...sources) => {
 	return target;
 };
 
+const mergeOptions = (...sources) => {
+	sources = sources.map(source => source || {});
+	const merged = merge({}, ...sources);
+
+	const hooks = {};
+	for (const hook of knownHookEvents) {
+		hooks[hook] = [];
+	}
+
+	for (const source of sources) {
+		if (source.hooks) {
+			for (const hook of knownHookEvents) {
+				hooks[hook] = hooks[hook].concat(source.hooks[hook]);
+			}
+		}
+	}
+
+	merged.hooks = hooks;
+
+	return merged;
+};
+
+const mergeInstances = (instances, methods) => {
+	const handlers = instances.map(instance => instance.defaults.handler);
+	const size = instances.length - 1;
+
+	return {
+		methods,
+		options: mergeOptions(...instances.map(instance => instance.defaults.options)),
+		handler: (options, next) => {
+			let iteration = -1;
+			const iterate = options => handlers[++iteration](options, iteration === size ? next : iterate);
+
+			return iterate(options);
+		}
+	};
+};
+
 module.exports = merge;
+module.exports.options = mergeOptions;
+module.exports.instances = mergeInstances;

package/source/normalize-arguments.js

@@ -1,64 +1,102 @@
 'use strict';
 const {URL, URLSearchParams} = require('url'); // TODO: Use the `URL` global when targeting Node.js 10
+const urlLib = require('url');
 const is = require('@sindresorhus/is');
-const toReadableStream = require('to-readable-stream');
 const urlParseLax = require('url-parse-lax');
-const isRetryOnNetworkErrorAllowed = require('./is-retry-on-network-error-allowed');
-const urlToOptions = require('./url-to-options');
-const isFormData = require('./is-form-data');
-const knownHookEvents = require('./known-hook-events');
+const lowercaseKeys = require('lowercase-keys');
+const isRetryOnNetworkErrorAllowed = require('./utils/is-retry-on-network-error-allowed');
+const urlToOptions = require('./utils/url-to-options');
+const isFormData = require('./utils/is-form-data');
 const merge = require('./merge');
+const knownHookEvents = require('./known-hook-events');
 
 const retryAfterStatusCodes = new Set([413, 429, 503]);
 
-// `preNormalize` handles things related to static options, like `baseUrl`, `followRedirect`, `hooks`, etc.
-// While `normalize` does `preNormalize` + handles things related to dynamic options, like URL, headers, body, etc.
-const preNormalize = options => {
-	options = {
-		headers: {},
-		...options
-	};
+// `preNormalize` handles static things (lowercasing headers; normalizing baseUrl, timeout, retry)
+// While `normalize` does `preNormalize` + handles things which need to be reworked when user changes them
+const preNormalize = (options, defaults) => {
+	if (is.nullOrUndefined(options.headers)) {
+		options.headers = {};
+	} else {
+		options.headers = lowercaseKeys(options.headers);
+	}
 
 	if (options.baseUrl && !options.baseUrl.toString().endsWith('/')) {
 		options.baseUrl += '/';
 	}
 
-	if (is.undefined(options.followRedirect)) {
-		options.followRedirect = true;
+	if (options.stream) {
+		options.json = false;
 	}
 
 	if (is.nullOrUndefined(options.hooks)) {
 		options.hooks = {};
+	} else if (!is.object(options.hooks)) {
+		throw new TypeError(`Parameter \`hooks\` must be an object, not ${is(options.hooks)}`);
 	}
-	if (is.object(options.hooks)) {
-		for (const hookEvent of knownHookEvents) {
-			const hooks = options.hooks[hookEvent];
-			if (is.nullOrUndefined(hooks)) {
-				options.hooks[hookEvent] = [];
-			} else if (is.array(hooks)) {
-				for (const [index, hook] of hooks.entries()) {
-					if (!is.function(hook)) {
-						throw new TypeError(
-							`Parameter \`hooks.${hookEvent}[${index}]\` must be a function, not ${is(hook)}`
-						);
-					}
-				}
+
+	for (const event of knownHookEvents) {
+		if (is.nullOrUndefined(options.hooks[event])) {
+			if (defaults) {
+				options.hooks[event] = [...defaults.hooks[event]];
 			} else {
-				throw new TypeError(`Parameter \`hooks.${hookEvent}\` must be an array, not ${is(hooks)}`);
+				options.hooks[event] = [];
 			}
 		}
-	} else {
-		throw new TypeError(`Parameter \`hooks\` must be an object, not ${is(options.hooks)}`);
+	}
+
+	if (is.number(options.timeout)) {
+		options.gotTimeout = {request: options.timeout};
+	} else if (is.object(options.timeout)) {
+		options.gotTimeout = options.timeout;
+	}
+	delete options.timeout;
+
+	const {retry} = options;
+	options.retry = {
+		retries: 0,
+		methods: [],
+		statusCodes: []
+	};
+
+	if (is.nonEmptyObject(defaults) && retry !== false) {
+		options.retry = {...defaults.retry};
+	}
+
+	if (retry !== false) {
+		if (is.number(retry)) {
+			options.retry.retries = retry;
+		} else {
+			options.retry = {...options.retry, ...retry};
+		}
+	}
+
+	if (options.gotTimeout) {
+		options.retry.maxRetryAfter = Math.min(...[options.gotTimeout.request, options.gotTimeout.connection].filter(n => !is.nullOrUndefined(n)));
+	}
+
+	if (is.array(options.retry.methods)) {
+		options.retry.methods = new Set(options.retry.methods.map(method => method.toUpperCase()));
+	}
+
+	if (is.array(options.retry.statusCodes)) {
+		options.retry.statusCodes = new Set(options.retry.statusCodes);
 	}
 
 	return options;
 };
 
-module.exports = (url, options, defaults) => {
-	options = merge({}, defaults.options, options ? preNormalize(options) : {});
+const normalize = (url, options, defaults) => {
+	if (is.plainObject(url)) {
+		options = {...url, ...options};
+		url = options.url || {};
+		delete options.url;
+	}
 
-	if (Reflect.has(options, 'url') || (is.object(url) && Reflect.has(url, 'url'))) {
-		throw new TypeError('Parameter `url` is not an option. Use got(url, options)');
+	if (defaults) {
+		options = merge({}, defaults.options, options ? preNormalize(options, defaults.options) : {});
+	} else {
+		options = merge({}, options ? preNormalize(options) : {});
 	}
 
 	if (!is.string(url) && !is.object(url)) {
@@ -75,12 +113,6 @@ module.exports = (url, options, defaults) => {
 		} else {
 			url = url.replace(/^unix:/, 'http://$&');
 
-			try {
-				decodeURI(url);
-			} catch (_) {
-				throw new Error('Parameter `url` must contain valid UTF-8 character sequences');
-			}
-
 			url = urlParseLax(url);
 			if (url.auth) {
 				throw new Error('Basic authentication must be done with the `auth` option');
@@ -90,12 +122,8 @@ module.exports = (url, options, defaults) => {
 		url = urlToOptions(url);
 	}
 
-	options = {
-		path: '',
-		...url,
-		protocol: url.protocol || 'http:', // Override both null/undefined with default protocol
-		...options
-	};
+	// Override both null/undefined with default protocol
+	options = merge({path: ''}, url, {protocol: url.protocol || 'https:'}, options);
 
 	const {baseUrl} = options;
 	Object.defineProperty(options, 'baseUrl', {
@@ -106,7 +134,7 @@ module.exports = (url, options, defaults) => {
 	});
 
 	const {query} = options;
-	if (!is.empty(query) || query instanceof URLSearchParams) {
+	if (is.nonEmptyString(query) || is.nonEmptyObject(query) || query instanceof URLSearchParams) {
 		if (!is.string(query)) {
 			options.query = (new URLSearchParams(query)).toString();
 		}
@@ -114,12 +142,18 @@ module.exports = (url, options, defaults) => {
 		delete options.query;
 	}
 
-	if (options.stream && options.json) {
-		options.json = false;
-	}
+	if (options.hostname === 'unix') {
+		const matches = /(.+?):(.+)/.exec(options.path);
 
-	if (options.json && is.undefined(options.headers.accept)) {
-		options.headers.accept = 'application/json';
+		if (matches) {
+			const [, socketPath, path] = matches;
+			options = {
+				...options,
+				socketPath,
+				path,
+				host: null
+			};
+		}
 	}
 
 	const {headers} = options;
@@ -129,11 +163,19 @@ module.exports = (url, options, defaults) => {
 		}
 	}
 
+	if (options.json && is.undefined(headers.accept)) {
+		headers.accept = 'application/json';
+	}
+
+	if (options.decompress && is.undefined(headers['accept-encoding'])) {
+		headers['accept-encoding'] = 'gzip, deflate';
+	}
+
 	const {body} = options;
 	if (is.nullOrUndefined(body)) {
-		options.method = options.method || 'GET';
+		options.method = options.method ? options.method.toUpperCase() : 'GET';
 	} else {
-		const isObject = is.object(body) && !Buffer.isBuffer(body) && !is.nodeStream(body);
+		const isObject = is.object(body) && !is.buffer(body) && !is.nodeStream(body);
 		if (!is.nodeStream(body) && !is.string(body) && !is.buffer(body) && !(options.form || options.json)) {
 			throw new TypeError('The `body` option must be a stream.Readable, string or Buffer');
 		}
@@ -157,103 +199,52 @@ module.exports = (url, options, defaults) => {
 			options.body = JSON.stringify(body);
 		}
 
-		// Convert buffer to stream to receive upload progress events (#322)
-		if (is.buffer(body)) {
-			options.body = toReadableStream(body);
-			options.body._buffer = body;
-		}
-
-		options.method = options.method || 'POST';
-	}
-
-	options.method = options.method.toUpperCase();
-
-	if (options.decompress && is.undefined(options.headers['accept-encoding'])) {
-		options.headers['accept-encoding'] = 'gzip, deflate';
+		options.method = options.method ? options.method.toUpperCase() : 'POST';
 	}
 
-	if (options.hostname === 'unix') {
-		const matches = /(.+?):(.+)/.exec(options.path);
+	if (!is.function(options.retry.retries)) {
+		const {retries} = options.retry;
 
-		if (matches) {
-			const [, socketPath, path] = matches;
-			options = {
-				...options,
-				socketPath,
-				path,
-				host: null
-			};
-		}
-	}
-
-	options.gotRetry = {retries: 0, methods: [], statusCodes: []};
-	if (options.retry !== false) {
-		if (is.number(options.retry)) {
-			if (is.object(defaults.options.retry)) {
-				options.gotRetry = {...defaults.options.retry, retries: options.retry};
-			} else {
-				options.gotRetry.retries = options.retry;
+		options.retry.retries = (iteration, error) => {
+			if (iteration > retries) {
+				return 0;
 			}
-		} else {
-			options.gotRetry = {...options.gotRetry, ...options.retry};
-		}
-		delete options.retry;
-	}
-
-	options.gotRetry.methods = new Set(options.gotRetry.methods.map(method => method.toUpperCase()));
-	options.gotRetry.statusCodes = new Set(options.gotRetry.statusCodes);
-
-	if (!options.gotRetry.maxRetryAfter && Reflect.has(options, 'timeout')) {
-		if (is.number(options.timeout)) {
-			options.gotRetry.maxRetryAfter = options.timeout;
-		} else {
-			options.gotRetry.maxRetryAfter = Math.min(...[options.timeout.request, options.timeout.connection].filter(n => !is.nullOrUndefined(n)));
-		}
-	}
 
-	if (is.number(options.timeout) || is.object(options.timeout)) {
-		if (is.number(options.timeout)) {
-			options.gotTimeout = {request: options.timeout};
-		} else {
-			options.gotTimeout = options.timeout;
-		}
-		delete options.timeout;
-	}
+			if (error !== null) {
+				if (!isRetryOnNetworkErrorAllowed(error) && (!options.retry.methods.has(error.method) || !options.retry.statusCodes.has(error.statusCode))) {
+					return 0;
+				}
 
-	if (!is.function(options.gotRetry.retries)) {
-		const {retries} = options.gotRetry;
+				if (Reflect.has(error, 'headers') && Reflect.has(error.headers, 'retry-after') && retryAfterStatusCodes.has(error.statusCode)) {
+					let after = Number(error.headers['retry-after']);
+					if (is.nan(after)) {
+						after = Date.parse(error.headers['retry-after']) - Date.now();
+					} else {
+						after *= 1000;
+					}
 
-		options.gotRetry.retries = (iteration, error) => {
-			if (iteration > retries || (!isRetryOnNetworkErrorAllowed(error) && (!options.gotRetry.methods.has(error.method) || !options.gotRetry.statusCodes.has(error.statusCode)))) {
-				return 0;
-			}
+					if (after > options.retry.maxRetryAfter) {
+						return 0;
+					}
 
-			if (Reflect.has(error, 'headers') && Reflect.has(error.headers, 'retry-after') && retryAfterStatusCodes.has(error.statusCode)) {
-				let after = Number(error.headers['retry-after']);
-				if (is.nan(after)) {
-					after = Date.parse(error.headers['retry-after']) - Date.now();
-				} else {
-					after *= 1000;
+					return after;
 				}
 
-				if (after > options.gotRetry.maxRetryAfter) {
+				if (error.statusCode === 413) {
 					return 0;
 				}
-
-				return after;
-			}
-
-			if (error.statusCode === 413) {
-				return 0;
 			}
 
 			const noise = Math.random() * 100;
-
-			return ((1 << iteration) * 1000) + noise;
+			return ((2 ** (iteration - 1)) * 1000) + noise;
 		};
 	}
 
 	return options;
 };
 
+const reNormalize = options => normalize(urlLib.format(options), options);
+
+module.exports = normalize;
 module.exports.preNormalize = preNormalize;
+module.exports.reNormalize = reNormalize;

package/source/progress.js

@@ -1,6 +1,40 @@
 'use strict';
+const {Transform} = require('stream');
 
 module.exports = {
+	download(response, emitter, downloadBodySize) {
+		let downloaded = 0;
+
+		return new Transform({
+			transform(chunk, encoding, callback) {
+				downloaded += chunk.length;
+
+				const percent = downloadBodySize ? downloaded / downloadBodySize : 0;
+
+				// Let `flush()` be responsible for emitting the last event
+				if (percent < 1) {
+					emitter.emit('downloadProgress', {
+						percent,
+						transferred: downloaded,
+						total: downloadBodySize
+					});
+				}
+
+				callback(null, chunk);
+			},
+
+			flush(callback) {
+				emitter.emit('downloadProgress', {
+					percent: 1,
+					transferred: downloaded,
+					total: downloadBodySize
+				});
+
+				callback();
+			}
+		});
+	},
+
 	upload(request, emitter, uploadBodySize) {
 		const uploadEventFrequency = 150;
 		let uploaded = 0;
@@ -34,11 +68,6 @@ module.exports = {
 					const headersSize = request._header ? Buffer.byteLength(request._header) : 0;
 					uploaded = socket.bytesWritten - headersSize;
 
-					/* istanbul ignore next: see https://github.com/sindresorhus/got/pull/322#pullrequestreview-51647813 (no proof) */
-					if (uploadBodySize && uploaded > uploadBodySize) {
-						uploaded = uploadBodySize;
-					}
-
 					// Don't emit events with unchanged progress and
 					// prevent last event from being emitted, because
 					// it's emitted when `response` is emitted