got 8.3.2 → 9.0.0

package/source/as-stream.js

@@ -0,0 +1,115 @@
+'use strict';
+const {PassThrough} = require('stream');
+const duplexer3 = require('duplexer3');
+const is = require('@sindresorhus/is');
+const requestAsEventEmitter = require('./request-as-event-emitter');
+const {HTTPError, ReadError} = require('./errors');
+
+module.exports = options => {
+	const input = new PassThrough();
+	const output = new PassThrough();
+	const proxy = duplexer3(input, output);
+	const piped = new Set();
+	let isFinished = false;
+
+	options.gotRetry.retries = () => 0;
+
+	if (options.body) {
+		proxy.write = () => {
+			throw new Error('Got\'s stream is not writable when the `body` option is used');
+		};
+	}
+
+	const emitter = requestAsEventEmitter(options);
+
+	emitter.on('request', request => {
+		proxy.emit('request', request);
+		const uploadComplete = () => {
+			request.emit('upload-complete');
+		};
+
+		if (is.nodeStream(options.body)) {
+			options.body.once('end', uploadComplete);
+			options.body.pipe(request);
+			return;
+		}
+
+		if (options.body) {
+			request.end(options.body, uploadComplete);
+			return;
+		}
+
+		if (options.method === 'POST' || options.method === 'PUT' || options.method === 'PATCH') {
+			input.once('end', uploadComplete);
+			input.pipe(request);
+			return;
+		}
+
+		request.end(uploadComplete);
+	});
+
+	emitter.on('response', response => {
+		const {statusCode} = response;
+
+		response.on('error', error => {
+			proxy.emit('error', new ReadError(error, options));
+		});
+
+		if (options.throwHttpErrors && statusCode !== 304 && (statusCode < 200 || statusCode > 299)) {
+			proxy.emit('error', new HTTPError(statusCode, response.statusMessage, response.headers, options), null, response);
+			return;
+		}
+
+		isFinished = true;
+
+		response.pipe(output);
+
+		for (const destination of piped) {
+			if (destination.headersSent) {
+				continue;
+			}
+
+			for (const [key, value] of Object.entries(response.headers)) {
+				// Got gives *uncompressed* data. Overriding `content-encoding` header would result in an error.
+				// It's not possible to decompress uncompressed data, is it?
+				const allowed = options.decompress ? key !== 'content-encoding' : true;
+				if (allowed) {
+					destination.setHeader(key, value);
+				}
+			}
+
+			destination.statusCode = response.statusCode;
+		}
+
+		proxy.emit('response', response);
+	});
+
+	[
+		'error',
+		'redirect',
+		'uploadProgress',
+		'downloadProgress'
+	].forEach(event => emitter.on(event, (...args) => proxy.emit(event, ...args)));
+
+	const pipe = proxy.pipe.bind(proxy);
+	const unpipe = proxy.unpipe.bind(proxy);
+	proxy.pipe = (destination, options) => {
+		if (isFinished) {
+			throw new Error('Failed to pipe. The response has been emitted already.');
+		}
+
+		const result = pipe(destination, options);
+
+		if (Reflect.has(destination, 'setHeader')) {
+			piped.add(destination);
+		}
+
+		return result;
+	};
+	proxy.unpipe = stream => {
+		piped.delete(stream);
+		return unpipe(stream);
+	};
+
+	return proxy;
+};

package/source/create.js

@@ -0,0 +1,56 @@
+'use strict';
+const errors = require('./errors');
+const asStream = require('./as-stream');
+const asPromise = require('./as-promise');
+const normalizeArguments = require('./normalize-arguments');
+const merge = require('./merge');
+const deepFreeze = require('./deep-freeze');
+
+const next = options => options.stream ? asStream(options) : asPromise(options);
+const mergeOptions = (defaults, options = {}) => merge({}, defaults, options);
+
+const create = defaults => {
+	defaults = merge({}, defaults);
+	if (!defaults.handler) {
+		defaults.handler = next;
+	}
+
+	function got(url, options) {
+		try {
+			options = mergeOptions(defaults.options, options);
+			return defaults.handler(normalizeArguments(url, options, defaults), next);
+		} catch (error) {
+			return Promise.reject(error);
+		}
+	}
+
+	got.create = create;
+	got.extend = (options = {}) => create({
+		options: mergeOptions(defaults.options, options),
+		methods: defaults.methods,
+		handler: defaults.handler
+	});
+
+	got.stream = (url, options) => {
+		options = mergeOptions(defaults.options, options);
+		options.stream = true;
+		return defaults.handler(normalizeArguments(url, options, defaults), next);
+	};
+
+	for (const method of defaults.methods) {
+		got[method] = (url, options) => got(url, {...options, method});
+		got.stream[method] = (url, options) => got.stream(url, {...options, method});
+	}
+
+	Object.assign(got, {...errors, mergeOptions});
+	Object.defineProperty(got, 'defaults', {
+		value: deepFreeze(defaults),
+		writable: false,
+		enumerable: true,
+		configurable: true
+	});
+
+	return got;
+};
+
+module.exports = create;

package/source/deep-freeze.js

@@ -0,0 +1,12 @@
+'use strict';
+const is = require('@sindresorhus/is');
+
+module.exports = function deepFreeze(object) {
+	for (const [key, value] of Object.entries(object)) {
+		if (is.object(value)) {
+			deepFreeze(object[key]);
+		}
+	}
+
+	return Object.freeze(object);
+};

package/{errors.js → source/errors.js}

@@ -19,6 +19,7 @@ class GotError extends Error {
 			hostname: opts.hostname,
 			method: opts.method,
 			path: opts.path,
+			socketPath: opts.socketPath,
 			protocol: opts.protocol,
 			url: opts.href
 		});
@@ -89,4 +90,12 @@ module.exports.UnsupportedProtocolError = class extends GotError {
 	}
 };
 
+module.exports.TimeoutError = class extends GotError {
+	constructor(threshold, event, opts) {
+		super(`Timeout awaiting '${event}' for ${threshold}ms`, {code: 'ETIMEDOUT'}, opts);
+		this.name = 'TimeoutError';
+		this.event = event;
+	}
+};
+
 module.exports.CancelError = PCancelable.CancelError;

package/source/get-body-size.js

@@ -0,0 +1,40 @@
+'use strict';
+const fs = require('fs');
+const util = require('util');
+const is = require('@sindresorhus/is');
+const isFormData = require('./is-form-data');
+
+module.exports = async options => {
+	const {body} = options;
+
+	if (options.headers['content-length']) {
+		return Number(options.headers['content-length']);
+	}
+
+	if (!body && !options.stream) {
+		return 0;
+	}
+
+	if (is.string(body)) {
+		return Buffer.byteLength(body);
+	}
+
+	if (is.buffer(body)) {
+		return body.length;
+	}
+
+	if (isFormData(body)) {
+		return util.promisify(body.getLength.bind(body))();
+	}
+
+	if (body instanceof fs.ReadStream) {
+		const {size} = await util.promisify(fs.stat)(body.path);
+		return size;
+	}
+
+	if (is.nodeStream(body) && is.buffer(body._buffer)) {
+		return body._buffer.length;
+	}
+
+	return null;
+};

package/source/get-response.js

@@ -0,0 +1,60 @@
+'use strict';
+const {Transform} = require('stream');
+const decompressResponse = require('decompress-response');
+const is = require('@sindresorhus/is');
+const mimicResponse = require('mimic-response');
+
+module.exports = (response, options, emitter, redirects) => {
+	const downloadBodySize = Number(response.headers['content-length']) || null;
+	let downloaded = 0;
+
+	const progressStream = new Transform({
+		transform(chunk, encoding, callback) {
+			downloaded += chunk.length;
+
+			const percent = downloadBodySize ? downloaded / downloadBodySize : 0;
+
+			// Let `flush()` be responsible for emitting the last event
+			if (percent < 1) {
+				emitter.emit('downloadProgress', {
+					percent,
+					transferred: downloaded,
+					total: downloadBodySize
+				});
+			}
+
+			callback(null, chunk);
+		},
+
+		flush(callback) {
+			emitter.emit('downloadProgress', {
+				percent: 1,
+				transferred: downloaded,
+				total: downloadBodySize
+			});
+
+			callback();
+		}
+	});
+
+	mimicResponse(response, progressStream);
+	progressStream.redirectUrls = redirects;
+
+	const newResponse = options.decompress === true &&
+		is.function(decompressResponse) &&
+		options.method !== 'HEAD' ? decompressResponse(progressStream) : progressStream;
+
+	if (!options.decompress && ['gzip', 'deflate'].includes(response.headers['content-encoding'])) {
+		options.encoding = null;
+	}
+
+	emitter.emit('response', newResponse);
+
+	emitter.emit('downloadProgress', {
+		percent: 0,
+		transferred: 0,
+		total: downloadBodySize
+	});
+
+	response.pipe(progressStream);
+};

package/source/index.js

@@ -0,0 +1,49 @@
+'use strict';
+const pkg = require('../package.json');
+const create = require('./create');
+
+const defaults = {
+	methods: [
+		'get',
+		'post',
+		'put',
+		'patch',
+		'head',
+		'delete'
+	],
+	options: {
+		retry: {
+			retries: 2,
+			methods: [
+				'get',
+				'put',
+				'head',
+				'delete',
+				'options',
+				'trace'
+			],
+			statusCodes: [
+				408,
+				413,
+				429,
+				502,
+				503,
+				504
+			]
+		},
+		cache: false,
+		decompress: true,
+		useElectronNet: false,
+		throwHttpErrors: true,
+		headers: {
+			'user-agent': `${pkg.name}/${pkg.version} (https://github.com/sindresorhus/got)`
+		},
+		hooks: {
+			beforeRequest: []
+		}
+	}
+};
+
+const got = create(defaults);
+
+module.exports = got;

package/source/is-form-data.js

@@ -0,0 +1,4 @@
+'use strict';
+const is = require('@sindresorhus/is');
+
+module.exports = body => is.nodeStream(body) && is.function(body.getBoundary);

package/source/is-retry-on-network-error-allowed.js

@@ -0,0 +1,17 @@
+'use strict';
+
+const WHITELIST = new Set([
+	'ETIMEDOUT',
+	'ECONNRESET',
+	'EADDRINUSE',
+	'ECONNREFUSED',
+	'EPIPE'
+]);
+
+module.exports = error => {
+	if (error && WHITELIST.has(error.code)) {
+		return true;
+	}
+
+	return false;
+};

package/source/merge.js

@@ -0,0 +1,32 @@
+'use strict';
+const {URL} = require('url');
+const is = require('@sindresorhus/is');
+
+const merge = (target, ...sources) => {
+	for (const source of sources) {
+		for (const [key, sourceValue] of Object.entries(source)) {
+			if (is.undefined(sourceValue)) {
+				continue;
+			}
+
+			const targetValue = target[key];
+			if (is.urlInstance(targetValue) && (is.urlInstance(sourceValue) || is.string(sourceValue))) {
+				target[key] = new URL(sourceValue, targetValue);
+			} else if (is.plainObject(sourceValue)) {
+				if (is.plainObject(targetValue)) {
+					target[key] = merge({}, targetValue, sourceValue);
+				} else {
+					target[key] = merge({}, sourceValue);
+				}
+			} else if (is.array(sourceValue)) {
+				target[key] = merge([], sourceValue);
+			} else {
+				target[key] = sourceValue;
+			}
+		}
+	}
+
+	return target;
+};
+
+module.exports = merge;

package/source/normalize-arguments.js

@@ -0,0 +1,230 @@
+'use strict';
+/* istanbul ignore next: compatibility reason */
+const URLGlobal = typeof URL === 'undefined' ? require('url').URL : URL; // TODO: Use the `URL` global when targeting Node.js 10
+/* istanbul ignore next: compatibility reason */
+const URLSearchParamsGlobal = typeof URLSearchParams === 'undefined' ? require('url').URLSearchParams : URLSearchParams;
+const is = require('@sindresorhus/is');
+const toReadableStream = require('to-readable-stream');
+const urlParseLax = require('url-parse-lax');
+const isRetryOnNetworkErrorAllowed = require('./is-retry-on-network-error-allowed');
+const urlToOptions = require('./url-to-options');
+const isFormData = require('./is-form-data');
+
+const retryAfterStatusCodes = new Set([413, 429, 503]);
+const knownHookEvents = ['beforeRequest'];
+
+module.exports = (url, options, defaults) => {
+	if (Reflect.has(options, 'url') || (is.object(url) && Reflect.has(url, 'url'))) {
+		throw new TypeError('Parameter `url` is not an option. Use got(url, options)');
+	}
+
+	if (!is.string(url) && !is.object(url)) {
+		throw new TypeError(`Parameter \`url\` must be a string or object, not ${is(url)}`);
+	}
+
+	if (is.string(url)) {
+		if (options.baseUrl) {
+			url = urlToOptions(new URLGlobal(url, options.baseUrl));
+		} else {
+			url = url.replace(/^unix:/, 'http://$&');
+
+			try {
+				decodeURI(url);
+			} catch (_) {
+				throw new Error('Parameter `url` must contain valid UTF-8 character sequences');
+			}
+
+			url = urlParseLax(url);
+			if (url.auth) {
+				throw new Error('Basic authentication must be done with the `auth` option');
+			}
+		}
+	} else if (is(url) === 'URL') {
+		url = urlToOptions(url);
+	}
+
+	options = {
+		path: '',
+		headers: {},
+		...url,
+		protocol: url.protocol || 'http:', // Override both null/undefined with default protocol
+		...options
+	};
+
+	if (options.stream && options.json) {
+		options.json = false;
+	}
+
+	if (options.decompress && is.undefined(options.headers['accept-encoding'])) {
+		options.headers['accept-encoding'] = 'gzip, deflate';
+	}
+
+	const {query} = options;
+	if (query) {
+		if (!is.string(query)) {
+			options.query = (new URLSearchParamsGlobal(query)).toString();
+		}
+
+		options.path = `${options.path.split('?')[0]}?${options.query}`;
+		delete options.query;
+	}
+
+	if (options.json && is.undefined(options.headers.accept)) {
+		options.headers.accept = 'application/json';
+	}
+
+	const {headers} = options;
+	for (const [key, value] of Object.entries(headers)) {
+		if (is.nullOrUndefined(value)) {
+			delete headers[key];
+		}
+	}
+
+	const {body} = options;
+	if (is.nullOrUndefined(body)) {
+		options.method = (options.method || 'GET').toUpperCase();
+	} else {
+		const isObject = is.object(body) && !Buffer.isBuffer(body) && !is.nodeStream(body);
+		if (!is.nodeStream(body) && !is.string(body) && !is.buffer(body) && !(options.form || options.json)) {
+			throw new TypeError('The `body` option must be a stream.Readable, string or Buffer');
+		}
+
+		if (options.json && !(isObject || is.array(body))) {
+			throw new TypeError('The `body` option must be an Object or Array when the `json` option is used');
+		}
+
+		if (options.form && !isObject) {
+			throw new TypeError('The `body` option must be an Object when the `form` option is used');
+		}
+
+		if (isFormData(body)) {
+			// Special case for https://github.com/form-data/form-data
+			headers['content-type'] = headers['content-type'] || `multipart/form-data; boundary=${body.getBoundary()}`;
+		} else if (options.form) {
+			headers['content-type'] = headers['content-type'] || 'application/x-www-form-urlencoded';
+			options.body = (new URLSearchParamsGlobal(body)).toString();
+		} else if (options.json) {
+			headers['content-type'] = headers['content-type'] || 'application/json';
+			options.body = JSON.stringify(body);
+		}
+
+		// Convert buffer to stream to receive upload progress events (#322)
+		if (is.buffer(body)) {
+			options.body = toReadableStream(body);
+			options.body._buffer = body;
+		}
+
+		options.method = (options.method || 'POST').toUpperCase();
+	}
+
+	if (options.hostname === 'unix') {
+		const matches = /(.+?):(.+)/.exec(options.path);
+
+		if (matches) {
+			const [, socketPath, path] = matches;
+			options = {
+				...options,
+				socketPath,
+				path,
+				host: null
+			};
+		}
+	}
+
+	options.gotRetry = {retries: 0, methods: [], statusCodes: []};
+	if (options.retry !== false) {
+		if (is.number(options.retry)) {
+			if (is.object(defaults.options.retry)) {
+				options.gotRetry = {...defaults.options.retry, retries: options.retry};
+			} else {
+				options.gotRetry.retries = options.retry;
+			}
+		} else {
+			options.gotRetry = {...options.gotRetry, ...options.retry};
+		}
+		delete options.retry;
+	}
+
+	options.gotRetry.methods = new Set(options.gotRetry.methods.map(method => method.toUpperCase()));
+	options.gotRetry.statusCodes = new Set(options.gotRetry.statusCodes);
+
+	if (!options.gotRetry.maxRetryAfter && Reflect.has(options, 'timeout')) {
+		if (is.number(options.timeout)) {
+			options.gotRetry.maxRetryAfter = options.timeout;
+		} else {
+			options.gotRetry.maxRetryAfter = Math.min(...[options.timeout.request, options.timeout.connection].filter(n => !is.nullOrUndefined(n)));
+		}
+	}
+
+	if (!is.function(options.gotRetry.retries)) {
+		const {retries} = options.gotRetry;
+
+		options.gotRetry.retries = (iteration, error) => {
+			if (iteration > retries || (!isRetryOnNetworkErrorAllowed(error) && (!options.gotRetry.methods.has(error.method) || !options.gotRetry.statusCodes.has(error.statusCode)))) {
+				return 0;
+			}
+
+			if (Reflect.has(error, 'headers') && Reflect.has(error.headers, 'retry-after') && retryAfterStatusCodes.has(error.statusCode)) {
+				let after = Number(error.headers['retry-after']);
+				if (is.nan(after)) {
+					after = Date.parse(error.headers['retry-after']) - Date.now();
+				} else {
+					after *= 1000;
+				}
+
+				if (after > options.gotRetry.maxRetryAfter) {
+					return 0;
+				}
+
+				return after;
+			}
+
+			if (error.statusCode === 413) {
+				return 0;
+			}
+
+			const noise = Math.random() * 100;
+
+			return ((1 << iteration) * 1000) + noise;
+		};
+	}
+
+	if (is.undefined(options.followRedirect)) {
+		options.followRedirect = true;
+	}
+
+	if (is.number(options.timeout) || is.object(options.timeout)) {
+		if (is.number(options.timeout)) {
+			options.gotTimeout = {request: options.timeout};
+		} else {
+			options.gotTimeout = options.timeout;
+		}
+		delete options.timeout;
+	}
+
+	if (is.nullOrUndefined(options.hooks)) {
+		options.hooks = {};
+	}
+	if (is.object(options.hooks)) {
+		for (const hookEvent of knownHookEvents) {
+			const hooks = options.hooks[hookEvent];
+			if (is.nullOrUndefined(hooks)) {
+				options.hooks[hookEvent] = [];
+			} else if (is.array(hooks)) {
+				for (const [index, hook] of hooks.entries()) {
+					if (!is.function(hook)) {
+						throw new TypeError(
+							`Parameter \`hooks.${hookEvent}[${index}]\` must be a function, not ${is(hook)}`
+						);
+					}
+				}
+			} else {
+				throw new TypeError(`Parameter \`hooks.${hookEvent}\` must be an array, not ${is(hooks)}`);
+			}
+		}
+	} else {
+		throw new TypeError(`Parameter \`hooks\` must be an object, not ${is(options.hooks)}`);
+	}
+
+	return options;
+};