got 8.2.0 → 9.0.0

package/source/normalize-arguments.js

@@ -0,0 +1,230 @@
+'use strict';
+/* istanbul ignore next: compatibility reason */
+const URLGlobal = typeof URL === 'undefined' ? require('url').URL : URL; // TODO: Use the `URL` global when targeting Node.js 10
+/* istanbul ignore next: compatibility reason */
+const URLSearchParamsGlobal = typeof URLSearchParams === 'undefined' ? require('url').URLSearchParams : URLSearchParams;
+const is = require('@sindresorhus/is');
+const toReadableStream = require('to-readable-stream');
+const urlParseLax = require('url-parse-lax');
+const isRetryOnNetworkErrorAllowed = require('./is-retry-on-network-error-allowed');
+const urlToOptions = require('./url-to-options');
+const isFormData = require('./is-form-data');
+
+const retryAfterStatusCodes = new Set([413, 429, 503]);
+const knownHookEvents = ['beforeRequest'];
+
+module.exports = (url, options, defaults) => {
+	if (Reflect.has(options, 'url') || (is.object(url) && Reflect.has(url, 'url'))) {
+		throw new TypeError('Parameter `url` is not an option. Use got(url, options)');
+	}
+
+	if (!is.string(url) && !is.object(url)) {
+		throw new TypeError(`Parameter \`url\` must be a string or object, not ${is(url)}`);
+	}
+
+	if (is.string(url)) {
+		if (options.baseUrl) {
+			url = urlToOptions(new URLGlobal(url, options.baseUrl));
+		} else {
+			url = url.replace(/^unix:/, 'http://$&');
+
+			try {
+				decodeURI(url);
+			} catch (_) {
+				throw new Error('Parameter `url` must contain valid UTF-8 character sequences');
+			}
+
+			url = urlParseLax(url);
+			if (url.auth) {
+				throw new Error('Basic authentication must be done with the `auth` option');
+			}
+		}
+	} else if (is(url) === 'URL') {
+		url = urlToOptions(url);
+	}
+
+	options = {
+		path: '',
+		headers: {},
+		...url,
+		protocol: url.protocol || 'http:', // Override both null/undefined with default protocol
+		...options
+	};
+
+	if (options.stream && options.json) {
+		options.json = false;
+	}
+
+	if (options.decompress && is.undefined(options.headers['accept-encoding'])) {
+		options.headers['accept-encoding'] = 'gzip, deflate';
+	}
+
+	const {query} = options;
+	if (query) {
+		if (!is.string(query)) {
+			options.query = (new URLSearchParamsGlobal(query)).toString();
+		}
+
+		options.path = `${options.path.split('?')[0]}?${options.query}`;
+		delete options.query;
+	}
+
+	if (options.json && is.undefined(options.headers.accept)) {
+		options.headers.accept = 'application/json';
+	}
+
+	const {headers} = options;
+	for (const [key, value] of Object.entries(headers)) {
+		if (is.nullOrUndefined(value)) {
+			delete headers[key];
+		}
+	}
+
+	const {body} = options;
+	if (is.nullOrUndefined(body)) {
+		options.method = (options.method || 'GET').toUpperCase();
+	} else {
+		const isObject = is.object(body) && !Buffer.isBuffer(body) && !is.nodeStream(body);
+		if (!is.nodeStream(body) && !is.string(body) && !is.buffer(body) && !(options.form || options.json)) {
+			throw new TypeError('The `body` option must be a stream.Readable, string or Buffer');
+		}
+
+		if (options.json && !(isObject || is.array(body))) {
+			throw new TypeError('The `body` option must be an Object or Array when the `json` option is used');
+		}
+
+		if (options.form && !isObject) {
+			throw new TypeError('The `body` option must be an Object when the `form` option is used');
+		}
+
+		if (isFormData(body)) {
+			// Special case for https://github.com/form-data/form-data
+			headers['content-type'] = headers['content-type'] || `multipart/form-data; boundary=${body.getBoundary()}`;
+		} else if (options.form) {
+			headers['content-type'] = headers['content-type'] || 'application/x-www-form-urlencoded';
+			options.body = (new URLSearchParamsGlobal(body)).toString();
+		} else if (options.json) {
+			headers['content-type'] = headers['content-type'] || 'application/json';
+			options.body = JSON.stringify(body);
+		}
+
+		// Convert buffer to stream to receive upload progress events (#322)
+		if (is.buffer(body)) {
+			options.body = toReadableStream(body);
+			options.body._buffer = body;
+		}
+
+		options.method = (options.method || 'POST').toUpperCase();
+	}
+
+	if (options.hostname === 'unix') {
+		const matches = /(.+?):(.+)/.exec(options.path);
+
+		if (matches) {
+			const [, socketPath, path] = matches;
+			options = {
+				...options,
+				socketPath,
+				path,
+				host: null
+			};
+		}
+	}
+
+	options.gotRetry = {retries: 0, methods: [], statusCodes: []};
+	if (options.retry !== false) {
+		if (is.number(options.retry)) {
+			if (is.object(defaults.options.retry)) {
+				options.gotRetry = {...defaults.options.retry, retries: options.retry};
+			} else {
+				options.gotRetry.retries = options.retry;
+			}
+		} else {
+			options.gotRetry = {...options.gotRetry, ...options.retry};
+		}
+		delete options.retry;
+	}
+
+	options.gotRetry.methods = new Set(options.gotRetry.methods.map(method => method.toUpperCase()));
+	options.gotRetry.statusCodes = new Set(options.gotRetry.statusCodes);
+
+	if (!options.gotRetry.maxRetryAfter && Reflect.has(options, 'timeout')) {
+		if (is.number(options.timeout)) {
+			options.gotRetry.maxRetryAfter = options.timeout;
+		} else {
+			options.gotRetry.maxRetryAfter = Math.min(...[options.timeout.request, options.timeout.connection].filter(n => !is.nullOrUndefined(n)));
+		}
+	}
+
+	if (!is.function(options.gotRetry.retries)) {
+		const {retries} = options.gotRetry;
+
+		options.gotRetry.retries = (iteration, error) => {
+			if (iteration > retries || (!isRetryOnNetworkErrorAllowed(error) && (!options.gotRetry.methods.has(error.method) || !options.gotRetry.statusCodes.has(error.statusCode)))) {
+				return 0;
+			}
+
+			if (Reflect.has(error, 'headers') && Reflect.has(error.headers, 'retry-after') && retryAfterStatusCodes.has(error.statusCode)) {
+				let after = Number(error.headers['retry-after']);
+				if (is.nan(after)) {
+					after = Date.parse(error.headers['retry-after']) - Date.now();
+				} else {
+					after *= 1000;
+				}
+
+				if (after > options.gotRetry.maxRetryAfter) {
+					return 0;
+				}
+
+				return after;
+			}
+
+			if (error.statusCode === 413) {
+				return 0;
+			}
+
+			const noise = Math.random() * 100;
+
+			return ((1 << iteration) * 1000) + noise;
+		};
+	}
+
+	if (is.undefined(options.followRedirect)) {
+		options.followRedirect = true;
+	}
+
+	if (is.number(options.timeout) || is.object(options.timeout)) {
+		if (is.number(options.timeout)) {
+			options.gotTimeout = {request: options.timeout};
+		} else {
+			options.gotTimeout = options.timeout;
+		}
+		delete options.timeout;
+	}
+
+	if (is.nullOrUndefined(options.hooks)) {
+		options.hooks = {};
+	}
+	if (is.object(options.hooks)) {
+		for (const hookEvent of knownHookEvents) {
+			const hooks = options.hooks[hookEvent];
+			if (is.nullOrUndefined(hooks)) {
+				options.hooks[hookEvent] = [];
+			} else if (is.array(hooks)) {
+				for (const [index, hook] of hooks.entries()) {
+					if (!is.function(hook)) {
+						throw new TypeError(
+							`Parameter \`hooks.${hookEvent}[${index}]\` must be a function, not ${is(hook)}`
+						);
+					}
+				}
+			} else {
+				throw new TypeError(`Parameter \`hooks.${hookEvent}\` must be an array, not ${is(hooks)}`);
+			}
+		}
+	} else {
+		throw new TypeError(`Parameter \`hooks\` must be an object, not ${is(options.hooks)}`);
+	}
+
+	return options;
+};

package/source/progress.js

@@ -0,0 +1,71 @@
+'use strict';
+module.exports = {
+	upload(request, emitter, uploadBodySize) {
+		const uploadEventFrequency = 150;
+		let uploaded = 0;
+		let progressInterval;
+
+		emitter.emit('uploadProgress', {
+			percent: 0,
+			transferred: 0,
+			total: uploadBodySize
+		});
+
+		request.once('error', () => {
+			clearInterval(progressInterval);
+		});
+
+		request.once('response', () => {
+			clearInterval(progressInterval);
+
+			emitter.emit('uploadProgress', {
+				percent: 1,
+				transferred: uploaded,
+				total: uploadBodySize
+			});
+		});
+
+		request.once('socket', socket => {
+			const onSocketConnect = () => {
+				progressInterval = setInterval(() => {
+					/* istanbul ignore next: hard to test */
+					if (socket.destroyed) {
+						clearInterval(progressInterval);
+						return;
+					}
+
+					const lastUploaded = uploaded;
+					/* istanbul ignore next: see #490 (occurs randomly!) */
+					const headersSize = request._header ? Buffer.byteLength(request._header) : 0;
+					uploaded = socket.bytesWritten - headersSize;
+
+					/* istanbul ignore next: see https://github.com/sindresorhus/got/pull/322#pullrequestreview-51647813 (no proof) */
+					if (uploadBodySize && uploaded > uploadBodySize) {
+						uploaded = uploadBodySize;
+					}
+
+					// Don't emit events with unchanged progress and
+					// prevent last event from being emitted, because
+					// it's emitted when `response` is emitted
+					if (uploaded === lastUploaded || uploaded === uploadBodySize) {
+						return;
+					}
+
+					emitter.emit('uploadProgress', {
+						percent: uploadBodySize ? uploaded / uploadBodySize : 0,
+						transferred: uploaded,
+						total: uploadBodySize
+					});
+				}, uploadEventFrequency);
+			};
+
+			if (socket.connecting) {
+				socket.once('connect', onSocketConnect);
+			} else {
+				// The socket is being reused from pool,
+				// so the connect event will not be emitted
+				onSocketConnect();
+			}
+		});
+	}
+};

package/source/request-as-event-emitter.js

@@ -0,0 +1,185 @@
+'use strict';
+/* istanbul ignore next: compatibility reason */
+const URLGlobal = typeof URL === 'undefined' ? require('url').URL : URL; // TODO: Use the `URL` global when targeting Node.js 10
+const EventEmitter = require('events');
+const http = require('http');
+const https = require('https');
+const urlLib = require('url');
+const CacheableRequest = require('cacheable-request');
+const is = require('@sindresorhus/is');
+const timedOut = require('./timed-out');
+const getBodySize = require('./get-body-size');
+const getResponse = require('./get-response');
+const progress = require('./progress');
+const {GotError, CacheError, UnsupportedProtocolError, MaxRedirectsError, RequestError} = require('./errors');
+
+const getMethodRedirectCodes = new Set([300, 301, 302, 303, 304, 305, 307, 308]);
+const allMethodRedirectCodes = new Set([300, 303, 307, 308]);
+
+module.exports = options => {
+	const emitter = new EventEmitter();
+	const requestUrl = options.href || (new URLGlobal(options.path, urlLib.format(options))).toString();
+	const redirects = [];
+	const agents = is.object(options.agent) ? options.agent : null;
+	let retryCount = 0;
+	let retryTries = 0;
+	let redirectUrl;
+	let uploadBodySize;
+
+	const get = options => {
+		if (options.protocol !== 'http:' && options.protocol !== 'https:') {
+			emitter.emit('error', new UnsupportedProtocolError(options));
+			return;
+		}
+
+		let fn = options.protocol === 'https:' ? https : http;
+
+		if (agents) {
+			const protocolName = options.protocol === 'https:' ? 'https' : 'http';
+			options.agent = agents[protocolName] || options.agent;
+		}
+
+		/* istanbul ignore next: electron.net is broken */
+		if (options.useElectronNet && process.versions.electron) {
+			const electron = global['require']('electron'); // eslint-disable-line dot-notation
+			fn = electron.net || electron.remote.net;
+		}
+
+		const cacheableRequest = new CacheableRequest(fn.request, options.cache);
+		const cacheReq = cacheableRequest(options, response => {
+			const {statusCode} = response;
+			response.retryCount = retryCount;
+			response.url = redirectUrl || requestUrl;
+			response.requestUrl = requestUrl;
+
+			const followRedirect = options.followRedirect && 'location' in response.headers;
+			const redirectGet = followRedirect && getMethodRedirectCodes.has(statusCode);
+			const redirectAll = followRedirect && allMethodRedirectCodes.has(statusCode);
+
+			if (redirectAll || (redirectGet && (options.method === 'GET' || options.method === 'HEAD'))) {
+				response.resume();
+
+				if (statusCode === 303) {
+					// Server responded with "see other", indicating that the resource exists at another location,
+					// and the client should request it from that location via GET or HEAD.
+					options.method = 'GET';
+				}
+
+				if (redirects.length >= 10) {
+					emitter.emit('error', new MaxRedirectsError(statusCode, redirects, options), null, response);
+					return;
+				}
+
+				const bufferString = Buffer.from(response.headers.location, 'binary').toString();
+				redirectUrl = (new URLGlobal(bufferString, urlLib.format(options))).toString();
+
+				try {
+					redirectUrl = decodeURI(redirectUrl);
+				} catch (error) {
+					emitter.emit('error', error);
+					return;
+				}
+
+				redirects.push(redirectUrl);
+
+				const redirectOpts = {
+					...options,
+					...urlLib.parse(redirectUrl)
+				};
+
+				emitter.emit('redirect', response, redirectOpts);
+
+				get(redirectOpts);
+				return;
+			}
+
+			try {
+				getResponse(response, options, emitter, redirects);
+			} catch (error) {
+				emitter.emit('error', error);
+			}
+		});
+
+		cacheReq.on('error', error => {
+			if (error instanceof CacheableRequest.RequestError) {
+				emitter.emit('error', new RequestError(error, options));
+			} else {
+				emitter.emit('error', new CacheError(error, options));
+			}
+		});
+
+		cacheReq.once('request', request => {
+			let aborted = false;
+			request.once('abort', _ => {
+				aborted = true;
+			});
+
+			request.once('error', error => {
+				if (aborted) {
+					return;
+				}
+
+				if (!(error instanceof GotError)) {
+					error = new RequestError(error, options);
+				}
+				emitter.emit('retry', error, retried => {
+					if (!retried) {
+						emitter.emit('error', error);
+					}
+				});
+			});
+
+			progress.upload(request, emitter, uploadBodySize);
+
+			if (options.gotTimeout) {
+				timedOut(request, options);
+			}
+
+			emitter.emit('request', request);
+		});
+	};
+
+	emitter.on('retry', (error, cb) => {
+		let backoff;
+		try {
+			backoff = options.gotRetry.retries(++retryTries, error);
+		} catch (error) {
+			emitter.emit('error', error);
+			return;
+		}
+
+		if (backoff) {
+			retryCount++;
+			setTimeout(get, backoff, options);
+			cb(true);
+			return;
+		}
+
+		cb(false);
+	});
+
+	setImmediate(async () => {
+		try {
+			uploadBodySize = await getBodySize(options);
+
+			if (
+				uploadBodySize > 0 &&
+				is.undefined(options.headers['content-length']) &&
+				is.undefined(options.headers['transfer-encoding'])
+			) {
+				options.headers['content-length'] = uploadBodySize;
+			}
+
+			for (const hook of options.hooks.beforeRequest) {
+				// eslint-disable-next-line no-await-in-loop
+				await hook(options);
+			}
+
+			get(options);
+		} catch (error) {
+			emitter.emit('error', error);
+		}
+	});
+
+	return emitter;
+};

package/source/timed-out.js

@@ -0,0 +1,160 @@
+'use strict';
+const net = require('net');
+const {TimeoutError} = require('./errors');
+
+const reentry = Symbol('reentry');
+
+function addTimeout(delay, callback, ...args) {
+	// Event loop order is timers, poll, immediates.
+	// The timed event may emit during the current tick poll phase, so
+	// defer calling the handler until the poll phase completes.
+	let immediate;
+	const timeout = setTimeout(
+		() => {
+			immediate = setImmediate(callback, delay, ...args);
+			/* istanbul ignore next: added in node v9.7.0 */
+			if (immediate.unref) {
+				immediate.unref();
+			}
+		},
+		delay
+	);
+	timeout.unref();
+	return () => {
+		clearTimeout(timeout);
+		clearImmediate(immediate);
+	};
+}
+
+module.exports = (request, options) => {
+	/* istanbul ignore next: this makes sure timed-out isn't called twice */
+	if (request[reentry]) {
+		return;
+	}
+
+	request[reentry] = true;
+	const {gotTimeout: delays, host, hostname} = options;
+	const timeoutHandler = (delay, event) => {
+		request.abort();
+		request.emit('error', new TimeoutError(delay, event, options));
+	};
+
+	const cancelers = [];
+	const cancelTimeouts = () => {
+		cancelers.forEach(cancelTimeout => cancelTimeout());
+	};
+
+	request.on('error', cancelTimeouts);
+	request.once('response', response => {
+		response.once('end', cancelTimeouts);
+	});
+
+	if (delays.request !== undefined) {
+		const cancelTimeout = addTimeout(
+			delays.request,
+			timeoutHandler,
+			'request'
+		);
+		cancelers.push(cancelTimeout);
+	}
+
+	if (delays.socket !== undefined) {
+		request.setTimeout(
+			delays.socket,
+			() => {
+				timeoutHandler(delays.socket, 'socket');
+			}
+		);
+	}
+
+	if (delays.lookup !== undefined && !request.socketPath && !net.isIP(hostname || host)) {
+		request.once('socket', socket => {
+			/* istanbul ignore next: hard to test */
+			if (socket.connecting) {
+				const cancelTimeout = addTimeout(
+					delays.lookup,
+					timeoutHandler,
+					'lookup'
+				);
+				cancelers.push(cancelTimeout);
+				socket.once('lookup', cancelTimeout);
+			}
+		});
+	}
+
+	if (delays.connect !== undefined) {
+		request.once('socket', socket => {
+			/* istanbul ignore next: hard to test */
+			if (socket.connecting) {
+				const timeConnect = () => {
+					const cancelTimeout = addTimeout(
+						delays.connect,
+						timeoutHandler,
+						'connect'
+					);
+					cancelers.push(cancelTimeout);
+					return cancelTimeout;
+				};
+
+				if (request.socketPath || net.isIP(hostname || host)) {
+					socket.once('connect', timeConnect());
+				} else {
+					socket.once('lookup', () => {
+						socket.once('connect', timeConnect());
+					});
+				}
+			}
+		});
+	}
+
+	if (delays.secureConnect !== undefined && options.protocol === 'https:') {
+		request.once('socket', socket => {
+			/* istanbul ignore next: hard to test */
+			if (socket.connecting) {
+				socket.once('connect', () => {
+					const cancelTimeout = addTimeout(
+						delays.secureConnect,
+						timeoutHandler,
+						'secureConnect'
+					);
+					cancelers.push(cancelTimeout);
+					socket.once('secureConnect', cancelTimeout);
+				});
+			}
+		});
+	}
+
+	if (delays.send !== undefined) {
+		request.once('socket', socket => {
+			const timeRequest = () => {
+				const cancelTimeout = addTimeout(
+					delays.send,
+					timeoutHandler,
+					'send'
+				);
+				cancelers.push(cancelTimeout);
+				return cancelTimeout;
+			};
+			/* istanbul ignore next: hard to test */
+			if (socket.connecting) {
+				socket.once('connect', () => {
+					request.once('upload-complete', timeRequest());
+				});
+			} else {
+				request.once('upload-complete', timeRequest());
+			}
+		});
+	}
+
+	if (delays.response !== undefined) {
+		request.once('upload-complete', () => {
+			const cancelTimeout = addTimeout(
+				delays.response,
+				timeoutHandler,
+				'response'
+			);
+			cancelers.push(cancelTimeout);
+			request.once('response', cancelTimeout);
+		});
+	}
+};

package/source/url-to-options.js

@@ -0,0 +1,25 @@
+'use strict';
+const is = require('@sindresorhus/is');
+
+module.exports = url => {
+	const options = {
+		protocol: url.protocol,
+		hostname: url.hostname.startsWith('[') ? url.hostname.slice(1, -1) : url.hostname,
+		hash: url.hash,
+		search: url.search,
+		pathname: url.pathname,
+		href: url.href
+	};
+
+	if (is.string(url.port) && url.port.length > 0) {
+		options.port = Number(url.port);
+	}
+
+	if (url.username || url.password) {
+		options.auth = `${url.username}:${url.password}`;
+	}
+
+	options.path = is.null(url.search) ? url.pathname : `${url.pathname}${url.search}`;
+
+	return options;
+};