got 9.2.1 → 9.3.2

package/source/request-as-event-emitter.js

@@ -6,38 +6,42 @@ const http = require('http');
 const https = require('https');
 const urlLib = require('url');
 const CacheableRequest = require('cacheable-request');
+const toReadableStream = require('to-readable-stream');
 const is = require('@sindresorhus/is');
 const timer = require('@szmarczak/http-timer');
-const timedOut = require('./timed-out');
-const getBodySize = require('./get-body-size');
+const timedOut = require('./utils/timed-out');
+const getBodySize = require('./utils/get-body-size');
 const getResponse = require('./get-response');
 const progress = require('./progress');
-const {GotError, CacheError, UnsupportedProtocolError, MaxRedirectsError, RequestError} = require('./errors');
+const {CacheError, UnsupportedProtocolError, MaxRedirectsError, RequestError, TimeoutError} = require('./errors');
+const urlToOptions = require('./utils/url-to-options');
 
 const getMethodRedirectCodes = new Set([300, 301, 302, 303, 304, 305, 307, 308]);
 const allMethodRedirectCodes = new Set([300, 303, 307, 308]);
 
-module.exports = options => {
+module.exports = (options, input) => {
 	const emitter = new EventEmitter();
-	const requestUrl = options.href || (new URL(options.path, urlLib.format(options))).toString();
 	const redirects = [];
-	const agents = is.object(options.agent) ? options.agent : null;
-	let retryCount = 0;
-	let retryTries = 0;
-	let redirectUrl;
+	let currentRequest;
+	let requestUrl;
+	let redirectString;
 	let uploadBodySize;
+	let retryCount = 0;
+	let shouldAbort = false;
 
 	const setCookie = options.cookieJar ? util.promisify(options.cookieJar.setCookie.bind(options.cookieJar)) : null;
 	const getCookieString = options.cookieJar ? util.promisify(options.cookieJar.getCookieString.bind(options.cookieJar)) : null;
+	const agents = is.object(options.agent) ? options.agent : null;
 
 	const get = async options => {
-		const currentUrl = redirectUrl || requestUrl;
+		const currentUrl = redirectString || requestUrl;
 
 		if (options.protocol !== 'http:' && options.protocol !== 'https:') {
-			emitter.emit('error', new UnsupportedProtocolError(options));
-			return;
+			throw new UnsupportedProtocolError(options);
 		}
 
+		decodeURI(currentUrl);
+
 		let fn;
 		if (is.function(options.request)) {
 			fn = {request: options.request};
@@ -58,124 +62,109 @@ module.exports = options => {
 		}
 
 		if (options.cookieJar) {
-			try {
-				const cookieString = await getCookieString(currentUrl, {});
+			const cookieString = await getCookieString(currentUrl, {});
 
-				if (!is.empty(cookieString)) {
-					options.headers.cookie = cookieString;
-				}
-			} catch (error) {
-				emitter.emit('error', error);
+			if (is.nonEmptyString(cookieString)) {
+				options.headers.cookie = cookieString;
 			}
 		}
 
 		let timings;
-		const cacheableRequest = new CacheableRequest(fn.request, options.cache);
-		const cacheReq = cacheableRequest(options, async response => {
-			/* istanbul ignore next: fixes https://github.com/electron/electron/blob/cbb460d47628a7a146adf4419ed48550a98b2923/lib/browser/api/net.js#L59-L65 */
-			if (options.useElectronNet) {
-				response = new Proxy(response, {
-					get: (target, name) => {
-						if (name === 'trailers' || name === 'rawTrailers') {
-							return [];
+		const handleResponse = async response => {
+			try {
+				/* istanbul ignore next: fixes https://github.com/electron/electron/blob/cbb460d47628a7a146adf4419ed48550a98b2923/lib/browser/api/net.js#L59-L65 */
+				if (options.useElectronNet) {
+					response = new Proxy(response, {
+						get: (target, name) => {
+							if (name === 'trailers' || name === 'rawTrailers') {
+								return [];
+							}
+
+							const value = target[name];
+							return is.function(value) ? value.bind(target) : value;
 						}
+					});
+				}
 
-						const value = target[name];
-						return is.function(value) ? value.bind(target) : value;
-					}
-				});
-			}
-
-			const {statusCode} = response;
-			response.url = currentUrl;
-			response.requestUrl = requestUrl;
-			response.retryCount = retryCount;
-			response.timings = timings;
+				const {statusCode} = response;
+				response.url = currentUrl;
+				response.requestUrl = requestUrl;
+				response.retryCount = retryCount;
+				response.timings = timings;
+				response.redirectUrls = redirects;
 
-			const rawCookies = response.headers['set-cookie'];
-			if (options.cookieJar && rawCookies) {
-				try {
+				const rawCookies = response.headers['set-cookie'];
+				if (options.cookieJar && rawCookies) {
 					await Promise.all(rawCookies.map(rawCookie => setCookie(rawCookie, response.url)));
-				} catch (error) {
-					emitter.emit('error', error);
 				}
-			}
-
-			const followRedirect = options.followRedirect && 'location' in response.headers;
-			const redirectGet = followRedirect && getMethodRedirectCodes.has(statusCode);
-			const redirectAll = followRedirect && allMethodRedirectCodes.has(statusCode);
 
-			if (redirectAll || (redirectGet && (options.method === 'GET' || options.method === 'HEAD'))) {
-				response.resume();
+				if (options.followRedirect && 'location' in response.headers) {
+					if (allMethodRedirectCodes.has(statusCode) || (getMethodRedirectCodes.has(statusCode) && (options.method === 'GET' || options.method === 'HEAD'))) {
+						response.resume(); // We're being redirected, we don't care about the response.
 
-				if (statusCode === 303) {
-					// Server responded with "see other", indicating that the resource exists at another location,
-					// and the client should request it from that location via GET or HEAD.
-					options.method = 'GET';
-				}
+						if (statusCode === 303) {
+							// Server responded with "see other", indicating that the resource exists at another location,
+							// and the client should request it from that location via GET or HEAD.
+							options.method = 'GET';
+						}
 
-				if (redirects.length >= 10) {
-					emitter.emit('error', new MaxRedirectsError(statusCode, redirects, options), null, response);
-					return;
-				}
+						if (redirects.length >= 10) {
+							throw new MaxRedirectsError(statusCode, redirects, options);
+						}
 
-				const bufferString = Buffer.from(response.headers.location, 'binary').toString();
-				redirectUrl = (new URL(bufferString, urlLib.format(options))).toString();
+						// Handles invalid URLs. See https://github.com/sindresorhus/got/issues/604
+						const redirectBuffer = Buffer.from(response.headers.location, 'binary').toString();
+						const redirectURL = new URL(redirectBuffer, currentUrl);
+						redirectString = redirectURL.toString();
 
-				try {
-					decodeURI(redirectUrl);
-				} catch (error) {
-					emitter.emit('error', error);
-					return;
-				}
+						redirects.push(redirectString);
 
-				redirects.push(redirectUrl);
+						const redirectOpts = {
+							...options,
+							...urlToOptions(redirectURL)
+						};
 
-				const redirectOpts = {
-					...options,
-					...urlLib.parse(redirectUrl)
-				};
+						for (const hook of options.hooks.beforeRedirect) {
+							// eslint-disable-next-line no-await-in-loop
+							await hook(redirectOpts);
+						}
 
-				emitter.emit('redirect', response, redirectOpts);
+						emitter.emit('redirect', response, redirectOpts);
 
-				await get(redirectOpts);
-				return;
-			}
+						await get(redirectOpts);
+						return;
+					}
+				}
 
-			try {
-				getResponse(response, options, emitter, redirects);
+				getResponse(response, options, emitter);
 			} catch (error) {
 				emitter.emit('error', error);
 			}
-		});
+		};
 
-		cacheReq.on('error', error => {
-			if (error instanceof CacheableRequest.RequestError) {
-				emitter.emit('error', new RequestError(error, options));
-			} else {
-				emitter.emit('error', new CacheError(error, options));
+		const handleRequest = request => {
+			if (shouldAbort) {
+				request.once('error', () => {});
+				request.abort();
+				return;
 			}
-		});
 
-		cacheReq.once('request', request => {
-			let aborted = false;
-			request.once('abort', _ => {
-				aborted = true;
-			});
+			currentRequest = request;
 
 			request.once('error', error => {
-				if (aborted) {
+				if (request.aborted) {
 					return;
 				}
 
-				if (!(error instanceof GotError)) {
+				if (error instanceof timedOut.TimeoutError) {
+					error = new TimeoutError(error, options);
+				} else {
 					error = new RequestError(error, options);
 				}
-				emitter.emit('retry', error, retried => {
-					if (!retried) {
-						emitter.emit('error', error);
-					}
-				});
+
+				if (emitter.retry(error) === false) {
+					emitter.emit('error', error);
+				}
 			});
 
 			timings = timer(request);
@@ -183,38 +172,109 @@ module.exports = options => {
 			progress.upload(request, emitter, uploadBodySize);
 
 			if (options.gotTimeout) {
-				timedOut(request, options);
+				timedOut(request, options.gotTimeout, options);
 			}
 
 			emitter.emit('request', request);
-		});
+
+			const uploadComplete = () => {
+				request.emit('upload-complete');
+			};
+
+			try {
+				if (is.nodeStream(options.body)) {
+					options.body.once('end', uploadComplete);
+					options.body.pipe(request);
+					options.body = undefined;
+				} else if (options.body) {
+					request.end(options.body, uploadComplete);
+				} else if (input && (options.method === 'POST' || options.method === 'PUT' || options.method === 'PATCH')) {
+					input.once('end', uploadComplete);
+					input.pipe(request);
+				} else {
+					request.end(uploadComplete);
+				}
+			} catch (error) {
+				emitter.emit('error', new RequestError(error, options));
+			}
+		};
+
+		if (options.cache) {
+			const cacheableRequest = new CacheableRequest(fn.request, options.cache);
+			const cacheReq = cacheableRequest(options, handleResponse);
+
+			cacheReq.once('error', error => {
+				if (error instanceof CacheableRequest.RequestError) {
+					emitter.emit('error', new RequestError(error, options));
+				} else {
+					emitter.emit('error', new CacheError(error, options));
+				}
+			});
+
+			cacheReq.once('request', handleRequest);
+		} else {
+			// Catches errors thrown by calling fn.request(...)
+			try {
+				handleRequest(fn.request(options, handleResponse));
+			} catch (error) {
+				emitter.emit('error', new RequestError(error, options));
+			}
+		}
 	};
 
-	emitter.on('retry', (error, cb) => {
+	emitter.retry = error => {
 		let backoff;
+
 		try {
-			backoff = options.gotRetry.retries(++retryTries, error);
+			backoff = options.retry.retries(++retryCount, error);
 		} catch (error2) {
 			emitter.emit('error', error2);
 			return;
 		}
 
 		if (backoff) {
-			retryCount++;
-			setTimeout(get, backoff, {...options, forceRefresh: true});
-			cb(true);
-			return;
+			const retry = async options => {
+				try {
+					for (const hook of options.hooks.beforeRetry) {
+						// eslint-disable-next-line no-await-in-loop
+						await hook(options, error, retryCount);
+					}
+
+					await get(options);
+				} catch (error) {
+					emitter.emit('error', error);
+				}
+			};
+
+			setTimeout(retry, backoff, {...options, forceRefresh: true});
+			return true;
 		}
 
-		cb(false);
-	});
+		return false;
+	};
+
+	emitter.abort = () => {
+		if (currentRequest) {
+			currentRequest.once('error', () => {});
+			currentRequest.abort();
+		} else {
+			shouldAbort = true;
+		}
+	};
 
 	setImmediate(async () => {
 		try {
-			uploadBodySize = await getBodySize(options);
+			// Convert buffer to stream to receive upload progress events (#322)
+			const {body} = options;
+			if (is.buffer(body)) {
+				options.body = toReadableStream(body);
+				uploadBodySize = body.length;
+			} else {
+				uploadBodySize = await getBodySize(options);
+			}
 
 			if (is.undefined(options.headers['content-length']) && is.undefined(options.headers['transfer-encoding'])) {
-				if (uploadBodySize > 0 || options.method === 'PUT') {
+				if ((uploadBodySize > 0 || options.method === 'PUT') && !is.null(uploadBodySize)) {
 					options.headers['content-length'] = uploadBodySize;
 				}
 			}
@@ -224,6 +284,8 @@ module.exports = options => {
 				await hook(options);
 			}
 
+			requestUrl = options.href || (new URL(options.path, urlLib.format(options))).toString();
+
 			await get(options);
 		} catch (error) {
 			emitter.emit('error', error);

package/source/{get-body-size.js → utils/get-body-size.js}

@@ -28,9 +28,5 @@ module.exports = async options => {
 		return size;
 	}
 
-	if (is.nodeStream(body) && is.buffer(body._buffer)) {
-		return body._buffer.length;
-	}
-
 	return null;
 };

package/source/{is-retry-on-network-error-allowed.js → utils/is-retry-on-network-error-allowed.js}

@@ -5,7 +5,10 @@ const WHITELIST = new Set([
 	'ECONNRESET',
 	'EADDRINUSE',
 	'ECONNREFUSED',
-	'EPIPE'
+	'EPIPE',
+	'ENOTFOUND',
+	'ENETUNREACH',
+	'EAI_AGAIN'
 ]);
 
 module.exports = error => {

package/source/utils/timed-out.js

@@ -0,0 +1,153 @@
+'use strict';
+const net = require('net');
+
+class TimeoutError extends Error {
+	constructor(threshold, event) {
+		super(`Timeout awaiting '${event}' for ${threshold}ms`);
+		this.name = 'TimeoutError';
+		this.code = 'ETIMEDOUT';
+		this.event = event;
+	}
+}
+
+const reentry = Symbol('reentry');
+
+const noop = () => {};
+
+module.exports = (request, delays, options) => {
+	/* istanbul ignore next: this makes sure timed-out isn't called twice */
+	if (request[reentry]) {
+		return;
+	}
+
+	request[reentry] = true;
+
+	let stopNewTimeouts = false;
+
+	const addTimeout = (delay, callback, ...args) => {
+		// An error had been thrown before. Going further would result in uncaught errors.
+		// See https://github.com/sindresorhus/got/issues/631#issuecomment-435675051
+		if (stopNewTimeouts) {
+			return noop;
+		}
+
+		// Event loop order is timers, poll, immediates.
+		// The timed event may emit during the current tick poll phase, so
+		// defer calling the handler until the poll phase completes.
+		let immediate;
+		const timeout = setTimeout(() => {
+			immediate = setImmediate(callback, delay, ...args);
+			/* istanbul ignore next: added in node v9.7.0 */
+			if (immediate.unref) {
+				immediate.unref();
+			}
+		}, delay);
+
+		/* istanbul ignore next: in order to support electron renderer */
+		if (timeout.unref) {
+			timeout.unref();
+		}
+
+		const cancel = () => {
+			clearTimeout(timeout);
+			clearImmediate(immediate);
+		};
+
+		cancelers.push(cancel);
+
+		return cancel;
+	};
+
+	const {host, hostname} = options;
+	const timeoutHandler = (delay, event) => {
+		request.emit('error', new TimeoutError(delay, event));
+		request.once('error', () => {}); // Ignore the `socket hung up` error made by request.abort()
+
+		request.abort();
+	};
+
+	const cancelers = [];
+	const cancelTimeouts = () => {
+		stopNewTimeouts = true;
+		cancelers.forEach(cancelTimeout => cancelTimeout());
+	};
+
+	request.once('error', cancelTimeouts);
+	request.once('response', response => {
+		response.once('end', cancelTimeouts);
+	});
+
+	if (delays.request !== undefined) {
+		addTimeout(delays.request, timeoutHandler, 'request');
+	}
+
+	if (delays.socket !== undefined) {
+		request.setTimeout(delays.socket, () => {
+			timeoutHandler(delays.socket, 'socket');
+		});
+	}
+
+	if (delays.lookup !== undefined && !request.socketPath && !net.isIP(hostname || host)) {
+		request.once('socket', socket => {
+			/* istanbul ignore next: hard to test */
+			if (socket.connecting) {
+				const cancelTimeout = addTimeout(delays.lookup, timeoutHandler, 'lookup');
+				socket.once('lookup', cancelTimeout);
+			}
+		});
+	}
+
+	if (delays.connect !== undefined) {
+		request.once('socket', socket => {
+			/* istanbul ignore next: hard to test */
+			if (socket.connecting) {
+				const timeConnect = () => addTimeout(delays.connect, timeoutHandler, 'connect');
+
+				if (request.socketPath || net.isIP(hostname || host)) {
+					socket.once('connect', timeConnect());
+				} else {
+					socket.once('lookup', error => {
+						if (error === null) {
+							socket.once('connect', timeConnect());
+						}
+					});
+				}
+			}
+		});
+	}
+
+	if (delays.secureConnect !== undefined && options.protocol === 'https:') {
+		request.once('socket', socket => {
+			/* istanbul ignore next: hard to test */
+			if (socket.connecting) {
+				socket.once('connect', () => {
+					const cancelTimeout = addTimeout(delays.secureConnect, timeoutHandler, 'secureConnect');
+					socket.once('secureConnect', cancelTimeout);
+				});
+			}
+		});
+	}
+
+	if (delays.send !== undefined) {
+		request.once('socket', socket => {
+			const timeRequest = () => addTimeout(delays.send, timeoutHandler, 'send');
+			/* istanbul ignore next: hard to test */
+			if (socket.connecting) {
+				socket.once('connect', () => {
+					request.once('upload-complete', timeRequest());
+				});
+			} else {
+				request.once('upload-complete', timeRequest());
+			}
+		});
+	}
+
+	if (delays.response !== undefined) {
+		request.once('upload-complete', () => {
+			const cancelTimeout = addTimeout(delays.response, timeoutHandler, 'response');
+			request.once('response', cancelTimeout);
+		});
+	}
+};
+
+module.exports.TimeoutError = TimeoutError;

package/source/merge-instances.js

@@ -1,36 +0,0 @@
-'use strict';
-const merge = require('./merge');
-const knownHookEvents = require('./known-hook-events');
-
-module.exports = (instances, methods) => {
-	const handlers = instances.map(instance => instance.defaults.handler);
-	const size = instances.length - 1;
-
-	let options = {};
-	const hooks = {};
-	for (const instance of instances) {
-		options = merge({}, options, instance.defaults.options);
-
-		const instanceHooks = instance.defaults.options.hooks;
-		for (const name of knownHookEvents) {
-			if (hooks[name]) {
-				hooks[name] = hooks[name].concat(instanceHooks[name]);
-			} else {
-				hooks[name] = [...instanceHooks[name]];
-			}
-		}
-	}
-
-	options.hooks = hooks;
-
-	return {
-		methods,
-		options,
-		handler: (options, next) => {
-			let iteration = -1;
-			const iterate = options => handlers[++iteration](options, iteration === size ? next : iterate);
-
-			return iterate(options);
-		}
-	};
-};