got 6.7.0 → 8.0.0

package/index.js

@@ -3,47 +3,136 @@ const EventEmitter = require('events');
 const http = require('http');
 const https = require('https');
 const PassThrough = require('stream').PassThrough;
+const Transform = require('stream').Transform;
 const urlLib = require('url');
+const fs = require('fs');
 const querystring = require('querystring');
+const CacheableRequest = require('cacheable-request');
 const duplexer3 = require('duplexer3');
+const intoStream = require('into-stream');
 const isStream = require('is-stream');
 const getStream = require('get-stream');
 const timedOut = require('timed-out');
 const urlParseLax = require('url-parse-lax');
+const urlToOptions = require('url-to-options');
 const lowercaseKeys = require('lowercase-keys');
-const isRedirect = require('is-redirect');
-const unzipResponse = require('unzip-response');
-const createErrorClass = require('create-error-class');
+const decompressResponse = require('decompress-response');
+const mimicResponse = require('mimic-response');
 const isRetryAllowed = require('is-retry-allowed');
+const isURL = require('isurl');
+const isPlainObj = require('is-plain-obj');
+const PCancelable = require('p-cancelable');
+const pTimeout = require('p-timeout');
+const pify = require('pify');
 const Buffer = require('safe-buffer').Buffer;
-const pkg = require('./package');
+const pkg = require('./package.json');
+
+const getMethodRedirectCodes = new Set([300, 301, 302, 303, 304, 305, 307, 308]);
+const allMethodRedirectCodes = new Set([300, 303, 307, 308]);
+
+const isFormData = body => isStream(body) && typeof body.getBoundary === 'function';
+
+const getBodySize = opts => {
+	const body = opts.body;
+
+	if (opts.headers['content-length']) {
+		return Number(opts.headers['content-length']);
+	}
+
+	if (!body && !opts.stream) {
+		return 0;
+	}
+
+	if (typeof body === 'string') {
+		return Buffer.byteLength(body);
+	}
+
+	if (isFormData(body)) {
+		return pify(body.getLength.bind(body))();
+	}
+
+	if (body instanceof fs.ReadStream) {
+		return pify(fs.stat)(body.path).then(stat => stat.size);
+	}
+
+	if (isStream(body) && Buffer.isBuffer(body._buffer)) {
+		return body._buffer.length;
+	}
+
+	return null;
+};
 
 function requestAsEventEmitter(opts) {
 	opts = opts || {};
 
 	const ee = new EventEmitter();
 	const requestUrl = opts.href || urlLib.resolve(urlLib.format(opts), opts.path);
-	let redirectCount = 0;
+	const redirects = [];
+	const agents = typeof opts.agent === 'object' ? opts.agent : null;
 	let retryCount = 0;
 	let redirectUrl;
+	let uploadBodySize;
+	let uploaded = 0;
 
 	const get = opts => {
-		const fn = opts.protocol === 'https:' ? https : http;
+		if (opts.protocol !== 'http:' && opts.protocol !== 'https:') {
+			ee.emit('error', new got.UnsupportedProtocolError(opts));
+			return;
+		}
+
+		let fn = opts.protocol === 'https:' ? https : http;
+
+		if (agents) {
+			const protocolName = opts.protocol === 'https:' ? 'https' : 'http';
+			opts.agent = agents[protocolName] || opts.agent;
+		}
+
+		if (opts.useElectronNet && process.versions.electron) {
+			const electron = require('electron');
+			fn = electron.net || electron.remote.net;
+		}
+
+		let progressInterval;
+
+		const cacheableRequest = new CacheableRequest(fn.request, opts.cache);
+		const cacheReq = cacheableRequest(opts, res => {
+			clearInterval(progressInterval);
+
+			ee.emit('uploadProgress', {
+				percent: 1,
+				transferred: uploaded,
+				total: uploadBodySize
+			});
 
-		const req = fn.request(opts, res => {
 			const statusCode = res.statusCode;
 
-			if (isRedirect(statusCode) && opts.followRedirect && 'location' in res.headers && (opts.method === 'GET' || opts.method === 'HEAD')) {
+			res.url = redirectUrl || requestUrl;
+			res.requestUrl = requestUrl;
+
+			const followRedirect = opts.followRedirect && 'location' in res.headers;
+			const redirectGet = followRedirect && getMethodRedirectCodes.has(statusCode);
+			const redirectAll = followRedirect && allMethodRedirectCodes.has(statusCode);
+
+			if (redirectAll || (redirectGet && (opts.method === 'GET' || opts.method === 'HEAD'))) {
 				res.resume();
 
-				if (++redirectCount > 10) {
-					ee.emit('error', new got.MaxRedirectsError(statusCode, opts), null, res);
+				if (statusCode === 303) {
+					// Server responded with "see other", indicating that the resource exists at another location,
+					// and the client should request it from that location via GET or HEAD.
+					opts.method = 'GET';
+				}
+
+				if (redirects.length >= 10) {
+					ee.emit('error', new got.MaxRedirectsError(statusCode, redirects, opts), null, res);
 					return;
 				}
 
 				const bufferString = Buffer.from(res.headers.location, 'binary').toString();
 
 				redirectUrl = urlLib.resolve(urlLib.format(opts), bufferString);
+
+				redirects.push(redirectUrl);
+
 				const redirectOpts = Object.assign({}, opts, urlLib.parse(redirectUrl));
 
 				ee.emit('redirect', res, redirectOpts);
@@ -53,44 +142,169 @@ function requestAsEventEmitter(opts) {
 				return;
 			}
 
+			const downloadBodySize = Number(res.headers['content-length']) || null;
+			let downloaded = 0;
+
 			setImmediate(() => {
-				const response = typeof unzipResponse === 'function' && req.method !== 'HEAD' ? unzipResponse(res) : res;
-				response.url = redirectUrl || requestUrl;
-				response.requestUrl = requestUrl;
+				const progressStream = new Transform({
+					transform(chunk, encoding, callback) {
+						downloaded += chunk.length;
+
+						const percent = downloadBodySize ? downloaded / downloadBodySize : 0;
+
+						// Let flush() be responsible for emitting the last event
+						if (percent < 1) {
+							ee.emit('downloadProgress', {
+								percent,
+								transferred: downloaded,
+								total: downloadBodySize
+							});
+						}
+
+						callback(null, chunk);
+					},
+
+					flush(callback) {
+						ee.emit('downloadProgress', {
+							percent: 1,
+							transferred: downloaded,
+							total: downloadBodySize
+						});
+
+						callback();
+					}
+				});
+
+				mimicResponse(res, progressStream);
+				progressStream.redirectUrls = redirects;
+
+				const response = opts.decompress === true &&
+					typeof decompressResponse === 'function' &&
+					opts.method !== 'HEAD' ? decompressResponse(progressStream) : progressStream;
+
+				if (!opts.decompress && ['gzip', 'deflate'].indexOf(res.headers['content-encoding']) !== -1) {
+					opts.encoding = null;
+				}
 
 				ee.emit('response', response);
+
+				ee.emit('downloadProgress', {
+					percent: 0,
+					transferred: 0,
+					total: downloadBodySize
+				});
+
+				res.pipe(progressStream);
 			});
 		});
 
-		req.once('error', err => {
-			const backoff = opts.retries(++retryCount, err);
-
-			if (backoff) {
-				setTimeout(get, backoff, opts);
-				return;
+		cacheReq.on('error', err => {
+			if (err instanceof CacheableRequest.RequestError) {
+				ee.emit('error', new got.RequestError(err, opts));
+			} else {
+				ee.emit('error', new got.CacheError(err, opts));
 			}
-
-			ee.emit('error', new got.RequestError(err, opts));
 		});
 
-		if (opts.timeout) {
-			timedOut(req, opts.timeout);
-		}
+		cacheReq.once('request', req => {
+			req.once('error', err => {
+				clearInterval(progressInterval);
+
+				const backoff = opts.retries(++retryCount, err);
+
+				if (backoff) {
+					setTimeout(get, backoff, opts);
+					return;
+				}
+
+				ee.emit('error', new got.RequestError(err, opts));
+			});
+
+			ee.once('request', req => {
+				ee.emit('uploadProgress', {
+					percent: 0,
+					transferred: 0,
+					total: uploadBodySize
+				});
+
+				req.connection.once('connect', () => {
+					const uploadEventFrequency = 150;
+
+					progressInterval = setInterval(() => {
+						const lastUploaded = uploaded;
+						const headersSize = Buffer.byteLength(req._header);
+						uploaded = req.connection.bytesWritten - headersSize;
+
+						// Prevent the known issue of `bytesWritten` being larger than body size
+						if (uploadBodySize && uploaded > uploadBodySize) {
+							uploaded = uploadBodySize;
+						}
+
+						// Don't emit events with unchanged progress and
+						// prevent last event from being emitted, because
+						// it's emitted when `response` is emitted
+						if (uploaded === lastUploaded || uploaded === uploadBodySize) {
+							return;
+						}
 
-		setImmediate(() => {
-			ee.emit('request', req);
+						ee.emit('uploadProgress', {
+							percent: uploadBodySize ? uploaded / uploadBodySize : 0,
+							transferred: uploaded,
+							total: uploadBodySize
+						});
+					}, uploadEventFrequency);
+				});
+			});
+
+			if (opts.gotTimeout) {
+				clearInterval(progressInterval);
+				timedOut(req, opts.gotTimeout);
+			}
+
+			setImmediate(() => {
+				ee.emit('request', req);
+			});
 		});
 	};
 
-	get(opts);
+	setImmediate(() => {
+		Promise.resolve(getBodySize(opts))
+			.then(size => {
+				uploadBodySize = size;
+				get(opts);
+			})
+			.catch(err => {
+				ee.emit('error', err);
+			});
+	});
+
 	return ee;
 }
 
 function asPromise(opts) {
-	return new Promise((resolve, reject) => {
+	const timeoutFn = requestPromise => opts.gotTimeout && opts.gotTimeout.request ?
+		pTimeout(requestPromise, opts.gotTimeout.request, new got.RequestError({message: 'Request timed out', code: 'ETIMEDOUT'}, opts)) :
+		requestPromise;
+
+	const proxy = new EventEmitter();
+
+	const cancelable = new PCancelable((onCancel, resolve, reject) => {
 		const ee = requestAsEventEmitter(opts);
+		let cancelOnRequest = false;
+
+		onCancel(() => {
+			cancelOnRequest = true;
+		});
 
 		ee.on('request', req => {
+			if (cancelOnRequest) {
+				req.abort();
+			}
+
+			onCancel(() => {
+				req.abort();
+			});
+
 			if (isStream(opts.body)) {
 				opts.body.pipe(req);
 				opts.body = undefined;
@@ -114,13 +328,15 @@ function asPromise(opts) {
 					if (opts.json && res.body) {
 						try {
 							res.body = JSON.parse(res.body);
-						} catch (e) {
-							throw new got.ParseError(e, statusCode, opts, data);
+						} catch (err) {
+							if (statusCode >= 200 && statusCode < 300) {
+								throw new got.ParseError(err, statusCode, opts, data);
+							}
 						}
 					}
 
-					if (statusCode < 200 || statusCode > limitStatusCode) {
-						throw new got.HTTPError(statusCode, opts);
+					if (statusCode !== 304 && (statusCode < 200 || statusCode > limitStatusCode)) {
+						throw new got.HTTPError(statusCode, res.statusMessage, res.headers, opts);
 					}
 
 					resolve(res);
@@ -131,22 +347,45 @@ function asPromise(opts) {
 				});
 		});
 
-		ee.on('error', reject);
+		ee.once('error', reject);
+		ee.on('redirect', proxy.emit.bind(proxy, 'redirect'));
+		ee.on('uploadProgress', proxy.emit.bind(proxy, 'uploadProgress'));
+		ee.on('downloadProgress', proxy.emit.bind(proxy, 'downloadProgress'));
 	});
+
+	const promise = timeoutFn(cancelable);
+
+	promise.cancel = cancelable.cancel.bind(cancelable);
+
+	promise.on = (name, fn) => {
+		proxy.on(name, fn);
+		return promise;
+	};
+
+	return promise;
 }
 
 function asStream(opts) {
+	opts.stream = true;
+
 	const input = new PassThrough();
 	const output = new PassThrough();
 	const proxy = duplexer3(input, output);
+	let timeout;
+
+	if (opts.gotTimeout && opts.gotTimeout.request) {
+		timeout = setTimeout(() => {
+			proxy.emit('error', new got.RequestError({message: 'Request timed out', code: 'ETIMEDOUT'}, opts));
+		}, opts.gotTimeout.request);
+	}
 
 	if (opts.json) {
-		throw new Error('got can not be used as stream when options.json is used');
+		throw new Error('Got can not be used as a stream when the `json` option is used');
 	}
 
 	if (opts.body) {
 		proxy.write = () => {
-			throw new Error('got\'s stream is not writable when options.body is used');
+			throw new Error('Got\'s stream is not writable when the `body` option is used');
 		};
 	}
 
@@ -174,52 +413,67 @@ function asStream(opts) {
 	});
 
 	ee.on('response', res => {
+		clearTimeout(timeout);
+
 		const statusCode = res.statusCode;
 
 		res.pipe(output);
 
-		if (statusCode < 200 || statusCode > 299) {
-			proxy.emit('error', new got.HTTPError(statusCode, opts), null, res);
+		if (statusCode !== 304 && (statusCode < 200 || statusCode > 299)) {
+			proxy.emit('error', new got.HTTPError(statusCode, res.statusMessage, res.headers, opts), null, res);
 			return;
 		}
 
 		proxy.emit('response', res);
 	});
 
-	ee.on('redirect', proxy.emit.bind(proxy, 'redirect'));
 	ee.on('error', proxy.emit.bind(proxy, 'error'));
+	ee.on('redirect', proxy.emit.bind(proxy, 'redirect'));
+	ee.on('uploadProgress', proxy.emit.bind(proxy, 'uploadProgress'));
+	ee.on('downloadProgress', proxy.emit.bind(proxy, 'downloadProgress'));
 
 	return proxy;
 }
 
 function normalizeArguments(url, opts) {
 	if (typeof url !== 'string' && typeof url !== 'object') {
-		throw new Error(`Parameter \`url\` must be a string or object, not ${typeof url}`);
-	}
-
-	if (typeof url === 'string') {
+		throw new TypeError(`Parameter \`url\` must be a string or object, not ${typeof url}`);
+	} else if (typeof url === 'string') {
 		url = url.replace(/^unix:/, 'http://$&');
 		url = urlParseLax(url);
-
 		if (url.auth) {
-			throw new Error('Basic authentication must be done with auth option');
+			throw new Error('Basic authentication must be done with the `auth` option');
 		}
+	} else if (isURL.lenient(url)) {
+		url = urlToOptions(url);
 	}
 
 	opts = Object.assign(
 		{
-			protocol: 'http:',
 			path: '',
-			retries: 5
+			retries: 2,
+			cache: false,
+			decompress: true,
+			useElectronNet: false
 		},
 		url,
+		{
+			protocol: url.protocol || 'http:' // Override both null/undefined with default protocol
+		},
 		opts
 	);
 
+	const headers = lowercaseKeys(opts.headers);
+	for (const key of Object.keys(headers)) {
+		if (headers[key] === null || headers[key] === undefined) {
+			delete headers[key];
+		}
+	}
+
 	opts.headers = Object.assign({
 		'user-agent': `${pkg.name}/${pkg.version} (https://github.com/sindresorhus/got)`,
 		'accept-encoding': 'gzip,deflate'
-	}, lowercaseKeys(opts.headers));
+	}, headers);
 
 	const query = opts.query;
 
@@ -236,33 +490,48 @@ function normalizeArguments(url, opts) {
 		opts.headers.accept = 'application/json';
 	}
 
-	let body = opts.body;
-
-	if (body) {
-		if (typeof body !== 'string' && !(body !== null && typeof body === 'object')) {
-			throw new Error('options.body must be a ReadableStream, string, Buffer or plain Object');
+	const body = opts.body;
+	if (body !== null && body !== undefined) {
+		const headers = opts.headers;
+		if (!isStream(body) && typeof body !== 'string' && !Buffer.isBuffer(body) && !(opts.form || opts.json)) {
+			throw new TypeError('The `body` option must be a stream.Readable, string, Buffer or plain Object');
 		}
 
-		opts.method = opts.method || 'POST';
+		const canBodyBeStringified = isPlainObj(body) || Array.isArray(body);
+		if ((opts.form || opts.json) && !canBodyBeStringified) {
+			throw new TypeError('The `body` option must be a plain Object or Array when the `form` or `json` option is used');
+		}
 
-		if (isStream(body) && typeof body.getBoundary === 'function') {
+		if (isFormData(body)) {
 			// Special case for https://github.com/form-data/form-data
-			opts.headers['content-type'] = opts.headers['content-type'] || `multipart/form-data; boundary=${body.getBoundary()}`;
-		} else if (body !== null && typeof body === 'object' && !Buffer.isBuffer(body) && !isStream(body)) {
-			opts.headers['content-type'] = opts.headers['content-type'] || 'application/x-www-form-urlencoded';
-			body = opts.body = querystring.stringify(body);
+			headers['content-type'] = headers['content-type'] || `multipart/form-data; boundary=${body.getBoundary()}`;
+		} else if (opts.form && canBodyBeStringified) {
+			headers['content-type'] = headers['content-type'] || 'application/x-www-form-urlencoded';
+			opts.body = querystring.stringify(body);
+		} else if (opts.json && canBodyBeStringified) {
+			headers['content-type'] = headers['content-type'] || 'application/json';
+			opts.body = JSON.stringify(body);
 		}
 
-		if (opts.headers['content-length'] === undefined && opts.headers['transfer-encoding'] === undefined && !isStream(body)) {
-			const length = typeof body === 'string' ? Buffer.byteLength(body) : body.length;
-			opts.headers['content-length'] = length;
+		if (headers['content-length'] === undefined && headers['transfer-encoding'] === undefined && !isStream(body)) {
+			const length = typeof opts.body === 'string' ? Buffer.byteLength(opts.body) : opts.body.length;
+			headers['content-length'] = length;
+		}
+
+		// Convert buffer to stream to receive upload progress events
+		// see https://github.com/sindresorhus/got/pull/322
+		if (Buffer.isBuffer(body)) {
+			opts.body = intoStream(body);
+			opts.body._buffer = body;
 		}
-	}
 
-	opts.method = (opts.method || 'GET').toUpperCase();
+		opts.method = (opts.method || 'POST').toUpperCase();
+	} else {
+		opts.method = (opts.method || 'GET').toUpperCase();
+	}
 
 	if (opts.hostname === 'unix') {
-		const matches = /(.+):(.+)/.exec(opts.path);
+		const matches = /(.+?):(.+)/.exec(opts.path);
 
 		if (matches) {
 			opts.socketPath = matches[1];
@@ -289,6 +558,15 @@ function normalizeArguments(url, opts) {
 		opts.followRedirect = true;
 	}
 
+	if (opts.timeout) {
+		if (typeof opts.timeout === 'number') {
+			opts.gotTimeout = {request: opts.timeout};
+		} else {
+			opts.gotTimeout = opts.timeout;
+		}
+		delete opts.timeout;
+	}
+
 	return opts;
 }
 
@@ -300,7 +578,9 @@ function got(url, opts) {
 	}
 }
 
-const helpers = [
+got.stream = (url, opts) => asStream(normalizeArguments(url, opts));
+
+const methods = [
 	'get',
 	'post',
 	'put',
@@ -309,51 +589,94 @@ const helpers = [
 	'delete'
 ];
 
-helpers.forEach(el => {
-	got[el] = (url, opts) => got(url, Object.assign({}, opts, {method: el}));
-});
+for (const method of methods) {
+	got[method] = (url, opts) => got(url, Object.assign({}, opts, {method}));
+	got.stream[method] = (url, opts) => got.stream(url, Object.assign({}, opts, {method}));
+}
+
+class StdError extends Error {
+	constructor(message, error, opts) {
+		super(message);
+		Error.captureStackTrace(this, this.constructor);
+		this.name = 'StdError';
 
-got.stream = (url, opts) => asStream(normalizeArguments(url, opts));
+		if (error.code !== undefined) {
+			this.code = error.code;
+		}
 
-for (const el of helpers) {
-	got.stream[el] = (url, opts) => got.stream(url, Object.assign({}, opts, {method: el}));
+		Object.assign(this, {
+			host: opts.host,
+			hostname: opts.hostname,
+			method: opts.method,
+			path: opts.path,
+			protocol: opts.protocol,
+			url: opts.href
+		});
+	}
 }
 
-function stdError(error, opts) {
-	if (error.code !== undefined) {
-		this.code = error.code;
+got.CacheError = class extends StdError {
+	constructor(error, opts) {
+		super(error.message, error, opts);
+		this.name = 'CacheError';
 	}
+};
 
-	Object.assign(this, {
-		message: error.message,
-		host: opts.host,
-		hostname: opts.hostname,
-		method: opts.method,
-		path: opts.path
-	});
-}
+got.RequestError = class extends StdError {
+	constructor(error, opts) {
+		super(error.message, error, opts);
+		this.name = 'RequestError';
+	}
+};
+
+got.ReadError = class extends StdError {
+	constructor(error, opts) {
+		super(error.message, error, opts);
+		this.name = 'ReadError';
+	}
+};
+
+got.ParseError = class extends StdError {
+	constructor(error, statusCode, opts, data) {
+		super(`${error.message} in "${urlLib.format(opts)}": \n${data.slice(0, 77)}...`, error, opts);
+		this.name = 'ParseError';
+		this.statusCode = statusCode;
+		this.statusMessage = http.STATUS_CODES[this.statusCode];
+	}
+};
+
+got.HTTPError = class extends StdError {
+	constructor(statusCode, statusMessage, headers, opts) {
+		if (statusMessage) {
+			statusMessage = statusMessage.replace(/\r?\n/g, ' ').trim();
+		} else {
+			statusMessage = http.STATUS_CODES[statusCode];
+		}
+		super(`Response code ${statusCode} (${statusMessage})`, {}, opts);
+		this.name = 'HTTPError';
+		this.statusCode = statusCode;
+		this.statusMessage = statusMessage;
+		this.headers = headers;
+	}
+};
+
+got.MaxRedirectsError = class extends StdError {
+	constructor(statusCode, redirectUrls, opts) {
+		super('Redirected 10 times. Aborting.', {}, opts);
+		this.name = 'MaxRedirectsError';
+		this.statusCode = statusCode;
+		this.statusMessage = http.STATUS_CODES[this.statusCode];
+		this.redirectUrls = redirectUrls;
+	}
+};
+
+got.UnsupportedProtocolError = class extends StdError {
+	constructor(opts) {
+		super(`Unsupported protocol "${opts.protocol}"`, {}, opts);
+		this.name = 'UnsupportedProtocolError';
+	}
+};
 
-got.RequestError = createErrorClass('RequestError', stdError);
-got.ReadError = createErrorClass('ReadError', stdError);
-got.ParseError = createErrorClass('ParseError', function (e, statusCode, opts, data) {
-	stdError.call(this, e, opts);
-	this.statusCode = statusCode;
-	this.statusMessage = http.STATUS_CODES[this.statusCode];
-	this.message = `${e.message} in "${urlLib.format(opts)}": \n${data.slice(0, 77)}...`;
-});
-
-got.HTTPError = createErrorClass('HTTPError', function (statusCode, opts) {
-	stdError.call(this, {}, opts);
-	this.statusCode = statusCode;
-	this.statusMessage = http.STATUS_CODES[this.statusCode];
-	this.message = `Response code ${this.statusCode} (${this.statusMessage})`;
-});
-
-got.MaxRedirectsError = createErrorClass('MaxRedirectsError', function (statusCode, opts) {
-	stdError.call(this, {}, opts);
-	this.statusCode = statusCode;
-	this.statusMessage = http.STATUS_CODES[this.statusCode];
-	this.message = 'Redirected 10 times. Aborting.';
-});
+got.CancelError = PCancelable.CancelError;
 
 module.exports = got;