@anmiles/downloader 2.0.0

package/.eslintrc.js

@@ -0,0 +1,93 @@
+module.exports = {
+	root    : true,
+	extends : [
+		'eslint:recommended',
+		'plugin:jest/recommended',
+	],
+	parser        : '@typescript-eslint/parser',
+	parserOptions : {
+		ecmaVersion : 2019,
+		sourceType  : 'module',
+	},
+	plugins : [
+		'@typescript-eslint',
+		'align-assignments',
+		'import',
+		'jest',
+	],
+	env : {
+		node : true,
+		jest : true,
+	},
+	ignorePatterns : [
+		'**/node_modules/',
+		'coverage/',
+		'dist/',
+	],
+	rules : {
+		'no-unused-vars'                      : [ 'off' ],
+		'@typescript-eslint/no-redeclare'     : [ 'error' ],
+		'@typescript-eslint/no-unused-vars'   : [ 'error' ],
+		'align-assignments/align-assignments' : [ 'error' ],
+		'array-bracket-spacing'               : [ 'error', 'always' ],
+		'arrow-body-style'                    : [ 'error' ],
+		'arrow-parens'                        : [ 'error' ],
+		'arrow-spacing'                       : [ 'error' ],
+		'block-spacing'                       : [ 'error' ],
+		'brace-style'                         : [ 'error' ],
+		'camelcase'                           : [ 'error' ],
+		'comma-dangle'                        : [ 'error', 'always-multiline' ],
+		'comma-spacing'                       : [ 'error' ],
+		'comma-style'                         : [ 'error' ],
+		'complexity'                          : [ 'error' ],
+		'computed-property-spacing'           : [ 'error', 'never' ],
+		'curly'                               : [ 'error' ],
+		'dot-location'                        : [ 'error', 'property' ],
+		'eol-last'                            : [ 'error' ],
+		'func-call-spacing'                   : [ 'error' ],
+		'func-style'                          : [ 'error', 'declaration', { allowArrowFunctions : true } ],
+		'generator-star-spacing'              : [ 'error', 'neither' ],
+		'import/order'                        : [ 'error', { groups : [ 'builtin', 'external', 'unknown', 'internal', 'parent', 'sibling', 'index' ] } ],
+		'indent'                              : [ 'error', 'tab', { SwitchCase : 1 } ],
+		'jest/no-standalone-expect'           : [ 'error' ],
+		'key-spacing'                         : [ 'error', { beforeColon : true, afterColon : true, align : 'colon' } ],
+		'keyword-spacing'                     : [ 'error' ],
+		'linebreak-style'                     : [ 'error', 'unix' ],
+		'max-params'                          : [ 'error', { max : 5 } ],
+		'new-parens'                          : [ 'error' ],
+		'no-eval'                             : [ 'error' ],
+		'no-extra-bind'                       : [ 'error' ],
+		'no-floating-decimal'                 : [ 'error' ],
+		'no-implied-eval'                     : [ 'error' ],
+		'no-loop-func'                        : [ 'error' ],
+		'no-mixed-spaces-and-tabs'            : [ 'error', 'smart-tabs' ],
+		'no-multiple-empty-lines'             : [ 'error', { max : 1, maxEOF : 1, maxBOF : 0 } ],
+		'no-redeclare'                        : [ 'off' ],
+		'no-return-await'                     : [ 'error' ],
+		'no-trailing-spaces'                  : [ 'error' ],
+		'no-useless-rename'                   : [ 'error' ],
+		'no-var'                              : [ 'error' ],
+		'no-whitespace-before-property'       : [ 'error' ],
+		'object-curly-spacing'                : [ 'error', 'always' ],
+		'object-property-newline'             : [ 'error', { allowMultiplePropertiesPerLine : true } ],
+		'object-shorthand'                    : [ 'error' ],
+		'operator-linebreak'                  : [ 'error', 'before' ],
+		'prefer-const'                        : [ 'error' ],
+		'prefer-numeric-literals'             : [ 'error' ],
+		'prefer-spread'                       : [ 'error' ],
+		'prefer-template'                     : [ 'error' ],
+		'quote-props'                         : [ 'error', 'consistent-as-needed' ],
+		'quotes'                              : [ 'error', 'single', { avoidEscape : true } ],
+		'semi-spacing'                        : [ 'error' ],
+		'semi'                                : [ 'error' ],
+		'space-before-blocks'                 : [ 'error' ],
+		'space-before-function-paren'         : [ 'error', { anonymous : 'never', named : 'never', asyncArrow : 'always' } ],
+		'space-in-parens'                     : [ 'error' ],
+		'space-infix-ops'                     : [ 'error' ],
+		'space-unary-ops'                     : [ 'error' ],
+		'spaced-comment'                      : [ 'error' ],
+		'template-curly-spacing'              : [ 'error' ],
+		'yield-star-spacing'                  : [ 'error' ],
+		'yoda'                                : [ 'error' ],
+	},
+};

package/.gitlab-ci.yml

@@ -0,0 +1,117 @@
+default:
+  image: node:18.14
+
+stages:
+  - setup
+  - build
+  - test
+  - coverage
+  - deploy
+
+install:
+  stage: setup
+  cache:
+    key:
+      files:
+        - package-lock.json
+      prefix: ${CI_COMMIT_REF_SLUG}
+    paths:
+      - node_modules
+    policy: pull-push
+  script:
+    - npm ci
+  only:
+    - main
+    - merge_requests
+
+build:
+  stage: build
+  cache:
+    key:
+      files:
+        - package-lock.json
+      prefix: ${CI_COMMIT_REF_SLUG}
+    paths:
+      - node_modules
+    policy: pull
+  script:
+    - npm run build
+  artifacts:
+    paths:
+      - dist/
+  only:
+    - main
+    - merge_requests
+
+lint:
+  stage: test
+  cache:
+    key:
+      files:
+        - package-lock.json
+      prefix: ${CI_COMMIT_REF_SLUG}
+    paths:
+      - node_modules
+    policy: pull
+  script:
+    - npm run lint
+  only:
+    - main
+    - merge_requests
+
+test:
+  stage: test
+  cache:
+    - key:
+        files:
+          - package-lock.json
+        prefix: ${CI_COMMIT_REF_SLUG}
+      paths:
+        - node_modules
+      policy: pull
+    - key: ${CI_COMMIT_REF_SLUG}_coverage
+      paths:
+        - coverage/**/*
+      policy: push
+  script:
+    - npm run test:ci
+  only:
+    - main
+    - merge_requests
+
+coverage:
+  stage: coverage
+  cache:
+    - key:
+        files:
+          - package-lock.json
+        prefix: ${CI_COMMIT_REF_SLUG}
+      paths:
+        - node_modules
+      policy: pull
+    - key: ${CI_COMMIT_REF_SLUG}_coverage
+      paths:
+        - coverage/**/*
+      policy: pull
+  script:
+    - npm run test:report:coverage
+  only:
+    - main
+    - merge_requests
+
+publish:
+  stage: deploy
+  cache:
+    key:
+      files:
+        - package-lock.json
+      prefix: ${CI_COMMIT_REF_SLUG}
+    paths:
+      - node_modules
+    policy: pull
+  script:
+    - npm publish
+  when: manual
+  only:
+    - main
+    - merge_requests

package/.vscode/settings.json

@@ -0,0 +1,10 @@
+{
+	"cSpell.words": [
+		"anmiles",
+		"cicd",
+		"colorette",
+		"KHTML",
+		"linebreak",
+		"nycrc"
+	]
+}

package/CHANGELOG.md

@@ -0,0 +1,14 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [1.0.0](../../tags/v1.0.0) - 2023-05-03
+### Changed
+- Ready to release
+
+## 0.0.1 - 2023-05-03
+### Added
+- Initial commit

package/LICENSE.md

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Anatoliy Oblaukhov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,46 @@
+# @anmiles/lib/downloader
+
+Wrapper for downloading data as string, buffer or complex types
+
+----
+
+## Installation
+
+1. Install dependencies
+`npm install`
+1. Build
+`npm run build`
+1. Test everything
+`npm test`
+
+## Usage examples
+
+```js
+import { download } from '@anmiles/downloader';
+const buffer = await download('http://url/page');
+```
+
+```js
+import { download } from '@anmiles/downloader';
+await download('http://url/file', '/path/to/file');
+```
+
+```js
+import { downloadString } from '@anmiles/downloader';
+const str = downloadString('http://url/string');
+```
+
+```js
+import { downloadString } from '@anmiles/downloader';
+const str = downloadString('http://url/base64-string', 'base64');
+```
+
+```js
+import { downloadJSON } from '@anmiles/downloader';
+const str = downloadJSON('http://url/json');
+```
+
+```js
+import { downloadJSON } from '@anmiles/downloader';
+const str = downloadJSON('http://url/base64-encoded-json', 'base64');
+```

package/coverage.config.js

@@ -0,0 +1,8 @@
+module.exports = {
+	'check-coverage' : true,
+	'statements'     : 100,
+	'branches'       : 100,
+	'lines'          : 100,
+	'functions'      : 100,
+	'reporter'       : [ 'html', 'text' ],
+};

package/dist/index.d.ts

@@ -0,0 +1 @@
+export { download, downloadString, downloadJSON } from './lib/downloader';

package/dist/index.js

@@ -0,0 +1,8 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.downloadJSON = exports.downloadString = exports.download = void 0;
+var downloader_1 = require("./lib/downloader");
+Object.defineProperty(exports, "download", { enumerable: true, get: function () { return downloader_1.download; } });
+Object.defineProperty(exports, "downloadString", { enumerable: true, get: function () { return downloader_1.downloadString; } });
+Object.defineProperty(exports, "downloadJSON", { enumerable: true, get: function () { return downloader_1.downloadJSON; } });
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,+CAA0E;AAAjE,sGAAA,QAAQ,OAAA;AAAE,4GAAA,cAAc,OAAA;AAAE,0GAAA,YAAY,OAAA"}

package/dist/lib/downloader.d.ts

@@ -0,0 +1,12 @@
+/// <reference types="node" />
+export { download, downloadString, downloadJSON };
+declare const _default: {
+    download: typeof download;
+    downloadString: typeof downloadString;
+    downloadJSON: typeof downloadJSON;
+};
+export default _default;
+declare function download(url: string): Promise<Buffer>;
+declare function download(url: string, file: string): Promise<void>;
+declare function downloadString(url: string, encoding?: Parameters<Buffer['toString']>[0]): Promise<string>;
+declare function downloadJSON(url: string, encoding?: Parameters<Buffer['toString']>[0]): Promise<any>;

package/dist/lib/downloader.js

@@ -0,0 +1,69 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.downloadJSON = exports.downloadString = exports.download = void 0;
+const fs_1 = __importDefault(require("fs"));
+const http_1 = __importDefault(require("http"));
+const https_1 = __importDefault(require("https"));
+const iconv_lite_1 = __importDefault(require("iconv-lite"));
+const downloader_1 = __importDefault(require("./downloader"));
+exports.default = { download, downloadString, downloadJSON };
+function download(url, file) {
+    return new Promise((resolve, reject) => {
+        let protocol;
+        if (url.startsWith('https://')) {
+            protocol = https_1.default;
+        }
+        else if (url.startsWith('http://')) {
+            protocol = http_1.default;
+        }
+        else {
+            throw `Unknown protocol in url ${url}, expected one of "http" or "https"`;
+        }
+        const options = {
+            headers: {
+                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36',
+            },
+        };
+        protocol.get(url, options, function (res) {
+            if (res.statusCode !== 200) {
+                reject(`Request to ${url} returned with status code: ${res.statusCode}`);
+                res.resume();
+            }
+            const chunks = [];
+            if (typeof file === 'undefined') {
+                res.on('data', function (chunk) {
+                    chunks.push(chunk);
+                });
+                res.on('end', function () {
+                    resolve(Buffer.concat(chunks));
+                });
+            }
+            else {
+                res.pipe(fs_1.default.createWriteStream(file));
+                res.on('end', function () {
+                    resolve();
+                });
+            }
+        }).on('error', (e) => {
+            reject(`Request to ${url} failed with error: ${e.message}`);
+        });
+    });
+}
+exports.download = download;
+async function downloadString(url, encoding = 'utf8') {
+    if (!Buffer.isEncoding(encoding)) {
+        throw `Unknown encoding ${encoding}`;
+    }
+    const buffer = await downloader_1.default.download(url);
+    return iconv_lite_1.default.decode(buffer, encoding);
+}
+exports.downloadString = downloadString;
+async function downloadJSON(url, encoding = 'utf8') {
+    const json = await downloader_1.default.downloadString(url, encoding);
+    return JSON.parse(json);
+}
+exports.downloadJSON = downloadJSON;
+//# sourceMappingURL=downloader.js.map

package/dist/lib/downloader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"downloader.js","sourceRoot":"","sources":["../../src/lib/downloader.ts"],"names":[],"mappings":";;;;;;AAAA,4CAAoB;AACpB,gDAAwB;AACxB,kDAA0B;AAC1B,4DAA+B;AAE/B,8DAAsC;AAGtC,kBAAe,EAAE,QAAQ,EAAE,cAAc,EAAE,YAAY,EAAE,CAAC;AAI1D,SAAS,QAAQ,CAAC,GAAW,EAAE,IAAa;IAC3C,OAAO,IAAI,OAAO,CAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrD,IAAI,QAAqC,CAAC;QAE1C,IAAI,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE;YAC/B,QAAQ,GAAG,eAAK,CAAC;SACjB;aAAM,IAAI,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;YACrC,QAAQ,GAAG,cAAI,CAAC;SAChB;aAAM;YACN,MAAM,2BAA2B,GAAG,qCAAqC,CAAC;SAC1E;QAED,MAAM,OAAO,GAAG;YACf,OAAO,EAAG;gBACT,YAAY,EAAG,iHAAiH;aAChI;SACD,CAAC;QAEF,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,EAAE,UAAS,GAAG;YACtC,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG,EAAE;gBAC3B,MAAM,CAAC,cAAc,GAAG,+BAA+B,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;gBACzE,GAAG,CAAC,MAAM,EAAE,CAAC;aACb;YAED,MAAM,MAAM,GAAiB,EAAE,CAAC;YAEhC,IAAI,OAAO,IAAI,KAAK,WAAW,EAAE;gBAChC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,UAAS,KAAK;oBAC5B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACpB,CAAC,CAAC,CAAC;gBAEH,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE;oBACb,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;gBAChC,CAAC,CAAC,CAAC;aACH;iBAAM;gBACN,GAAG,CAAC,IAAI,CAAC,YAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC;gBAErC,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE;oBACb,OAAO,EAAE,CAAC;gBACX,CAAC,CAAC,CAAC;aACH;QACF,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE;YACpB,MAAM,CAAC,cAAc,GAAG,uBAAuB,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7D,CAAC,CAAC,CAAC;IACJ,CAAC,CAAC,CAAC;AACJ,CAAC;AAlDQ,4BAAQ;AAoDjB,KAAK,UAAU,cAAc,CAAC,GAAW,EAAE,WAA8C,MAAM;IAC9F,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE;QACjC,MAAM,oBAAoB,QAAQ,EAAE,CAAC;KACrC;IAED,MAAM,MAAM,GAAG,MAAM,oBAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC9C,OAAO,oBAAK,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;AACvC,CAAC;AA3DkB,wCAAc;AA6DjC,KAAK,UAAU,YAAY,CAAC,GAAW,EAAE,WAA8C,MAAM;IAC5F,MAAM,IAAI,GAAG,MAAM,oBAAU,CAAC,cAAc,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC5D,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AACzB,CAAC;AAhEkC,oCAAY"}

package/jest.config.js

@@ -0,0 +1,22 @@
+module.exports = {
+	preset     : 'ts-jest',
+	clearMocks : true,
+
+	roots     : [ '<rootDir>/src' ],
+	testMatch : [ '<rootDir>/src/**/__tests__/*.test.ts' ],
+	transform : {
+		'^.+\\.ts$' : 'ts-jest',
+	},
+	collectCoverageFrom : [
+		'<rootDir>/src/**/*.ts',
+		'!<rootDir>/src/**/*.d.ts',
+		'!<rootDir>/src/*.ts',
+		'!<rootDir>/src/types/*.ts',
+
+		'!**/node_modules/**',
+		'!**/__tests__/**',
+
+		'!<rootDir>/coverage/**',
+		'!<rootDir>/dist/**',
+	],
+};

package/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "@anmiles/downloader",
+  "version": "2.0.0",
+  "description": "Wrapper for downloading data as string, buffer or complex types",
+  "keywords": [
+    "download",
+    "http",
+    "request"
+  ],
+  "author": "Anatoliy Oblaukhov",
+  "homepage": "https://gitlab.com/anmiles/lib/downloader",
+  "repository": "gitlab:anmiles/lib/downloader",
+  "license": "MIT",
+  "engines": {
+    "node": ">=18.14.2"
+  },
+  "main": "dist/index.js",
+  "scripts": {
+    "build": "rimraf dist && tsc",
+    "lint": "eslint --ext .js,.ts .",
+    "lint:fix": "npm run lint -- --fix",
+    "test": "jest --verbose",
+    "test:coverage": "npm test -- --coverage",
+    "test:ci": "npm test -- --ci --coverage",
+    "test:watch": "npm test -- --watch",
+    "test:watch:coverage": "npm test -- --watch --coverage",
+    "test:report:coverage": "nyc report --nycrc-path ./coverage.config.js -t ./coverage --report-dir ./coverage"
+  },
+  "dependencies": {
+    "colorette": "^2.0.19",
+    "iconv-lite": "^0.6.3"
+  },
+  "devDependencies": {
+    "@types/jest": "^29.5.1",
+    "@typescript-eslint/eslint-plugin": "^5.59.2",
+    "@typescript-eslint/parser": "^5.59.2",
+    "eslint": "^8.39.0",
+    "eslint-plugin-align-assignments": "^1.1.2",
+    "eslint-plugin-import": "^2.27.5",
+    "eslint-plugin-jest": "^27.2.1",
+    "jest": "^29.5.0",
+    "nyc": "^15.1.0",
+    "rimraf": "^5.0.0",
+    "ts-jest": "^29.1.0",
+    "typescript": "^5.0.4"
+  }
+}

package/src/index.ts

@@ -0,0 +1 @@
+export { download, downloadString, downloadJSON } from './lib/downloader';

package/src/lib/__tests__/downloader.test.ts

@@ -0,0 +1,177 @@
+import fs from 'fs';
+import http from 'http';
+import https from 'https';
+import iconv from 'iconv-lite';
+
+import downloader from '../downloader';
+const original = jest.requireActual('../downloader').default as typeof downloader;
+
+jest.mock<Partial<typeof downloader>>('../downloader', () => ({
+	download       : jest.fn().mockImplementation(() => downloaded),
+	downloadString : jest.fn().mockImplementation((...args: Parameters<typeof original.downloadString>) => original.downloadString(...args)),
+}));
+
+const request = {} as http.ClientRequest;
+request.on    = jest.fn().mockImplementation((ev: string, listener: () => void) => {
+	switch (ev) {
+		case 'error':
+			requestError = listener;
+			break;
+	}
+});
+
+const response = {} as http.IncomingMessage;
+
+let data: (data: Uint8Array) => void = () => {};
+let end: () => void                  = () => {};
+let requestError: (e: Error) => void = () => {};
+
+function get(url: string | URL, options: https.RequestOptions, callback?: ((res: http.IncomingMessage) => void) | undefined): http.ClientRequest {
+	if (callback) {
+		callback(response);
+	}
+
+	return request;
+}
+
+beforeEach(() => {
+	response.statusCode = 200;
+	response.on         = jest.fn().mockImplementation((ev: string, listener: () => void) => {
+		switch (ev) {
+			case 'data':
+				data = listener;
+				break;
+			case 'end':
+				end = listener;
+				break;
+		}
+	});
+	response.pipe   = jest.fn();
+	response.resume = jest.fn();
+});
+
+let httpGetSpy: jest.SpyInstance;
+let httpsGetSpy: jest.SpyInstance;
+let downloaded: Buffer;
+
+beforeAll(() => {
+	httpGetSpy  = jest.spyOn(http, 'get');
+	httpsGetSpy = jest.spyOn(https, 'get');
+});
+
+beforeEach(() => {
+	httpGetSpy.mockImplementation(get);
+	httpsGetSpy.mockImplementation(get);
+});
+
+afterAll(() => {
+	httpGetSpy.mockRestore();
+});
+
+describe('src/lib/downloader', () => {
+	describe('download', () => {
+		it('should throw if url protocol is not supported', async () => {
+			await expect(() => original.download('ftp://url')).rejects.toEqual('Unknown protocol in url ftp://url, expected one of "http" or "https"');
+		});
+
+		it('should call http.get if url protocol is http', async () => {
+			const promise = original.download('http://url');
+			end();
+			await promise;
+			expect(httpGetSpy.mock.calls[0][0]).toEqual('http://url');
+		});
+
+		it('should call https.get if url protocol is https', async () => {
+			const promise = original.download('https://url');
+			end();
+			await promise;
+			expect(httpsGetSpy.mock.calls[0][0]).toEqual('https://url');
+		});
+
+		it('should pass user-agent in options', async () => {
+			const promise = original.download('http://url');
+			end();
+			await promise;
+			expect(httpGetSpy.mock.calls[0][1]).toEqual(expect.objectContaining({
+				headers : {
+					'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36',
+				},
+			}));
+		});
+
+		it('should reject and resume response if status code is not 200', async () => {
+			response.statusCode = 404;
+			await expect(() => original.download('http://url')).rejects.toEqual('Request to http://url returned with status code: 404');
+			expect(response.resume).toHaveBeenCalled();
+		});
+
+		it('should reject if response errored', async () => {
+			const promise = original.download('http://url');
+			requestError(new Error('request error'));
+			await expect(() => promise).rejects.toEqual('Request to http://url failed with error: request error');
+		});
+
+		it('should concat and resolve received data if no file specified', async () => {
+			const promise = original.download('http://url');
+			data(new Uint8Array([ 10, 11, 12 ]));
+			data(new Uint8Array([ 20, 21, 22 ]));
+			data(new Uint8Array([ 30, 31, 32 ]));
+			end();
+			const result = await promise;
+			expect(result).toEqual(Buffer.from([ 10, 11, 12, 20, 21, 22, 30, 31, 32 ]));
+		});
+
+		it('should pipe response stream to file if specified', async () => {
+			const stream               = {} as fs.WriteStream;
+			const createWriteStreamSpy = jest.spyOn(fs, 'createWriteStream').mockReturnValue(stream);
+			const promise              = original.download('http://url', 'file');
+			data(new Uint8Array([ 10, 11, 12 ]));
+			data(new Uint8Array([ 20, 21, 22 ]));
+			data(new Uint8Array([ 30, 31, 32 ]));
+			end();
+			const result = await promise;
+			expect(createWriteStreamSpy).toHaveBeenCalledWith('file');
+			expect(response.pipe).toHaveBeenCalledWith(stream);
+			expect(result).toEqual(undefined);
+			createWriteStreamSpy.mockRestore();
+		});
+	});
+
+	describe('downloadString', () => {
+		it('should throw if unknown encoding specified', async () => {
+			await expect(() => original.downloadString('http://url', 'wrong_encoding' as any)).rejects.toEqual('Unknown encoding wrong_encoding');
+		});
+
+		it('should return string decoded with utf8', async () => {
+			downloaded   = iconv.encode('test', 'utf8');
+			const result = await original.downloadString('http://url');
+			expect(downloader.download).toHaveBeenCalledWith('http://url');
+			expect(result).toEqual('test');
+		});
+
+		it('should return string decoded with specified encoding', async () => {
+			downloaded   = iconv.encode('test', 'base64');
+			const result = await original.downloadString('http://url', 'base64');
+			expect(downloader.download).toHaveBeenCalledWith('http://url');
+			expect(result).toEqual('test');
+		});
+	});
+
+	describe('downloadJSON', () => {
+		it('should JSON.parse downloaded string decoded with utf8', async () => {
+			downloaded   = iconv.encode('{"key1": "value", "key2": 5}', 'utf8');
+			const result = await original.downloadJSON('http://url');
+			expect(downloader.download).toHaveBeenCalledWith('http://url');
+			expect(downloader.downloadString).toHaveBeenCalledWith('http://url', 'utf8');
+			expect(result).toEqual({ key1 : 'value', key2 : 5 });
+		});
+
+		it('should JSON.parse downloaded string decoded with specified encoding', async () => {
+			downloaded   = iconv.encode('{"key1": "value", "key2": 5}', 'ucs2');
+			const result = await original.downloadJSON('http://url', 'ucs2');
+			expect(downloader.download).toHaveBeenCalledWith('http://url');
+			expect(downloader.downloadString).toHaveBeenCalledWith('http://url', 'ucs2');
+			expect(result).toEqual({ key1 : 'value', key2 : 5 });
+		});
+	});
+});

package/src/lib/downloader.ts

@@ -0,0 +1,72 @@
+import fs from 'fs';
+import http from 'http';
+import https from 'https';
+import iconv from 'iconv-lite';
+
+import downloader from './downloader';
+
+export { download, downloadString, downloadJSON };
+export default { download, downloadString, downloadJSON };
+
+function download(url: string): Promise<Buffer>;
+function download(url: string, file: string): Promise<void>;
+function download(url: string, file?: string): Promise<Buffer | void> {
+	return new Promise<Buffer | void>((resolve, reject) => {
+		let protocol : typeof https | typeof http;
+
+		if (url.startsWith('https://')) {
+			protocol = https;
+		} else if (url.startsWith('http://')) {
+			protocol = http;
+		} else {
+			throw `Unknown protocol in url ${url}, expected one of "http" or "https"`;
+		}
+
+		const options = {
+			headers : {
+				'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36',
+			},
+		};
+
+		protocol.get(url, options, function(res) {
+			if (res.statusCode !== 200) {
+				reject(`Request to ${url} returned with status code: ${res.statusCode}`);
+				res.resume();
+			}
+
+			const chunks: Uint8Array[] = [];
+
+			if (typeof file === 'undefined') {
+				res.on('data', function(chunk) {
+					chunks.push(chunk);
+				});
+
+				res.on('end', function() {
+					resolve(Buffer.concat(chunks));
+				});
+			} else {
+				res.pipe(fs.createWriteStream(file));
+
+				res.on('end', function() {
+					resolve();
+				});
+			}
+		}).on('error', (e) => {
+			reject(`Request to ${url} failed with error: ${e.message}`);
+		});
+	});
+}
+
+async function downloadString(url: string, encoding: Parameters<Buffer['toString']>[0] = 'utf8') {
+	if (!Buffer.isEncoding(encoding)) {
+		throw `Unknown encoding ${encoding}`;
+	}
+
+	const buffer = await downloader.download(url);
+	return iconv.decode(buffer, encoding);
+}
+
+async function downloadJSON(url: string, encoding: Parameters<Buffer['toString']>[0] = 'utf8') {
+	const json = await downloader.downloadString(url, encoding);
+	return JSON.parse(json);
+}

package/tsconfig.json

@@ -0,0 +1,26 @@
+{
+	"compilerOptions": {
+		"module": "CommonJS",
+		"target": "ES2019",
+
+		"moduleResolution": "node",
+		"resolveJsonModule": true,
+
+		"rootDir": "./src",
+		"outDir": "./dist",
+
+		"allowJs": false,
+		"sourceMap": true,
+		"declaration": true,
+		"noImplicitAny": true,
+		"esModuleInterop": true,
+		"strictNullChecks": true,
+		"allowSyntheticDefaultImports": true,
+	},
+	"exclude": [
+		"**/node_modules/",
+		"**/__tests__/",
+		"coverage/",
+		"dist/",
+	],
+}