@k03mad/request 1.0.0

package/.editorconfig

@@ -0,0 +1,12 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+indent_size = 4
+indent_style = space
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.json]
+indent_size = 2

package/.eslintrc.json

@@ -0,0 +1,3 @@
+{
+  "extends": "@k03mad"
+}

package/.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "npm"
+    directory: "/"
+    schedule:
+      interval: "monthly"

package/.husky/pre-commit

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+. "$(dirname -- "$0")/_/husky.sh"
+
+npm run lint

package/.nvmrc

@@ -0,0 +1 @@
+20

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Kirill Molchanov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1 @@
+Adds queues and logging to the `Got` library

package/app/index.js

@@ -0,0 +1 @@
+export * from './lib/request.js';

package/app/lib/curl.js

@@ -0,0 +1,89 @@
+import chalk from 'chalk';
+import prettyBytes from 'pretty-bytes';
+
+const {bgWhite, black, blue, dim, green, magenta, red, white, yellow} = chalk;
+
+/**
+ * Сформировать curl-строку из опций got
+ * @param {string} url
+ * @param {object} [opts]
+ * @param {string} [opts.method]
+ * @param {object} [opts.headers]
+ * @param {string} [opts.body]
+ * @param {object} [opts.json]
+ * @param {object} [opts.form]
+ * @param {object} [opts.searchParams]
+ * @param {string} [opts.username]
+ * @param {string} [opts.password]
+ * @param {object} res
+ * @returns {string}
+ */
+export default (url, {
+    body, form,
+    headers = {}, json, method = 'GET', password,
+    searchParams, username,
+}, res) => {
+    const msg = [];
+
+    // если третий параметр — объект ошибки, то все необходимое на уровень ниже, в ключе response
+    if (res?.response) {
+        res = res.response;
+    }
+
+    // 200 [800 ms] [3.15 kB]
+    if (res?.statusCode) {
+        msg.push(bgWhite(black(res.statusCode)));
+    }
+
+    if (res?.timings) {
+        msg.push(`[${res.timings.phases.total} ms]`);
+    }
+
+    if (res?.headers?.['content-length']) {
+        msg.push(`[${prettyBytes(Number(res.headers['content-length']))}]`);
+    }
+
+    if (searchParams) {
+        url += `?${new URLSearchParams(searchParams).toString()}`;
+    }
+
+    msg.push(white('curl -v -X'), green(method));
+
+    if (username && password) {
+        msg.push(dim(red(`-u ${username}:${password}`)));
+    }
+
+    msg.push(blue(url));
+
+    let bodyParams;
+
+    if (json) {
+        headers['content-type'] = 'application/json';
+        bodyParams = `-d '${JSON.stringify(json)}'`;
+    } else if (form) {
+        headers['content-type'] = 'application/x-www-form-urlencoded';
+        bodyParams = `-d '${new URLSearchParams(form).toString()}'`;
+    } else if (body) {
+        bodyParams = `-d '${body}'`;
+    }
+
+    const headersParams = Object.entries(headers);
+
+    if (headersParams.length > 0) {
+        msg.push(dim(magenta(headersParams.map(([key, value]) => `-H "${key}: ${value}"`).join(' '))));
+    }
+
+    if (bodyParams) {
+        msg.push(dim(yellow(bodyParams)));
+    }
+
+    if (res) {
+        const message = JSON.stringify(res.body || res.message);
+
+        if (message?.length < 1000) {
+            msg.push(`\n${dim(green(message))}`);
+        }
+    }
+
+    return msg.join(' ');
+};

package/app/lib/queue.js

@@ -0,0 +1,86 @@
+import _debug from 'debug';
+import PQueue from 'p-queue';
+
+const debug = _debug('mad:queue');
+
+// const rps = num => ({intervalCap: num, interval: 1000});
+const concurrency = num => ({concurrency: num});
+
+// первый уровень — хост (* — любой, кроме уже перечисленных)
+// второй уровень — метод (* — любой, кроме уже перечисленных)
+// третий уровень — настройки очереди из p-queue
+const requestQueue = {
+    '*': {
+        '*': concurrency(3),
+    },
+};
+
+/**
+ * Поставить логирование и вернуть очередь
+ * @param {string} host
+ * @param {string} method
+ * @param {object} opts
+ * @returns {object}
+ */
+const getLoggedQueue = (host, method, opts) => {
+    const queue = requestQueue[host][method];
+
+    queue.on('active', () => {
+        const {pending, size} = queue;
+        const {concurrency: concurrent, interval, intervalCap} = opts;
+
+        const parallel = concurrent
+            ? `${concurrent} concurrent`
+            : `${intervalCap} rp ${interval} ms`;
+
+        const logMessage = `[${
+            method === '*' ? '' : `${method}: `
+        }${host}] ${parallel} | queue: ${size} | running: ${pending}`;
+
+        debug(logMessage);
+    });
+
+    return queue;
+};
+
+/**
+ * Получить очередь по хосту и методу
+ * @param {string} host
+ * @param {string} method
+ * @returns {object}
+ */
+export default (host, method = 'GET') => {
+    // проверка на предустановленные настройки очереди для хоста и метода
+    for (const elem of [method, '*']) {
+        // очередь уже проинициализирована
+        if (requestQueue[host]?.[elem]?._events) {
+            return requestQueue[host][elem];
+        }
+
+        // очередь нужно проинициализировать
+        if (requestQueue[host]?.[elem]) {
+            const opts = requestQueue[host][elem];
+            requestQueue[host][elem] = new PQueue(opts);
+            return getLoggedQueue(host, elem, opts);
+        }
+    }
+
+    // инициализация очереди для хоста без текущего метода в предустановках
+    if (requestQueue[host]) {
+        const opts = requestQueue['*']['*'];
+        requestQueue[host]['*'] = new PQueue(opts);
+        return getLoggedQueue(host, '*', opts);
+    }
+
+    // инициализация очереди для хоста с методом из предустановок для всех очередей
+    if (requestQueue['*'][method]) {
+        const opts = requestQueue['*'][method];
+        requestQueue[host] = {[method]: new PQueue(opts)};
+        return getLoggedQueue(host, method, opts);
+    }
+
+    // нет ни хоста не метода в предустановках
+    const opts = requestQueue['*']['*'];
+    requestQueue[host] = {'*': new PQueue(opts)};
+    return getLoggedQueue(host, '*', opts);
+};

package/app/lib/request.js

@@ -0,0 +1,136 @@
+import chalk from 'chalk';
+import _debug from 'debug';
+import got from 'got';
+
+import getCurl from './curl.js';
+import getQueue from './queue.js';
+
+const {blue, cyan, dim, green, red, yellow} = chalk;
+const debug = _debug('mad:request');
+
+/**
+ * @param {object} opts
+ * @returns {object}
+ */
+const prepareRequestOpts = (opts = {}) => {
+    const UA = 'curl/8.1.2';
+
+    const preparedOpts = {
+        timeout: {request: 15_000},
+        ...opts,
+    };
+
+    if (!preparedOpts.headers) {
+        preparedOpts.headers = {'user-agent': UA};
+    } else if (!preparedOpts.headers['user-agent']) {
+        preparedOpts.headers['user-agent'] = UA;
+    }
+
+    return preparedOpts;
+};
+
+/**
+ * Отправить запрос
+ * @param {string} url
+ * @param {object} opts
+ * @returns {object}
+ */
+const sendRequest = async (url, opts) => {
+    const preparedOpts = prepareRequestOpts(opts);
+
+    try {
+        const response = await got(url, preparedOpts);
+
+        if (!preparedOpts.responseType) {
+            try {
+                response.body = JSON.parse(response.body);
+            } catch {}
+        }
+
+        debug(getCurl(url, preparedOpts, response));
+        return response;
+    } catch (err) {
+        debug(getCurl(url, preparedOpts, err));
+        throw err;
+    }
+};
+
+export const cache = new Map();
+
+/**
+ * Отправить запрос c выбором использования очереди
+ * @param {string} url
+ * @param {object} [opts]
+ * @param {object} [params]
+ * @param {boolean} [params.skipQueue]
+ * @returns {Promise<object>}
+ */
+export const request = (url, opts = {}, {skipQueue} = {}) => {
+    if (skipQueue) {
+        return sendRequest(url, opts);
+    }
+
+    const queue = getQueue(new URL(url).host, opts.method);
+    return queue.add(() => sendRequest(url, opts));
+};
+
+/**
+ * @param {string} url
+ * @param {object} [opts]
+ * @param {object} [params]
+ * @param {number} [params.expire] seconds
+ * @param {object} [params.cacheBy]
+ * @returns {Promise<object>}
+ */
+export const requestCache = (url, opts = {}, {cacheBy, expire = 43_200} = {}) => {
+    const queue = getQueue(new URL(url).host, opts.method);
+
+    return queue.add(async () => {
+        const preparedOpts = prepareRequestOpts(opts);
+
+        const cacheGotResponseKeys = [
+            'body',
+            'headers',
+            'method',
+            'statusCode',
+            'statusMessage',
+            'timings',
+        ];
+
+        const cacheKey = `${url}::${JSON.stringify(cacheBy || preparedOpts)}`;
+        const log = `${blue(url)}\n${dim(cacheKey)}`;
+
+        try {
+            if (cache.has(cacheKey)) {
+                const {cachedResponse, date} = cache.get(cacheKey);
+
+                const measurement = 'seconds';
+                const currentDiff = (Date.now() - date) * 1000;
+
+                if (currentDiff < expire) {
+                    debug(`${green('FROM CACHE')} :: ${currentDiff}/${expire} ${measurement} left :: ${log}`);
+                    return {cacheKey, ...cachedResponse};
+                }
+
+                debug(`${red('CACHE EXPIRED')} :: ${currentDiff}/${expire} ${measurement} left :: ${log}`);
+            } else {
+                debug(`${yellow('CACHE NOT FOUND')} :: ${log}`);
+            }
+        } catch (err) {
+            debug(`${red('CACHE ERROR')} :: ${dim(err)} :: ${log}`);
+        }
+
+        const res = await request(url, preparedOpts, {skipQueue: true});
+
+        const cachedResponse = {};
+
+        cacheGotResponseKeys.forEach(key => {
+            cachedResponse[key] = res[key];
+        });
+
+        cache.set(cacheKey, {date: Date.now(), cachedResponse});
+        debug(`${cyan('CACHE SAVED')} :: ${log}`);
+
+        return res;
+    });
+};

package/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@k03mad/request",
+  "version": "1.0.0",
+  "description": "Request library",
+  "maintainers": [
+    "Kirill Molchanov <k03.mad@gmail.com"
+  ],
+  "main": "app/index.js",
+  "repository": "k03mad/request",
+  "license": "MIT",
+  "type": "module",
+  "engines": {
+    "node": ">=20"
+  },
+  "dependencies": {
+    "chalk": "5.3.0",
+    "debug": "4.3.4",
+    "got": "13.0.0",
+    "p-queue": "7.3.4",
+    "pretty-bytes": "6.1.1"
+  },
+  "devDependencies": {
+    "@k03mad/eslint-config": "12.0.4",
+    "eslint": "8.45.0",
+    "eslint-plugin-import": "2.27.5",
+    "eslint-plugin-jsdoc": "46.4.4",
+    "eslint-plugin-n": "16.0.1",
+    "eslint-plugin-simple-import-sort": "10.0.0",
+    "eslint-plugin-sort-destructure-keys": "1.5.0",
+    "eslint-plugin-unicorn": "48.0.0",
+    "husky": "8.0.3"
+  },
+  "scripts": {
+    "lint": "eslint ./ --report-unused-disable-directives",
+    "clean": "rm -rfv ./node_modules || true",
+    "setup": "npm run clean && npm i",
+    "prepare": "husky install"
+  }
+}