@k03mad/request 2.1.0 → 3.0.0

package/app/lib/curl.js

@@ -4,7 +4,6 @@ import prettyBytes from 'pretty-bytes';
 const {bgWhite, black, blue, dim, green, magenta, red, white, yellow} = chalk;
 
 /**
- * Сформировать curl-строку из опций got
  * @param {string} url
  * @param {object} [opts]
  * @param {string} [opts.method]
@@ -25,12 +24,10 @@ export default (url, {
 }, res) => {
     const msg = [];
 
-    // если третий параметр — объект ошибки, то все необходимое на уровень ниже, в ключе response
     if (res?.response) {
         res = res.response;
     }
 
-    // 200 [800 ms] [3.15 kB]
     if (res?.statusCode) {
         msg.push(bgWhite(black(res.statusCode)));
     }

package/app/lib/queue.js

@@ -3,27 +3,24 @@ import PQueue from 'p-queue';
 
 const debug = _debug('mad:queue');
 
-// const rps = num => ({intervalCap: num, interval: 1000});
-const concurrency = num => ({concurrency: num});
+const intervals = {
+    concurrency: num => ({concurrency: num}),
+    rpm: num => ({intervalCap: num, interval: 60_000}),
+    rps: num => ({intervalCap: num, interval: 1000}),
+};
 
-// первый уровень — хост (* — любой, кроме уже перечисленных)
-// второй уровень — метод (* — любой, кроме уже перечисленных)
-// третий уровень — настройки очереди из p-queue
+// host: p-queue options
 const requestQueue = {
-    '*': {
-        '*': concurrency(3),
-    },
+    '*': intervals.concurrency(3),
 };
 
 /**
- * Поставить логирование и вернуть очередь
  * @param {string} host
- * @param {string} method
  * @param {object} opts
  * @returns {object}
  */
-const getLoggedQueue = (host, method, opts) => {
-    const queue = requestQueue[host][method];
+const getLoggedQueue = (host, opts) => {
+    const queue = requestQueue[host];
 
     queue.on('active', () => {
         const {pending, size} = queue;
@@ -33,9 +30,7 @@ const getLoggedQueue = (host, method, opts) => {
             ? `${concurrent} concurrent`
             : `${intervalCap} rp ${interval} ms`;
 
-        const logMessage = `[${
-            method === '*' ? '' : `${method}: `
-        }${host}] ${parallel} | queue: ${size} | running: ${pending}`;
+        const logMessage = `[${host}] ${parallel} | queue: ${size} | running: ${pending}`;
 
         debug(logMessage);
     });
@@ -44,43 +39,34 @@ const getLoggedQueue = (host, method, opts) => {
 };
 
 /**
- * Получить очередь по хосту и методу
  * @param {string} host
- * @param {string} method
+ * @param {object} params
+ * @param {number} params.concurrency
+ * @param {number} params.rpm
+ * @param {number} params.rps
  * @returns {object}
  */
-export default (host, method = 'GET') => {
-    // проверка на предустановленные настройки очереди для хоста и метода
-    for (const elem of [method, '*']) {
-        // очередь уже проинициализирована
-        if (requestQueue[host]?.[elem]?._events) {
-            return requestQueue[host][elem];
-        }
+export default (host, params) => {
+    if (requestQueue[host]?._events) {
+        return requestQueue[host];
+    }
 
-        // очередь нужно проинициализировать
-        if (requestQueue[host]?.[elem]) {
-            const opts = requestQueue[host][elem];
-            requestQueue[host][elem] = new PQueue(opts);
-            return getLoggedQueue(host, elem, opts);
+    if (Object.keys(params).length > 0) {
+        for (const [key, value] of Object.entries(params)) {
+            if (intervals[key]) {
+                requestQueue[host] = intervals[key](value);
+                break;
+            }
         }
     }
 
-    // инициализация очереди для хоста без текущего метода в предустановках
     if (requestQueue[host]) {
-        const opts = requestQueue['*']['*'];
-        requestQueue[host]['*'] = new PQueue(opts);
-        return getLoggedQueue(host, '*', opts);
-    }
-
-    // инициализация очереди для хоста с методом из предустановок для всех очередей
-    if (requestQueue['*'][method]) {
-        const opts = requestQueue['*'][method];
-        requestQueue[host] = {[method]: new PQueue(opts)};
-        return getLoggedQueue(host, method, opts);
+        const opts = requestQueue[host];
+        requestQueue[host] = new PQueue(opts);
+        return getLoggedQueue(host, opts);
     }
 
-    // нет ни хоста не метода в предустановках
-    const opts = requestQueue['*']['*'];
-    requestQueue[host] = {'*': new PQueue(opts)};
-    return getLoggedQueue(host, '*', opts);
+    const opts = requestQueue['*'];
+    requestQueue[host] = new PQueue(opts);
+    return getLoggedQueue(host, opts);
 };

package/app/lib/request.js

@@ -8,21 +8,37 @@ import getQueue from './queue.js';
 const {blue, cyan, dim, green, red, yellow} = chalk;
 const debug = _debug('mad:request');
 
-const gotDefault = got.extend({
+const gotCache = new Map();
+
+const cacheGotResponseKeys = [
+    'body',
+    'headers',
+    'method',
+    'statusCode',
+    'statusMessage',
+    'timings',
+];
+
+const gotDefaultOpts = got.extend({
     dnsCache: true,
     timeout: {request: 15_000},
-    headers: {'user-agent': 'curl/8.1.2'},
+    headers: {'user-agent': 'curl/7.81.0'},
 });
 
+const cacheDebug = msgArr => {
+    if (process.env.DEBUG) {
+        debug(msgArr.join(' :: '));
+    }
+};
+
 /**
- * Отправить запрос
  * @param {string} url
  * @param {object} opts
  * @returns {object}
  */
 const sendRequest = async (url, opts) => {
     try {
-        const response = await gotDefault(url, opts);
+        const response = await gotDefaultOpts(url, opts);
 
         if (!opts.responseType) {
             try {
@@ -65,22 +81,17 @@ const sendRequest = async (url, opts) => {
     }
 };
 
-export const cache = new Map();
-
 /**
- * Отправить запрос c выбором использования очереди
  * @param {string} url
  * @param {object} [opts]
  * @param {object} [params]
- * @param {boolean} [params.skipQueue]
+ * @param {number} [params.concurrency]
+ * @param {number} [params.rpm]
+ * @param {number} [params.rps]
  * @returns {Promise<object>}
  */
-export const request = (url, opts = {}, {skipQueue} = {}) => {
-    if (skipQueue) {
-        return sendRequest(url, opts);
-    }
-
-    const queue = getQueue(new URL(url).host, opts.method);
+export const request = (url, opts = {}, params = {}) => {
+    const queue = getQueue(new URL(url).host, params);
     return queue.add(() => sendRequest(url, opts));
 };
 
@@ -90,45 +101,40 @@ export const request = (url, opts = {}, {skipQueue} = {}) => {
  * @param {object} [params]
  * @param {number} [params.expire] seconds
  * @param {object} [params.cacheBy]
+ * @param {number} [params.concurrency]
+ * @param {number} [params.rpm]
+ * @param {number} [params.rps]
  * @returns {Promise<object>}
  */
-export const requestCache = (url, opts = {}, {cacheBy, expire = 43_200} = {}) => {
-    const queue = getQueue(new URL(url).host, opts.method);
+export const requestCache = (url, opts = {}, {cacheBy, expire = 43_200, ...params} = {}) => {
+    const queue = getQueue(new URL(url).host, params);
 
     return queue.add(async () => {
-        const cacheGotResponseKeys = [
-            'body',
-            'headers',
-            'method',
-            'statusCode',
-            'statusMessage',
-            'timings',
-        ];
-
         const cacheKey = `${url}::${JSON.stringify(cacheBy || opts)}`;
-        const log = `${blue(url)}\n${dim(cacheKey)}`;
+        const urlLog = `${blue(url)}\n${dim(cacheKey)}`;
 
         try {
-            if (cache.has(cacheKey)) {
-                const {cachedResponse, date} = cache.get(cacheKey);
+            if (gotCache.has(cacheKey)) {
+                const {cachedResponse, date} = gotCache.get(cacheKey);
 
                 const measurement = 'seconds';
                 const currentDiff = Math.round((Date.now() - date) / 1000);
+                const diffLog = `${currentDiff}/${expire} ${measurement} left`;
 
                 if (currentDiff < expire) {
-                    debug(`${green('FROM CACHE')} :: ${currentDiff}/${expire} ${measurement} left :: ${log}`);
+                    cacheDebug([green('FROM CACHE'), diffLog, urlLog]);
                     return {cacheKey, ...cachedResponse};
                 }
 
-                debug(`${red('CACHE EXPIRED')} :: ${currentDiff}/${expire} ${measurement} left :: ${log}`);
+                cacheDebug([yellow('CACHE EXPIRED'), diffLog, urlLog]);
             } else {
-                debug(`${yellow('CACHE NOT FOUND')} :: ${log}`);
+                cacheDebug([blue('CACHE NOT FOUND'), urlLog]);
             }
         } catch (err) {
-            debug(`${red('CACHE ERROR')} :: ${dim(err)} :: ${log}`);
+            cacheDebug([red('CACHE ERROR'), dim(err), urlLog]);
         }
 
-        const res = await request(url, opts, {skipQueue: true});
+        const res = await sendRequest(url, opts);
 
         const cachedResponse = {};
 
@@ -136,8 +142,8 @@ export const requestCache = (url, opts = {}, {cacheBy, expire = 43_200} = {}) =>
             cachedResponse[key] = res[key];
         });
 
-        cache.set(cacheKey, {date: Date.now(), cachedResponse});
-        debug(`${cyan('CACHE SAVED')} :: ${log}`);
+        gotCache.set(cacheKey, {date: Date.now(), cachedResponse});
+        cacheDebug([cyan('CACHE SAVED'), urlLog]);
 
         return res;
     });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@k03mad/request",
-  "version": "2.1.0",
+  "version": "3.0.0",
   "description": "Request library",
   "maintainers": [
     "Kirill Molchanov <k03.mad@gmail.com"
@@ -20,7 +20,7 @@
     "pretty-bytes": "6.1.1"
   },
   "devDependencies": {
-    "@k03mad/eslint-config": "12.0.5",
+    "@k03mad/eslint-config": "12.0.6",
     "@microsoft/eslint-formatter-sarif": "3.0.0",
     "eslint": "8.46.0",
     "eslint-plugin-import": "2.28.0",