@naturalcycles/nodejs-lib 13.28.1 → 13.29.0

package/dist/index.d.ts

@@ -42,6 +42,7 @@ export * from './stream/transform/transformSplit';
 export * from './stream/transform/transformTap';
 export * from './stream/transform/transformToArray';
 export * from './stream/transform/transformTee';
+export * from './stream/transform/transformThrottle';
 export * from './stream/transform/worker/baseWorkerClass';
 export * from './stream/transform/worker/transformMultiThreaded';
 export * from './stream/transform/worker/transformMultiThreaded.model';

package/dist/index.js

@@ -46,6 +46,7 @@ tslib_1.__exportStar(require("./stream/transform/transformSplit"), exports);
 tslib_1.__exportStar(require("./stream/transform/transformTap"), exports);
 tslib_1.__exportStar(require("./stream/transform/transformToArray"), exports);
 tslib_1.__exportStar(require("./stream/transform/transformTee"), exports);
+tslib_1.__exportStar(require("./stream/transform/transformThrottle"), exports);
 tslib_1.__exportStar(require("./stream/transform/worker/baseWorkerClass"), exports);
 tslib_1.__exportStar(require("./stream/transform/worker/transformMultiThreaded"), exports);
 tslib_1.__exportStar(require("./stream/transform/worker/transformMultiThreaded.model"), exports);

package/dist/stream/transform/transformThrottle.d.ts

@@ -0,0 +1,31 @@
+import { NumberOfSeconds, PositiveInteger } from '@naturalcycles/js-lib';
+import { TransformTyped } from '../stream.model';
+export interface TransformThrottleOptions {
+    /**
+     * How many items to allow per `interval` of seconds.
+     */
+    throughput: PositiveInteger;
+    /**
+     * How long is the interval (in seconds) where number of items should not exceed `throughput`.
+     */
+    interval: NumberOfSeconds;
+    debug?: boolean;
+}
+/**
+ * Allows to throttle the throughput of the stream.
+ * For example, when you have an API with rate limit of 5000 requests per minute,
+ * `transformThrottle` can help you utilize it most efficiently.
+ * You can define it as:
+ *
+ * _pipeline([
+ *   // ...
+ *   transformThrottle({
+ *     throughput: 5000,
+ *     interval: 60,
+ *   }),
+ *   // ...
+ * ])
+ *
+ * @experimental
+ */
+export declare function transformThrottle<T>(opt: TransformThrottleOptions): TransformTyped<T, T>;

package/dist/stream/transform/transformThrottle.js

@@ -0,0 +1,74 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.transformThrottle = transformThrottle;
+const node_stream_1 = require("node:stream");
+const js_lib_1 = require("@naturalcycles/js-lib");
+/**
+ * Allows to throttle the throughput of the stream.
+ * For example, when you have an API with rate limit of 5000 requests per minute,
+ * `transformThrottle` can help you utilize it most efficiently.
+ * You can define it as:
+ *
+ * _pipeline([
+ *   // ...
+ *   transformThrottle({
+ *     throughput: 5000,
+ *     interval: 60,
+ *   }),
+ *   // ...
+ * ])
+ *
+ * @experimental
+ */
+function transformThrottle(opt) {
+    const { throughput, interval, debug } = opt;
+    let count = 0;
+    let start;
+    let paused;
+    let timeout;
+    return new node_stream_1.Transform({
+        objectMode: true,
+        async transform(item, _, cb) {
+            // console.log('incoming', item, { paused: !!paused, count })
+            if (!start) {
+                start = Date.now();
+                timeout = setTimeout(() => onInterval(this), interval * 1000);
+                if (debug) {
+                    console.log(`${js_lib_1.localTime.now().toPretty()} transformThrottle started with`, {
+                        throughput,
+                        interval,
+                        rps: Math.round(throughput / interval),
+                    });
+                }
+            }
+            if (paused) {
+                // console.log('awaiting pause', {item, count})
+                await paused;
+            }
+            if (++count >= throughput) {
+                // console.log('pausing now after', {item, count})
+                paused = (0, js_lib_1.pDefer)();
+                if (debug) {
+                    console.log(`${js_lib_1.localTime.now().toPretty()} transformThrottle activated: ${count} items passed in ${(0, js_lib_1._ms)(interval * 1000)}, will pause for ${(0, js_lib_1._ms)(interval * 1000 - (Date.now() - start))}`);
+                }
+            }
+            cb(null, item); // pass the item through
+        },
+        final(cb) {
+            clearTimeout(timeout);
+            cb();
+        },
+    });
+    function onInterval(transform) {
+        if (!paused)
+            return;
+        if (debug) {
+            console.log(`${js_lib_1.localTime.now().toPretty()} transformThrottle resumed`);
+        }
+        count = 0;
+        start = Date.now();
+        timeout = setTimeout(() => onInterval(transform), interval * 1000);
+        paused.resolve();
+        paused = undefined;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/nodejs-lib",
-  "version": "13.28.1",
+  "version": "13.29.0",
   "scripts": {
     "prepare": "husky",
     "docs-serve": "vuepress dev docs",

package/src/index.ts

@@ -52,6 +52,7 @@ export * from './stream/transform/transformSplit'
 export * from './stream/transform/transformTap'
 export * from './stream/transform/transformToArray'
 export * from './stream/transform/transformTee'
+export * from './stream/transform/transformThrottle'
 export * from './stream/transform/worker/baseWorkerClass'
 export * from './stream/transform/worker/transformMultiThreaded'
 export * from './stream/transform/worker/transformMultiThreaded.model'

package/src/stream/transform/transformThrottle.ts

@@ -0,0 +1,104 @@
+import { Transform } from 'node:stream'
+import {
+  _ms,
+  DeferredPromise,
+  localTime,
+  NumberOfMilliseconds,
+  NumberOfSeconds,
+  pDefer,
+  PositiveInteger,
+} from '@naturalcycles/js-lib'
+import { TransformTyped } from '../stream.model'
+
+export interface TransformThrottleOptions {
+  /**
+   * How many items to allow per `interval` of seconds.
+   */
+  throughput: PositiveInteger
+
+  /**
+   * How long is the interval (in seconds) where number of items should not exceed `throughput`.
+   */
+  interval: NumberOfSeconds
+
+  debug?: boolean
+}
+
+/**
+ * Allows to throttle the throughput of the stream.
+ * For example, when you have an API with rate limit of 5000 requests per minute,
+ * `transformThrottle` can help you utilize it most efficiently.
+ * You can define it as:
+ *
+ * _pipeline([
+ *   // ...
+ *   transformThrottle({
+ *     throughput: 5000,
+ *     interval: 60,
+ *   }),
+ *   // ...
+ * ])
+ *
+ * @experimental
+ */
+export function transformThrottle<T>(opt: TransformThrottleOptions): TransformTyped<T, T> {
+  const { throughput, interval, debug } = opt
+
+  let count = 0
+  let start: NumberOfMilliseconds
+  let paused: DeferredPromise | undefined
+  let timeout: NodeJS.Timeout | undefined
+
+  return new Transform({
+    objectMode: true,
+    async transform(item: T, _, cb) {
+      // console.log('incoming', item, { paused: !!paused, count })
+      if (!start) {
+        start = Date.now()
+        timeout = setTimeout(() => onInterval(this), interval * 1000)
+        if (debug) {
+          console.log(`${localTime.now().toPretty()} transformThrottle started with`, {
+            throughput,
+            interval,
+            rps: Math.round(throughput / interval),
+          })
+        }
+      }
+
+      if (paused) {
+        // console.log('awaiting pause', {item, count})
+        await paused
+      }
+
+      if (++count >= throughput) {
+        // console.log('pausing now after', {item, count})
+        paused = pDefer()
+        if (debug) {
+          console.log(
+            `${localTime.now().toPretty()} transformThrottle activated: ${count} items passed in ${_ms(interval * 1000)}, will pause for ${_ms(interval * 1000 - (Date.now() - start))}`,
+          )
+        }
+      }
+
+      cb(null, item) // pass the item through
+    },
+    final(cb) {
+      clearTimeout(timeout)
+      cb()
+    },
+  })
+
+  function onInterval(transform: Transform): void {
+    if (!paused) return
+
+    if (debug) {
+      console.log(`${localTime.now().toPretty()} transformThrottle resumed`)
+    }
+
+    count = 0
+    start = Date.now()
+    timeout = setTimeout(() => onInterval(transform), interval * 1000)
+    paused.resolve()
+    paused = undefined
+  }
+}