@naturalcycles/nodejs-lib 15.101.0 → 15.103.0

package/dist/script/runScript.js

@@ -1,5 +1,5 @@
 import os from 'node:os';
-import { pDelay } from '@naturalcycles/js-lib/promise/pDelay.js';
+import { AsyncManager } from '@naturalcycles/js-lib';
 import { setGlobalStringifyFunction } from '@naturalcycles/js-lib/string/stringify.js';
 import { dimGrey } from '../colors/colors.js';
 import { loadEnvFileIfExists } from '../node.util.js';
@@ -45,9 +45,10 @@ export function runScript(fn, opt = {}) {
     void (async () => {
         try {
             await fn();
-            await pDelay(); // to ensure all async operations are completed
             if (DEBUG_RUN_SCRIPT)
                 logger.log(`runScript promise resolved`);
+            // to ensure all async operations are completed (with a timeout)
+            await AsyncManager.allDone(600_000);
             if (!noExit) {
                 setImmediate(() => process.exit(0));
             }

package/dist/stream/index.d.ts

@@ -23,6 +23,7 @@ export * from './transform/transformOffset.js';
 export * from './transform/transformSplit.js';
 export * from './transform/transformTap.js';
 export * from './transform/transformThrottle.js';
+export * from './transform/transformThrottleByRSS.js';
 export * from './transform/transformWarmup.js';
 export * from './transform/worker/baseWorkerClass.js';
 export * from './transform/worker/transformMultiThreaded.js';

package/dist/stream/index.js

@@ -23,6 +23,7 @@ export * from './transform/transformOffset.js';
 export * from './transform/transformSplit.js';
 export * from './transform/transformTap.js';
 export * from './transform/transformThrottle.js';
+export * from './transform/transformThrottleByRSS.js';
 export * from './transform/transformWarmup.js';
 export * from './transform/worker/baseWorkerClass.js';
 export * from './transform/worker/transformMultiThreaded.js';

package/dist/stream/pipeline.d.ts

@@ -9,6 +9,7 @@ import type { TransformMapSimpleOptions } from './transform/transformMapSimple.j
 import type { TransformMapSyncOptions } from './transform/transformMapSync.js';
 import type { TransformOffsetOptions } from './transform/transformOffset.js';
 import type { TransformThrottleOptions } from './transform/transformThrottle.js';
+import type { TransformThrottleByRSSOptions } from './transform/transformThrottleByRSS.js';
 import type { TransformWarmupOptions } from './transform/transformWarmup.js';
 export declare class Pipeline<T = unknown> {
     private readonly source;
@@ -70,6 +71,7 @@ export declare class Pipeline<T = unknown> {
     tap(fn: AsyncIndexedMapper<T, any>, opt?: TransformOptions): this;
     tapSync(fn: IndexedMapper<T, any>, opt?: TransformOptions): this;
     throttle(opt: TransformThrottleOptions): this;
+    throttleByRSS(opt: TransformThrottleByRSSOptions): this;
     /**
      * @experimental to be removed after transformMap2 is stable
      */

package/dist/stream/pipeline.js

@@ -24,6 +24,7 @@ import { transformOffset } from './transform/transformOffset.js';
 import { transformSplitOnNewline } from './transform/transformSplit.js';
 import { transformTap, transformTapSync } from './transform/transformTap.js';
 import { transformThrottle } from './transform/transformThrottle.js';
+import { transformThrottleByRSS } from './transform/transformThrottleByRSS.js';
 import { transformWarmup } from './transform/transformWarmup.js';
 import { writablePushToArray } from './writable/writablePushToArray.js';
 import { writableVoid } from './writable/writableVoid.js';
@@ -175,6 +176,10 @@ export class Pipeline {
         this.transforms.push(transformThrottle(opt));
         return this;
     }
+    throttleByRSS(opt) {
+        this.transforms.push(transformThrottleByRSS(opt));
+        return this;
+    }
     /**
      * @experimental to be removed after transformMap2 is stable
      */

package/dist/stream/transform/transformThrottleByRSS.d.ts

@@ -0,0 +1,43 @@
+import type { Integer, NumberOfMilliseconds } from '@naturalcycles/js-lib/types';
+import type { TransformOptions, TransformTyped } from '../stream.model.js';
+export interface TransformThrottleByRSSOptions extends TransformOptions {
+    /**
+     * Maximum RSS (Resident Set Size) in megabytes.
+     * When process RSS exceeds this value, the stream will pause
+     * until RSS drops below the threshold.
+     */
+    maxRSS: Integer;
+    /**
+     * How often to re-check RSS (in milliseconds) while paused.
+     *
+     * @default 5000
+     */
+    pollInterval?: NumberOfMilliseconds;
+    /**
+     * If this timeout is reached while RSS is above the limit -
+     * the transform will "give up", log the bold warning, and "open the gateways".
+     * Things will likely OOM after that, but at least it will not "hang forever".
+     *
+     * @default 30 minutes
+     */
+    pollTimeout?: NumberOfMilliseconds;
+    /**
+     * What to do if pollTimeout is reached.
+     * 'open-the-floodgates' will disable this throttle completely (YOLO).
+     * 'throw' will throw an error, which will destroy the stream/Pipeline.
+     *
+     * @default 'open-the-floodgates'
+     */
+    onPollTimeout?: 'open-the-floodgates' | 'throw';
+}
+/**
+ * Throttles the stream based on process memory (RSS) usage.
+ * When RSS exceeds `maxRSS` (in megabytes), the stream pauses
+ * and periodically re-checks until RSS drops below the threshold.
+ *
+ * Useful for pipelines that process large amounts of data and
+ * may cause memory pressure (e.g. database imports, file processing).
+ *
+ * @experimental
+ */
+export declare function transformThrottleByRSS<T>(opt: TransformThrottleByRSSOptions): TransformTyped<T, T>;

package/dist/stream/transform/transformThrottleByRSS.js

@@ -0,0 +1,89 @@
+import { Transform } from 'node:stream';
+import { _mb } from '@naturalcycles/js-lib';
+import { _ms, localTime } from '@naturalcycles/js-lib/datetime';
+import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
+import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
+/**
+ * Throttles the stream based on process memory (RSS) usage.
+ * When RSS exceeds `maxRSS` (in megabytes), the stream pauses
+ * and periodically re-checks until RSS drops below the threshold.
+ *
+ * Useful for pipelines that process large amounts of data and
+ * may cause memory pressure (e.g. database imports, file processing).
+ *
+ * @experimental
+ */
+export function transformThrottleByRSS(opt) {
+    const { maxRSS, pollInterval = 5000, pollTimeout = 30 * 60_000, // 30 min
+    onPollTimeout = 'open-the-floodgates', objectMode = true, highWaterMark, } = opt;
+    const maxRSSBytes = maxRSS * 1024 * 1024;
+    let lock;
+    let pollTimer;
+    let rssCheckTimer;
+    let lastRSS = 0;
+    let pausedSince = 0;
+    let disabled = false;
+    const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
+    return new Transform({
+        objectMode,
+        highWaterMark,
+        async transform(item, _, cb) {
+            if (lock) {
+                try {
+                    await lock;
+                }
+                catch (err) {
+                    cb(err);
+                    return;
+                }
+            }
+            if (!disabled && lastRSS > maxRSSBytes && !lock) {
+                lock = pDefer();
+                pausedSince = Date.now();
+                logger.log(`${localTime.now().toPretty()} transformThrottleByRSS paused: RSS ${_mb(lastRSS)} > ${maxRSS} MB`);
+                pollTimer = setTimeout(() => pollRSS(), pollInterval);
+            }
+            cb(null, item);
+        },
+        construct(cb) {
+            // Start periodic RSS checking
+            checkRSS();
+            cb();
+        },
+        final(cb) {
+            clearTimeout(pollTimer);
+            clearTimeout(rssCheckTimer);
+            cb();
+        },
+    });
+    function checkRSS() {
+        lastRSS = process.memoryUsage.rss();
+        rssCheckTimer = setTimeout(() => checkRSS(), pollInterval);
+    }
+    function pollRSS() {
+        const rss = lastRSS;
+        if (rss <= maxRSSBytes) {
+            logger.log(`${localTime.now().toPretty()} transformThrottleByRSS resumed: RSS ${_mb(rss)} <= ${maxRSS} MB`);
+            lock.resolve();
+            lock = undefined;
+        }
+        else if (pollTimeout && Date.now() - pausedSince >= pollTimeout) {
+            clearTimeout(rssCheckTimer);
+            if (onPollTimeout === 'throw') {
+                lock.reject(new Error(`transformThrottleByRSS pollTimeout of ${_ms(pollTimeout)} reached, RSS ${_mb(rss)} still > ${maxRSS} MB`));
+                lock = undefined;
+            }
+            else {
+                // open-the-floodgates
+                logger.error(`${localTime.now().toPretty()} transformThrottleByRSS: pollTimeout of ${_ms(pollTimeout)} reached, RSS ${_mb(rss)} still > ${maxRSS} MB — DISABLING THROTTLE`);
+                disabled = true;
+                lock.resolve();
+                lock = undefined;
+            }
+        }
+        else {
+            logger.log(`${localTime.now().toPretty()} transformThrottleByRSS still paused: RSS ${_mb(rss)} > ${maxRSS} MB, rechecking in ${_ms(pollInterval)}`);
+            pollTimer = setTimeout(() => pollRSS(), pollInterval);
+        }
+    }
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.101.0",
+  "version": "15.103.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@standard-schema/spec": "^1",

package/src/script/runScript.ts

@@ -1,6 +1,6 @@
 import os from 'node:os'
+import { AsyncManager } from '@naturalcycles/js-lib'
 import type { CommonLogger } from '@naturalcycles/js-lib/log'
-import { pDelay } from '@naturalcycles/js-lib/promise/pDelay.js'
 import { setGlobalStringifyFunction } from '@naturalcycles/js-lib/string/stringify.js'
 import type { AnyObject } from '@naturalcycles/js-lib/types'
 import { dimGrey } from '../colors/colors.js'
@@ -76,10 +76,11 @@ export function runScript(fn: (...args: any[]) => any, opt: RunScriptOptions = {
     try {
       await fn()
 
-      await pDelay() // to ensure all async operations are completed
-
       if (DEBUG_RUN_SCRIPT) logger.log(`runScript promise resolved`)
 
+      // to ensure all async operations are completed (with a timeout)
+      await AsyncManager.allDone(600_000)
+
       if (!noExit) {
         setImmediate(() => process.exit(0))
       }

package/src/stream/index.ts

@@ -23,6 +23,7 @@ export * from './transform/transformOffset.js'
 export * from './transform/transformSplit.js'
 export * from './transform/transformTap.js'
 export * from './transform/transformThrottle.js'
+export * from './transform/transformThrottleByRSS.js'
 export * from './transform/transformWarmup.js'
 export * from './transform/worker/baseWorkerClass.js'
 export * from './transform/worker/transformMultiThreaded.js'

package/src/stream/pipeline.ts

@@ -51,6 +51,8 @@ import { transformSplitOnNewline } from './transform/transformSplit.js'
 import { transformTap, transformTapSync } from './transform/transformTap.js'
 import { transformThrottle } from './transform/transformThrottle.js'
 import type { TransformThrottleOptions } from './transform/transformThrottle.js'
+import { transformThrottleByRSS } from './transform/transformThrottleByRSS.js'
+import type { TransformThrottleByRSSOptions } from './transform/transformThrottleByRSS.js'
 import { transformWarmup } from './transform/transformWarmup.js'
 import type { TransformWarmupOptions } from './transform/transformWarmup.js'
 import { writablePushToArray } from './writable/writablePushToArray.js'
@@ -249,6 +251,11 @@ export class Pipeline<T = unknown> {
     return this
   }
 
+  throttleByRSS(opt: TransformThrottleByRSSOptions): this {
+    this.transforms.push(transformThrottleByRSS(opt))
+    return this
+  }
+
   /**
    * @experimental to be removed after transformMap2 is stable
    */

package/src/stream/transform/transformThrottleByRSS.ts

@@ -0,0 +1,150 @@
+import { Transform } from 'node:stream'
+import { _mb } from '@naturalcycles/js-lib'
+import { _ms, localTime } from '@naturalcycles/js-lib/datetime'
+import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
+import type { DeferredPromise } from '@naturalcycles/js-lib/promise'
+import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js'
+import type { Integer, NumberOfMilliseconds } from '@naturalcycles/js-lib/types'
+import type { TransformOptions, TransformTyped } from '../stream.model.js'
+
+export interface TransformThrottleByRSSOptions extends TransformOptions {
+  /**
+   * Maximum RSS (Resident Set Size) in megabytes.
+   * When process RSS exceeds this value, the stream will pause
+   * until RSS drops below the threshold.
+   */
+  maxRSS: Integer
+
+  /**
+   * How often to re-check RSS (in milliseconds) while paused.
+   *
+   * @default 5000
+   */
+  pollInterval?: NumberOfMilliseconds
+
+  /**
+   * If this timeout is reached while RSS is above the limit -
+   * the transform will "give up", log the bold warning, and "open the gateways".
+   * Things will likely OOM after that, but at least it will not "hang forever".
+   *
+   * @default 30 minutes
+   */
+  pollTimeout?: NumberOfMilliseconds
+
+  /**
+   * What to do if pollTimeout is reached.
+   * 'open-the-floodgates' will disable this throttle completely (YOLO).
+   * 'throw' will throw an error, which will destroy the stream/Pipeline.
+   *
+   * @default 'open-the-floodgates'
+   */
+  onPollTimeout?: 'open-the-floodgates' | 'throw'
+}
+
+/**
+ * Throttles the stream based on process memory (RSS) usage.
+ * When RSS exceeds `maxRSS` (in megabytes), the stream pauses
+ * and periodically re-checks until RSS drops below the threshold.
+ *
+ * Useful for pipelines that process large amounts of data and
+ * may cause memory pressure (e.g. database imports, file processing).
+ *
+ * @experimental
+ */
+export function transformThrottleByRSS<T>(
+  opt: TransformThrottleByRSSOptions,
+): TransformTyped<T, T> {
+  const {
+    maxRSS,
+    pollInterval = 5000,
+    pollTimeout = 30 * 60_000, // 30 min
+    onPollTimeout = 'open-the-floodgates',
+    objectMode = true,
+    highWaterMark,
+  } = opt
+
+  const maxRSSBytes = maxRSS * 1024 * 1024
+  let lock: DeferredPromise | undefined
+  let pollTimer: NodeJS.Timeout | undefined
+  let rssCheckTimer: NodeJS.Timeout | undefined
+  let lastRSS = 0
+  let pausedSince = 0
+  let disabled = false
+  const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
+
+  return new Transform({
+    objectMode,
+    highWaterMark,
+    async transform(item: T, _, cb) {
+      if (lock) {
+        try {
+          await lock
+        } catch (err) {
+          cb(err as Error)
+          return
+        }
+      }
+
+      if (!disabled && lastRSS > maxRSSBytes && !lock) {
+        lock = pDefer()
+        pausedSince = Date.now()
+        logger.log(
+          `${localTime.now().toPretty()} transformThrottleByRSS paused: RSS ${_mb(lastRSS)} > ${maxRSS} MB`,
+        )
+        pollTimer = setTimeout(() => pollRSS(), pollInterval)
+      }
+
+      cb(null, item)
+    },
+    construct(cb) {
+      // Start periodic RSS checking
+      checkRSS()
+      cb()
+    },
+    final(cb) {
+      clearTimeout(pollTimer)
+      clearTimeout(rssCheckTimer)
+      cb()
+    },
+  })
+
+  function checkRSS(): void {
+    lastRSS = process.memoryUsage.rss()
+    rssCheckTimer = setTimeout(() => checkRSS(), pollInterval)
+  }
+
+  function pollRSS(): void {
+    const rss = lastRSS
+
+    if (rss <= maxRSSBytes) {
+      logger.log(
+        `${localTime.now().toPretty()} transformThrottleByRSS resumed: RSS ${_mb(rss)} <= ${maxRSS} MB`,
+      )
+      lock!.resolve()
+      lock = undefined
+    } else if (pollTimeout && Date.now() - pausedSince >= pollTimeout) {
+      clearTimeout(rssCheckTimer)
+      if (onPollTimeout === 'throw') {
+        lock!.reject(
+          new Error(
+            `transformThrottleByRSS pollTimeout of ${_ms(pollTimeout)} reached, RSS ${_mb(rss)} still > ${maxRSS} MB`,
+          ),
+        )
+        lock = undefined
+      } else {
+        // open-the-floodgates
+        logger.error(
+          `${localTime.now().toPretty()} transformThrottleByRSS: pollTimeout of ${_ms(pollTimeout)} reached, RSS ${_mb(rss)} still > ${maxRSS} MB — DISABLING THROTTLE`,
+        )
+        disabled = true
+        lock!.resolve()
+        lock = undefined
+      }
+    } else {
+      logger.log(
+        `${localTime.now().toPretty()} transformThrottleByRSS still paused: RSS ${_mb(rss)} > ${maxRSS} MB, rechecking in ${_ms(pollInterval)}`,
+      )
+      pollTimer = setTimeout(() => pollRSS(), pollInterval)
+    }
+  }
+}