@naturalcycles/nodejs-lib 15.102.0 → 15.104.0

package/dist/stream/index.d.ts

@@ -23,6 +23,7 @@ export * from './transform/transformOffset.js';
 export * from './transform/transformSplit.js';
 export * from './transform/transformTap.js';
 export * from './transform/transformThrottle.js';
+export * from './transform/transformThrottleByRSS.js';
 export * from './transform/transformWarmup.js';
 export * from './transform/worker/baseWorkerClass.js';
 export * from './transform/worker/transformMultiThreaded.js';

package/dist/stream/index.js

@@ -23,6 +23,7 @@ export * from './transform/transformOffset.js';
 export * from './transform/transformSplit.js';
 export * from './transform/transformTap.js';
 export * from './transform/transformThrottle.js';
+export * from './transform/transformThrottleByRSS.js';
 export * from './transform/transformWarmup.js';
 export * from './transform/worker/baseWorkerClass.js';
 export * from './transform/worker/transformMultiThreaded.js';

package/dist/stream/pipeline.d.ts

@@ -9,6 +9,7 @@ import type { TransformMapSimpleOptions } from './transform/transformMapSimple.j
 import type { TransformMapSyncOptions } from './transform/transformMapSync.js';
 import type { TransformOffsetOptions } from './transform/transformOffset.js';
 import type { TransformThrottleOptions } from './transform/transformThrottle.js';
+import type { TransformThrottleByRSSOptions } from './transform/transformThrottleByRSS.js';
 import type { TransformWarmupOptions } from './transform/transformWarmup.js';
 export declare class Pipeline<T = unknown> {
     private readonly source;
@@ -70,6 +71,7 @@ export declare class Pipeline<T = unknown> {
     tap(fn: AsyncIndexedMapper<T, any>, opt?: TransformOptions): this;
     tapSync(fn: IndexedMapper<T, any>, opt?: TransformOptions): this;
     throttle(opt: TransformThrottleOptions): this;
+    throttleByRSS(opt: TransformThrottleByRSSOptions): this;
     /**
      * @experimental to be removed after transformMap2 is stable
      */

package/dist/stream/pipeline.js

@@ -24,6 +24,7 @@ import { transformOffset } from './transform/transformOffset.js';
 import { transformSplitOnNewline } from './transform/transformSplit.js';
 import { transformTap, transformTapSync } from './transform/transformTap.js';
 import { transformThrottle } from './transform/transformThrottle.js';
+import { transformThrottleByRSS } from './transform/transformThrottleByRSS.js';
 import { transformWarmup } from './transform/transformWarmup.js';
 import { writablePushToArray } from './writable/writablePushToArray.js';
 import { writableVoid } from './writable/writableVoid.js';
@@ -175,6 +176,10 @@ export class Pipeline {
         this.transforms.push(transformThrottle(opt));
         return this;
     }
+    throttleByRSS(opt) {
+        this.transforms.push(transformThrottleByRSS(opt));
+        return this;
+    }
     /**
      * @experimental to be removed after transformMap2 is stable
      */

package/dist/stream/transform/transformChunk.js

@@ -11,6 +11,7 @@ export function transformChunk(chunkSize, opt) {
     let buf = [];
     return new Transform({
         objectMode: true,
+        highWaterMark: 1,
         ...opt,
         transform(chunk, _, cb) {
             buf.push(chunk);

package/dist/stream/transform/transformFilter.js

@@ -16,6 +16,7 @@ export function transformFilterSync(predicate, opt = {}) {
     let index = 0;
     return new Transform({
         objectMode: true,
+        highWaterMark: 1,
         ...opt,
         transform(chunk, _, cb) {
             try {

package/dist/stream/transform/transformFlatten.js

@@ -2,6 +2,7 @@ import { Transform } from 'node:stream';
 export function transformFlatten() {
     return new Transform({
         objectMode: true,
+        highWaterMark: 1,
         transform(chunk, _, cb) {
             for (const item of chunk) {
                 this.push(item);
@@ -13,6 +14,7 @@ export function transformFlatten() {
 export function transformFlattenIfNeeded() {
     return new Transform({
         objectMode: true,
+        highWaterMark: 1,
         transform(chunk, _, cb) {
             if (Array.isArray(chunk)) {
                 for (const item of chunk) {

package/dist/stream/transform/transformFork.js

@@ -12,7 +12,7 @@ import { createReadable } from '../readable/createReadable.js';
  * @experimental
  */
 export function transformFork(fn, opt = {}) {
-    const { objectMode = true, highWaterMark } = opt;
+    const { objectMode = true, highWaterMark = 1 } = opt;
     const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
     let lock;
     const fork = createReadable([], {}, () => {

package/dist/stream/transform/transformLimit.js

@@ -2,7 +2,7 @@ import { Transform } from 'node:stream';
 import { PIPELINE_GRACEFUL_ABORT } from '../stream.util.js';
 import { transformNoOp } from './transformNoOp.js';
 export function transformLimit(opt) {
-    const { limit, signal, objectMode = true, highWaterMark } = opt;
+    const { limit, signal, objectMode = true, highWaterMark = 1 } = opt;
     if (!limit) {
         return transformNoOp();
     }

package/dist/stream/transform/transformMapSimple.js

@@ -11,7 +11,7 @@ import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js';
  */
 export function transformMapSimple(mapper, opt = {}) {
     let index = -1;
-    const { errorMode = ErrorMode.THROW_IMMEDIATELY, logger = console, objectMode = true, highWaterMark, } = opt;
+    const { errorMode = ErrorMode.THROW_IMMEDIATELY, logger = console, objectMode = true, highWaterMark = 1, } = opt;
     return new Transform({
         objectMode,
         highWaterMark,

package/dist/stream/transform/transformMapSync.js

@@ -20,6 +20,7 @@ export function transformMapSync(mapper, opt = {}) {
     const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
     return new Transform({
         objectMode,
+        highWaterMark: 1,
         ...opt,
         transform(chunk, _, cb) {
             // Stop processing if isSettled

package/dist/stream/transform/transformNoOp.js

@@ -7,6 +7,7 @@ import { Transform } from 'node:stream';
 export function transformNoOp() {
     return new Transform({
         objectMode: true,
+        highWaterMark: 1,
         transform(chunk, _, cb) {
             cb(null, chunk);
         },

package/dist/stream/transform/transformOffset.js

@@ -9,6 +9,7 @@ export function transformOffset(opt) {
     let i = 0; // so we start first chunk with 1
     return new Transform({
         objectMode: true,
+        highWaterMark: 1,
         ...opt,
         transform(chunk, _, cb) {
             if (++i <= offset) {

package/dist/stream/transform/transformThrottle.js

@@ -20,7 +20,7 @@ import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
  * @experimental
  */
 export function transformThrottle(opt) {
-    const { throughput, interval, objectMode = true, highWaterMark } = opt;
+    const { throughput, interval, objectMode = true, highWaterMark = 1 } = opt;
     let count = 0;
     let start;
     let lock;

package/dist/stream/transform/transformThrottleByRSS.d.ts

@@ -0,0 +1,43 @@
+import type { Integer, NumberOfMilliseconds } from '@naturalcycles/js-lib/types';
+import type { TransformOptions, TransformTyped } from '../stream.model.js';
+export interface TransformThrottleByRSSOptions extends TransformOptions {
+    /**
+     * Maximum RSS (Resident Set Size) in megabytes.
+     * When process RSS exceeds this value, the stream will pause
+     * until RSS drops below the threshold.
+     */
+    maxRSS: Integer;
+    /**
+     * How often to re-check RSS (in milliseconds) while paused.
+     *
+     * @default 5000
+     */
+    pollInterval?: NumberOfMilliseconds;
+    /**
+     * If this timeout is reached while RSS is above the limit -
+     * the transform will "give up", log the bold warning, and "open the gateways".
+     * Things will likely OOM after that, but at least it will not "hang forever".
+     *
+     * @default 30 minutes
+     */
+    pollTimeout?: NumberOfMilliseconds;
+    /**
+     * What to do if pollTimeout is reached.
+     * 'open-the-floodgates' will disable this throttle completely (YOLO).
+     * 'throw' will throw an error, which will destroy the stream/Pipeline.
+     *
+     * @default 'open-the-floodgates'
+     */
+    onPollTimeout?: 'open-the-floodgates' | 'throw';
+}
+/**
+ * Throttles the stream based on process memory (RSS) usage.
+ * When RSS exceeds `maxRSS` (in megabytes), the stream pauses
+ * and periodically re-checks until RSS drops below the threshold.
+ *
+ * Useful for pipelines that process large amounts of data and
+ * may cause memory pressure (e.g. database imports, file processing).
+ *
+ * @experimental
+ */
+export declare function transformThrottleByRSS<T>(opt: TransformThrottleByRSSOptions): TransformTyped<T, T>;

package/dist/stream/transform/transformThrottleByRSS.js

@@ -0,0 +1,89 @@
+import { Transform } from 'node:stream';
+import { _mb } from '@naturalcycles/js-lib';
+import { _ms, localTime } from '@naturalcycles/js-lib/datetime';
+import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
+import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
+/**
+ * Throttles the stream based on process memory (RSS) usage.
+ * When RSS exceeds `maxRSS` (in megabytes), the stream pauses
+ * and periodically re-checks until RSS drops below the threshold.
+ *
+ * Useful for pipelines that process large amounts of data and
+ * may cause memory pressure (e.g. database imports, file processing).
+ *
+ * @experimental
+ */
+export function transformThrottleByRSS(opt) {
+    const { maxRSS, pollInterval = 5000, pollTimeout = 30 * 60_000, // 30 min
+    onPollTimeout = 'open-the-floodgates', objectMode = true, highWaterMark = 1, } = opt;
+    const maxRSSBytes = maxRSS * 1024 * 1024;
+    let lock;
+    let pollTimer;
+    let rssCheckTimer;
+    let lastRSS = 0;
+    let pausedSince = 0;
+    let disabled = false;
+    const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
+    return new Transform({
+        objectMode,
+        highWaterMark,
+        async transform(item, _, cb) {
+            if (lock) {
+                try {
+                    await lock;
+                }
+                catch (err) {
+                    cb(err);
+                    return;
+                }
+            }
+            if (!disabled && lastRSS > maxRSSBytes && !lock) {
+                lock = pDefer();
+                pausedSince = Date.now();
+                logger.log(`${localTime.now().toPretty()} transformThrottleByRSS paused: RSS ${_mb(lastRSS)} > ${maxRSS} MB`);
+                pollTimer = setTimeout(() => pollRSS(), pollInterval);
+            }
+            cb(null, item);
+        },
+        construct(cb) {
+            // Start periodic RSS checking
+            checkRSS();
+            cb();
+        },
+        final(cb) {
+            clearTimeout(pollTimer);
+            clearTimeout(rssCheckTimer);
+            cb();
+        },
+    });
+    function checkRSS() {
+        lastRSS = process.memoryUsage.rss();
+        rssCheckTimer = setTimeout(() => checkRSS(), pollInterval);
+    }
+    function pollRSS() {
+        const rss = lastRSS;
+        if (rss <= maxRSSBytes) {
+            logger.log(`${localTime.now().toPretty()} transformThrottleByRSS resumed: RSS ${_mb(rss)} <= ${maxRSS} MB`);
+            lock.resolve();
+            lock = undefined;
+        }
+        else if (pollTimeout && Date.now() - pausedSince >= pollTimeout) {
+            clearTimeout(rssCheckTimer);
+            if (onPollTimeout === 'throw') {
+                lock.reject(new Error(`transformThrottleByRSS pollTimeout of ${_ms(pollTimeout)} reached, RSS ${_mb(rss)} still > ${maxRSS} MB`));
+                lock = undefined;
+            }
+            else {
+                // open-the-floodgates
+                logger.error(`${localTime.now().toPretty()} transformThrottleByRSS: pollTimeout of ${_ms(pollTimeout)} reached, RSS ${_mb(rss)} still > ${maxRSS} MB — DISABLING THROTTLE`);
+                disabled = true;
+                lock.resolve();
+                lock = undefined;
+            }
+        }
+        else {
+            logger.log(`${localTime.now().toPretty()} transformThrottleByRSS still paused: RSS ${_mb(rss)} > ${maxRSS} MB, rechecking in ${_ms(pollInterval)}`);
+            pollTimer = setTimeout(() => pollRSS(), pollInterval);
+        }
+    }
+}

package/dist/stream/transform/transformWarmup.js

@@ -12,7 +12,7 @@ import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
  * @experimental
  */
 export function transformWarmup(opt) {
-    const { concurrency, warmupSeconds, objectMode = true, highWaterMark } = opt;
+    const { concurrency, warmupSeconds, objectMode = true, highWaterMark = 1 } = opt;
     const warmupMs = warmupSeconds * 1000;
     const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
     let startTime = 0;

package/dist/zip/zip.util.d.ts

@@ -1,11 +1,13 @@
 import type { ZlibOptions, ZstdOptions } from 'node:zlib';
 import type { Integer } from '@naturalcycles/js-lib/types';
 export declare function decompressZstdOrInflateToString(buf: Buffer): Promise<string>;
+export declare function decompressZstdOrInflateToStringSync(buf: Buffer): string;
 /**
  * Detects if Buffer is zstd-compressed.
  * Otherwise attempts to Inflate.
  */
 export declare function decompressZstdOrInflate(buf: Buffer): Promise<Buffer<ArrayBuffer>>;
+export declare function decompressZstdOrInflateSync(buf: Buffer): Buffer<ArrayBuffer>;
 /**
  * deflateBuffer uses `deflate`.
  * It's 9 bytes shorter than `gzip`.

package/dist/zip/zip.util.js

@@ -9,6 +9,9 @@ const zstdDecompressAsync = promisify(zlib.zstdDecompress.bind(zlib));
 export async function decompressZstdOrInflateToString(buf) {
     return (await decompressZstdOrInflate(buf)).toString();
 }
+export function decompressZstdOrInflateToStringSync(buf) {
+    return decompressZstdOrInflateSync(buf).toString();
+}
 /**
  * Detects if Buffer is zstd-compressed.
  * Otherwise attempts to Inflate.
@@ -19,6 +22,12 @@ export async function decompressZstdOrInflate(buf) {
     }
     return await inflate(buf);
 }
+export function decompressZstdOrInflateSync(buf) {
+    if (isZstdBuffer(buf)) {
+        return zlib.zstdDecompressSync(buf);
+    }
+    return zlib.inflateSync(buf);
+}
 /**
  * deflateBuffer uses `deflate`.
  * It's 9 bytes shorter than `gzip`.

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.102.0",
+  "version": "15.104.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@standard-schema/spec": "^1",

package/src/stream/index.ts

@@ -23,6 +23,7 @@ export * from './transform/transformOffset.js'
 export * from './transform/transformSplit.js'
 export * from './transform/transformTap.js'
 export * from './transform/transformThrottle.js'
+export * from './transform/transformThrottleByRSS.js'
 export * from './transform/transformWarmup.js'
 export * from './transform/worker/baseWorkerClass.js'
 export * from './transform/worker/transformMultiThreaded.js'

package/src/stream/pipeline.ts

@@ -51,6 +51,8 @@ import { transformSplitOnNewline } from './transform/transformSplit.js'
 import { transformTap, transformTapSync } from './transform/transformTap.js'
 import { transformThrottle } from './transform/transformThrottle.js'
 import type { TransformThrottleOptions } from './transform/transformThrottle.js'
+import { transformThrottleByRSS } from './transform/transformThrottleByRSS.js'
+import type { TransformThrottleByRSSOptions } from './transform/transformThrottleByRSS.js'
 import { transformWarmup } from './transform/transformWarmup.js'
 import type { TransformWarmupOptions } from './transform/transformWarmup.js'
 import { writablePushToArray } from './writable/writablePushToArray.js'
@@ -249,6 +251,11 @@ export class Pipeline<T = unknown> {
     return this
   }
 
+  throttleByRSS(opt: TransformThrottleByRSSOptions): this {
+    this.transforms.push(transformThrottleByRSS(opt))
+    return this
+  }
+
   /**
    * @experimental to be removed after transformMap2 is stable
    */

package/src/stream/transform/transformChunk.ts

@@ -18,6 +18,7 @@ export function transformChunk<IN = any>(
 
   return new Transform({
     objectMode: true,
+    highWaterMark: 1,
     ...opt,
     transform(chunk, _, cb) {
       buf.push(chunk)

package/src/stream/transform/transformFilter.ts

@@ -28,6 +28,7 @@ export function transformFilterSync<IN = any>(
 
   return new Transform({
     objectMode: true,
+    highWaterMark: 1,
     ...opt,
     transform(chunk: IN, _, cb) {
       try {

package/src/stream/transform/transformFlatten.ts

@@ -4,6 +4,7 @@ import type { TransformTyped } from '../stream.model.js'
 export function transformFlatten<T>(): TransformTyped<T[], T> {
   return new Transform({
     objectMode: true,
+    highWaterMark: 1,
     transform(chunk: T[], _, cb) {
       for (const item of chunk) {
         this.push(item)
@@ -16,6 +17,7 @@ export function transformFlatten<T>(): TransformTyped<T[], T> {
 export function transformFlattenIfNeeded<T>(): TransformTyped<T[], T> {
   return new Transform({
     objectMode: true,
+    highWaterMark: 1,
     transform(chunk: T[], _, cb) {
       if (Array.isArray(chunk)) {
         for (const item of chunk) {

package/src/stream/transform/transformFork.ts

@@ -18,7 +18,7 @@ export function transformFork<T>(
   fn: (pipeline: Pipeline<T>) => Promise<void>,
   opt: TransformOptions = {},
 ): TransformTyped<T, T> {
-  const { objectMode = true, highWaterMark } = opt
+  const { objectMode = true, highWaterMark = 1 } = opt
   const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
 
   let lock: DeferredPromise | undefined

package/src/stream/transform/transformLimit.ts

@@ -18,7 +18,7 @@ export interface TransformLimitOptions extends TransformOptions {
 }
 
 export function transformLimit<IN>(opt: TransformLimitOptions): TransformTyped<IN, IN> {
-  const { limit, signal, objectMode = true, highWaterMark } = opt
+  const { limit, signal, objectMode = true, highWaterMark = 1 } = opt
 
   if (!limit) {
     return transformNoOp()

package/src/stream/transform/transformMapSimple.ts

@@ -30,7 +30,7 @@ export function transformMapSimple<IN = any, OUT = IN>(
     errorMode = ErrorMode.THROW_IMMEDIATELY,
     logger = console,
     objectMode = true,
-    highWaterMark,
+    highWaterMark = 1,
   } = opt
 
   return new Transform({

package/src/stream/transform/transformMapSync.ts

@@ -88,6 +88,7 @@ export function transformMapSync<IN = any, OUT = IN>(
 
   return new Transform({
     objectMode,
+    highWaterMark: 1,
     ...opt,
     transform(chunk: IN, _, cb) {
       // Stop processing if isSettled

package/src/stream/transform/transformNoOp.ts

@@ -9,6 +9,7 @@ import type { TransformTyped } from '../stream.model.js'
 export function transformNoOp<T = any>(): TransformTyped<T, T> {
   return new Transform({
     objectMode: true,
+    highWaterMark: 1,
     transform(chunk: T, _, cb) {
       cb(null, chunk)
     },

package/src/stream/transform/transformOffset.ts

@@ -22,6 +22,7 @@ export function transformOffset<IN>(opt: TransformOffsetOptions): TransformTyped
   let i = 0 // so we start first chunk with 1
   return new Transform({
     objectMode: true,
+    highWaterMark: 1,
     ...opt,
     transform(chunk: IN, _, cb) {
       if (++i <= offset) {

package/src/stream/transform/transformThrottle.ts

@@ -40,7 +40,7 @@ export interface TransformThrottleOptions extends TransformOptions {
  * @experimental
  */
 export function transformThrottle<T>(opt: TransformThrottleOptions): TransformTyped<T, T> {
-  const { throughput, interval, objectMode = true, highWaterMark } = opt
+  const { throughput, interval, objectMode = true, highWaterMark = 1 } = opt
 
   let count = 0
   let start: UnixTimestampMillis

package/src/stream/transform/transformThrottleByRSS.ts

@@ -0,0 +1,150 @@
+import { Transform } from 'node:stream'
+import { _mb } from '@naturalcycles/js-lib'
+import { _ms, localTime } from '@naturalcycles/js-lib/datetime'
+import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
+import type { DeferredPromise } from '@naturalcycles/js-lib/promise'
+import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js'
+import type { Integer, NumberOfMilliseconds } from '@naturalcycles/js-lib/types'
+import type { TransformOptions, TransformTyped } from '../stream.model.js'
+
+export interface TransformThrottleByRSSOptions extends TransformOptions {
+  /**
+   * Maximum RSS (Resident Set Size) in megabytes.
+   * When process RSS exceeds this value, the stream will pause
+   * until RSS drops below the threshold.
+   */
+  maxRSS: Integer
+
+  /**
+   * How often to re-check RSS (in milliseconds) while paused.
+   *
+   * @default 5000
+   */
+  pollInterval?: NumberOfMilliseconds
+
+  /**
+   * If this timeout is reached while RSS is above the limit -
+   * the transform will "give up", log the bold warning, and "open the gateways".
+   * Things will likely OOM after that, but at least it will not "hang forever".
+   *
+   * @default 30 minutes
+   */
+  pollTimeout?: NumberOfMilliseconds
+
+  /**
+   * What to do if pollTimeout is reached.
+   * 'open-the-floodgates' will disable this throttle completely (YOLO).
+   * 'throw' will throw an error, which will destroy the stream/Pipeline.
+   *
+   * @default 'open-the-floodgates'
+   */
+  onPollTimeout?: 'open-the-floodgates' | 'throw'
+}
+
+/**
+ * Throttles the stream based on process memory (RSS) usage.
+ * When RSS exceeds `maxRSS` (in megabytes), the stream pauses
+ * and periodically re-checks until RSS drops below the threshold.
+ *
+ * Useful for pipelines that process large amounts of data and
+ * may cause memory pressure (e.g. database imports, file processing).
+ *
+ * @experimental
+ */
+export function transformThrottleByRSS<T>(
+  opt: TransformThrottleByRSSOptions,
+): TransformTyped<T, T> {
+  const {
+    maxRSS,
+    pollInterval = 5000,
+    pollTimeout = 30 * 60_000, // 30 min
+    onPollTimeout = 'open-the-floodgates',
+    objectMode = true,
+    highWaterMark = 1,
+  } = opt
+
+  const maxRSSBytes = maxRSS * 1024 * 1024
+  let lock: DeferredPromise | undefined
+  let pollTimer: NodeJS.Timeout | undefined
+  let rssCheckTimer: NodeJS.Timeout | undefined
+  let lastRSS = 0
+  let pausedSince = 0
+  let disabled = false
+  const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
+
+  return new Transform({
+    objectMode,
+    highWaterMark,
+    async transform(item: T, _, cb) {
+      if (lock) {
+        try {
+          await lock
+        } catch (err) {
+          cb(err as Error)
+          return
+        }
+      }
+
+      if (!disabled && lastRSS > maxRSSBytes && !lock) {
+        lock = pDefer()
+        pausedSince = Date.now()
+        logger.log(
+          `${localTime.now().toPretty()} transformThrottleByRSS paused: RSS ${_mb(lastRSS)} > ${maxRSS} MB`,
+        )
+        pollTimer = setTimeout(() => pollRSS(), pollInterval)
+      }
+
+      cb(null, item)
+    },
+    construct(cb) {
+      // Start periodic RSS checking
+      checkRSS()
+      cb()
+    },
+    final(cb) {
+      clearTimeout(pollTimer)
+      clearTimeout(rssCheckTimer)
+      cb()
+    },
+  })
+
+  function checkRSS(): void {
+    lastRSS = process.memoryUsage.rss()
+    rssCheckTimer = setTimeout(() => checkRSS(), pollInterval)
+  }
+
+  function pollRSS(): void {
+    const rss = lastRSS
+
+    if (rss <= maxRSSBytes) {
+      logger.log(
+        `${localTime.now().toPretty()} transformThrottleByRSS resumed: RSS ${_mb(rss)} <= ${maxRSS} MB`,
+      )
+      lock!.resolve()
+      lock = undefined
+    } else if (pollTimeout && Date.now() - pausedSince >= pollTimeout) {
+      clearTimeout(rssCheckTimer)
+      if (onPollTimeout === 'throw') {
+        lock!.reject(
+          new Error(
+            `transformThrottleByRSS pollTimeout of ${_ms(pollTimeout)} reached, RSS ${_mb(rss)} still > ${maxRSS} MB`,
+          ),
+        )
+        lock = undefined
+      } else {
+        // open-the-floodgates
+        logger.error(
+          `${localTime.now().toPretty()} transformThrottleByRSS: pollTimeout of ${_ms(pollTimeout)} reached, RSS ${_mb(rss)} still > ${maxRSS} MB — DISABLING THROTTLE`,
+        )
+        disabled = true
+        lock!.resolve()
+        lock = undefined
+      }
+    } else {
+      logger.log(
+        `${localTime.now().toPretty()} transformThrottleByRSS still paused: RSS ${_mb(rss)} > ${maxRSS} MB, rechecking in ${_ms(pollInterval)}`,
+      )
+      pollTimer = setTimeout(() => pollRSS(), pollInterval)
+    }
+  }
+}

package/src/stream/transform/transformWarmup.ts

@@ -29,7 +29,7 @@ export interface TransformWarmupOptions extends TransformOptions {
  * @experimental
  */
 export function transformWarmup<T>(opt: TransformWarmupOptions): TransformTyped<T, T> {
-  const { concurrency, warmupSeconds, objectMode = true, highWaterMark } = opt
+  const { concurrency, warmupSeconds, objectMode = true, highWaterMark = 1 } = opt
   const warmupMs = warmupSeconds * 1000
   const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
 

package/src/zip/zip.util.ts

@@ -14,6 +14,10 @@ export async function decompressZstdOrInflateToString(buf: Buffer): Promise<stri
   return (await decompressZstdOrInflate(buf)).toString()
 }
 
+export function decompressZstdOrInflateToStringSync(buf: Buffer): string {
+  return decompressZstdOrInflateSync(buf).toString()
+}
+
 /**
  * Detects if Buffer is zstd-compressed.
  * Otherwise attempts to Inflate.
@@ -25,6 +29,13 @@ export async function decompressZstdOrInflate(buf: Buffer): Promise<Buffer<Array
   return await inflate(buf)
 }
 
+export function decompressZstdOrInflateSync(buf: Buffer): Buffer<ArrayBuffer> {
+  if (isZstdBuffer(buf)) {
+    return zlib.zstdDecompressSync(buf)
+  }
+  return zlib.inflateSync(buf)
+}
+
 /**
  * deflateBuffer uses `deflate`.
  * It's 9 bytes shorter than `gzip`.