@naturalcycles/nodejs-lib 15.69.0 → 15.70.0

package/dist/stream/pipeline.d.ts

@@ -62,11 +62,8 @@ export declare class Pipeline<T = unknown> {
     flatten<TO>(this: Pipeline<readonly TO[]>): Pipeline<TO>;
     flattenIfNeeded(): Pipeline<T extends readonly (infer TO)[] ? TO : T>;
     logProgress(opt?: TransformLogProgressOptions): this;
-    map<TO>(mapper: AbortableAsyncMapper<T, TO | typeof SKIP | typeof END>, opt?: TransformMapOptions<T, TO>): Pipeline<TO>;
-    /**
-     * @experimental if proven to be stable - will replace transformMap
-     */
-    map2<TO>(mapper: AbortableAsyncMapper<T, TO | typeof SKIP | typeof END>, opt?: TransformMap2Options<T, TO>): Pipeline<TO>;
+    mapLegacy<TO>(mapper: AbortableAsyncMapper<T, TO | typeof SKIP | typeof END>, opt?: TransformMapOptions<T, TO>): Pipeline<TO>;
+    map<TO>(mapper: AbortableAsyncMapper<T, TO | typeof SKIP | typeof END>, opt?: TransformMap2Options<T, TO>): Pipeline<TO>;
     mapSync<TO>(mapper: IndexedMapper<T, TO | typeof SKIP | typeof END>, opt?: TransformMapSyncOptions): Pipeline<TO>;
     mapSimple<TO>(mapper: IndexedMapper<T, TO>, opt?: TransformMapSimpleOptions): Pipeline<TO>;
     filter(asyncPredicate: AsyncPredicate<T>, opt?: TransformMapOptions): this;

package/dist/stream/pipeline.js

@@ -128,17 +128,14 @@ export class Pipeline {
         this.transforms.push(transformLogProgress(opt));
         return this;
     }
-    map(mapper, opt) {
+    mapLegacy(mapper, opt) {
         this.transforms.push(transformMap(mapper, {
             ...opt,
             signal: this.abortableSignal,
         }));
         return this;
     }
-    /**
-     * @experimental if proven to be stable - will replace transformMap
-     */
-    map2(mapper, opt) {
+    map(mapper, opt) {
         this.transforms.push(transformMap2(mapper, {
             ...opt,
             signal: this.abortableSignal,
@@ -157,7 +154,7 @@ export class Pipeline {
         return this;
     }
     filter(asyncPredicate, opt) {
-        this.transforms.push(transformMap(v => v, {
+        this.transforms.push(transformMap2(v => v, {
             asyncPredicate,
             ...opt,
             signal: this.abortableSignal,
@@ -314,7 +311,7 @@ export class Pipeline {
         await this.run();
     }
     async forEach(fn, opt = {}) {
-        this.transforms.push(transformMap(fn, {
+        this.transforms.push(transformMap2(fn, {
             predicate: opt.logEvery ? _passthroughPredicate : undefined, // for the logger to work
             ...opt,
             signal: this.abortableSignal,

package/dist/stream/transform/transformFilter.js

@@ -1,10 +1,10 @@
 import { Transform } from 'node:stream';
-import { transformMap } from './transformMap.js';
+import { transformMap2 } from './transformMap2.js';
 /**
  * Just a convenience wrapper around `transformMap` that has built-in predicate filtering support.
  */
 export function transformFilter(asyncPredicate, opt = {}) {
-    return transformMap(v => v, {
+    return transformMap2(v => v, {
         asyncPredicate,
         ...opt,
     });

package/dist/stream/transform/transformMap2.js

@@ -1,10 +1,12 @@
 import { Transform } from 'node:stream';
+import { _since } from '@naturalcycles/js-lib/datetime';
 import { _anyToError, _assert, ErrorMode } from '@naturalcycles/js-lib/error';
 import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
 import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
 import { END, SKIP, } from '@naturalcycles/js-lib/types';
 import { yellow } from '../../colors/colors.js';
 import { PIPELINE_GRACEFUL_ABORT } from '../stream.util.js';
+const WARMUP_CHECK_INTERVAL_MS = 1000;
 /**
  * Like transformMap, but with native concurrency control (no through2-concurrent dependency)
  * and support for gradual warmup.
@@ -12,73 +14,62 @@ import { PIPELINE_GRACEFUL_ABORT } from '../stream.util.js';
  * @experimental
  */
 export function transformMap2(mapper, opt = {}) {
-    const { concurrency = 16, warmupSeconds = 0, predicate, asyncPredicate, errorMode = ErrorMode.THROW_IMMEDIATELY, onError, onDone, metric = 'stream', signal, objectMode = true, highWaterMark = 64, } = opt;
+    const { concurrency: maxConcurrency = 16, warmupSeconds = 0, predicate, asyncPredicate, errorMode = ErrorMode.THROW_IMMEDIATELY, onError, onDone, metric = 'stream', signal, objectMode = true, highWaterMark = 64, } = opt;
     const warmupMs = warmupSeconds * 1000;
     const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
     // Stats
-    const started = Date.now();
+    let started = 0;
     let index = -1;
     let countOut = 0;
     let isSettled = false;
     let ok = true;
     let errors = 0;
     const collectedErrors = [];
-    // Concurrency control
-    let startTime = 0;
-    let warmupComplete = warmupSeconds <= 0 || concurrency <= 1;
+    // Concurrency control - single counter, single callback for backpressure
     let inFlight = 0;
-    const waiters = [];
-    // Track pending operations for proper flush
-    let pendingOperations = 0;
+    let blockedCallback = null;
+    let flushBlocked = null;
+    // Warmup - cached concurrency to reduce Date.now() syscalls
+    let warmupComplete = warmupSeconds <= 0 || maxConcurrency <= 1;
+    let concurrency = warmupComplete ? maxConcurrency : 1;
+    let lastWarmupCheck = 0;
     return new Transform({
         objectMode,
         readableHighWaterMark: highWaterMark,
         writableHighWaterMark: highWaterMark,
         async transform(chunk, _, cb) {
             // Initialize start time on first item
-            if (startTime === 0) {
-                startTime = Date.now();
+            if (started === 0) {
+                started = Date.now();
+                lastWarmupCheck = started;
             }
-            // Stop processing if isSettled
             if (isSettled)
                 return cb();
             const currentIndex = ++index;
-            const currentConcurrency = getCurrentConcurrency();
-            // Wait for a slot if at capacity
-            if (inFlight >= currentConcurrency) {
-                const waiter = pDefer();
-                waiters.push(waiter);
-                await waiter;
+            inFlight++;
+            if (!warmupComplete) {
+                updateConcurrency();
+            }
+            // Apply backpressure if at capacity, otherwise request more input
+            if (inFlight < concurrency) {
+                cb();
             }
             else {
-                inFlight++;
+                blockedCallback = cb;
             }
-            // Signal that we're ready for more input
-            cb();
-            // Track this operation
-            pendingOperations++;
-            // Process the item asynchronously
             try {
                 const res = await mapper(chunk, currentIndex);
-                if (isSettled) {
-                    release();
-                    pendingOperations--;
+                if (isSettled)
                     return;
-                }
                 if (res === END) {
                     isSettled = true;
                     logger.log(`transformMap2 END received at index ${currentIndex}`);
                     _assert(signal, 'signal is required when using END');
                     signal.abort(new Error(PIPELINE_GRACEFUL_ABORT));
-                    release();
-                    pendingOperations--;
                     return;
                 }
-                if (res === SKIP) {
-                    release();
-                    pendingOperations--;
+                if (res === SKIP)
                     return;
-                }
                 let shouldPush = true;
                 if (predicate) {
                     shouldPush = predicate(res, currentIndex);
@@ -104,31 +95,33 @@ export function transformMap2(mapper, opt = {}) {
                 if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
                     isSettled = true;
                     ok = false;
-                    // Call onDone before destroying, since flush won't be called
                     await callOnDone();
                     this.destroy(_anyToError(err));
+                    return;
                 }
-                else if (errorMode === ErrorMode.THROW_AGGREGATED) {
+                if (errorMode === ErrorMode.THROW_AGGREGATED) {
                     collectedErrors.push(_anyToError(err));
                 }
             }
             finally {
-                release();
-                pendingOperations--;
+                inFlight--;
+                // Release blocked callback if we now have capacity
+                if (blockedCallback && inFlight < concurrency) {
+                    const pendingCb = blockedCallback;
+                    blockedCallback = null;
+                    pendingCb();
+                }
+                // Trigger flush completion if all done
+                if (inFlight === 0 && flushBlocked) {
+                    flushBlocked.resolve();
+                }
             }
         },
         async flush(cb) {
-            // Wait for all pending operations to complete
-            // Polling is simple and race-condition-free
-            // Timeout prevents infinite loop if something goes wrong
-            const flushStart = Date.now();
-            const flushTimeoutMs = 60_000;
-            while (pendingOperations > 0) {
-                await new Promise(resolve => setImmediate(resolve));
-                if (Date.now() - flushStart > flushTimeoutMs) {
-                    logger.error(`transformMap2 flush timeout: ${pendingOperations} operations still pending after ${flushTimeoutMs}ms`);
-                    break;
-                }
+            // Wait for all in-flight operations to complete
+            if (inFlight > 0) {
+                flushBlocked = pDefer();
+                await flushBlocked;
             }
             logErrorStats(true);
             await callOnDone();
@@ -140,25 +133,20 @@ export function transformMap2(mapper, opt = {}) {
             }
         },
     });
-    function getCurrentConcurrency() {
-        if (warmupComplete)
-            return concurrency;
-        const elapsed = Date.now() - startTime;
+    function updateConcurrency() {
+        const now = Date.now();
+        if (now - lastWarmupCheck < WARMUP_CHECK_INTERVAL_MS)
+            return;
+        lastWarmupCheck = now;
+        const elapsed = now - started;
         if (elapsed >= warmupMs) {
             warmupComplete = true;
-            logger.debug('warmup complete');
-            return concurrency;
+            concurrency = maxConcurrency;
+            logger.log(`transformMap2: warmup complete in ${_since(started)}`);
+            return;
         }
         const progress = elapsed / warmupMs;
-        return Math.max(1, Math.floor(1 + (concurrency - 1) * progress));
-    }
-    function release() {
-        inFlight--;
-        const currentConcurrency = getCurrentConcurrency();
-        while (waiters.length && inFlight < currentConcurrency) {
-            inFlight++;
-            waiters.shift().resolve();
-        }
+        concurrency = Math.max(1, Math.floor(1 + (maxConcurrency - 1) * progress));
     }
     function logErrorStats(final = false) {
         if (!errors)

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.69.0",
+  "version": "15.70.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@types/js-yaml": "^4",

package/src/stream/pipeline.ts

@@ -185,7 +185,7 @@ export class Pipeline<T = unknown> {
     return this
   }
 
-  map<TO>(
+  mapLegacy<TO>(
     mapper: AbortableAsyncMapper<T, TO | typeof SKIP | typeof END>,
     opt?: TransformMapOptions<T, TO>,
   ): Pipeline<TO> {
@@ -198,10 +198,7 @@ export class Pipeline<T = unknown> {
     return this as any
   }
 
-  /**
-   * @experimental if proven to be stable - will replace transformMap
-   */
-  map2<TO>(
+  map<TO>(
     mapper: AbortableAsyncMapper<T, TO | typeof SKIP | typeof END>,
     opt?: TransformMap2Options<T, TO>,
   ): Pipeline<TO> {
@@ -234,7 +231,7 @@ export class Pipeline<T = unknown> {
 
   filter(asyncPredicate: AsyncPredicate<T>, opt?: TransformMapOptions): this {
     this.transforms.push(
-      transformMap(v => v, {
+      transformMap2(v => v, {
         asyncPredicate,
         ...opt,
         signal: this.abortableSignal,
@@ -433,7 +430,7 @@ export class Pipeline<T = unknown> {
     opt: TransformMapOptions<T, void> & TransformLogProgressOptions<T> = {},
   ): Promise<void> {
     this.transforms.push(
-      transformMap(fn, {
+      transformMap2(fn, {
         predicate: opt.logEvery ? _passthroughPredicate : undefined, // for the logger to work
         ...opt,
         signal: this.abortableSignal,

package/src/stream/transform/transformFilter.ts

@@ -2,7 +2,7 @@ import { Transform } from 'node:stream'
 import type { AsyncPredicate, Predicate } from '@naturalcycles/js-lib/types'
 import type { TransformOptions, TransformTyped } from '../stream.model.js'
 import type { TransformMapOptions } from './transformMap.js'
-import { transformMap } from './transformMap.js'
+import { transformMap2 } from './transformMap2.js'
 
 /**
  * Just a convenience wrapper around `transformMap` that has built-in predicate filtering support.
@@ -11,7 +11,7 @@ export function transformFilter<IN = any>(
   asyncPredicate: AsyncPredicate<IN>,
   opt: TransformMapOptions = {},
 ): TransformTyped<IN, IN> {
-  return transformMap(v => v, {
+  return transformMap2(v => v, {
     asyncPredicate,
     ...opt,
   })

package/src/stream/transform/transformMap2.ts

@@ -1,5 +1,6 @@
 import { Transform } from 'node:stream'
 import type { AbortableSignal } from '@naturalcycles/js-lib'
+import { _since } from '@naturalcycles/js-lib/datetime'
 import { _anyToError, _assert, ErrorMode } from '@naturalcycles/js-lib/error'
 import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
 import type { DeferredPromise } from '@naturalcycles/js-lib/promise'
@@ -83,6 +84,8 @@ export interface TransformMap2Options<IN = any, OUT = IN> extends TransformOptio
   signal?: AbortableSignal
 }
 
+const WARMUP_CHECK_INTERVAL_MS = 1000
+
 /**
  * Like transformMap, but with native concurrency control (no through2-concurrent dependency)
  * and support for gradual warmup.
@@ -94,7 +97,7 @@ export function transformMap2<IN = any, OUT = IN>(
   opt: TransformMap2Options<IN, OUT> = {},
 ): TransformTyped<IN, OUT> {
   const {
-    concurrency = 16,
+    concurrency: maxConcurrency = 16,
     warmupSeconds = 0,
     predicate,
     asyncPredicate,
@@ -111,7 +114,7 @@ export function transformMap2<IN = any, OUT = IN>(
   const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
 
   // Stats
-  const started = Date.now() as UnixTimestampMillis
+  let started = 0 as UnixTimestampMillis
   let index = -1
   let countOut = 0
   let isSettled = false
@@ -119,14 +122,15 @@ export function transformMap2<IN = any, OUT = IN>(
   let errors = 0
   const collectedErrors: Error[] = []
 
-  // Concurrency control
-  let startTime = 0
-  let warmupComplete = warmupSeconds <= 0 || concurrency <= 1
+  // Concurrency control - single counter, single callback for backpressure
   let inFlight = 0
-  const waiters: DeferredPromise[] = []
+  let blockedCallback: (() => void) | null = null
+  let flushBlocked: DeferredPromise | null = null
 
-  // Track pending operations for proper flush
-  let pendingOperations = 0
+  // Warmup - cached concurrency to reduce Date.now() syscalls
+  let warmupComplete = warmupSeconds <= 0 || maxConcurrency <= 1
+  let concurrency = warmupComplete ? maxConcurrency : 1
+  let lastWarmupCheck = 0
 
   return new Transform({
     objectMode,
@@ -134,56 +138,40 @@ export function transformMap2<IN = any, OUT = IN>(
     writableHighWaterMark: highWaterMark,
     async transform(this: Transform, chunk: IN, _, cb) {
       // Initialize start time on first item
-      if (startTime === 0) {
-        startTime = Date.now()
+      if (started === 0) {
+        started = Date.now() as UnixTimestampMillis
+        lastWarmupCheck = started
       }
 
-      // Stop processing if isSettled
       if (isSettled) return cb()
 
       const currentIndex = ++index
-      const currentConcurrency = getCurrentConcurrency()
+      inFlight++
+      if (!warmupComplete) {
+        updateConcurrency()
+      }
 
-      // Wait for a slot if at capacity
-      if (inFlight >= currentConcurrency) {
-        const waiter = pDefer()
-        waiters.push(waiter)
-        await waiter
+      // Apply backpressure if at capacity, otherwise request more input
+      if (inFlight < concurrency) {
+        cb()
       } else {
-        inFlight++
+        blockedCallback = cb
       }
 
-      // Signal that we're ready for more input
-      cb()
-
-      // Track this operation
-      pendingOperations++
-
-      // Process the item asynchronously
       try {
         const res: OUT | typeof SKIP | typeof END = await mapper(chunk, currentIndex)
 
-        if (isSettled) {
-          release()
-          pendingOperations--
-          return
-        }
+        if (isSettled) return
 
         if (res === END) {
           isSettled = true
           logger.log(`transformMap2 END received at index ${currentIndex}`)
           _assert(signal, 'signal is required when using END')
           signal.abort(new Error(PIPELINE_GRACEFUL_ABORT))
-          release()
-          pendingOperations--
           return
         }
 
-        if (res === SKIP) {
-          release()
-          pendingOperations--
-          return
-        }
+        if (res === SKIP) return
 
         let shouldPush = true
         if (predicate) {
@@ -210,31 +198,34 @@ export function transformMap2<IN = any, OUT = IN>(
         if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
           isSettled = true
           ok = false
-          // Call onDone before destroying, since flush won't be called
           await callOnDone()
           this.destroy(_anyToError(err))
-        } else if (errorMode === ErrorMode.THROW_AGGREGATED) {
+          return
+        }
+        if (errorMode === ErrorMode.THROW_AGGREGATED) {
           collectedErrors.push(_anyToError(err))
         }
       } finally {
-        release()
-        pendingOperations--
+        inFlight--
+
+        // Release blocked callback if we now have capacity
+        if (blockedCallback && inFlight < concurrency) {
+          const pendingCb = blockedCallback
+          blockedCallback = null
+          pendingCb()
+        }
+
+        // Trigger flush completion if all done
+        if (inFlight === 0 && flushBlocked) {
+          flushBlocked.resolve()
+        }
       }
     },
     async flush(cb) {
-      // Wait for all pending operations to complete
-      // Polling is simple and race-condition-free
-      // Timeout prevents infinite loop if something goes wrong
-      const flushStart = Date.now()
-      const flushTimeoutMs = 60_000
-      while (pendingOperations > 0) {
-        await new Promise(resolve => setImmediate(resolve))
-        if (Date.now() - flushStart > flushTimeoutMs) {
-          logger.error(
-            `transformMap2 flush timeout: ${pendingOperations} operations still pending after ${flushTimeoutMs}ms`,
-          )
-          break
-        }
+      // Wait for all in-flight operations to complete
+      if (inFlight > 0) {
+        flushBlocked = pDefer()
+        await flushBlocked
       }
 
       logErrorStats(true)
@@ -253,27 +244,21 @@ export function transformMap2<IN = any, OUT = IN>(
     },
   })
 
-  function getCurrentConcurrency(): number {
-    if (warmupComplete) return concurrency
+  function updateConcurrency(): void {
+    const now = Date.now()
+    if (now - lastWarmupCheck < WARMUP_CHECK_INTERVAL_MS) return
+    lastWarmupCheck = now
 
-    const elapsed = Date.now() - startTime
+    const elapsed = now - started
     if (elapsed >= warmupMs) {
       warmupComplete = true
-      logger.debug('warmup complete')
-      return concurrency
+      concurrency = maxConcurrency
+      logger.log(`transformMap2: warmup complete in ${_since(started)}`)
+      return
     }
 
     const progress = elapsed / warmupMs
-    return Math.max(1, Math.floor(1 + (concurrency - 1) * progress))
-  }
-
-  function release(): void {
-    inFlight--
-    const currentConcurrency = getCurrentConcurrency()
-    while (waiters.length && inFlight < currentConcurrency) {
-      inFlight++
-      waiters.shift()!.resolve()
-    }
+    concurrency = Math.max(1, Math.floor(1 + (maxConcurrency - 1) * progress))
   }
 
   function logErrorStats(final = false): void {