@naturalcycles/nodejs-lib 15.68.0 → 15.69.1

package/dist/stream/pipeline.d.ts

@@ -109,6 +109,7 @@ export declare class Pipeline<T = unknown> {
     toNDJsonFile(outputFilePath: string): Promise<void>;
     to(destination: WritableTyped<T>): Promise<void>;
     forEach(fn: AsyncIndexedMapper<T, void>, opt?: TransformMapOptions<T, void> & TransformLogProgressOptions<T>): Promise<void>;
+    forEach2(fn: AsyncIndexedMapper<T, void>, opt?: TransformMap2Options<T, void> & TransformLogProgressOptions<T>): Promise<void>;
     forEachSync(fn: IndexedMapper<T, void>, opt?: TransformMapSyncOptions<T, void> & TransformLogProgressOptions<T>): Promise<void>;
     run(): Promise<void>;
 }

package/dist/stream/pipeline.js

@@ -324,6 +324,17 @@ export class Pipeline {
         }
         await this.run();
     }
+    async forEach2(fn, opt = {}) {
+        this.transforms.push(transformMap2(fn, {
+            predicate: opt.logEvery ? _passthroughPredicate : undefined, // for the logger to work
+            ...opt,
+            signal: this.abortableSignal,
+        }));
+        if (opt.logEvery) {
+            this.transforms.push(transformLogProgress(opt));
+        }
+        await this.run();
+    }
     async forEachSync(fn, opt = {}) {
         this.transforms.push(transformMapSync(fn, {
             predicate: opt.logEvery ? _passthroughPredicate : undefined, // for the logger to work

package/dist/stream/transform/transformMap2.js

@@ -1,4 +1,5 @@
 import { Transform } from 'node:stream';
+import { _since } from '@naturalcycles/js-lib/datetime';
 import { _anyToError, _assert, ErrorMode } from '@naturalcycles/js-lib/error';
 import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
 import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
@@ -16,7 +17,7 @@ export function transformMap2(mapper, opt = {}) {
     const warmupMs = warmupSeconds * 1000;
     const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
     // Stats
-    const started = Date.now();
+    let started = 0;
     let index = -1;
     let countOut = 0;
     let isSettled = false;
@@ -24,7 +25,6 @@ export function transformMap2(mapper, opt = {}) {
     let errors = 0;
     const collectedErrors = [];
     // Concurrency control
-    let startTime = 0;
     let warmupComplete = warmupSeconds <= 0 || concurrency <= 1;
     let inFlight = 0;
     const waiters = [];
@@ -36,8 +36,8 @@ export function transformMap2(mapper, opt = {}) {
         writableHighWaterMark: highWaterMark,
         async transform(chunk, _, cb) {
             // Initialize start time on first item
-            if (startTime === 0) {
-                startTime = Date.now();
+            if (started === 0) {
+                started = Date.now();
             }
             // Stop processing if isSettled
             if (isSettled)
@@ -143,10 +143,10 @@ export function transformMap2(mapper, opt = {}) {
     function getCurrentConcurrency() {
         if (warmupComplete)
             return concurrency;
-        const elapsed = Date.now() - startTime;
+        const elapsed = Date.now() - started;
         if (elapsed >= warmupMs) {
             warmupComplete = true;
-            logger.debug('warmup complete');
+            logger.log(`transformMap2: warmup complete in ${_since(started)}`);
             return concurrency;
         }
         const progress = elapsed / warmupMs;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.68.0",
+  "version": "15.69.1",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@types/js-yaml": "^4",

package/src/stream/pipeline.ts

@@ -445,6 +445,23 @@ export class Pipeline<T = unknown> {
     await this.run()
   }
 
+  async forEach2(
+    fn: AsyncIndexedMapper<T, void>,
+    opt: TransformMap2Options<T, void> & TransformLogProgressOptions<T> = {},
+  ): Promise<void> {
+    this.transforms.push(
+      transformMap2(fn, {
+        predicate: opt.logEvery ? _passthroughPredicate : undefined, // for the logger to work
+        ...opt,
+        signal: this.abortableSignal,
+      }),
+    )
+    if (opt.logEvery) {
+      this.transforms.push(transformLogProgress(opt))
+    }
+    await this.run()
+  }
+
   async forEachSync(
     fn: IndexedMapper<T, void>,
     opt: TransformMapSyncOptions<T, void> & TransformLogProgressOptions<T> = {},

package/src/stream/transform/transformMap2.ts

@@ -1,5 +1,6 @@
 import { Transform } from 'node:stream'
 import type { AbortableSignal } from '@naturalcycles/js-lib'
+import { _since } from '@naturalcycles/js-lib/datetime'
 import { _anyToError, _assert, ErrorMode } from '@naturalcycles/js-lib/error'
 import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
 import type { DeferredPromise } from '@naturalcycles/js-lib/promise'
@@ -111,7 +112,7 @@ export function transformMap2<IN = any, OUT = IN>(
   const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel)
 
   // Stats
-  const started = Date.now() as UnixTimestampMillis
+  let started = 0 as UnixTimestampMillis
   let index = -1
   let countOut = 0
   let isSettled = false
@@ -120,7 +121,6 @@ export function transformMap2<IN = any, OUT = IN>(
   const collectedErrors: Error[] = []
 
   // Concurrency control
-  let startTime = 0
   let warmupComplete = warmupSeconds <= 0 || concurrency <= 1
   let inFlight = 0
   const waiters: DeferredPromise[] = []
@@ -134,8 +134,8 @@ export function transformMap2<IN = any, OUT = IN>(
     writableHighWaterMark: highWaterMark,
     async transform(this: Transform, chunk: IN, _, cb) {
       // Initialize start time on first item
-      if (startTime === 0) {
-        startTime = Date.now()
+      if (started === 0) {
+        started = Date.now() as UnixTimestampMillis
       }
 
       // Stop processing if isSettled
@@ -256,10 +256,10 @@ export function transformMap2<IN = any, OUT = IN>(
   function getCurrentConcurrency(): number {
     if (warmupComplete) return concurrency
 
-    const elapsed = Date.now() - startTime
+    const elapsed = Date.now() - started
     if (elapsed >= warmupMs) {
       warmupComplete = true
-      logger.debug('warmup complete')
+      logger.log(`transformMap2: warmup complete in ${_since(started)}`)
       return concurrency
     }