@naturalcycles/nodejs-lib 15.28.0 → 15.30.0

package/dist/stream/pipeline.d.ts

@@ -1,7 +1,7 @@
 import { type Transform } from 'node:stream';
 import type { ReadableStream as WebReadableStream } from 'node:stream/web';
 import { type ZlibOptions } from 'node:zlib';
-import type { AbortableAsyncMapper, AsyncIndexedMapper, AsyncPredicate, END, IndexedMapper, NonNegativeInteger, PositiveInteger, Predicate, SKIP } from '@naturalcycles/js-lib/types';
+import { type AbortableAsyncMapper, type AsyncIndexedMapper, type AsyncPredicate, type END, type IndexedMapper, type NonNegativeInteger, type PositiveInteger, type Predicate, type SKIP } from '@naturalcycles/js-lib/types';
 import type { ReadableTyped, TransformOptions, TransformTyped, WritableTyped } from './stream.model.js';
 import { type TransformLogProgressOptions } from './transform/transformLogProgress.js';
 import { type TransformMapOptions } from './transform/transformMap.js';
@@ -63,7 +63,7 @@ export declare class Pipeline<T = unknown> {
     map<TO>(mapper: AbortableAsyncMapper<T, TO | typeof SKIP | typeof END>, opt?: TransformMapOptions<T, TO>): Pipeline<TO>;
     mapSync<TO>(mapper: IndexedMapper<T, TO | typeof SKIP | typeof END>, opt?: TransformMapSyncOptions): Pipeline<TO>;
     mapSimple<TO>(mapper: IndexedMapper<T, TO>, opt?: TransformMapSimpleOptions): Pipeline<TO>;
-    filter(predicate: AsyncPredicate<T>, opt?: TransformMapOptions): this;
+    filter(asyncPredicate: AsyncPredicate<T>, opt?: TransformMapOptions): this;
     filterSync(predicate: Predicate<T>, opt?: TransformOptions): this;
     offset(opt: TransformOffsetOptions): this;
     tap(fn: AsyncIndexedMapper<T, any>, opt?: TransformOptions): this;
@@ -96,7 +96,7 @@ export declare class Pipeline<T = unknown> {
     toFile(outputFilePath: string): Promise<void>;
     toNDJsonFile(outputFilePath: string): Promise<void>;
     to(destination: WritableTyped<T>): Promise<void>;
-    forEach(fn: AsyncIndexedMapper<T, void>, opt?: TransformMapOptions<T, void>): Promise<void>;
-    forEachSync(fn: IndexedMapper<T, void>, opt?: TransformMapSyncOptions<T, void>): Promise<void>;
+    forEach(fn: AsyncIndexedMapper<T, void>, opt?: TransformMapOptions<T, void> & TransformLogProgressOptions<T>): Promise<void>;
+    forEachSync(fn: IndexedMapper<T, void>, opt?: TransformMapSyncOptions<T, void> & TransformLogProgressOptions<T>): Promise<void>;
     run(): Promise<void>;
 }

package/dist/stream/pipeline.js

@@ -3,6 +3,7 @@ import { pipeline } from 'node:stream/promises';
 import { createUnzip } from 'node:zlib';
 import { createGzip } from 'node:zlib';
 import { createAbortableSignal } from '@naturalcycles/js-lib';
+import { _passthroughPredicate, } from '@naturalcycles/js-lib/types';
 import { fs2 } from '../fs/fs2.js';
 import { createReadStreamAsNDJson } from './ndjson/createReadStreamAsNDJson.js';
 import { transformJsonParse } from './ndjson/transformJsonParse.js';
@@ -100,6 +101,11 @@ export class Pipeline {
      * less clean than `.take()` on the source.
      */
     limit(limit) {
+        if (!this.transforms.length) {
+            console.warn(`Pipeline.limit was used as a very first Transfrom - please use Pipeline.limitSource instead`);
+            this.limitSource(limit);
+            return this;
+        }
         this.transforms.push(transformLimit({
             limit,
             signal: this.abortableSignal,
@@ -141,9 +147,9 @@ export class Pipeline {
         this.transforms.push(transformMapSimple(mapper, opt));
         return this;
     }
-    filter(predicate, opt) {
+    filter(asyncPredicate, opt) {
         this.transforms.push(transformMap(v => v, {
-            predicate,
+            asyncPredicate,
             ...opt,
             signal: this.abortableSignal,
         }));
@@ -270,18 +276,26 @@ export class Pipeline {
         this.destination = destination;
         await this.run();
     }
-    async forEach(fn, opt) {
+    async forEach(fn, opt = {}) {
         this.transforms.push(transformMap(fn, {
+            predicate: opt.logEvery ? _passthroughPredicate : undefined, // for the logger to work
             ...opt,
             signal: this.abortableSignal,
         }));
+        if (opt.logEvery) {
+            this.transforms.push(transformLogProgress(opt));
+        }
         await this.run();
     }
-    async forEachSync(fn, opt) {
+    async forEachSync(fn, opt = {}) {
         this.transforms.push(transformMapSync(fn, {
+            predicate: opt.logEvery ? _passthroughPredicate : undefined, // for the logger to work
             ...opt,
             signal: this.abortableSignal,
         }));
+        if (opt.logEvery) {
+            this.transforms.push(transformLogProgress(opt));
+        }
         await this.run();
     }
     async run() {

package/dist/stream/transform/transformFilter.d.ts

@@ -4,7 +4,7 @@ import type { TransformMapOptions } from './transformMap.js';
 /**
  * Just a convenience wrapper around `transformMap` that has built-in predicate filtering support.
  */
-export declare function transformFilter<IN = any>(predicate: AsyncPredicate<IN>, opt?: TransformMapOptions): TransformTyped<IN, IN>;
+export declare function transformFilter<IN = any>(asyncPredicate: AsyncPredicate<IN>, opt?: TransformMapOptions): TransformTyped<IN, IN>;
 /**
  * Sync version of `transformFilter`
  */

package/dist/stream/transform/transformFilter.js

@@ -3,9 +3,9 @@ import { transformMap } from './transformMap.js';
 /**
  * Just a convenience wrapper around `transformMap` that has built-in predicate filtering support.
  */
-export function transformFilter(predicate, opt = {}) {
+export function transformFilter(asyncPredicate, opt = {}) {
     return transformMap(v => v, {
-        predicate,
+        asyncPredicate,
         ...opt,
     });
 }

package/dist/stream/transform/transformLogProgress.js

@@ -4,7 +4,7 @@ import { progressLogger } from '../progressLogger.js';
  * Pass-through transform that optionally logs progress.
  */
 export function transformLogProgress(opt = {}) {
-    const { objectMode = true, highWaterMark } = opt;
+    const { objectMode = true, highWaterMark = 1 } = opt;
     const progress = progressLogger(opt);
     return new Transform({
         objectMode,

package/dist/stream/transform/transformMap.d.ts

@@ -10,7 +10,8 @@ export interface TransformMapOptions<IN = any, OUT = IN> extends TransformOption
      * Defaults to "pass everything" (including null, undefined, etc).
      * Simpler way to exclude certain cases is to return SKIP symbol from the mapper.
      */
-    predicate?: Predicate<OUT> | AsyncPredicate<OUT>;
+    predicate?: Predicate<OUT>;
+    asyncPredicate?: AsyncPredicate<OUT>;
     /**
      * Number of concurrently pending promises returned by `mapper`.
      *

package/dist/stream/transform/transformMap.js

@@ -23,7 +23,7 @@ import { PIPELINE_GRACEFUL_ABORT } from '../stream.util.js';
  */
 export function transformMap(mapper, opt = {}) {
     const { concurrency = 16, highWaterMark = 64, predicate, // we now default to "no predicate" (meaning pass-everything)
-    errorMode = ErrorMode.THROW_IMMEDIATELY, onError, onDone, metric = 'stream', signal, } = opt;
+    asyncPredicate, errorMode = ErrorMode.THROW_IMMEDIATELY, onError, onDone, metric = 'stream', signal, } = opt;
     const started = Date.now();
     let index = -1;
     let countOut = 0;
@@ -94,8 +94,20 @@ export function transformMap(mapper, opt = {}) {
                 // do nothing, don't push
                 return cb();
             }
-            if (!predicate || ((await predicate(res, currentIndex)) && !isSettled)) {
-                // isSettled could have happened in parallel, hence the extra check
+            if (predicate) {
+                if (predicate(res, currentIndex)) {
+                    countOut++;
+                    this.push(res);
+                }
+            }
+            else if (asyncPredicate) {
+                if ((await asyncPredicate(res, currentIndex)) && !isSettled) {
+                    // isSettled could have happened in parallel, hence the extra check
+                    countOut++;
+                    this.push(res);
+                }
+            }
+            else {
                 countOut++;
                 this.push(res);
             }

package/dist/stream/writable/writableVoid.js

@@ -5,9 +5,10 @@ import { Writable } from 'node:stream';
  * Put it in the end of your pipeline in case it ends with Transform that needs a consumer.
  */
 export function writableVoid(opt = {}) {
+    const { objectMode = true, highWaterMark = 1 } = opt;
     return new Writable({
-        objectMode: true,
-        ...opt,
+        objectMode,
+        highWaterMark,
         write(_chunk, _, cb) {
             cb();
         },

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.28.0",
+  "version": "15.30.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@types/js-yaml": "^4",
@@ -23,7 +23,7 @@
   },
   "devDependencies": {
     "@types/through2-concurrent": "^2",
-    "@naturalcycles/dev-lib": "19.37.0"
+    "@naturalcycles/dev-lib": "18.4.2"
   },
   "exports": {
     ".": "./dist/index.js",

package/src/stream/pipeline.ts

@@ -4,17 +4,18 @@ import type { ReadableStream as WebReadableStream } from 'node:stream/web'
 import { createUnzip, type ZlibOptions } from 'node:zlib'
 import { createGzip } from 'node:zlib'
 import { createAbortableSignal } from '@naturalcycles/js-lib'
-import type {
-  AbortableAsyncMapper,
-  AsyncIndexedMapper,
-  AsyncPredicate,
-  END,
-  IndexedMapper,
-  Integer,
-  NonNegativeInteger,
-  PositiveInteger,
-  Predicate,
-  SKIP,
+import {
+  _passthroughPredicate,
+  type AbortableAsyncMapper,
+  type AsyncIndexedMapper,
+  type AsyncPredicate,
+  type END,
+  type IndexedMapper,
+  type Integer,
+  type NonNegativeInteger,
+  type PositiveInteger,
+  type Predicate,
+  type SKIP,
 } from '@naturalcycles/js-lib/types'
 import { fs2 } from '../fs/fs2.js'
 import { createReadStreamAsNDJson } from './ndjson/createReadStreamAsNDJson.js'
@@ -139,6 +140,14 @@ export class Pipeline<T = unknown> {
    * less clean than `.take()` on the source.
    */
   limit(limit: NonNegativeInteger | undefined): this {
+    if (!this.transforms.length) {
+      console.warn(
+        `Pipeline.limit was used as a very first Transfrom - please use Pipeline.limitSource instead`,
+      )
+      this.limitSource(limit)
+      return this
+    }
+
     this.transforms.push(
       transformLimit({
         limit,
@@ -200,10 +209,10 @@ export class Pipeline<T = unknown> {
     return this as any
   }
 
-  filter(predicate: AsyncPredicate<T>, opt?: TransformMapOptions): this {
+  filter(asyncPredicate: AsyncPredicate<T>, opt?: TransformMapOptions): this {
     this.transforms.push(
       transformMap(v => v, {
-        predicate,
+        asyncPredicate,
         ...opt,
         signal: this.abortableSignal,
       }),
@@ -362,27 +371,35 @@ export class Pipeline<T = unknown> {
 
   async forEach(
     fn: AsyncIndexedMapper<T, void>,
-    opt?: TransformMapOptions<T, void>,
+    opt: TransformMapOptions<T, void> & TransformLogProgressOptions<T> = {},
   ): Promise<void> {
     this.transforms.push(
       transformMap(fn, {
+        predicate: opt.logEvery ? _passthroughPredicate : undefined, // for the logger to work
         ...opt,
         signal: this.abortableSignal,
       }),
     )
+    if (opt.logEvery) {
+      this.transforms.push(transformLogProgress(opt))
+    }
     await this.run()
   }
 
   async forEachSync(
     fn: IndexedMapper<T, void>,
-    opt?: TransformMapSyncOptions<T, void>,
+    opt: TransformMapSyncOptions<T, void> & TransformLogProgressOptions<T> = {},
   ): Promise<void> {
     this.transforms.push(
       transformMapSync(fn, {
+        predicate: opt.logEvery ? _passthroughPredicate : undefined, // for the logger to work
         ...opt,
         signal: this.abortableSignal,
       }),
     )
+    if (opt.logEvery) {
+      this.transforms.push(transformLogProgress(opt))
+    }
     await this.run()
   }
 

package/src/stream/transform/transformFilter.ts

@@ -8,11 +8,11 @@ import { transformMap } from './transformMap.js'
  * Just a convenience wrapper around `transformMap` that has built-in predicate filtering support.
  */
 export function transformFilter<IN = any>(
-  predicate: AsyncPredicate<IN>,
+  asyncPredicate: AsyncPredicate<IN>,
   opt: TransformMapOptions = {},
 ): TransformTyped<IN, IN> {
   return transformMap(v => v, {
-    predicate,
+    asyncPredicate,
     ...opt,
   })
 }

package/src/stream/transform/transformLogProgress.ts

@@ -13,7 +13,7 @@ export interface TransformLogProgressOptions<IN = any>
 export function transformLogProgress<IN = any>(
   opt: TransformLogProgressOptions = {},
 ): TransformTyped<IN, IN> {
-  const { objectMode = true, highWaterMark } = opt
+  const { objectMode = true, highWaterMark = 1 } = opt
   const progress = progressLogger(opt)
 
   return new Transform({

package/src/stream/transform/transformMap.ts

@@ -27,7 +27,9 @@ export interface TransformMapOptions<IN = any, OUT = IN> extends TransformOption
    * Defaults to "pass everything" (including null, undefined, etc).
    * Simpler way to exclude certain cases is to return SKIP symbol from the mapper.
    */
-  predicate?: Predicate<OUT> | AsyncPredicate<OUT>
+  predicate?: Predicate<OUT>
+
+  asyncPredicate?: AsyncPredicate<OUT>
 
   /**
    * Number of concurrently pending promises returned by `mapper`.
@@ -138,6 +140,7 @@ export function transformMap<IN = any, OUT = IN>(
     concurrency = 16,
     highWaterMark = 64,
     predicate, // we now default to "no predicate" (meaning pass-everything)
+    asyncPredicate,
     errorMode = ErrorMode.THROW_IMMEDIATELY,
     onError,
     onDone,
@@ -227,8 +230,18 @@ export function transformMap<IN = any, OUT = IN>(
           return cb()
         }
 
-        if (!predicate || ((await predicate(res, currentIndex)) && !isSettled)) {
-          // isSettled could have happened in parallel, hence the extra check
+        if (predicate) {
+          if (predicate(res, currentIndex)) {
+            countOut++
+            this.push(res)
+          }
+        } else if (asyncPredicate) {
+          if ((await asyncPredicate(res, currentIndex)) && !isSettled) {
+            // isSettled could have happened in parallel, hence the extra check
+            countOut++
+            this.push(res)
+          }
+        } else {
           countOut++
           this.push(res)
         }

package/src/stream/writable/writableVoid.ts

@@ -7,9 +7,10 @@ import type { TransformOptions } from '../stream.model.js'
  * Put it in the end of your pipeline in case it ends with Transform that needs a consumer.
  */
 export function writableVoid(opt: TransformOptions = {}): Writable {
+  const { objectMode = true, highWaterMark = 1 } = opt
   return new Writable({
-    objectMode: true,
-    ...opt,
+    objectMode,
+    highWaterMark,
     write(_chunk, _, cb) {
       cb()
     },