@naturalcycles/nodejs-lib 15.27.3 → 15.29.0

package/dist/stream/index.d.ts

@@ -5,8 +5,8 @@ export * from './ndjson/transformJsonParse.js';
 export * from './ndjson/transformToNDJson.js';
 export * from './pipeline.js';
 export * from './progressLogger.js';
+export * from './readable/createReadable.js';
 export * from './readable/readableCombined.js';
-export * from './readable/readableCreate.js';
 export * from './readable/readableFromArray.js';
 export * from './stream.model.js';
 export * from './transform/transformChunk.js';

package/dist/stream/index.js

@@ -5,8 +5,8 @@ export * from './ndjson/transformJsonParse.js';
 export * from './ndjson/transformToNDJson.js';
 export * from './pipeline.js';
 export * from './progressLogger.js';
+export * from './readable/createReadable.js';
 export * from './readable/readableCombined.js';
-export * from './readable/readableCreate.js';
 export * from './readable/readableFromArray.js';
 export * from './stream.model.js';
 export * from './transform/transformChunk.js';

package/dist/stream/pipeline.d.ts

@@ -1,16 +1,15 @@
 import { type Transform } from 'node:stream';
 import type { ReadableStream as WebReadableStream } from 'node:stream/web';
 import { type ZlibOptions } from 'node:zlib';
-import type { AbortableAsyncMapper, AsyncIndexedMapper, AsyncPredicate, END, IndexedMapper, NonNegativeInteger, PositiveInteger, Predicate, SKIP } from '@naturalcycles/js-lib/types';
+import { type AbortableAsyncMapper, type AsyncIndexedMapper, type AsyncPredicate, type END, type IndexedMapper, type NonNegativeInteger, type PositiveInteger, type Predicate, type SKIP } from '@naturalcycles/js-lib/types';
 import type { ReadableTyped, TransformOptions, TransformTyped, WritableTyped } from './stream.model.js';
 import { type TransformLogProgressOptions } from './transform/transformLogProgress.js';
 import { type TransformMapOptions } from './transform/transformMap.js';
 import { type TransformMapSimpleOptions } from './transform/transformMapSimple.js';
 import { type TransformMapSyncOptions } from './transform/transformMapSync.js';
 import { type TransformOffsetOptions } from './transform/transformOffset.js';
-import { type TransformTapOptions } from './transform/transformTap.js';
 import { type TransformThrottleOptions } from './transform/transformThrottle.js';
-export declare class Pipeline<T> {
+export declare class Pipeline<T = unknown> {
     private readonly source;
     private transforms;
     private destination?;
@@ -19,6 +18,12 @@ export declare class Pipeline<T> {
     private abortableSignal;
     private constructor();
     static from<T>(source: ReadableTyped<T>): Pipeline<T>;
+    /**
+     * Useful in cases when Readable is not immediately available,
+     * but only available after an async operation is completed.
+     * Implemented via a proxy Transform, which should be transparent.
+     */
+    static fromAsyncReadable<T = unknown>(fn: () => Promise<ReadableTyped<T>>): Pipeline<T>;
     static fromWeb<T>(webReadableStream: WebReadableStream<T>): Pipeline<T>;
     /**
      * Technically same as `fromIterable` (since Array is Iterable),
@@ -58,10 +63,11 @@ export declare class Pipeline<T> {
     map<TO>(mapper: AbortableAsyncMapper<T, TO | typeof SKIP | typeof END>, opt?: TransformMapOptions<T, TO>): Pipeline<TO>;
     mapSync<TO>(mapper: IndexedMapper<T, TO | typeof SKIP | typeof END>, opt?: TransformMapSyncOptions): Pipeline<TO>;
     mapSimple<TO>(mapper: IndexedMapper<T, TO>, opt?: TransformMapSimpleOptions): Pipeline<TO>;
-    filter(predicate: AsyncPredicate<T>, opt?: TransformMapOptions): this;
+    filter(asyncPredicate: AsyncPredicate<T>, opt?: TransformMapOptions): this;
     filterSync(predicate: Predicate<T>, opt?: TransformOptions): this;
     offset(opt: TransformOffsetOptions): this;
-    tap(fn: AsyncIndexedMapper<T, any>, opt?: TransformTapOptions): this;
+    tap(fn: AsyncIndexedMapper<T, any>, opt?: TransformOptions): this;
+    tapSync(fn: IndexedMapper<T, any>, opt?: TransformOptions): this;
     throttle(opt: TransformThrottleOptions): this;
     transform<TO>(transform: TransformTyped<T, TO>): Pipeline<TO>;
     /**
@@ -90,7 +96,7 @@ export declare class Pipeline<T> {
     toFile(outputFilePath: string): Promise<void>;
     toNDJsonFile(outputFilePath: string): Promise<void>;
     to(destination: WritableTyped<T>): Promise<void>;
-    forEach(fn: AsyncIndexedMapper<T, void>, opt?: TransformMapOptions<T, void>): Promise<void>;
-    forEachSync(fn: IndexedMapper<T, void>, opt?: TransformMapSyncOptions<T, void>): Promise<void>;
+    forEach(fn: AsyncIndexedMapper<T, void>, opt?: TransformMapOptions<T, void> & TransformLogProgressOptions<T>): Promise<void>;
+    forEachSync(fn: IndexedMapper<T, void>, opt?: TransformMapSyncOptions<T, void> & TransformLogProgressOptions<T>): Promise<void>;
     run(): Promise<void>;
 }

package/dist/stream/pipeline.js

@@ -3,10 +3,12 @@ import { pipeline } from 'node:stream/promises';
 import { createUnzip } from 'node:zlib';
 import { createGzip } from 'node:zlib';
 import { createAbortableSignal } from '@naturalcycles/js-lib';
+import { _passthroughPredicate, } from '@naturalcycles/js-lib/types';
 import { fs2 } from '../fs/fs2.js';
 import { createReadStreamAsNDJson } from './ndjson/createReadStreamAsNDJson.js';
 import { transformJsonParse } from './ndjson/transformJsonParse.js';
 import { transformToNDJson } from './ndjson/transformToNDJson.js';
+import { createReadableFromAsync } from './readable/createReadable.js';
 import { PIPELINE_GRACEFUL_ABORT } from './stream.util.js';
 import { transformChunk } from './transform/transformChunk.js';
 import { transformFilterSync } from './transform/transformFilter.js';
@@ -19,7 +21,7 @@ import { transformMapSimple, } from './transform/transformMapSimple.js';
 import { transformMapSync } from './transform/transformMapSync.js';
 import { transformOffset } from './transform/transformOffset.js';
 import { transformSplitOnNewline } from './transform/transformSplit.js';
-import { transformTap } from './transform/transformTap.js';
+import { transformTap, transformTapSync } from './transform/transformTap.js';
 import { transformThrottle } from './transform/transformThrottle.js';
 import { writablePushToArray } from './writable/writablePushToArray.js';
 import { writableVoid } from './writable/writableVoid.js';
@@ -39,6 +41,14 @@ export class Pipeline {
     static from(source) {
         return new Pipeline(source);
     }
+    /**
+     * Useful in cases when Readable is not immediately available,
+     * but only available after an async operation is completed.
+     * Implemented via a proxy Transform, which should be transparent.
+     */
+    static fromAsyncReadable(fn) {
+        return new Pipeline(createReadableFromAsync(fn));
+    }
     static fromWeb(webReadableStream) {
         return new Pipeline(Readable.fromWeb(webReadableStream));
     }
@@ -132,9 +142,9 @@ export class Pipeline {
         this.transforms.push(transformMapSimple(mapper, opt));
         return this;
     }
-    filter(predicate, opt) {
+    filter(asyncPredicate, opt) {
         this.transforms.push(transformMap(v => v, {
-            predicate,
+            asyncPredicate,
             ...opt,
             signal: this.abortableSignal,
         }));
@@ -152,6 +162,10 @@ export class Pipeline {
         this.transforms.push(transformTap(fn, opt));
         return this;
     }
+    tapSync(fn, opt) {
+        this.transforms.push(transformTapSync(fn, opt));
+        return this;
+    }
     throttle(opt) {
         this.transforms.push(transformThrottle(opt));
         return this;
@@ -257,18 +271,26 @@ export class Pipeline {
         this.destination = destination;
         await this.run();
     }
-    async forEach(fn, opt) {
+    async forEach(fn, opt = {}) {
         this.transforms.push(transformMap(fn, {
+            predicate: opt.logEvery ? _passthroughPredicate : undefined, // for the logger to work
             ...opt,
             signal: this.abortableSignal,
         }));
+        if (opt.logEvery) {
+            this.transforms.push(transformLogProgress(opt));
+        }
         await this.run();
     }
-    async forEachSync(fn, opt) {
+    async forEachSync(fn, opt = {}) {
         this.transforms.push(transformMapSync(fn, {
+            predicate: opt.logEvery ? _passthroughPredicate : undefined, // for the logger to work
             ...opt,
             signal: this.abortableSignal,
         }));
+        if (opt.logEvery) {
+            this.transforms.push(transformLogProgress(opt));
+        }
         await this.run();
     }
     async run() {

package/dist/stream/readable/{readableCreate.d.ts → createReadable.d.ts}

@@ -1,4 +1,4 @@
-import type { ReadableOptions } from 'node:stream';
+import { type ReadableOptions } from 'node:stream';
 import type { ReadableTyped } from '../stream.model.js';
 /**
  * Convenience function to create a Readable that can be pushed into (similar to RxJS Subject).
@@ -12,8 +12,14 @@ import type { ReadableTyped } from '../stream.model.js';
  * if read() will be called AFTER everything was pushed and Readable is closed (by pushing `null`).
  * Beware of it when e.g doing unit testing! Jest prefers to hang (not exit-0).
  */
-export declare function readableCreate<T>(items?: Iterable<T>, opt?: ReadableOptions, onRead?: () => void): ReadableTyped<T>;
+export declare function createReadable<T>(items?: Iterable<T>, opt?: ReadableOptions, onRead?: () => void): ReadableTyped<T>;
 /**
  * Convenience type-safe wrapper around Readable.from() that infers the Type of input.
  */
-export declare function readableFrom<T>(iterable: Iterable<T> | AsyncIterable<T>, opt?: ReadableOptions): ReadableTyped<T>;
+export declare function createReadableFrom<T>(iterable: Iterable<T> | AsyncIterable<T>, opt?: ReadableOptions): ReadableTyped<T>;
+/**
+ * Allows to "create Readable asynchronously".
+ * Implemented via a proxy Transform, which is created (and returned) eagerly,
+ * and later (when source Readable is created) serves as a pass-through proxy.
+ */
+export declare function createReadableFromAsync<T>(fn: () => Promise<ReadableTyped<T>>): ReadableTyped<T>;

package/dist/stream/readable/{readableCreate.js → createReadable.js}

@@ -1,3 +1,4 @@
+import { Transform } from 'node:stream';
 import { Readable } from 'node:stream';
 /**
  * Convenience function to create a Readable that can be pushed into (similar to RxJS Subject).
@@ -11,7 +12,7 @@ import { Readable } from 'node:stream';
  * if read() will be called AFTER everything was pushed and Readable is closed (by pushing `null`).
  * Beware of it when e.g doing unit testing! Jest prefers to hang (not exit-0).
  */
-export function readableCreate(items = [], opt, onRead) {
+export function createReadable(items = [], opt, onRead) {
     const readable = new Readable({
         objectMode: true,
         ...opt,
@@ -27,6 +28,26 @@ export function readableCreate(items = [], opt, onRead) {
 /**
  * Convenience type-safe wrapper around Readable.from() that infers the Type of input.
  */
-export function readableFrom(iterable, opt) {
+export function createReadableFrom(iterable, opt) {
     return Readable.from(iterable, opt);
 }
+/**
+ * Allows to "create Readable asynchronously".
+ * Implemented via a proxy Transform, which is created (and returned) eagerly,
+ * and later (when source Readable is created) serves as a pass-through proxy.
+ */
+export function createReadableFromAsync(fn) {
+    const transform = new Transform({
+        objectMode: true,
+        highWaterMark: 1,
+        transform: (chunk, _encoding, cb) => {
+            cb(null, chunk);
+        },
+    });
+    void fn()
+        .then(readable => {
+        readable.on('error', err => transform.destroy(err)).pipe(transform);
+    })
+        .catch(err => transform.destroy(err));
+    return transform;
+}

package/dist/stream/stream.model.d.ts

@@ -11,11 +11,11 @@ export interface ReadableArrayOptions {
     /** allows destroying the stream if the signal is aborted. */
     signal?: AbortSignal;
 }
-export type ReadableMapper<IN, OUT> = (data: IN, opt?: ReadableSignalOptions) => Promisable<OUT>;
-export type ReadableFlatMapper<IN, OUT> = (data: IN, opt?: ReadableSignalOptions) => Promisable<OUT[]>;
+export type ReadableMapper<IN, OUT = unknown> = (data: IN, opt?: ReadableSignalOptions) => Promisable<OUT>;
+export type ReadableFlatMapper<IN, OUT = unknown> = (data: IN, opt?: ReadableSignalOptions) => Promisable<OUT[]>;
 export type ReadableVoidMapper<IN> = (data: IN, opt?: ReadableSignalOptions) => void | Promise<void>;
 export type ReadablePredicate<IN> = (data: IN, opt?: ReadableSignalOptions) => boolean | Promise<boolean>;
-export interface ReadableTyped<T> extends Readable {
+export interface ReadableTyped<T = unknown> extends Readable {
     toArray: (opt?: ReadableSignalOptions) => Promise<T[]>;
     map: <OUT>(mapper: ReadableMapper<T, OUT>, opt?: ReadableArrayOptions) => ReadableTyped<OUT>;
     flatMap: <OUT>(mapper: ReadableFlatMapper<T, OUT>, opt?: ReadableArrayOptions) => ReadableTyped<OUT>;
@@ -26,17 +26,7 @@ export interface ReadableTyped<T> extends Readable {
 }
 export interface WritableTyped<T> extends Writable {
 }
-/**
- * Type alias that indicates that the Readable is not in objectMode,
- * e.g returns a binary stream (like a gzip stream).
- */
-export type ReadableBinary = Readable;
-/**
- * Type alias that indicates that the Writable is not in objectMode,
- * e.g reads a binary stream (like a gzip stream).
- */
-export type WritableBinary = Writable;
-export interface TransformTyped<IN, OUT> extends Transform {
+export interface TransformTyped<IN = unknown, OUT = unknown> extends Transform {
 }
 export interface TransformOptions {
     /**

package/dist/stream/transform/transformFilter.d.ts

@@ -4,7 +4,7 @@ import type { TransformMapOptions } from './transformMap.js';
 /**
  * Just a convenience wrapper around `transformMap` that has built-in predicate filtering support.
  */
-export declare function transformFilter<IN = any>(predicate: AsyncPredicate<IN>, opt?: TransformMapOptions): TransformTyped<IN, IN>;
+export declare function transformFilter<IN = any>(asyncPredicate: AsyncPredicate<IN>, opt?: TransformMapOptions): TransformTyped<IN, IN>;
 /**
  * Sync version of `transformFilter`
  */

package/dist/stream/transform/transformFilter.js

@@ -3,9 +3,9 @@ import { transformMap } from './transformMap.js';
 /**
  * Just a convenience wrapper around `transformMap` that has built-in predicate filtering support.
  */
-export function transformFilter(predicate, opt = {}) {
+export function transformFilter(asyncPredicate, opt = {}) {
     return transformMap(v => v, {
-        predicate,
+        asyncPredicate,
         ...opt,
     });
 }

package/dist/stream/transform/transformFork.js

@@ -2,7 +2,7 @@ import { Transform } from 'node:stream';
 import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
 import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
 import { Pipeline } from '../pipeline.js';
-import { readableCreate } from '../readable/readableCreate.js';
+import { createReadable } from '../readable/createReadable.js';
 /**
  * Allows to "fork" away from the "main pipeline" into the "forked pipeline".
  *
@@ -14,12 +14,12 @@ export function transformFork(fn, opt = {}) {
     const { objectMode = true, highWaterMark } = opt;
     const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
     let lock;
-    const fork = readableCreate([], {}, () => {
+    const fork = createReadable([], {}, () => {
         // `_read` is called
         if (!lock)
             return;
         // We had a lock - let's Resume
-        logger.log(`TransformFork: resume`);
+        logger.debug(`TransformFork: resume`);
         const lockCopy = lock;
         lock = undefined;
         lockCopy.resolve();
@@ -45,7 +45,7 @@ export function transformFork(fn, opt = {}) {
             if (!shouldContinue && !lock) {
                 // Forked pipeline indicates that we should Pause
                 lock = pDefer();
-                logger.log(`TransformFork: pause`);
+                logger.debug(`TransformFork: pause`);
             }
             // acknowledge that we've finished processing the input chunk
             cb();

package/dist/stream/transform/transformLogProgress.js

@@ -4,7 +4,7 @@ import { progressLogger } from '../progressLogger.js';
  * Pass-through transform that optionally logs progress.
  */
 export function transformLogProgress(opt = {}) {
-    const { objectMode = true, highWaterMark } = opt;
+    const { objectMode = true, highWaterMark = 1 } = opt;
     const progress = progressLogger(opt);
     return new Transform({
         objectMode,

package/dist/stream/transform/transformMap.d.ts

@@ -1,6 +1,6 @@
 import { type AbortableSignal } from '@naturalcycles/js-lib';
 import { ErrorMode } from '@naturalcycles/js-lib/error';
-import { type AbortableAsyncMapper, type AsyncPredicate, END, type PositiveInteger, type Promisable, SKIP, type StringMap, type UnixTimestampMillis } from '@naturalcycles/js-lib/types';
+import { type AbortableAsyncMapper, type AsyncPredicate, END, type PositiveInteger, type Predicate, type Promisable, SKIP, type StringMap, type UnixTimestampMillis } from '@naturalcycles/js-lib/types';
 import type { TransformOptions, TransformTyped } from '../stream.model.js';
 export interface TransformMapOptions<IN = any, OUT = IN> extends TransformOptions {
     /**
@@ -10,7 +10,8 @@ export interface TransformMapOptions<IN = any, OUT = IN> extends TransformOption
      * Defaults to "pass everything" (including null, undefined, etc).
      * Simpler way to exclude certain cases is to return SKIP symbol from the mapper.
      */
-    predicate?: AsyncPredicate<OUT>;
+    predicate?: Predicate<OUT>;
+    asyncPredicate?: AsyncPredicate<OUT>;
     /**
      * Number of concurrently pending promises returned by `mapper`.
      *

package/dist/stream/transform/transformMap.js

@@ -23,7 +23,7 @@ import { PIPELINE_GRACEFUL_ABORT } from '../stream.util.js';
  */
 export function transformMap(mapper, opt = {}) {
     const { concurrency = 16, highWaterMark = 64, predicate, // we now default to "no predicate" (meaning pass-everything)
-    errorMode = ErrorMode.THROW_IMMEDIATELY, onError, onDone, metric = 'stream', signal, } = opt;
+    asyncPredicate, errorMode = ErrorMode.THROW_IMMEDIATELY, onError, onDone, metric = 'stream', signal, } = opt;
     const started = Date.now();
     let index = -1;
     let countOut = 0;
@@ -94,8 +94,20 @@ export function transformMap(mapper, opt = {}) {
                 // do nothing, don't push
                 return cb();
             }
-            if (!predicate || ((await predicate(res, currentIndex)) && !isSettled)) {
-                // isSettled could have happened in parallel, hence the extra check
+            if (predicate) {
+                if (predicate(res, currentIndex)) {
+                    countOut++;
+                    this.push(res);
+                }
+            }
+            else if (asyncPredicate) {
+                if ((await asyncPredicate(res, currentIndex)) && !isSettled) {
+                    // isSettled could have happened in parallel, hence the extra check
+                    countOut++;
+                    this.push(res);
+                }
+            }
+            else {
                 countOut++;
                 this.push(res);
             }

package/dist/stream/transform/transformTap.d.ts

@@ -1,13 +1,13 @@
-import type { CommonLogger } from '@naturalcycles/js-lib/log';
-import type { AsyncIndexedMapper } from '@naturalcycles/js-lib/types';
+import type { AsyncIndexedMapper, IndexedMapper } from '@naturalcycles/js-lib/types';
 import type { TransformOptions, TransformTyped } from '../stream.model.js';
-export interface TransformTapOptions extends TransformOptions {
-    logger?: CommonLogger;
-}
 /**
  * Similar to RxJS `tap` - allows to run a function for each stream item, without affecting the result.
  * Item is passed through to the output.
  *
  * Can also act as a counter, since `index` is passed to `fn`
  */
-export declare function transformTap<IN>(fn: AsyncIndexedMapper<IN, any>, opt?: TransformTapOptions): TransformTyped<IN, IN>;
+export declare function transformTap<IN>(fn: AsyncIndexedMapper<IN, any>, opt?: TransformOptions): TransformTyped<IN, IN>;
+/**
+ * Sync version of transformTap
+ */
+export declare function transformTapSync<IN>(fn: IndexedMapper<IN, any>, opt?: TransformOptions): TransformTyped<IN, IN>;

package/dist/stream/transform/transformTap.js

@@ -6,13 +6,12 @@ import { Transform } from 'node:stream';
  * Can also act as a counter, since `index` is passed to `fn`
  */
 export function transformTap(fn, opt = {}) {
-    const { logger = console } = opt;
+    const { logger = console, highWaterMark = 1 } = opt;
     let index = -1;
     return new Transform({
         objectMode: true,
-        ...opt,
+        highWaterMark,
         async transform(chunk, _, cb) {
-            // console.log('tap', chunk)
             try {
                 await fn(chunk, ++index);
             }
@@ -20,7 +19,28 @@ export function transformTap(fn, opt = {}) {
                 logger.error(err);
                 // suppressed error
             }
-            cb(null, chunk); // pass through the item
+            cb(null, chunk);
+        },
+    });
+}
+/**
+ * Sync version of transformTap
+ */
+export function transformTapSync(fn, opt = {}) {
+    const { logger = console, highWaterMark = 1 } = opt;
+    let index = -1;
+    return new Transform({
+        objectMode: true,
+        highWaterMark,
+        transform(chunk, _, cb) {
+            try {
+                fn(chunk, ++index);
+            }
+            catch (err) {
+                logger.error(err);
+                // suppressed error
+            }
+            cb(null, chunk);
         },
     });
 }

package/dist/stream/writable/writableVoid.js

@@ -5,9 +5,10 @@ import { Writable } from 'node:stream';
  * Put it in the end of your pipeline in case it ends with Transform that needs a consumer.
  */
 export function writableVoid(opt = {}) {
+    const { objectMode = true, highWaterMark = 1 } = opt;
     return new Writable({
-        objectMode: true,
-        ...opt,
+        objectMode,
+        highWaterMark,
         write(_chunk, _, cb) {
             cb();
         },

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.27.3",
+  "version": "15.29.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@types/js-yaml": "^4",

package/src/stream/index.ts

@@ -5,8 +5,8 @@ export * from './ndjson/transformJsonParse.js'
 export * from './ndjson/transformToNDJson.js'
 export * from './pipeline.js'
 export * from './progressLogger.js'
+export * from './readable/createReadable.js'
 export * from './readable/readableCombined.js'
-export * from './readable/readableCreate.js'
 export * from './readable/readableFromArray.js'
 export * from './stream.model.js'
 export * from './transform/transformChunk.js'

package/src/stream/pipeline.ts

@@ -4,22 +4,24 @@ import type { ReadableStream as WebReadableStream } from 'node:stream/web'
 import { createUnzip, type ZlibOptions } from 'node:zlib'
 import { createGzip } from 'node:zlib'
 import { createAbortableSignal } from '@naturalcycles/js-lib'
-import type {
-  AbortableAsyncMapper,
-  AsyncIndexedMapper,
-  AsyncPredicate,
-  END,
-  IndexedMapper,
-  Integer,
-  NonNegativeInteger,
-  PositiveInteger,
-  Predicate,
-  SKIP,
+import {
+  _passthroughPredicate,
+  type AbortableAsyncMapper,
+  type AsyncIndexedMapper,
+  type AsyncPredicate,
+  type END,
+  type IndexedMapper,
+  type Integer,
+  type NonNegativeInteger,
+  type PositiveInteger,
+  type Predicate,
+  type SKIP,
 } from '@naturalcycles/js-lib/types'
 import { fs2 } from '../fs/fs2.js'
 import { createReadStreamAsNDJson } from './ndjson/createReadStreamAsNDJson.js'
 import { transformJsonParse } from './ndjson/transformJsonParse.js'
 import { transformToNDJson } from './ndjson/transformToNDJson.js'
+import { createReadableFromAsync } from './readable/createReadable.js'
 import type {
   ReadableTyped,
   TransformOptions,
@@ -44,12 +46,12 @@ import {
 import { transformMapSync, type TransformMapSyncOptions } from './transform/transformMapSync.js'
 import { transformOffset, type TransformOffsetOptions } from './transform/transformOffset.js'
 import { transformSplitOnNewline } from './transform/transformSplit.js'
-import { transformTap, type TransformTapOptions } from './transform/transformTap.js'
+import { transformTap, transformTapSync } from './transform/transformTap.js'
 import { transformThrottle, type TransformThrottleOptions } from './transform/transformThrottle.js'
 import { writablePushToArray } from './writable/writablePushToArray.js'
 import { writableVoid } from './writable/writableVoid.js'
 
-export class Pipeline<T> {
+export class Pipeline<T = unknown> {
   // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
   private readonly source: Readable
   private transforms: NodeJS.ReadWriteStream[] = []
@@ -68,6 +70,15 @@ export class Pipeline<T> {
     return new Pipeline(source)
   }
 
+  /**
+   * Useful in cases when Readable is not immediately available,
+   * but only available after an async operation is completed.
+   * Implemented via a proxy Transform, which should be transparent.
+   */
+  static fromAsyncReadable<T = unknown>(fn: () => Promise<ReadableTyped<T>>): Pipeline<T> {
+    return new Pipeline(createReadableFromAsync(fn))
+  }
+
   static fromWeb<T>(webReadableStream: WebReadableStream<T>): Pipeline<T> {
     return new Pipeline(Readable.fromWeb(webReadableStream))
   }
@@ -190,10 +201,10 @@ export class Pipeline<T> {
     return this as any
   }
 
-  filter(predicate: AsyncPredicate<T>, opt?: TransformMapOptions): this {
+  filter(asyncPredicate: AsyncPredicate<T>, opt?: TransformMapOptions): this {
     this.transforms.push(
       transformMap(v => v, {
-        predicate,
+        asyncPredicate,
         ...opt,
         signal: this.abortableSignal,
       }),
@@ -211,11 +222,16 @@ export class Pipeline<T> {
     return this
   }
 
-  tap(fn: AsyncIndexedMapper<T, any>, opt?: TransformTapOptions): this {
+  tap(fn: AsyncIndexedMapper<T, any>, opt?: TransformOptions): this {
     this.transforms.push(transformTap(fn, opt))
     return this
   }
 
+  tapSync(fn: IndexedMapper<T, any>, opt?: TransformOptions): this {
+    this.transforms.push(transformTapSync(fn, opt))
+    return this
+  }
+
   throttle(opt: TransformThrottleOptions): this {
     this.transforms.push(transformThrottle(opt))
     return this
@@ -347,27 +363,35 @@ export class Pipeline<T> {
 
   async forEach(
     fn: AsyncIndexedMapper<T, void>,
-    opt?: TransformMapOptions<T, void>,
+    opt: TransformMapOptions<T, void> & TransformLogProgressOptions<T> = {},
   ): Promise<void> {
     this.transforms.push(
       transformMap(fn, {
+        predicate: opt.logEvery ? _passthroughPredicate : undefined, // for the logger to work
         ...opt,
         signal: this.abortableSignal,
       }),
     )
+    if (opt.logEvery) {
+      this.transforms.push(transformLogProgress(opt))
+    }
     await this.run()
   }
 
   async forEachSync(
     fn: IndexedMapper<T, void>,
-    opt?: TransformMapSyncOptions<T, void>,
+    opt: TransformMapSyncOptions<T, void> & TransformLogProgressOptions<T> = {},
   ): Promise<void> {
     this.transforms.push(
       transformMapSync(fn, {
+        predicate: opt.logEvery ? _passthroughPredicate : undefined, // for the logger to work
         ...opt,
         signal: this.abortableSignal,
       }),
     )
+    if (opt.logEvery) {
+      this.transforms.push(transformLogProgress(opt))
+    }
     await this.run()
   }
 

package/src/stream/readable/{readableCreate.ts → createReadable.ts}

@@ -1,4 +1,4 @@
-import type { ReadableOptions } from 'node:stream'
+import { type ReadableOptions, Transform } from 'node:stream'
 import { Readable } from 'node:stream'
 import type { ReadableTyped } from '../stream.model.js'
 
@@ -14,7 +14,7 @@ import type { ReadableTyped } from '../stream.model.js'
  * if read() will be called AFTER everything was pushed and Readable is closed (by pushing `null`).
  * Beware of it when e.g doing unit testing! Jest prefers to hang (not exit-0).
  */
-export function readableCreate<T>(
+export function createReadable<T>(
   items: Iterable<T> = [],
   opt?: ReadableOptions,
   onRead?: () => void, // read callback
@@ -35,9 +35,32 @@ export function readableCreate<T>(
 /**
  * Convenience type-safe wrapper around Readable.from() that infers the Type of input.
  */
-export function readableFrom<T>(
+export function createReadableFrom<T>(
   iterable: Iterable<T> | AsyncIterable<T>,
   opt?: ReadableOptions,
 ): ReadableTyped<T> {
   return Readable.from(iterable, opt)
 }
+
+/**
+ * Allows to "create Readable asynchronously".
+ * Implemented via a proxy Transform, which is created (and returned) eagerly,
+ * and later (when source Readable is created) serves as a pass-through proxy.
+ */
+export function createReadableFromAsync<T>(fn: () => Promise<ReadableTyped<T>>): ReadableTyped<T> {
+  const transform = new Transform({
+    objectMode: true,
+    highWaterMark: 1,
+    transform: (chunk, _encoding, cb) => {
+      cb(null, chunk)
+    },
+  })
+
+  void fn()
+    .then(readable => {
+      readable.on('error', err => transform.destroy(err)).pipe(transform)
+    })
+    .catch(err => transform.destroy(err))
+
+  return transform
+}

package/src/stream/stream.model.ts

@@ -14,9 +14,12 @@ export interface ReadableArrayOptions {
   signal?: AbortSignal
 }
 
-export type ReadableMapper<IN, OUT> = (data: IN, opt?: ReadableSignalOptions) => Promisable<OUT>
+export type ReadableMapper<IN, OUT = unknown> = (
+  data: IN,
+  opt?: ReadableSignalOptions,
+) => Promisable<OUT>
 
-export type ReadableFlatMapper<IN, OUT> = (
+export type ReadableFlatMapper<IN, OUT = unknown> = (
   data: IN,
   opt?: ReadableSignalOptions,
 ) => Promisable<OUT[]>
@@ -28,7 +31,7 @@ export type ReadablePredicate<IN> = (
   opt?: ReadableSignalOptions,
 ) => boolean | Promise<boolean>
 
-export interface ReadableTyped<T> extends Readable {
+export interface ReadableTyped<T = unknown> extends Readable {
   toArray: (opt?: ReadableSignalOptions) => Promise<T[]>
 
   map: <OUT>(mapper: ReadableMapper<T, OUT>, opt?: ReadableArrayOptions) => ReadableTyped<OUT>
@@ -49,19 +52,8 @@ export interface ReadableTyped<T> extends Readable {
 // biome-ignore lint/correctness/noUnusedVariables: ok
 export interface WritableTyped<T> extends Writable {}
 
-/**
- * Type alias that indicates that the Readable is not in objectMode,
- * e.g returns a binary stream (like a gzip stream).
- */
-export type ReadableBinary = Readable
-/**
- * Type alias that indicates that the Writable is not in objectMode,
- * e.g reads a binary stream (like a gzip stream).
- */
-export type WritableBinary = Writable
-
 // biome-ignore lint/correctness/noUnusedVariables: ok
-export interface TransformTyped<IN, OUT> extends Transform {}
+export interface TransformTyped<IN = unknown, OUT = unknown> extends Transform {}
 
 export interface TransformOptions {
   /**

package/src/stream/transform/transformFilter.ts

@@ -8,11 +8,11 @@ import { transformMap } from './transformMap.js'
  * Just a convenience wrapper around `transformMap` that has built-in predicate filtering support.
  */
 export function transformFilter<IN = any>(
-  predicate: AsyncPredicate<IN>,
+  asyncPredicate: AsyncPredicate<IN>,
   opt: TransformMapOptions = {},
 ): TransformTyped<IN, IN> {
   return transformMap(v => v, {
-    predicate,
+    asyncPredicate,
     ...opt,
   })
 }

package/src/stream/transform/transformFork.ts

@@ -2,7 +2,7 @@ import { Transform } from 'node:stream'
 import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log'
 import { type DeferredPromise, pDefer } from '@naturalcycles/js-lib/promise/pDefer.js'
 import { Pipeline } from '../pipeline.js'
-import { readableCreate } from '../readable/readableCreate.js'
+import { createReadable } from '../readable/createReadable.js'
 import type { TransformOptions, TransformTyped } from '../stream.model.js'
 
 /**
@@ -21,11 +21,11 @@ export function transformFork<T>(
 
   let lock: DeferredPromise | undefined
 
-  const fork = readableCreate<T>([], {}, () => {
+  const fork = createReadable<T>([], {}, () => {
     // `_read` is called
     if (!lock) return
     // We had a lock - let's Resume
-    logger.log(`TransformFork: resume`)
+    logger.debug(`TransformFork: resume`)
     const lockCopy = lock
     lock = undefined
     lockCopy.resolve()
@@ -54,7 +54,7 @@ export function transformFork<T>(
       if (!shouldContinue && !lock) {
         // Forked pipeline indicates that we should Pause
         lock = pDefer()
-        logger.log(`TransformFork: pause`)
+        logger.debug(`TransformFork: pause`)
       }
 
       // acknowledge that we've finished processing the input chunk

package/src/stream/transform/transformLogProgress.ts

@@ -13,7 +13,7 @@ export interface TransformLogProgressOptions<IN = any>
 export function transformLogProgress<IN = any>(
   opt: TransformLogProgressOptions = {},
 ): TransformTyped<IN, IN> {
-  const { objectMode = true, highWaterMark } = opt
+  const { objectMode = true, highWaterMark = 1 } = opt
   const progress = progressLogger(opt)
 
   return new Transform({

package/src/stream/transform/transformMap.ts

@@ -8,6 +8,7 @@ import {
   type AsyncPredicate,
   END,
   type PositiveInteger,
+  type Predicate,
   type Promisable,
   SKIP,
   type StringMap,
@@ -26,7 +27,9 @@ export interface TransformMapOptions<IN = any, OUT = IN> extends TransformOption
    * Defaults to "pass everything" (including null, undefined, etc).
    * Simpler way to exclude certain cases is to return SKIP symbol from the mapper.
    */
-  predicate?: AsyncPredicate<OUT>
+  predicate?: Predicate<OUT>
+
+  asyncPredicate?: AsyncPredicate<OUT>
 
   /**
    * Number of concurrently pending promises returned by `mapper`.
@@ -137,6 +140,7 @@ export function transformMap<IN = any, OUT = IN>(
     concurrency = 16,
     highWaterMark = 64,
     predicate, // we now default to "no predicate" (meaning pass-everything)
+    asyncPredicate,
     errorMode = ErrorMode.THROW_IMMEDIATELY,
     onError,
     onDone,
@@ -226,8 +230,18 @@ export function transformMap<IN = any, OUT = IN>(
           return cb()
         }
 
-        if (!predicate || ((await predicate(res, currentIndex)) && !isSettled)) {
-          // isSettled could have happened in parallel, hence the extra check
+        if (predicate) {
+          if (predicate(res, currentIndex)) {
+            countOut++
+            this.push(res)
+          }
+        } else if (asyncPredicate) {
+          if ((await asyncPredicate(res, currentIndex)) && !isSettled) {
+            // isSettled could have happened in parallel, hence the extra check
+            countOut++
+            this.push(res)
+          }
+        } else {
           countOut++
           this.push(res)
         }

package/src/stream/transform/transformTap.ts

@@ -1,12 +1,7 @@
 import { Transform } from 'node:stream'
-import type { CommonLogger } from '@naturalcycles/js-lib/log'
-import type { AsyncIndexedMapper } from '@naturalcycles/js-lib/types'
+import type { AsyncIndexedMapper, IndexedMapper } from '@naturalcycles/js-lib/types'
 import type { TransformOptions, TransformTyped } from '../stream.model.js'
 
-export interface TransformTapOptions extends TransformOptions {
-  logger?: CommonLogger
-}
-
 /**
  * Similar to RxJS `tap` - allows to run a function for each stream item, without affecting the result.
  * Item is passed through to the output.
@@ -15,17 +10,15 @@ export interface TransformTapOptions extends TransformOptions {
  */
 export function transformTap<IN>(
   fn: AsyncIndexedMapper<IN, any>,
-  opt: TransformTapOptions = {},
+  opt: TransformOptions = {},
 ): TransformTyped<IN, IN> {
-  const { logger = console } = opt
+  const { logger = console, highWaterMark = 1 } = opt
   let index = -1
 
   return new Transform({
     objectMode: true,
-    ...opt,
+    highWaterMark,
     async transform(chunk: IN, _, cb) {
-      // console.log('tap', chunk)
-
       try {
         await fn(chunk, ++index)
       } catch (err) {
@@ -33,7 +26,33 @@ export function transformTap<IN>(
         // suppressed error
       }
 
-      cb(null, chunk) // pass through the item
+      cb(null, chunk)
+    },
+  })
+}
+
+/**
+ * Sync version of transformTap
+ */
+export function transformTapSync<IN>(
+  fn: IndexedMapper<IN, any>,
+  opt: TransformOptions = {},
+): TransformTyped<IN, IN> {
+  const { logger = console, highWaterMark = 1 } = opt
+  let index = -1
+
+  return new Transform({
+    objectMode: true,
+    highWaterMark,
+    transform(chunk: IN, _, cb) {
+      try {
+        fn(chunk, ++index)
+      } catch (err) {
+        logger.error(err)
+        // suppressed error
+      }
+
+      cb(null, chunk)
     },
   })
 }

package/src/stream/writable/writableVoid.ts

@@ -7,9 +7,10 @@ import type { TransformOptions } from '../stream.model.js'
  * Put it in the end of your pipeline in case it ends with Transform that needs a consumer.
  */
 export function writableVoid(opt: TransformOptions = {}): Writable {
+  const { objectMode = true, highWaterMark = 1 } = opt
   return new Writable({
-    objectMode: true,
-    ...opt,
+    objectMode,
+    highWaterMark,
     write(_chunk, _, cb) {
       cb()
     },