@naturalcycles/nodejs-lib 15.25.0 → 15.27.0

package/dist/slack/slack.service.d.ts

@@ -28,6 +28,7 @@ export declare class SlackService<CTX = any> {
      */
     getCommonLogger(opt: {
         minLogLevel: CommonLogLevel;
+        debugChannel?: string;
         logChannel?: string;
         warnChannel?: string;
         errorChannel?: string;

package/dist/slack/slack.service.js

@@ -1,6 +1,6 @@
 import { localTime } from '@naturalcycles/js-lib/datetime/localTime.js';
 import { getFetcher } from '@naturalcycles/js-lib/http';
-import { commonLoggerMinLevel, } from '@naturalcycles/js-lib/log';
+import { createCommonLoggerAtLevel, } from '@naturalcycles/js-lib/log';
 import { _omit } from '@naturalcycles/js-lib/object/object.util.js';
 import { PQueue } from '@naturalcycles/js-lib/promise/pQueue.js';
 import { _inspect } from '../index.js';
@@ -117,12 +117,13 @@ export class SlackService {
      * Returns a CommonLogger implementation based on this SlackService instance.
      */
     getCommonLogger(opt) {
-        const { minLogLevel = 'log', logChannel, warnChannel, errorChannel } = opt;
+        const { minLogLevel = 'log', debugChannel, logChannel, warnChannel, errorChannel } = opt;
         const defaultChannel = this.cfg.defaults?.channel || DEFAULTS.channel;
         const q = new PQueue({
             concurrency: 1,
         });
-        return commonLoggerMinLevel({
+        return createCommonLoggerAtLevel({
+            debug: (...args) => q.push(() => this.send({ items: args, channel: debugChannel || defaultChannel })),
             log: (...args) => q.push(() => this.send({ items: args, channel: logChannel || defaultChannel })),
             warn: (...args) => q.push(() => this.send({ items: args, channel: warnChannel || defaultChannel })),
             error: (...args) => q.push(() => this.send({ items: args, channel: errorChannel || defaultChannel })),

package/dist/stream/index.d.ts

@@ -1,8 +1,6 @@
-export * from './ndjson/createReadStreamAsNDJSON.js';
-export * from './ndjson/createWriteStreamAsNDJSON.js';
+export * from './ndjson/createReadStreamAsNDJson.js';
 export * from './ndjson/ndjson.model.js';
 export * from './ndjson/ndjsonMap.js';
-export * from './ndjson/ndjsonStreamForEach.js';
 export * from './ndjson/transformJsonParse.js';
 export * from './ndjson/transformToNDJson.js';
 export * from './pipeline.js';
@@ -10,11 +8,11 @@ export * from './progressLogger.js';
 export * from './readable/readableCombined.js';
 export * from './readable/readableCreate.js';
 export * from './readable/readableFromArray.js';
-export * from './readable/readableToArray.js';
 export * from './stream.model.js';
 export * from './transform/transformChunk.js';
 export * from './transform/transformFilter.js';
 export * from './transform/transformFlatten.js';
+export * from './transform/transformFork.js';
 export * from './transform/transformLimit.js';
 export * from './transform/transformLogProgress.js';
 export * from './transform/transformMap.js';
@@ -24,13 +22,9 @@ export * from './transform/transformNoOp.js';
 export * from './transform/transformOffset.js';
 export * from './transform/transformSplit.js';
 export * from './transform/transformTap.js';
-export * from './transform/transformTee.js';
 export * from './transform/transformThrottle.js';
-export * from './transform/transformToArray.js';
 export * from './transform/worker/baseWorkerClass.js';
 export * from './transform/worker/transformMultiThreaded.js';
 export * from './transform/worker/transformMultiThreaded.model.js';
-export * from './writable/writableForEach.js';
-export * from './writable/writableFork.js';
 export * from './writable/writablePushToArray.js';
 export * from './writable/writableVoid.js';

package/dist/stream/index.js

@@ -1,8 +1,6 @@
-export * from './ndjson/createReadStreamAsNDJSON.js';
-export * from './ndjson/createWriteStreamAsNDJSON.js';
+export * from './ndjson/createReadStreamAsNDJson.js';
 export * from './ndjson/ndjson.model.js';
 export * from './ndjson/ndjsonMap.js';
-export * from './ndjson/ndjsonStreamForEach.js';
 export * from './ndjson/transformJsonParse.js';
 export * from './ndjson/transformToNDJson.js';
 export * from './pipeline.js';
@@ -10,11 +8,11 @@ export * from './progressLogger.js';
 export * from './readable/readableCombined.js';
 export * from './readable/readableCreate.js';
 export * from './readable/readableFromArray.js';
-export * from './readable/readableToArray.js';
 export * from './stream.model.js';
 export * from './transform/transformChunk.js';
 export * from './transform/transformFilter.js';
 export * from './transform/transformFlatten.js';
+export * from './transform/transformFork.js';
 export * from './transform/transformLimit.js';
 export * from './transform/transformLogProgress.js';
 export * from './transform/transformMap.js';
@@ -24,13 +22,9 @@ export * from './transform/transformNoOp.js';
 export * from './transform/transformOffset.js';
 export * from './transform/transformSplit.js';
 export * from './transform/transformTap.js';
-export * from './transform/transformTee.js';
 export * from './transform/transformThrottle.js';
-export * from './transform/transformToArray.js';
 export * from './transform/worker/baseWorkerClass.js';
 export * from './transform/worker/transformMultiThreaded.js';
 export * from './transform/worker/transformMultiThreaded.model.js';
-export * from './writable/writableForEach.js';
-export * from './writable/writableFork.js';
 export * from './writable/writablePushToArray.js';
 export * from './writable/writableVoid.js';

package/dist/stream/ndjson/createReadStreamAsNDJson.d.ts

@@ -0,0 +1,16 @@
+import type { ReadableTyped } from '../stream.model.js';
+/**
+ Returns a Readable of [already parsed] NDJSON objects.
+ 
+ Replaces a list of operations:
+ - requireFileToExist(inputPath)
+ - fs.createReadStream
+ - createUnzip (only if path ends with '.gz')
+ - transformSplitOnNewline
+ - transformJsonParse
+ 
+ To add a Limit or Offset: just add .take() or .drop(), example:
+ 
+ createReadStreamAsNDJson().take(100)
+ */
+export declare function createReadStreamAsNDJson<ROW = any>(inputPath: string): ReadableTyped<ROW>;

package/dist/stream/ndjson/{createReadStreamAsNDJSON.js → createReadStreamAsNDJson.js}

@@ -2,23 +2,20 @@ import { createUnzip } from 'node:zlib';
 import { fs2 } from '../../fs/fs2.js';
 import { transformSplitOnNewline } from '../transform/transformSplit.js';
 /**
-  Returns a Readable of [already parsed] NDJSON objects.
+ Returns a Readable of [already parsed] NDJSON objects.
  
-  Replaces a list of operations:
-  - requireFileToExist(inputPath)
-  - fs.createReadStream
-  - createUnzip (only if path ends with '.gz')
-  - transformSplitOnNewline
-  - transformJsonParse
+ Replaces a list of operations:
+ - requireFileToExist(inputPath)
+ - fs.createReadStream
+ - createUnzip (only if path ends with '.gz')
+ - transformSplitOnNewline
+ - transformJsonParse
  
-  To add a Limit or Offset: just add .take() or .drop(), example:
+ To add a Limit or Offset: just add .take() or .drop(), example:
  
-  _pipeline([
-    fs2.createReadStreamAsNDJSON().take(100),
-    transformX(),
-  ])
+ createReadStreamAsNDJson().take(100)
  */
-export function createReadStreamAsNDJSON(inputPath) {
+export function createReadStreamAsNDJson(inputPath) {
     fs2.requireFileToExist(inputPath);
     let stream = fs2
         .createReadStream(inputPath, {

package/dist/stream/ndjson/ndjsonMap.d.ts

@@ -13,5 +13,7 @@ export interface NDJSONMapOptions<IN = any, OUT = IN> extends TransformMapOption
 /**
  * Unzips input file automatically, if it ends with `.gz`.
  * Zips output file automatically, if it ends with `.gz`.
+ *
+ * @deprecated use Pipeline directly
  */
 export declare function ndjsonMap<IN = any, OUT = any>(mapper: AbortableAsyncMapper<IN, OUT>, opt: NDJSONMapOptions<IN, OUT>): Promise<void>;

package/dist/stream/ndjson/ndjsonMap.js

@@ -3,6 +3,8 @@ import { Pipeline } from '../pipeline.js';
 /**
  * Unzips input file automatically, if it ends with `.gz`.
  * Zips output file automatically, if it ends with `.gz`.
+ *
+ * @deprecated use Pipeline directly
  */
 export async function ndjsonMap(mapper, opt) {
     const { inputFilePath, outputFilePath, logEveryOutput = 100_000, limitInput, limitOutput } = opt;

package/dist/stream/pipeline.d.ts

@@ -26,8 +26,8 @@ export declare class Pipeline<T> {
      */
     static fromArray<T>(input: T[]): Pipeline<T>;
     static fromIterable<T>(input: Iterable<T> | AsyncIterable<T>): Pipeline<T>;
-    static fromFile(sourceFilePath: string): Pipeline<Uint8Array>;
     static fromNDJsonFile<T>(sourceFilePath: string): Pipeline<T>;
+    static fromFile(sourceFilePath: string): Pipeline<Uint8Array>;
     /**
      * Limits the source Readable, but using `.take(limit)` on it.
      * This is THE preferred way of limiting the source.
@@ -69,6 +69,7 @@ export declare class Pipeline<T> {
      * Not type safe! Prefer using singular `transform()` multiple times for type safety.
      */
     transformMany<TO>(transforms: Transform[]): Pipeline<TO>;
+    fork(fn: (pipeline: Pipeline<T>) => Pipeline<T>, opt?: TransformOptions): this;
     /**
      * Utility method just to conveniently type-cast the current Pipeline type.
      * No runtime effect.

package/dist/stream/pipeline.js

@@ -4,13 +4,14 @@ import { createUnzip } from 'node:zlib';
 import { createGzip } from 'node:zlib';
 import { createAbortableSignal } from '@naturalcycles/js-lib';
 import { fs2 } from '../fs/fs2.js';
-import { createReadStreamAsNDJSON } from './ndjson/createReadStreamAsNDJSON.js';
+import { createReadStreamAsNDJson } from './ndjson/createReadStreamAsNDJson.js';
 import { transformJsonParse } from './ndjson/transformJsonParse.js';
 import { transformToNDJson } from './ndjson/transformToNDJson.js';
 import { PIPELINE_GRACEFUL_ABORT } from './stream.util.js';
 import { transformChunk } from './transform/transformChunk.js';
 import { transformFilterSync } from './transform/transformFilter.js';
 import { transformFlatten, transformFlattenIfNeeded } from './transform/transformFlatten.js';
+import { transformFork } from './transform/transformFork.js';
 import { transformLimit } from './transform/transformLimit.js';
 import { transformLogProgress, } from './transform/transformLogProgress.js';
 import { transformMap } from './transform/transformMap.js';
@@ -51,11 +52,18 @@ export class Pipeline {
     static fromIterable(input) {
         return new Pipeline(Readable.from(input));
     }
-    static fromFile(sourceFilePath) {
-        return new Pipeline(fs2.createReadStream(sourceFilePath), false);
-    }
     static fromNDJsonFile(sourceFilePath) {
-        return new Pipeline(createReadStreamAsNDJSON(sourceFilePath));
+        // Important that createReadStreamAsNDJson function is used
+        // (and not Pipeline set of individual transforms),
+        // because createReadStreamAsNDJson returns a Readable,
+        // hence it allows to apply .take(limit) on it
+        // e.g like Pipeline.fromNDJsonFile().limitSource(limit)
+        return new Pipeline(createReadStreamAsNDJson(sourceFilePath));
+    }
+    static fromFile(sourceFilePath) {
+        return new Pipeline(fs2.createReadStream(sourceFilePath, {
+            highWaterMark: 64 * 1024, // no observed speedup
+        }), false);
     }
     /**
      * Limits the source Readable, but using `.take(limit)` on it.
@@ -148,7 +156,6 @@ export class Pipeline {
         this.transforms.push(transformThrottle(opt));
         return this;
     }
-    // todo: tee/fork
     transform(transform) {
         this.transforms.push(transform);
         return this;
@@ -161,6 +168,10 @@ export class Pipeline {
         this.transforms.push(...transforms);
         return this;
     }
+    fork(fn, opt) {
+        this.transforms.push(transformFork(fn, opt));
+        return this;
+    }
     /**
      * Utility method just to conveniently type-cast the current Pipeline type.
      * No runtime effect.
@@ -203,12 +214,18 @@ export class Pipeline {
         return this;
     }
     gzip(opt) {
-        this.transforms.push(createGzip(opt));
+        this.transforms.push(createGzip({
+            // chunkSize: 64 * 1024, // no observed speedup
+            ...opt,
+        }));
         this.objectMode = false;
         return this;
     }
     gunzip(opt) {
-        this.transforms.push(createUnzip(opt));
+        this.transforms.push(createUnzip({
+            chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
+            ...opt,
+        }));
         this.objectMode = false;
         return this;
     }

package/dist/stream/progressLogger.d.ts

@@ -1,5 +1,5 @@
 import type { CommonLogger } from '@naturalcycles/js-lib/log';
-import type { AnyObject } from '@naturalcycles/js-lib/types';
+import type { AnyObject, PositiveInteger } from '@naturalcycles/js-lib/types';
 import type { ReadableMapper } from './stream.model.js';
 export interface ProgressLoggerCfg<T = any> {
     /**
@@ -69,7 +69,7 @@ export interface ProgressLoggerCfg<T = any> {
      *
      * @default 1000
      */
-    logEvery?: number;
+    logEvery?: PositiveInteger;
     logger?: CommonLogger;
     /**
      * Function to return extra properties to the "progress object".
@@ -89,7 +89,7 @@ export interface ProgressLoggerCfg<T = any> {
      *
      * Defaults to 1.
      */
-    chunkSize?: number;
+    chunkSize?: PositiveInteger;
     /**
      * Experimental logging of item (shunk) sizes, when json-stringified.
      *

package/dist/stream/readable/readableCombined.d.ts

@@ -1,5 +1,6 @@
 import { Readable } from 'node:stream';
 import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
+import type { TransformOptions } from '../stream.model.js';
 /**
  * Allows to combine multiple Readables into 1 Readable.
  * As soon as any of the input Readables emit - the output Readable emits
@@ -12,8 +13,9 @@ import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
  */
 export declare class ReadableCombined<T> extends Readable implements ReadableTyped<T> {
     inputs: Readable[];
-    static create<T>(inputs: Readable[]): ReadableCombined<T>;
+    static create<T>(inputs: Readable[], opt?: TransformOptions): ReadableCombined<T>;
     private constructor();
+    private logger;
     /**
      * If defined - we are in Paused mode
      * and should await the lock to be resolved before proceeding.
@@ -24,7 +26,7 @@ export declare class ReadableCombined<T> extends Readable implements ReadableTyp
     private countIn;
     private countOut;
     private countReads;
-    private start;
+    private run;
     _read(): void;
     private logStats;
 }

package/dist/stream/readable/readableCombined.js

@@ -1,4 +1,5 @@
 import { Readable } from 'node:stream';
+import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
 import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
 import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
 /**
@@ -13,14 +14,17 @@ import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
  */
 export class ReadableCombined extends Readable {
     inputs;
-    static create(inputs) {
-        return new ReadableCombined(inputs);
+    static create(inputs, opt = {}) {
+        return new ReadableCombined(inputs, opt);
     }
-    constructor(inputs) {
-        super({ objectMode: true });
+    constructor(inputs, opt) {
+        const { objectMode = true, highWaterMark } = opt;
+        super({ objectMode, highWaterMark });
         this.inputs = inputs;
-        void this.start();
+        this.logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
+        void this.run();
     }
+    logger;
     /**
      * If defined - we are in Paused mode
      * and should await the lock to be resolved before proceeding.
@@ -34,7 +38,8 @@ export class ReadableCombined extends Readable {
     countOut = 0;
     // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
     countReads = 0;
-    async start() {
+    async run() {
+        const { logger } = this;
         await pMap(this.inputs, async (input, i) => {
             for await (const item of input) {
                 this.countIn++;
@@ -47,18 +52,18 @@ export class ReadableCombined extends Readable {
                 this.countOut++;
                 if (!shouldContinue && !this.lock) {
                     this.lock = pDefer();
-                    console.log(`ReadableCombined.push #${i} returned false, pausing the flow!`);
+                    logger.log(`ReadableCombined.push #${i} returned false, pausing the flow!`);
                 }
             }
-            console.log(`ReadableCombined: input #${i} done`);
+            logger.log(`ReadableCombined: input #${i} done`);
         });
-        console.log(`ReadableCombined: all inputs done!`);
+        logger.log(`ReadableCombined: all inputs done!`);
         this.push(null);
     }
     _read() {
         this.countReads++;
         if (this.lock) {
-            console.log(`ReadableCombined._read: resuming the flow!`);
+            this.logger.log(`ReadableCombined._read: resuming the flow!`);
             // calling it in this order is important!
             // this.lock should be undefined BEFORE we call lock.resolve()
             const { lock } = this;
@@ -68,7 +73,7 @@ export class ReadableCombined extends Readable {
     }
     logStats() {
         const { countIn, countOut, countReads } = this;
-        console.log({
+        this.logger.debug({
             countIn,
             countOut,
             countReads,

package/dist/stream/readable/readableCreate.d.ts

@@ -11,10 +11,8 @@ import type { ReadableTyped } from '../stream.model.js';
  * e.g the read() method doesn't return anything, so, it will hang the Node process (or cause it to process.exit(0))
  * if read() will be called AFTER everything was pushed and Readable is closed (by pushing `null`).
  * Beware of it when e.g doing unit testing! Jest prefers to hang (not exit-0).
- *
- * @deprecated because of the caution above
  */
-export declare function readableCreate<T>(items?: Iterable<T>, opt?: ReadableOptions): ReadableTyped<T>;
+export declare function readableCreate<T>(items?: Iterable<T>, opt?: ReadableOptions, onRead?: () => void): ReadableTyped<T>;
 /**
  * Convenience type-safe wrapper around Readable.from() that infers the Type of input.
  */

package/dist/stream/readable/readableCreate.js

@@ -10,14 +10,14 @@ import { Readable } from 'node:stream';
  * e.g the read() method doesn't return anything, so, it will hang the Node process (or cause it to process.exit(0))
  * if read() will be called AFTER everything was pushed and Readable is closed (by pushing `null`).
  * Beware of it when e.g doing unit testing! Jest prefers to hang (not exit-0).
- *
- * @deprecated because of the caution above
  */
-export function readableCreate(items = [], opt) {
+export function readableCreate(items = [], opt, onRead) {
     const readable = new Readable({
         objectMode: true,
         ...opt,
-        read() { }, // Caution, if this is called and Readable has not finished yet (null wasn't pushed) - it'll hang the process!
+        read() {
+            onRead?.();
+        },
     });
     for (const item of items) {
         readable.push(item);

package/dist/stream/stream.model.d.ts

@@ -1,4 +1,5 @@
 import type { Readable, Transform, Writable } from 'node:stream';
+import type { CommonLogger, CommonLogLevel } from '@naturalcycles/js-lib/log';
 import type { Promisable } from '@naturalcycles/js-lib/types';
 export interface ReadableSignalOptions {
     /** allows destroying the stream if the signal is aborted. */
@@ -46,4 +47,19 @@ export interface TransformOptions {
      * @default 16
      */
     highWaterMark?: number;
+    /**
+     * Defaults to `console`.
+     */
+    logger?: CommonLogger;
+    /**
+     * Not every Transform implements it.
+     * Can be one of:
+     * debug - most verbose, when debugging is needed
+     * log - default level
+     * error - logs errors and warnings only
+     *
+     * Default is 'log'.
+     *
+     */
+    logLevel?: CommonLogLevel;
 }

package/dist/stream/transform/transformFork.d.ts

@@ -0,0 +1,10 @@
+import { Pipeline } from '../pipeline.js';
+import type { TransformOptions, TransformTyped } from '../stream.model.js';
+/**
+ * Allows to "fork" away from the "main pipeline" into the "forked pipeline".
+ *
+ * Correctly keeps backpressure from both "downstreams" (main and forked).
+ *
+ * @experimental
+ */
+export declare function transformFork<T>(fn: (pipeline: Pipeline<T>) => Pipeline<T>, opt?: TransformOptions): TransformTyped<T, T>;

package/dist/stream/transform/transformFork.js

@@ -0,0 +1,62 @@
+import { Transform } from 'node:stream';
+import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
+import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
+import { Pipeline } from '../pipeline.js';
+import { readableCreate } from '../readable/readableCreate.js';
+/**
+ * Allows to "fork" away from the "main pipeline" into the "forked pipeline".
+ *
+ * Correctly keeps backpressure from both "downstreams" (main and forked).
+ *
+ * @experimental
+ */
+export function transformFork(fn, opt = {}) {
+    const { objectMode = true, highWaterMark } = opt;
+    const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
+    let lock;
+    const fork = readableCreate([], {}, () => {
+        // `_read` is called
+        if (!lock)
+            return;
+        // We had a lock - let's Resume
+        logger.log(`TransformFork: resume`);
+        const lockCopy = lock;
+        lock = undefined;
+        lockCopy.resolve();
+    });
+    const p = fn(Pipeline.from(fork));
+    void p.run().then(() => {
+        logger.log('TransformFork: done');
+    });
+    return new Transform({
+        objectMode,
+        highWaterMark,
+        async transform(chunk, _, cb) {
+            // pass through to the "main" pipeline
+            // Main pipeline should handle backpressure "automatically",
+            // so, we're not maintaining a Lock for it
+            this.push(chunk);
+            if (lock) {
+                // Forked pipeline is locked - let's wait for it to call _read
+                await lock;
+                // lock is undefined at this point
+            }
+            // pass to the "forked" pipeline
+            const shouldContinue = fork.push(chunk);
+            if (!shouldContinue && !lock) {
+                // Forked pipeline indicates that we should Pause
+                lock = pDefer();
+                logger.log(`TransformFork: pause`);
+            }
+            // acknowledge that we've finished processing the input chunk
+            cb();
+        },
+        async final(cb) {
+            logger.log('TransformFork: final');
+            // Pushing null "closes"/ends the secondary pipeline correctly
+            fork.push(null);
+            // Acknowledge that we've received `null` and passed it through to the fork
+            cb();
+        },
+    });
+}

package/dist/stream/transform/transformLimit.d.ts

@@ -1,10 +1,11 @@
 import type { AbortableSignal } from '@naturalcycles/js-lib';
+import type { NonNegativeInteger } from '@naturalcycles/js-lib/types';
 import type { TransformOptions, TransformTyped } from '../stream.model.js';
 export interface TransformLimitOptions extends TransformOptions {
     /**
      * Nullish value (e.g 0 or undefined) would mean "no limit"
      */
-    limit?: number;
+    limit?: NonNegativeInteger;
     /**
      * Allows to abort (gracefully stop) the stream from inside the Transform.
      */

package/dist/stream/transform/transformLimit.js

@@ -2,15 +2,15 @@ import { Transform } from 'node:stream';
 import { PIPELINE_GRACEFUL_ABORT } from '../stream.util.js';
 import { transformNoOp } from './transformNoOp.js';
 export function transformLimit(opt) {
-    const { limit, signal } = opt;
+    const { limit, signal, objectMode = true, highWaterMark } = opt;
     if (!limit) {
         return transformNoOp();
     }
     let i = 0; // so we start first chunk with 1
     let ended = false;
     return new Transform({
-        objectMode: true,
-        ...opt,
+        objectMode,
+        highWaterMark,
         transform(chunk, _, cb) {
             if (ended) {
                 return;

package/dist/stream/transform/transformLogProgress.js

@@ -4,10 +4,11 @@ import { progressLogger } from '../progressLogger.js';
  * Pass-through transform that optionally logs progress.
  */
 export function transformLogProgress(opt = {}) {
+    const { objectMode = true, highWaterMark } = opt;
     const progress = progressLogger(opt);
     return new Transform({
-        objectMode: true,
-        ...opt,
+        objectMode,
+        highWaterMark,
         transform(chunk, _, cb) {
             progress.log(chunk);
             cb(null, chunk); // pass-through

package/dist/stream/transform/transformMap.d.ts

@@ -1,9 +1,8 @@
 import { type AbortableSignal } from '@naturalcycles/js-lib';
 import { ErrorMode } from '@naturalcycles/js-lib/error';
-import type { CommonLogger } from '@naturalcycles/js-lib/log';
 import { type AbortableAsyncMapper, type AsyncPredicate, END, type PositiveInteger, type Promisable, SKIP, type StringMap, type UnixTimestampMillis } from '@naturalcycles/js-lib/types';
-import type { TransformTyped } from '../stream.model.js';
-export interface TransformMapOptions<IN = any, OUT = IN> {
+import type { TransformOptions, TransformTyped } from '../stream.model.js';
+export interface TransformMapOptions<IN = any, OUT = IN> extends TransformOptions {
     /**
      * Predicate to filter outgoing results (after mapper).
      * Allows to not emit all results.
@@ -57,7 +56,6 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
      * @default `stream`
      */
     metric?: string;
-    logger?: CommonLogger;
     /**
      * Allows to abort (gracefully stop) the stream from inside the Transform.
      */

package/dist/stream/transform/transformMap.js

@@ -1,6 +1,7 @@
 import { _hc } from '@naturalcycles/js-lib';
 import { _since } from '@naturalcycles/js-lib/datetime/time.util.js';
 import { _anyToError, _assert, ErrorMode } from '@naturalcycles/js-lib/error';
+import { createCommonLoggerAtLevel } from '@naturalcycles/js-lib/log';
 import { _stringify } from '@naturalcycles/js-lib/string/stringify.js';
 import { END, SKIP, } from '@naturalcycles/js-lib/types';
 import through2Concurrent from 'through2-concurrent';
@@ -22,7 +23,7 @@ import { PIPELINE_GRACEFUL_ABORT } from '../stream.util.js';
  */
 export function transformMap(mapper, opt = {}) {
     const { concurrency = 16, highWaterMark = 64, predicate, // we now default to "no predicate" (meaning pass-everything)
-    errorMode = ErrorMode.THROW_IMMEDIATELY, onError, onDone, metric = 'stream', logger = console, signal, } = opt;
+    errorMode = ErrorMode.THROW_IMMEDIATELY, onError, onDone, metric = 'stream', signal, } = opt;
     const started = Date.now();
     let index = -1;
     let countOut = 0;
@@ -30,12 +31,12 @@ export function transformMap(mapper, opt = {}) {
     let ok = true;
     let errors = 0;
     const collectedErrors = []; // only used if errorMode == THROW_AGGREGATED
+    const logger = createCommonLoggerAtLevel(opt.logger, opt.logLevel);
     return through2Concurrent.obj({
         maxConcurrency: concurrency,
         readableHighWaterMark: highWaterMark,
         writableHighWaterMark: highWaterMark,
         async final(cb) {
-            // console.log('transformMap final')
             logErrorStats(true);
             if (collectedErrors.length) {
                 try {

package/dist/stream/transform/transformMapSimple.d.ts

@@ -1,15 +1,13 @@
 import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js';
-import type { CommonLogger } from '@naturalcycles/js-lib/log';
 import type { IndexedMapper } from '@naturalcycles/js-lib/types';
-import type { TransformTyped } from '../stream.model.js';
-export interface TransformMapSimpleOptions {
+import type { TransformOptions, TransformTyped } from '../stream.model.js';
+export interface TransformMapSimpleOptions extends TransformOptions {
     /**
      * Only supports THROW_IMMEDIATELY (default) and SUPPRESS.
      *
      * @default ErrorMode.THROW_IMMEDIATELY
      */
     errorMode?: ErrorMode.THROW_IMMEDIATELY | ErrorMode.SUPPRESS;
-    logger?: CommonLogger;
 }
 /**
  * Simplest version of `transformMap`.