@naturalcycles/nodejs-lib 15.21.0 → 15.22.0

package/dist/stream/index.d.ts

@@ -7,6 +7,7 @@ export * from './ndjson/transformJsonParse.js';
 export * from './ndjson/transformToNDJson.js';
 export * from './pipeline/pipeline.js';
 export * from './progressLogger.js';
+export * from './readable/readableCombined.js';
 export * from './readable/readableCreate.js';
 export * from './readable/readableForEach.js';
 export * from './readable/readableFromArray.js';

package/dist/stream/index.js

@@ -7,6 +7,7 @@ export * from './ndjson/transformJsonParse.js';
 export * from './ndjson/transformToNDJson.js';
 export * from './pipeline/pipeline.js';
 export * from './progressLogger.js';
+export * from './readable/readableCombined.js';
 export * from './readable/readableCreate.js';
 export * from './readable/readableForEach.js';
 export * from './readable/readableFromArray.js';

package/dist/stream/ndjson/transformJsonParse.js

@@ -19,7 +19,6 @@ export function transformJsonParse(opt = {}) {
         writableObjectMode: false,
         readableObjectMode: true,
         // highWatermark increased, because it's proven to be faster: https://github.com/nodejs/node/pull/52037
-        // todo: it'll be default in Node 22, then we can remove this
         writableHighWaterMark: 64 * 1024,
         transform(chunk, _, cb) {
             try {

package/dist/stream/readable/readableCombined.d.ts

@@ -0,0 +1,30 @@
+import { Readable } from 'node:stream';
+import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
+/**
+ * Allows to combine multiple Readables into 1 Readable.
+ * As soon as any of the input Readables emit - the output Readable emits
+ * (passes through).
+ * Order is not preserved in any way, first come first served!
+ *
+ * Readable completes when all input Readables complete.
+ *
+ * @experimental
+ */
+export declare class ReadableCombined<T> extends Readable implements ReadableTyped<T> {
+    inputs: Readable[];
+    static create<T>(inputs: Readable[]): ReadableCombined<T>;
+    private constructor();
+    /**
+     * If defined - we are in Paused mode
+     * and should await the lock to be resolved before proceeding.
+     *
+     * If not defined - we are in Flowing mode, no limits in data flow.
+     */
+    private lock?;
+    private countIn;
+    private countOut;
+    private countReads;
+    private start;
+    _read(): void;
+    private logStats;
+}

package/dist/stream/readable/readableCombined.js

@@ -0,0 +1,77 @@
+import { Readable } from 'node:stream';
+import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
+import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
+/**
+ * Allows to combine multiple Readables into 1 Readable.
+ * As soon as any of the input Readables emit - the output Readable emits
+ * (passes through).
+ * Order is not preserved in any way, first come first served!
+ *
+ * Readable completes when all input Readables complete.
+ *
+ * @experimental
+ */
+export class ReadableCombined extends Readable {
+    inputs;
+    static create(inputs) {
+        return new ReadableCombined(inputs);
+    }
+    constructor(inputs) {
+        super({ objectMode: true });
+        this.inputs = inputs;
+        void this.start();
+    }
+    /**
+     * If defined - we are in Paused mode
+     * and should await the lock to be resolved before proceeding.
+     *
+     * If not defined - we are in Flowing mode, no limits in data flow.
+     */
+    lock;
+    // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
+    countIn = 0;
+    // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
+    countOut = 0;
+    // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
+    countReads = 0;
+    async start() {
+        await pMap(this.inputs, async (input, i) => {
+            for await (const item of input) {
+                this.countIn++;
+                this.logStats();
+                if (this.lock) {
+                    await this.lock;
+                    // lock is undefined at this point
+                }
+                const shouldContinue = this.push(item);
+                this.countOut++;
+                if (!shouldContinue && !this.lock) {
+                    this.lock = pDefer();
+                    console.log(`ReadableCombined.push #${i} returned false, pausing the flow!`);
+                }
+            }
+            console.log(`ReadableCombined: input #${i} done`);
+        });
+        console.log(`ReadableCombined: all inputs done!`);
+        this.push(null);
+    }
+    _read() {
+        this.countReads++;
+        if (this.lock) {
+            console.log(`ReadableCombined._read: resuming the flow!`);
+            // calling it in this order is important!
+            // this.lock should be undefined BEFORE we call lock.resolve()
+            const { lock } = this;
+            this.lock = undefined;
+            lock.resolve();
+        }
+    }
+    logStats() {
+        const { countIn, countOut, countReads } = this;
+        console.log({
+            countIn,
+            countOut,
+            countReads,
+        });
+    }
+}

package/dist/stream/transform/transformMap.d.ts

@@ -1,6 +1,6 @@
 import { ErrorMode } from '@naturalcycles/js-lib/error';
 import type { CommonLogger } from '@naturalcycles/js-lib/log';
-import { type AbortableAsyncMapper, type AsyncPredicate, END, type Promisable, SKIP, type StringMap, type UnixTimestampMillis } from '@naturalcycles/js-lib/types';
+import { type AbortableAsyncMapper, type AsyncPredicate, END, type PositiveInteger, type Promisable, SKIP, type StringMap, type UnixTimestampMillis } from '@naturalcycles/js-lib/types';
 import type { TransformTyped } from '../stream.model.js';
 export interface TransformMapOptions<IN = any, OUT = IN> {
     /**
@@ -21,7 +21,15 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
      * UPD: changed back from 32 to 16, "to be on a safe side", as 32 sometimes
      * causes "Datastore timeout errors".
      */
-    concurrency?: number;
+    concurrency?: PositiveInteger;
+    /**
+     * Defaults to 64 items.
+     * (objectMode default is 16, but we increased it)
+     *
+     * Affects both readable and writable highWaterMark (buffer).
+     * So, 64 means a total buffer of 128 (64 input and 64 output buffer).
+     */
+    highWaterMark?: PositiveInteger;
     /**
      * @default THROW_IMMEDIATELY
      */

package/dist/stream/transform/transformMap.js

@@ -21,7 +21,7 @@ import { pipelineClose } from '../stream.util.js';
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
  */
 export function transformMap(mapper, opt = {}) {
-    const { concurrency = 16, predicate, // we now default to "no predicate" (meaning pass-everything)
+    const { concurrency = 16, highWaterMark = 64, predicate, // we now default to "no predicate" (meaning pass-everything)
     errorMode = ErrorMode.THROW_IMMEDIATELY, onError, onDone, metric = 'stream', logger = console, } = opt;
     const started = Date.now();
     let index = -1;
@@ -31,6 +31,8 @@ export function transformMap(mapper, opt = {}) {
     const collectedErrors = []; // only used if errorMode == THROW_AGGREGATED
     return through2Concurrent.obj({
         maxConcurrency: concurrency,
+        readableHighWaterMark: highWaterMark,
+        writableHighWaterMark: highWaterMark,
         async final(cb) {
             // console.log('transformMap final')
             logErrorStats(true);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.21.0",
+  "version": "15.22.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@types/js-yaml": "^4",

package/src/stream/index.ts

@@ -7,6 +7,7 @@ export * from './ndjson/transformJsonParse.js'
 export * from './ndjson/transformToNDJson.js'
 export * from './pipeline/pipeline.js'
 export * from './progressLogger.js'
+export * from './readable/readableCombined.js'
 export * from './readable/readableCreate.js'
 export * from './readable/readableForEach.js'
 export * from './readable/readableFromArray.js'

package/src/stream/ndjson/transformJsonParse.ts

@@ -36,7 +36,6 @@ export function transformJsonParse<ROW = any>(
     writableObjectMode: false,
     readableObjectMode: true,
     // highWatermark increased, because it's proven to be faster: https://github.com/nodejs/node/pull/52037
-    // todo: it'll be default in Node 22, then we can remove this
     writableHighWaterMark: 64 * 1024,
     transform(chunk: string, _, cb) {
       try {

package/src/stream/readable/readableCombined.ts

@@ -0,0 +1,87 @@
+import { Readable } from 'node:stream'
+import { type DeferredPromise, pDefer } from '@naturalcycles/js-lib/promise/pDefer.js'
+import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
+import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
+
+/**
+ * Allows to combine multiple Readables into 1 Readable.
+ * As soon as any of the input Readables emit - the output Readable emits
+ * (passes through).
+ * Order is not preserved in any way, first come first served!
+ *
+ * Readable completes when all input Readables complete.
+ *
+ * @experimental
+ */
+export class ReadableCombined<T> extends Readable implements ReadableTyped<T> {
+  static create<T>(inputs: Readable[]): ReadableCombined<T> {
+    return new ReadableCombined<T>(inputs)
+  }
+
+  private constructor(public inputs: Readable[]) {
+    super({ objectMode: true })
+    void this.start()
+  }
+
+  /**
+   * If defined - we are in Paused mode
+   * and should await the lock to be resolved before proceeding.
+   *
+   * If not defined - we are in Flowing mode, no limits in data flow.
+   */
+  private lock?: DeferredPromise
+
+  // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
+  private countIn = 0
+  // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
+  private countOut = 0
+  // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
+  private countReads = 0
+
+  private async start(): Promise<void> {
+    await pMap(this.inputs, async (input, i) => {
+      for await (const item of input) {
+        this.countIn++
+        this.logStats()
+        if (this.lock) {
+          await this.lock
+          // lock is undefined at this point
+        }
+
+        const shouldContinue = this.push(item)
+        this.countOut++
+        if (!shouldContinue && !this.lock) {
+          this.lock = pDefer()
+          console.log(`ReadableCombined.push #${i} returned false, pausing the flow!`)
+        }
+      }
+
+      console.log(`ReadableCombined: input #${i} done`)
+    })
+
+    console.log(`ReadableCombined: all inputs done!`)
+    this.push(null)
+  }
+
+  override _read(): void {
+    this.countReads++
+
+    if (this.lock) {
+      console.log(`ReadableCombined._read: resuming the flow!`)
+      // calling it in this order is important!
+      // this.lock should be undefined BEFORE we call lock.resolve()
+      const { lock } = this
+      this.lock = undefined
+      lock.resolve()
+    }
+  }
+
+  private logStats(): void {
+    const { countIn, countOut, countReads } = this
+    console.log({
+      countIn,
+      countOut,
+      countReads,
+    })
+  }
+}

package/src/stream/transform/transformMap.ts

@@ -7,6 +7,7 @@ import {
   type AbortableAsyncMapper,
   type AsyncPredicate,
   END,
+  type PositiveInteger,
   type Promisable,
   SKIP,
   type StringMap,
@@ -38,7 +39,16 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
    * UPD: changed back from 32 to 16, "to be on a safe side", as 32 sometimes
    * causes "Datastore timeout errors".
    */
-  concurrency?: number
+  concurrency?: PositiveInteger
+
+  /**
+   * Defaults to 64 items.
+   * (objectMode default is 16, but we increased it)
+   *
+   * Affects both readable and writable highWaterMark (buffer).
+   * So, 64 means a total buffer of 128 (64 input and 64 output buffer).
+   */
+  highWaterMark?: PositiveInteger
 
   /**
    * @default THROW_IMMEDIATELY
@@ -123,6 +133,7 @@ export function transformMap<IN = any, OUT = IN>(
 ): TransformTyped<IN, OUT> {
   const {
     concurrency = 16,
+    highWaterMark = 64,
     predicate, // we now default to "no predicate" (meaning pass-everything)
     errorMode = ErrorMode.THROW_IMMEDIATELY,
     onError,
@@ -141,6 +152,8 @@ export function transformMap<IN = any, OUT = IN>(
   return through2Concurrent.obj(
     {
       maxConcurrency: concurrency,
+      readableHighWaterMark: highWaterMark,
+      writableHighWaterMark: highWaterMark,
       async final(cb) {
         // console.log('transformMap final')