@naturalcycles/nodejs-lib 15.20.2 → 15.22.0

package/dist/exec2/exec2.d.ts

@@ -1,3 +1,4 @@
+import { type StdioOptions } from 'node:child_process';
 import { AppError } from '@naturalcycles/js-lib/error/error.util.js';
 import type { AnyObject, NumberOfMilliseconds } from '@naturalcycles/js-lib/types';
 /**
@@ -164,6 +165,10 @@ export interface SpawnOptions {
      * Set to false or true to override.
      */
     forceColor?: boolean;
+    /**
+     * Defaults to "inherit"
+     */
+    stdio?: StdioOptions;
 }
 export interface ExecOptions {
     /**
@@ -192,5 +197,10 @@ export interface ExecOptions {
      * Set to true to pass `process.env` to the spawned process.
      */
     passProcessEnv?: boolean;
+    /**
+     * Defaults to undefined.
+     * beware that stdio: 'inherit', means we don't get the output returned.
+     */
+    stdio?: StdioOptions;
 }
 export {};

package/dist/exec2/exec2.js

@@ -40,7 +40,7 @@ class Exec2 {
      * log: true
      */
     spawn(cmd, opt = {}) {
-        const { shell = true, cwd, env, passProcessEnv = true, forceColor = hasColors } = opt;
+        const { shell = true, cwd, env, passProcessEnv = true, forceColor = hasColors, stdio = 'inherit', } = opt;
         opt.log ??= true; // by default log should be true, as we are printing the output
         opt.logStart ??= opt.log;
         opt.logFinish ??= opt.log;
@@ -48,7 +48,7 @@ class Exec2 {
         this.logStart(cmd, opt);
         const r = spawnSync(cmd, opt.args, {
             encoding: 'utf8',
-            stdio: 'inherit',
+            stdio,
             shell,
             cwd,
             env: {
@@ -81,7 +81,7 @@ class Exec2 {
      * log: false
      */
     exec(cmd, opt = {}) {
-        const { cwd, env, passProcessEnv = true, timeout } = opt;
+        const { cwd, env, passProcessEnv = true, timeout, stdio } = opt;
         opt.logStart ??= opt.log ?? false;
         opt.logFinish ??= opt.log ?? false;
         const started = Date.now();
@@ -89,8 +89,7 @@ class Exec2 {
         try {
             const s = execSync(cmd, {
                 encoding: 'utf8',
-                // stdio: 'inherit', // no, otherwise we don't get the output returned
-                stdio: undefined,
+                stdio,
                 // shell: undefined,
                 cwd,
                 timeout,

package/dist/stream/index.d.ts

@@ -7,6 +7,7 @@ export * from './ndjson/transformJsonParse.js';
 export * from './ndjson/transformToNDJson.js';
 export * from './pipeline/pipeline.js';
 export * from './progressLogger.js';
+export * from './readable/readableCombined.js';
 export * from './readable/readableCreate.js';
 export * from './readable/readableForEach.js';
 export * from './readable/readableFromArray.js';

package/dist/stream/index.js

@@ -7,6 +7,7 @@ export * from './ndjson/transformJsonParse.js';
 export * from './ndjson/transformToNDJson.js';
 export * from './pipeline/pipeline.js';
 export * from './progressLogger.js';
+export * from './readable/readableCombined.js';
 export * from './readable/readableCreate.js';
 export * from './readable/readableForEach.js';
 export * from './readable/readableFromArray.js';

package/dist/stream/ndjson/transformJsonParse.js

@@ -19,7 +19,6 @@ export function transformJsonParse(opt = {}) {
         writableObjectMode: false,
         readableObjectMode: true,
         // highWatermark increased, because it's proven to be faster: https://github.com/nodejs/node/pull/52037
-        // todo: it'll be default in Node 22, then we can remove this
         writableHighWaterMark: 64 * 1024,
         transform(chunk, _, cb) {
             try {

package/dist/stream/readable/readableCombined.d.ts

@@ -0,0 +1,30 @@
+import { Readable } from 'node:stream';
+import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream';
+/**
+ * Allows to combine multiple Readables into 1 Readable.
+ * As soon as any of the input Readables emit - the output Readable emits
+ * (passes through).
+ * Order is not preserved in any way, first come first served!
+ *
+ * Readable completes when all input Readables complete.
+ *
+ * @experimental
+ */
+export declare class ReadableCombined<T> extends Readable implements ReadableTyped<T> {
+    inputs: Readable[];
+    static create<T>(inputs: Readable[]): ReadableCombined<T>;
+    private constructor();
+    /**
+     * If defined - we are in Paused mode
+     * and should await the lock to be resolved before proceeding.
+     *
+     * If not defined - we are in Flowing mode, no limits in data flow.
+     */
+    private lock?;
+    private countIn;
+    private countOut;
+    private countReads;
+    private start;
+    _read(): void;
+    private logStats;
+}

package/dist/stream/readable/readableCombined.js

@@ -0,0 +1,77 @@
+import { Readable } from 'node:stream';
+import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
+import { pMap } from '@naturalcycles/js-lib/promise/pMap.js';
+/**
+ * Allows to combine multiple Readables into 1 Readable.
+ * As soon as any of the input Readables emit - the output Readable emits
+ * (passes through).
+ * Order is not preserved in any way, first come first served!
+ *
+ * Readable completes when all input Readables complete.
+ *
+ * @experimental
+ */
+export class ReadableCombined extends Readable {
+    inputs;
+    static create(inputs) {
+        return new ReadableCombined(inputs);
+    }
+    constructor(inputs) {
+        super({ objectMode: true });
+        this.inputs = inputs;
+        void this.start();
+    }
+    /**
+     * If defined - we are in Paused mode
+     * and should await the lock to be resolved before proceeding.
+     *
+     * If not defined - we are in Flowing mode, no limits in data flow.
+     */
+    lock;
+    // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
+    countIn = 0;
+    // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
+    countOut = 0;
+    // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
+    countReads = 0;
+    async start() {
+        await pMap(this.inputs, async (input, i) => {
+            for await (const item of input) {
+                this.countIn++;
+                this.logStats();
+                if (this.lock) {
+                    await this.lock;
+                    // lock is undefined at this point
+                }
+                const shouldContinue = this.push(item);
+                this.countOut++;
+                if (!shouldContinue && !this.lock) {
+                    this.lock = pDefer();
+                    console.log(`ReadableCombined.push #${i} returned false, pausing the flow!`);
+                }
+            }
+            console.log(`ReadableCombined: input #${i} done`);
+        });
+        console.log(`ReadableCombined: all inputs done!`);
+        this.push(null);
+    }
+    _read() {
+        this.countReads++;
+        if (this.lock) {
+            console.log(`ReadableCombined._read: resuming the flow!`);
+            // calling it in this order is important!
+            // this.lock should be undefined BEFORE we call lock.resolve()
+            const { lock } = this;
+            this.lock = undefined;
+            lock.resolve();
+        }
+    }
+    logStats() {
+        const { countIn, countOut, countReads } = this;
+        console.log({
+            countIn,
+            countOut,
+            countReads,
+        });
+    }
+}

package/dist/stream/transform/transformMap.d.ts

@@ -1,6 +1,6 @@
 import { ErrorMode } from '@naturalcycles/js-lib/error';
 import type { CommonLogger } from '@naturalcycles/js-lib/log';
-import { type AbortableAsyncMapper, type AsyncPredicate, END, type Promisable, SKIP, type StringMap, type UnixTimestampMillis } from '@naturalcycles/js-lib/types';
+import { type AbortableAsyncMapper, type AsyncPredicate, END, type PositiveInteger, type Promisable, SKIP, type StringMap, type UnixTimestampMillis } from '@naturalcycles/js-lib/types';
 import type { TransformTyped } from '../stream.model.js';
 export interface TransformMapOptions<IN = any, OUT = IN> {
     /**
@@ -21,7 +21,15 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
      * UPD: changed back from 32 to 16, "to be on a safe side", as 32 sometimes
      * causes "Datastore timeout errors".
      */
-    concurrency?: number;
+    concurrency?: PositiveInteger;
+    /**
+     * Defaults to 64 items.
+     * (objectMode default is 16, but we increased it)
+     *
+     * Affects both readable and writable highWaterMark (buffer).
+     * So, 64 means a total buffer of 128 (64 input and 64 output buffer).
+     */
+    highWaterMark?: PositiveInteger;
     /**
      * @default THROW_IMMEDIATELY
      */

package/dist/stream/transform/transformMap.js

@@ -21,7 +21,7 @@ import { pipelineClose } from '../stream.util.js';
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
  */
 export function transformMap(mapper, opt = {}) {
-    const { concurrency = 16, predicate, // we now default to "no predicate" (meaning pass-everything)
+    const { concurrency = 16, highWaterMark = 64, predicate, // we now default to "no predicate" (meaning pass-everything)
     errorMode = ErrorMode.THROW_IMMEDIATELY, onError, onDone, metric = 'stream', logger = console, } = opt;
     const started = Date.now();
     let index = -1;
@@ -31,6 +31,8 @@ export function transformMap(mapper, opt = {}) {
     const collectedErrors = []; // only used if errorMode == THROW_AGGREGATED
     return through2Concurrent.obj({
         maxConcurrency: concurrency,
+        readableHighWaterMark: highWaterMark,
+        writableHighWaterMark: highWaterMark,
         async final(cb) {
             // console.log('transformMap final')
             logErrorStats(true);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.20.2",
+  "version": "15.22.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@types/js-yaml": "^4",

package/src/exec2/exec2.ts

@@ -1,4 +1,4 @@
-import { execSync, spawn, spawnSync } from 'node:child_process'
+import { execSync, spawn, spawnSync, type StdioOptions } from 'node:child_process'
 import { _since } from '@naturalcycles/js-lib/datetime/time.util.js'
 import { AppError } from '@naturalcycles/js-lib/error/error.util.js'
 import { _substringAfterLast } from '@naturalcycles/js-lib/string/string.util.js'
@@ -46,7 +46,14 @@ class Exec2 {
    * log: true
    */
   spawn(cmd: string, opt: SpawnOptions = {}): void {
-    const { shell = true, cwd, env, passProcessEnv = true, forceColor = hasColors } = opt
+    const {
+      shell = true,
+      cwd,
+      env,
+      passProcessEnv = true,
+      forceColor = hasColors,
+      stdio = 'inherit',
+    } = opt
     opt.log ??= true // by default log should be true, as we are printing the output
     opt.logStart ??= opt.log
     opt.logFinish ??= opt.log
@@ -55,7 +62,7 @@ class Exec2 {
 
     const r = spawnSync(cmd, opt.args, {
       encoding: 'utf8',
-      stdio: 'inherit',
+      stdio,
       shell,
       cwd,
       env: {
@@ -91,7 +98,7 @@ class Exec2 {
    * log: false
    */
   exec(cmd: string, opt: ExecOptions = {}): string {
-    const { cwd, env, passProcessEnv = true, timeout } = opt
+    const { cwd, env, passProcessEnv = true, timeout, stdio } = opt
     opt.logStart ??= opt.log ?? false
     opt.logFinish ??= opt.log ?? false
     const started = Date.now() as UnixTimestampMillis
@@ -100,8 +107,7 @@ class Exec2 {
     try {
       const s = execSync(cmd, {
         encoding: 'utf8',
-        // stdio: 'inherit', // no, otherwise we don't get the output returned
-        stdio: undefined,
+        stdio,
         // shell: undefined,
         cwd,
         timeout,
@@ -398,6 +404,11 @@ export interface SpawnOptions {
    * Set to false or true to override.
    */
   forceColor?: boolean
+
+  /**
+   * Defaults to "inherit"
+   */
+  stdio?: StdioOptions
 }
 
 export interface ExecOptions {
@@ -429,4 +440,10 @@ export interface ExecOptions {
    * Set to true to pass `process.env` to the spawned process.
    */
   passProcessEnv?: boolean
+
+  /**
+   * Defaults to undefined.
+   * beware that stdio: 'inherit', means we don't get the output returned.
+   */
+  stdio?: StdioOptions
 }

package/src/stream/index.ts

@@ -7,6 +7,7 @@ export * from './ndjson/transformJsonParse.js'
 export * from './ndjson/transformToNDJson.js'
 export * from './pipeline/pipeline.js'
 export * from './progressLogger.js'
+export * from './readable/readableCombined.js'
 export * from './readable/readableCreate.js'
 export * from './readable/readableForEach.js'
 export * from './readable/readableFromArray.js'

package/src/stream/ndjson/transformJsonParse.ts

@@ -36,7 +36,6 @@ export function transformJsonParse<ROW = any>(
     writableObjectMode: false,
     readableObjectMode: true,
     // highWatermark increased, because it's proven to be faster: https://github.com/nodejs/node/pull/52037
-    // todo: it'll be default in Node 22, then we can remove this
     writableHighWaterMark: 64 * 1024,
     transform(chunk: string, _, cb) {
       try {

package/src/stream/readable/readableCombined.ts

@@ -0,0 +1,87 @@
+import { Readable } from 'node:stream'
+import { type DeferredPromise, pDefer } from '@naturalcycles/js-lib/promise/pDefer.js'
+import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
+import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
+
+/**
+ * Allows to combine multiple Readables into 1 Readable.
+ * As soon as any of the input Readables emit - the output Readable emits
+ * (passes through).
+ * Order is not preserved in any way, first come first served!
+ *
+ * Readable completes when all input Readables complete.
+ *
+ * @experimental
+ */
+export class ReadableCombined<T> extends Readable implements ReadableTyped<T> {
+  static create<T>(inputs: Readable[]): ReadableCombined<T> {
+    return new ReadableCombined<T>(inputs)
+  }
+
+  private constructor(public inputs: Readable[]) {
+    super({ objectMode: true })
+    void this.start()
+  }
+
+  /**
+   * If defined - we are in Paused mode
+   * and should await the lock to be resolved before proceeding.
+   *
+   * If not defined - we are in Flowing mode, no limits in data flow.
+   */
+  private lock?: DeferredPromise
+
+  // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
+  private countIn = 0
+  // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
+  private countOut = 0
+  // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
+  private countReads = 0
+
+  private async start(): Promise<void> {
+    await pMap(this.inputs, async (input, i) => {
+      for await (const item of input) {
+        this.countIn++
+        this.logStats()
+        if (this.lock) {
+          await this.lock
+          // lock is undefined at this point
+        }
+
+        const shouldContinue = this.push(item)
+        this.countOut++
+        if (!shouldContinue && !this.lock) {
+          this.lock = pDefer()
+          console.log(`ReadableCombined.push #${i} returned false, pausing the flow!`)
+        }
+      }
+
+      console.log(`ReadableCombined: input #${i} done`)
+    })
+
+    console.log(`ReadableCombined: all inputs done!`)
+    this.push(null)
+  }
+
+  override _read(): void {
+    this.countReads++
+
+    if (this.lock) {
+      console.log(`ReadableCombined._read: resuming the flow!`)
+      // calling it in this order is important!
+      // this.lock should be undefined BEFORE we call lock.resolve()
+      const { lock } = this
+      this.lock = undefined
+      lock.resolve()
+    }
+  }
+
+  private logStats(): void {
+    const { countIn, countOut, countReads } = this
+    console.log({
+      countIn,
+      countOut,
+      countReads,
+    })
+  }
+}

package/src/stream/transform/transformMap.ts

@@ -7,6 +7,7 @@ import {
   type AbortableAsyncMapper,
   type AsyncPredicate,
   END,
+  type PositiveInteger,
   type Promisable,
   SKIP,
   type StringMap,
@@ -38,7 +39,16 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
    * UPD: changed back from 32 to 16, "to be on a safe side", as 32 sometimes
    * causes "Datastore timeout errors".
    */
-  concurrency?: number
+  concurrency?: PositiveInteger
+
+  /**
+   * Defaults to 64 items.
+   * (objectMode default is 16, but we increased it)
+   *
+   * Affects both readable and writable highWaterMark (buffer).
+   * So, 64 means a total buffer of 128 (64 input and 64 output buffer).
+   */
+  highWaterMark?: PositiveInteger
 
   /**
    * @default THROW_IMMEDIATELY
@@ -123,6 +133,7 @@ export function transformMap<IN = any, OUT = IN>(
 ): TransformTyped<IN, OUT> {
   const {
     concurrency = 16,
+    highWaterMark = 64,
     predicate, // we now default to "no predicate" (meaning pass-everything)
     errorMode = ErrorMode.THROW_IMMEDIATELY,
     onError,
@@ -141,6 +152,8 @@ export function transformMap<IN = any, OUT = IN>(
   return through2Concurrent.obj(
     {
       maxConcurrency: concurrency,
+      readableHighWaterMark: highWaterMark,
+      writableHighWaterMark: highWaterMark,
       async final(cb) {
         // console.log('transformMap final')