@naturalcycles/nodejs-lib 15.21.0 → 15.23.0

package/dist/stream/transform/transformLimit.d.ts

@@ -1,6 +1,4 @@
-import type { Readable } from 'node:stream';
-import type { CommonLogger } from '@naturalcycles/js-lib/log';
-import { AbortableTransform } from '../pipeline/pipeline.js';
+import type { AbortableSignal } from '@naturalcycles/js-lib';
 import type { TransformOptions, TransformTyped } from '../stream.model.js';
 export interface TransformLimitOptions extends TransformOptions {
     /**
@@ -8,29 +6,8 @@ export interface TransformLimitOptions extends TransformOptions {
      */
     limit?: number;
     /**
-     * If provided (recommended!) - it will call readable.destroy() on limit.
-     * Without it - it will only stop the downstream consumers, but won't stop
-     * the Readable ("source" of the stream).
-     * It is almost always crucial to stop the Source too, so, please provide the Readable here!
+     * Allows to abort (gracefully stop) the stream from inside the Transform.
      */
-    sourceReadable?: Readable;
-    /**
-     * Please provide it (a Promise that resolves when the Stream is done, e.g finished consuming things)
-     * to be able to wait for Consumers before calling `readable.destroy`.
-     * Has no effect if `readable` is not provided.
-     */
-    streamDone?: Promise<void>;
-    logger?: CommonLogger;
-    /**
-     * Set to true to enable additional debug messages, e.g it'll log
-     * when readable still emits values after the limit is reached.
-     */
-    debug?: boolean;
-}
-/**
- * Class only exists to be able to do `instanceof TransformLimit`
- * and to set sourceReadable+streamDone to it in `_pipeline`.
- */
-export declare class TransformLimit extends AbortableTransform {
+    signal: AbortableSignal;
 }
 export declare function transformLimit<IN>(opt: TransformLimitOptions): TransformTyped<IN, IN>;

package/dist/stream/transform/transformLimit.js

@@ -1,41 +1,32 @@
-import { AbortableTransform } from '../pipeline/pipeline.js';
-import { pipelineClose } from '../stream.util.js';
+import { Transform } from 'node:stream';
+import { PIPELINE_GRACEFUL_ABORT } from '../stream.util.js';
 import { transformNoOp } from './transformNoOp.js';
-/**
- * Class only exists to be able to do `instanceof TransformLimit`
- * and to set sourceReadable+streamDone to it in `_pipeline`.
- */
-export class TransformLimit extends AbortableTransform {
-}
 export function transformLimit(opt) {
-    const { logger = console, limit, debug } = opt;
+    const { limit, signal } = opt;
     if (!limit) {
-        // No limit - returning pass-through transform
         return transformNoOp();
     }
     let i = 0; // so we start first chunk with 1
     let ended = false;
-    return new TransformLimit({
+    return new Transform({
         objectMode: true,
         ...opt,
         transform(chunk, _, cb) {
+            if (ended) {
+                return;
+            }
             i++;
             if (i === limit) {
                 ended = true;
-                logger.log(`transformLimit of ${limit} reached`);
                 this.push(chunk);
-                pipelineClose('transformLimit', this, opt.sourceReadable || this.sourceReadable, opt.streamDone || this.streamDone, logger);
-                cb(); // after pause
-            }
-            else if (!ended) {
-                cb(null, chunk);
-            }
-            else {
-                if (debug)
-                    logger.log(`transformLimit.transform after limit`, i);
-                // If we ever HANG (don't call cb) - Node will do process.exit(0) to us
-                cb(); // ended, don't emit anything
+                this.push(null); // tell downstream that we're done
+                cb();
+                queueMicrotask(() => {
+                    signal.abort(new Error(PIPELINE_GRACEFUL_ABORT));
+                });
+                return;
             }
+            cb(null, chunk);
         },
     });
 }

package/dist/stream/transform/transformMap.d.ts

@@ -1,6 +1,7 @@
+import { type AbortableSignal } from '@naturalcycles/js-lib';
 import { ErrorMode } from '@naturalcycles/js-lib/error';
 import type { CommonLogger } from '@naturalcycles/js-lib/log';
-import { type AbortableAsyncMapper, type AsyncPredicate, END, type Promisable, SKIP, type StringMap, type UnixTimestampMillis } from '@naturalcycles/js-lib/types';
+import { type AbortableAsyncMapper, type AsyncPredicate, END, type PositiveInteger, type Promisable, SKIP, type StringMap, type UnixTimestampMillis } from '@naturalcycles/js-lib/types';
 import type { TransformTyped } from '../stream.model.js';
 export interface TransformMapOptions<IN = any, OUT = IN> {
     /**
@@ -21,7 +22,15 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
      * UPD: changed back from 32 to 16, "to be on a safe side", as 32 sometimes
      * causes "Datastore timeout errors".
      */
-    concurrency?: number;
+    concurrency?: PositiveInteger;
+    /**
+     * Defaults to 64 items.
+     * (objectMode default is 16, but we increased it)
+     *
+     * Affects both readable and writable highWaterMark (buffer).
+     * So, 64 means a total buffer of 128 (64 input and 64 output buffer).
+     */
+    highWaterMark?: PositiveInteger;
     /**
      * @default THROW_IMMEDIATELY
      */
@@ -49,6 +58,10 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
      */
     metric?: string;
     logger?: CommonLogger;
+    /**
+     * Allows to abort (gracefully stop) the stream from inside the Transform.
+     */
+    signal?: AbortableSignal;
 }
 export interface TransformMapStats {
     /**

package/dist/stream/transform/transformMap.js

@@ -1,11 +1,11 @@
 import { _hc } from '@naturalcycles/js-lib';
 import { _since } from '@naturalcycles/js-lib/datetime/time.util.js';
-import { _anyToError, ErrorMode } from '@naturalcycles/js-lib/error';
+import { _anyToError, _assert, ErrorMode } from '@naturalcycles/js-lib/error';
 import { _stringify } from '@naturalcycles/js-lib/string/stringify.js';
 import { END, SKIP, } from '@naturalcycles/js-lib/types';
 import through2Concurrent from 'through2-concurrent';
 import { yellow } from '../../colors/colors.js';
-import { pipelineClose } from '../stream.util.js';
+import { PIPELINE_GRACEFUL_ABORT } from '../stream.util.js';
 // doesn't work, cause here we don't construct our Transform instance ourselves
 // export class TransformMap extends AbortableTransform {}
 /**
@@ -21,16 +21,19 @@ import { pipelineClose } from '../stream.util.js';
  * If an Array is returned by `mapper` - it will be flattened and multiple results will be emitted from it. Tested by Array.isArray().
  */
 export function transformMap(mapper, opt = {}) {
-    const { concurrency = 16, predicate, // we now default to "no predicate" (meaning pass-everything)
-    errorMode = ErrorMode.THROW_IMMEDIATELY, onError, onDone, metric = 'stream', logger = console, } = opt;
+    const { concurrency = 16, highWaterMark = 64, predicate, // we now default to "no predicate" (meaning pass-everything)
+    errorMode = ErrorMode.THROW_IMMEDIATELY, onError, onDone, metric = 'stream', logger = console, signal, } = opt;
     const started = Date.now();
     let index = -1;
     let countOut = 0;
     let isSettled = false;
+    let ok = true;
     let errors = 0;
     const collectedErrors = []; // only used if errorMode == THROW_AGGREGATED
     return through2Concurrent.obj({
         maxConcurrency: concurrency,
+        readableHighWaterMark: highWaterMark,
+        writableHighWaterMark: highWaterMark,
         async final(cb) {
             // console.log('transformMap final')
             logErrorStats(true);
@@ -55,7 +58,7 @@ export function transformMap(mapper, opt = {}) {
                 // emit no error
                 try {
                     await onDone?.({
-                        ok: true,
+                        ok,
                         collectedErrors,
                         countErrors: errors,
                         countIn: index + 1,
@@ -82,7 +85,8 @@ export function transformMap(mapper, opt = {}) {
             if (res === END) {
                 isSettled = true;
                 logger.log(`transformMap END received at index ${currentIndex}`);
-                pipelineClose('transformMap', this, this.sourceReadable, this.streamDone, logger);
+                _assert(signal, 'signal is required when using END');
+                signal.abort(new Error(PIPELINE_GRACEFUL_ABORT));
                 return cb();
             }
             if (res === SKIP) {
@@ -108,19 +112,21 @@ export function transformMap(mapper, opt = {}) {
             }
             if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
                 isSettled = true;
-                try {
-                    await onDone?.({
-                        ok: false,
-                        collectedErrors,
-                        countErrors: errors,
-                        countIn: index + 1,
-                        countOut,
-                        started,
-                    });
-                }
-                catch (err) {
-                    logger.error(err);
-                }
+                ok = false;
+                // Tests show that onDone is still called at `final` (second time),
+                // so, we no longer call it here
+                // try {
+                //   await onDone?.({
+                //     ok: false,
+                //     collectedErrors,
+                //     countErrors: errors,
+                //     countIn: index + 1,
+                //     countOut,
+                //     started,
+                //   })
+                // } catch (err) {
+                //   logger.error(err)
+                // }
                 return cb(err); // Emit error immediately
             }
             if (errorMode === ErrorMode.THROW_AGGREGATED) {

package/dist/stream/transform/transformMapSync.d.ts

@@ -1,8 +1,8 @@
+import type { AbortableSignal } from '@naturalcycles/js-lib';
 import { ErrorMode } from '@naturalcycles/js-lib/error';
 import type { CommonLogger } from '@naturalcycles/js-lib/log';
 import type { IndexedMapper, Predicate } from '@naturalcycles/js-lib/types';
 import { END, SKIP } from '@naturalcycles/js-lib/types';
-import { AbortableTransform } from '../pipeline/pipeline.js';
 import type { TransformTyped } from '../stream.model.js';
 import type { TransformMapStats } from './transformMap.js';
 export interface TransformMapSyncOptions<IN = any, OUT = IN> {
@@ -45,8 +45,10 @@ export interface TransformMapSyncOptions<IN = any, OUT = IN> {
      */
     metric?: string;
     logger?: CommonLogger;
-}
-export declare class TransformMapSync extends AbortableTransform {
+    /**
+     * Allows to abort (gracefully stop) the stream from inside the Transform.
+     */
+    signal?: AbortableSignal;
 }
 /**
  * Sync (not async) version of transformMap.

package/dist/stream/transform/transformMapSync.js

@@ -1,24 +1,22 @@
-import { _anyToError, ErrorMode } from '@naturalcycles/js-lib/error';
+import { Transform } from 'node:stream';
+import { _anyToError, _assert, ErrorMode } from '@naturalcycles/js-lib/error';
 import { END, SKIP } from '@naturalcycles/js-lib/types';
 import { yellow } from '../../colors/colors.js';
-import { AbortableTransform } from '../pipeline/pipeline.js';
-import { pipelineClose } from '../stream.util.js';
-export class TransformMapSync extends AbortableTransform {
-}
+import { PIPELINE_GRACEFUL_ABORT } from '../stream.util.js';
 /**
  * Sync (not async) version of transformMap.
  * Supposedly faster, for cases when async is not needed.
  */
 export function transformMapSync(mapper, opt = {}) {
     const { predicate, // defaults to "no predicate" (pass everything)
-    errorMode = ErrorMode.THROW_IMMEDIATELY, onError, onDone, metric = 'stream', objectMode = true, logger = console, } = opt;
+    errorMode = ErrorMode.THROW_IMMEDIATELY, onError, onDone, metric = 'stream', objectMode = true, logger = console, signal, } = opt;
     const started = Date.now();
     let index = -1;
     let countOut = 0;
     let isSettled = false;
     let errors = 0;
     const collectedErrors = []; // only used if errorMode == THROW_AGGREGATED
-    return new TransformMapSync({
+    return new Transform({
         objectMode,
         ...opt,
         transform(chunk, _, cb) {
@@ -32,7 +30,8 @@ export function transformMapSync(mapper, opt = {}) {
                 if (v === END) {
                     isSettled = true; // will be checked later
                     logger.log(`transformMapSync END received at index ${currentIndex}`);
-                    pipelineClose('transformMapSync', this, this.sourceReadable, this.streamDone, logger);
+                    _assert(signal, 'signal is required when using END');
+                    signal.abort(new Error(PIPELINE_GRACEFUL_ABORT));
                     return cb();
                 }
                 if (v === SKIP) {

package/dist/stream/transform/transformTee.js

@@ -1,5 +1,5 @@
 import { Transform } from 'node:stream';
-import { _pipeline } from '../pipeline/pipeline.js';
+import { pipeline } from 'node:stream/promises';
 import { readableCreate } from '../readable/readableCreate.js';
 /**
  * Allows to "tee"/"fork" away from the "main pipeline" into the "secondary pipeline".
@@ -12,10 +12,12 @@ import { readableCreate } from '../readable/readableCreate.js';
  */
 export function transformTee(streams) {
     const readable = readableCreate();
-    const secondPipelinePromise = _pipeline([readable, ...streams]);
+    const secondPipelinePromise = pipeline([readable, ...streams]);
     return new Transform({
         objectMode: true,
         transform(chunk, _, cb) {
+            // todo: it's possible to start respecting backpressure,
+            // if we start to listen to the boolean output of .push()
             // pass to the "secondary" pipeline
             readable.push(chunk);
             // pass through to the "main" pipeline

package/dist/stream/writable/writableForEach.d.ts

@@ -1,6 +1,7 @@
 import type { AsyncIndexedMapper, IndexedMapper } from '@naturalcycles/js-lib/types';
 import type { WritableTyped } from '../stream.model.js';
 import { type TransformMapOptions } from '../transform/transformMap.js';
+import { type TransformMapSyncOptions } from '../transform/transformMapSync.js';
 /**
  * Just an alias to transformMap that declares OUT as void.
  */
@@ -8,4 +9,4 @@ export declare function writableForEach<IN = any>(mapper: AsyncIndexedMapper<IN,
 /**
  * Just an alias to transformMap that declares OUT as void.
  */
-export declare function writableForEachSync<IN = any>(mapper: IndexedMapper<IN, void>, opt?: TransformMapOptions<IN, void>): WritableTyped<IN>;
+export declare function writableForEachSync<IN = any>(mapper: IndexedMapper<IN, void>, opt?: TransformMapSyncOptions<IN, void>): WritableTyped<IN>;

package/dist/stream/writable/writableFork.js

@@ -1,5 +1,5 @@
 import { Writable } from 'node:stream';
-import { _pipeline } from '../pipeline/pipeline.js';
+import { pipeline } from 'node:stream/promises';
 import { readableCreate } from '../readable/readableCreate.js';
 /**
  * Allows "forking" a stream inside pipeline into a number of pipeline chains (2 or more).
@@ -14,7 +14,7 @@ export function writableFork(chains, opt) {
     const allChainsDone = Promise.all(chains.map(async (chain) => {
         const readable = readableCreate();
         readables.push(readable);
-        return await _pipeline([readable, ...chain]);
+        return await pipeline([readable, ...chain]);
     })).catch(err => {
         console.error(err); // ensure the error is logged
         throw err;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.21.0",
+  "version": "15.23.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@types/js-yaml": "^4",

package/src/exec2/exec2.ts

@@ -288,6 +288,7 @@ class Exec2 {
         [
           '  ',
           dimGrey(envString),
+          // todo: only before first space
           white(_substringAfterLast(cmd, '/')),
           ...((opt as SpawnOptions).args || []),
         ]

package/src/stream/index.ts

@@ -5,10 +5,10 @@ export * from './ndjson/ndjsonMap.js'
 export * from './ndjson/ndjsonStreamForEach.js'
 export * from './ndjson/transformJsonParse.js'
 export * from './ndjson/transformToNDJson.js'
-export * from './pipeline/pipeline.js'
+export * from './pipeline.js'
 export * from './progressLogger.js'
+export * from './readable/readableCombined.js'
 export * from './readable/readableCreate.js'
-export * from './readable/readableForEach.js'
 export * from './readable/readableFromArray.js'
 export * from './readable/readableToArray.js'
 export * from './stream.model.js'

package/src/stream/ndjson/ndjsonMap.ts

@@ -1,13 +1,7 @@
 import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js'
 import type { AbortableAsyncMapper } from '@naturalcycles/js-lib/types'
-import {
-  createReadStreamAsNDJSON,
-  createWriteStreamAsNDJSON,
-  transformFlatten,
-  type TransformLogProgressOptions,
-  type TransformMapOptions,
-} from '../index.js'
-import { _pipeline, transformLimit, transformLogProgress, transformMap } from '../index.js'
+import type { TransformLogProgressOptions, TransformMapOptions } from '../index.js'
+import { Pipeline } from '../pipeline.js'
 
 export interface NDJSONMapOptions<IN = any, OUT = IN>
   extends TransformMapOptions<IN, OUT>,
@@ -39,20 +33,16 @@ export async function ndjsonMap<IN = any, OUT = any>(
     outputFilePath,
   })
 
-  const readable = createReadStreamAsNDJSON(inputFilePath).take(
-    limitInput || Number.POSITIVE_INFINITY,
-  )
-
-  await _pipeline([
-    readable,
-    transformLogProgress({ metric: 'read', ...opt }),
-    transformMap(mapper, {
+  await Pipeline.fromNDJsonFile<IN>(inputFilePath)
+    .limitSource(limitInput)
+    .logProgress({ metric: 'read', ...opt })
+    .map(mapper, {
       errorMode: ErrorMode.SUPPRESS,
       ...opt,
-    }),
-    transformFlatten(),
-    transformLimit({ limit: limitOutput, sourceReadable: readable }),
-    transformLogProgress({ metric: 'saved', logEvery: logEveryOutput }),
-    ...createWriteStreamAsNDJSON(outputFilePath),
-  ])
+    })
+    .flattenIfNeeded()
+    // .typeCastAs<OUT>()
+    .limit(limitOutput)
+    .logProgress({ metric: 'saved', logEvery: logEveryOutput })
+    .toNDJsonFile(outputFilePath)
 }

package/src/stream/ndjson/ndjsonStreamForEach.ts

@@ -1,13 +1,8 @@
 import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js'
 import type { AbortableAsyncMapper } from '@naturalcycles/js-lib/types'
-import { _pipeline } from '../pipeline/pipeline.js'
-import {
-  transformLogProgress,
-  type TransformLogProgressOptions,
-} from '../transform/transformLogProgress.js'
-import { transformMap, type TransformMapOptions } from '../transform/transformMap.js'
-import { writableVoid } from '../writable/writableVoid.js'
-import { createReadStreamAsNDJSON } from './createReadStreamAsNDJSON.js'
+import { Pipeline } from '../pipeline.js'
+import type { TransformLogProgressOptions } from '../transform/transformLogProgress.js'
+import type { TransformMapOptions } from '../transform/transformMap.js'
 
 export interface NDJSONStreamForEachOptions<IN = any>
   extends TransformMapOptions<IN, void>,
@@ -22,14 +17,12 @@ export async function ndjsonStreamForEach<T>(
   mapper: AbortableAsyncMapper<T, void>,
   opt: NDJSONStreamForEachOptions<T>,
 ): Promise<void> {
-  await _pipeline([
-    createReadStreamAsNDJSON(opt.inputFilePath),
-    transformMap<T, any>(mapper, {
+  await Pipeline.fromNDJsonFile<T>(opt.inputFilePath)
+    .map(mapper, {
       errorMode: ErrorMode.THROW_AGGREGATED,
       ...opt,
       predicate: () => true, // to log progress properly
-    }),
-    transformLogProgress(opt),
-    writableVoid(),
-  ])
+    })
+    .logProgress(opt)
+    .run()
 }

package/src/stream/ndjson/transformJsonParse.ts

@@ -36,7 +36,6 @@ export function transformJsonParse<ROW = any>(
     writableObjectMode: false,
     readableObjectMode: true,
     // highWatermark increased, because it's proven to be faster: https://github.com/nodejs/node/pull/52037
-    // todo: it'll be default in Node 22, then we can remove this
     writableHighWaterMark: 64 * 1024,
     transform(chunk: string, _, cb) {
       try {