@naturalcycles/nodejs-lib 15.21.0 → 15.23.0

package/src/stream/transform/transformMap.ts

@@ -1,12 +1,13 @@
-import { _hc } from '@naturalcycles/js-lib'
+import { _hc, type AbortableSignal } from '@naturalcycles/js-lib'
 import { _since } from '@naturalcycles/js-lib/datetime/time.util.js'
-import { _anyToError, ErrorMode } from '@naturalcycles/js-lib/error'
+import { _anyToError, _assert, ErrorMode } from '@naturalcycles/js-lib/error'
 import type { CommonLogger } from '@naturalcycles/js-lib/log'
 import { _stringify } from '@naturalcycles/js-lib/string/stringify.js'
 import {
   type AbortableAsyncMapper,
   type AsyncPredicate,
   END,
+  type PositiveInteger,
   type Promisable,
   SKIP,
   type StringMap,
@@ -14,9 +15,8 @@ import {
 } from '@naturalcycles/js-lib/types'
 import through2Concurrent from 'through2-concurrent'
 import { yellow } from '../../colors/colors.js'
-import type { AbortableTransform } from '../pipeline/pipeline.js'
 import type { TransformTyped } from '../stream.model.js'
-import { pipelineClose } from '../stream.util.js'
+import { PIPELINE_GRACEFUL_ABORT } from '../stream.util.js'
 
 export interface TransformMapOptions<IN = any, OUT = IN> {
   /**
@@ -38,7 +38,16 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
    * UPD: changed back from 32 to 16, "to be on a safe side", as 32 sometimes
    * causes "Datastore timeout errors".
    */
-  concurrency?: number
+  concurrency?: PositiveInteger
+
+  /**
+   * Defaults to 64 items.
+   * (objectMode default is 16, but we increased it)
+   *
+   * Affects both readable and writable highWaterMark (buffer).
+   * So, 64 means a total buffer of 128 (64 input and 64 output buffer).
+   */
+  highWaterMark?: PositiveInteger
 
   /**
    * @default THROW_IMMEDIATELY
@@ -71,6 +80,11 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
   metric?: string
 
   logger?: CommonLogger
+
+  /**
+   * Allows to abort (gracefully stop) the stream from inside the Transform.
+   */
+  signal?: AbortableSignal
 }
 
 export interface TransformMapStats {
@@ -123,24 +137,29 @@ export function transformMap<IN = any, OUT = IN>(
 ): TransformTyped<IN, OUT> {
   const {
     concurrency = 16,
+    highWaterMark = 64,
     predicate, // we now default to "no predicate" (meaning pass-everything)
     errorMode = ErrorMode.THROW_IMMEDIATELY,
     onError,
     onDone,
     metric = 'stream',
     logger = console,
+    signal,
   } = opt
 
   const started = Date.now() as UnixTimestampMillis
   let index = -1
   let countOut = 0
   let isSettled = false
+  let ok = true
   let errors = 0
   const collectedErrors: Error[] = [] // only used if errorMode == THROW_AGGREGATED
 
   return through2Concurrent.obj(
     {
       maxConcurrency: concurrency,
+      readableHighWaterMark: highWaterMark,
+      writableHighWaterMark: highWaterMark,
       async final(cb) {
         // console.log('transformMap final')
 
@@ -172,7 +191,7 @@ export function transformMap<IN = any, OUT = IN>(
 
           try {
             await onDone?.({
-              ok: true,
+              ok,
               collectedErrors,
               countErrors: errors,
               countIn: index + 1,
@@ -187,7 +206,7 @@ export function transformMap<IN = any, OUT = IN>(
         }
       },
     },
-    async function transformMapFn(this: AbortableTransform, chunk: IN, _, cb) {
+    async function transformMapFn(chunk: IN, _, cb) {
       // Stop processing if isSettled (either THROW_IMMEDIATELY was fired or END received)
       if (isSettled) return cb()
 
@@ -201,7 +220,8 @@ export function transformMap<IN = any, OUT = IN>(
         if (res === END) {
           isSettled = true
           logger.log(`transformMap END received at index ${currentIndex}`)
-          pipelineClose('transformMap', this, this.sourceReadable, this.streamDone, logger)
+          _assert(signal, 'signal is required when using END')
+          signal.abort(new Error(PIPELINE_GRACEFUL_ABORT))
           return cb()
         }
 
@@ -230,19 +250,23 @@ export function transformMap<IN = any, OUT = IN>(
 
         if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
           isSettled = true
-
-          try {
-            await onDone?.({
-              ok: false,
-              collectedErrors,
-              countErrors: errors,
-              countIn: index + 1,
-              countOut,
-              started,
-            })
-          } catch (err) {
-            logger.error(err)
-          }
+          ok = false
+
+          // Tests show that onDone is still called at `final` (second time),
+          // so, we no longer call it here
+
+          // try {
+          //   await onDone?.({
+          //     ok: false,
+          //     collectedErrors,
+          //     countErrors: errors,
+          //     countIn: index + 1,
+          //     countOut,
+          //     started,
+          //   })
+          // } catch (err) {
+          //   logger.error(err)
+          // }
 
           return cb(err) // Emit error immediately
         }

package/src/stream/transform/transformMapSync.ts

@@ -1,11 +1,12 @@
-import { _anyToError, ErrorMode } from '@naturalcycles/js-lib/error'
+import { Transform } from 'node:stream'
+import type { AbortableSignal } from '@naturalcycles/js-lib'
+import { _anyToError, _assert, ErrorMode } from '@naturalcycles/js-lib/error'
 import type { CommonLogger } from '@naturalcycles/js-lib/log'
 import type { IndexedMapper, Predicate, UnixTimestampMillis } from '@naturalcycles/js-lib/types'
 import { END, SKIP } from '@naturalcycles/js-lib/types'
 import { yellow } from '../../colors/colors.js'
-import { AbortableTransform } from '../pipeline/pipeline.js'
 import type { TransformTyped } from '../stream.model.js'
-import { pipelineClose } from '../stream.util.js'
+import { PIPELINE_GRACEFUL_ABORT } from '../stream.util.js'
 import type { TransformMapStats } from './transformMap.js'
 
 export interface TransformMapSyncOptions<IN = any, OUT = IN> {
@@ -54,9 +55,12 @@ export interface TransformMapSyncOptions<IN = any, OUT = IN> {
   metric?: string
 
   logger?: CommonLogger
-}
 
-export class TransformMapSync extends AbortableTransform {}
+  /**
+   * Allows to abort (gracefully stop) the stream from inside the Transform.
+   */
+  signal?: AbortableSignal
+}
 
 /**
  * Sync (not async) version of transformMap.
@@ -74,6 +78,7 @@ export function transformMapSync<IN = any, OUT = IN>(
     metric = 'stream',
     objectMode = true,
     logger = console,
+    signal,
   } = opt
 
   const started = Date.now() as UnixTimestampMillis
@@ -83,10 +88,10 @@ export function transformMapSync<IN = any, OUT = IN>(
   let errors = 0
   const collectedErrors: Error[] = [] // only used if errorMode == THROW_AGGREGATED
 
-  return new TransformMapSync({
+  return new Transform({
     objectMode,
     ...opt,
-    transform(this: AbortableTransform, chunk: IN, _, cb) {
+    transform(chunk: IN, _, cb) {
       // Stop processing if isSettled
       if (isSettled) return cb()
 
@@ -99,7 +104,8 @@ export function transformMapSync<IN = any, OUT = IN>(
         if (v === END) {
           isSettled = true // will be checked later
           logger.log(`transformMapSync END received at index ${currentIndex}`)
-          pipelineClose('transformMapSync', this, this.sourceReadable, this.streamDone, logger)
+          _assert(signal, 'signal is required when using END')
+          signal.abort(new Error(PIPELINE_GRACEFUL_ABORT))
           return cb()
         }
 

package/src/stream/transform/transformTee.ts

@@ -1,5 +1,5 @@
 import { Transform } from 'node:stream'
-import { _pipeline } from '../pipeline/pipeline.js'
+import { pipeline } from 'node:stream/promises'
 import { readableCreate } from '../readable/readableCreate.js'
 import type { TransformTyped } from '../stream.model.js'
 
@@ -17,11 +17,14 @@ type AnyStream = NodeJS.WritableStream | NodeJS.ReadWriteStream
 export function transformTee<T>(streams: AnyStream[]): TransformTyped<T, T> {
   const readable = readableCreate<T>()
 
-  const secondPipelinePromise = _pipeline([readable, ...streams])
+  const secondPipelinePromise = pipeline([readable, ...streams])
 
   return new Transform({
     objectMode: true,
     transform(chunk: T, _, cb) {
+      // todo: it's possible to start respecting backpressure,
+      // if we start to listen to the boolean output of .push()
+
       // pass to the "secondary" pipeline
       readable.push(chunk)
 

package/src/stream/writable/writableForEach.ts

@@ -2,7 +2,7 @@ import type { AsyncIndexedMapper, IndexedMapper } from '@naturalcycles/js-lib/ty
 import { _passNothingPredicate } from '@naturalcycles/js-lib/types'
 import type { WritableTyped } from '../stream.model.js'
 import { transformMap, type TransformMapOptions } from '../transform/transformMap.js'
-import { transformMapSync } from '../transform/transformMapSync.js'
+import { transformMapSync, type TransformMapSyncOptions } from '../transform/transformMapSync.js'
 
 /**
  * Just an alias to transformMap that declares OUT as void.
@@ -19,7 +19,7 @@ export function writableForEach<IN = any>(
  */
 export function writableForEachSync<IN = any>(
   mapper: IndexedMapper<IN, void>,
-  opt: TransformMapOptions<IN, void> = {},
+  opt: TransformMapSyncOptions<IN, void> = {},
 ): WritableTyped<IN> {
   return transformMapSync<IN, void>(mapper, { ...opt, predicate: _passNothingPredicate })
 }

package/src/stream/writable/writableFork.ts

@@ -1,5 +1,5 @@
 import { Writable } from 'node:stream'
-import { _pipeline } from '../pipeline/pipeline.js'
+import { pipeline } from 'node:stream/promises'
 import { readableCreate } from '../readable/readableCreate.js'
 import type { ReadableTyped, TransformOptions, WritableTyped } from '../stream.model.js'
 
@@ -22,7 +22,7 @@ export function writableFork<T>(
       const readable = readableCreate<T>()
       readables.push(readable)
 
-      return await _pipeline([readable, ...chain])
+      return await pipeline([readable, ...chain])
     }),
   ).catch(err => {
     console.error(err) // ensure the error is logged

package/dist/stream/pipeline/pipeline.d.ts

@@ -1,36 +0,0 @@
-import type { Readable } from 'node:stream';
-import { Transform } from 'node:stream';
-import type { DeferredPromise } from '@naturalcycles/js-lib/promise';
-type AnyStream = NodeJS.ReadableStream | NodeJS.WritableStream | NodeJS.ReadWriteStream;
-export interface PipelineOptions {
-    /**
-     * Set to true to allow ERR_STREAM_PREMATURE_CLOSE.
-     * Required to support graceful close when using transformLimit
-     */
-    allowClose?: boolean;
-    /**
-     * Set to true to allow graceful abort (via AbortSignal).
-     * "Graceful" means it'll swallow the AbortError and let the pipeline resolve normally.
-     *
-     * Default is false - AbortError will be thrown.
-     */
-    allowGracefulAbort?: boolean;
-    signal?: AbortSignal;
-}
-/**
- * Promisified `stream.pipeline`.
- *
- * Supports opt.allowClose, which allows transformLimit to work (to actually stop source Readable)
- * without throwing an error (ERR_STREAM_PREMATURE_CLOSE).
- */
-export declare function _pipeline(streams: AnyStream[], opt?: PipelineOptions): Promise<void>;
-/**
- * Convenience function to make _pipeline collect all items at the end of the stream (should be Transform, not Writeable!)
- * and return.
- */
-export declare function _pipelineToArray<T>(streams: AnyStream[], opt?: PipelineOptions): Promise<T[]>;
-export declare class AbortableTransform extends Transform {
-    sourceReadable?: Readable;
-    streamDone?: DeferredPromise;
-}
-export {};

package/dist/stream/pipeline/pipeline.js

@@ -1,82 +0,0 @@
-import { Transform } from 'node:stream';
-import { pipeline } from 'node:stream/promises';
-import { _last } from '@naturalcycles/js-lib/array/array.util.js';
-import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
-import { writablePushToArray } from '../writable/writablePushToArray.js';
-/**
- * Promisified `stream.pipeline`.
- *
- * Supports opt.allowClose, which allows transformLimit to work (to actually stop source Readable)
- * without throwing an error (ERR_STREAM_PREMATURE_CLOSE).
- */
-export async function _pipeline(streams, opt = {}) {
-    const first = streams[0];
-    const rest = streams.slice(1);
-    if (opt.allowClose) {
-        // Do the magic of making the pipeline "abortable"
-        //
-        // How does it work:
-        // It finds `sourceReadable` (basically, it's just first item in the passed array of streams)
-        // Finds last "writable" (last item), patches the `_final` method of it to detect when the whole pipeline is "done",
-        // sets the `streamDone` DeferredPromise that resolves when the pipeline is done.
-        // Scans through all passed items, finds those that are capable of "closing" the stream
-        // (currently its `transformLimit` or `transformMap`)
-        // Patches them by attaching `sourceReadable` and `streamDone`.
-        // These items (transformLimit and transformMap), when they need to "close the stream" - call `pipelineClose`.
-        // `pipelineClose` is the result of 2 sleepless nights of googling and experimentation:)
-        // It does:
-        // 1. Stops the "downstream" by doing `this.push(null)`.
-        // 2. Pauses the `sourceReadable` by calling sourceReadable.unpipe()
-        // 3. Waits for `streamDone` to ensure that downstream chunks are fully processed (e.g written to disk).
-        // 4. Calls `sourceReadable.destroy()`, which emits ERR_STREAM_PREMATURE_CLOSE
-        // 5. _pipeline (this function) catches that specific error and suppresses it (because it's expected and
-        // inevitable in this flow). Know a better way to close the stream? Tell me!
-        const streamDone = pDefer();
-        const sourceReadable = first;
-        const last = _last(streams);
-        const lastFinal = last._final?.bind(last) || ((cb) => cb());
-        last._final = cb => {
-            lastFinal(() => {
-                cb();
-                streamDone.resolve();
-            });
-        };
-        rest.forEach(s => {
-            // console.log(s)
-            if (s instanceof AbortableTransform || s.constructor.name === 'AbortableTransform') {
-                // console.log(`found ${s.constructor.name}, setting props`)
-                ;
-                s.sourceReadable = sourceReadable;
-                s.streamDone = streamDone;
-            }
-        });
-    }
-    try {
-        return await pipeline([first, ...rest], opt);
-    }
-    catch (err) {
-        if (opt.allowClose && err?.code === 'ERR_STREAM_PREMATURE_CLOSE') {
-            console.log('_pipeline closed (as expected)');
-            return;
-        }
-        if (opt.allowGracefulAbort && err?.name === 'AbortError') {
-            console.log('_pipeline closed via AbortSignal (as expected)');
-            return;
-        }
-        // console.log(`_pipeline error`, err)
-        throw err;
-    }
-}
-/**
- * Convenience function to make _pipeline collect all items at the end of the stream (should be Transform, not Writeable!)
- * and return.
- */
-export async function _pipelineToArray(streams, opt = {}) {
-    const a = [];
-    await _pipeline([...streams, writablePushToArray(a)], opt);
-    return a;
-}
-export class AbortableTransform extends Transform {
-    sourceReadable;
-    streamDone;
-}

package/dist/stream/readable/readableForEach.d.ts

@@ -1,19 +0,0 @@
-import type { AbortableAsyncMapper, IndexedMapper } from '@naturalcycles/js-lib/types';
-import type { ReadableTyped } from '../stream.model.js';
-import type { TransformMapOptions } from '../transform/transformMap.js';
-/**
- * Convenience function to do `.forEach` over a Readable.
- * Typed! (unlike default Readable).
- *
- * Try native readable.forEach() instead!
- *
- * @experimental
- */
-export declare function readableForEach<T>(readable: ReadableTyped<T>, mapper: AbortableAsyncMapper<T, void>, opt?: TransformMapOptions<T, void>): Promise<void>;
-/**
- * Convenience function to do `.forEach` over a Readable.
- * Typed! (unlike default Readable).
- *
- * @experimental
- */
-export declare function readableForEachSync<T>(readable: ReadableTyped<T>, mapper: IndexedMapper<T, void>): Promise<void>;

package/dist/stream/readable/readableForEach.js

@@ -1,30 +0,0 @@
-import { _passNothingPredicate } from '@naturalcycles/js-lib/types';
-import { _pipeline } from '../pipeline/pipeline.js';
-import { transformMap } from '../transform/transformMap.js';
-/**
- * Convenience function to do `.forEach` over a Readable.
- * Typed! (unlike default Readable).
- *
- * Try native readable.forEach() instead!
- *
- * @experimental
- */
-export async function readableForEach(readable, mapper, opt = {}) {
-    await _pipeline([
-        readable,
-        transformMap(mapper, { ...opt, predicate: _passNothingPredicate }),
-    ]);
-}
-/**
- * Convenience function to do `.forEach` over a Readable.
- * Typed! (unlike default Readable).
- *
- * @experimental
- */
-export async function readableForEachSync(readable, mapper) {
-    // async iteration
-    let index = 0;
-    for await (const item of readable) {
-        mapper(item, index++);
-    }
-}

package/src/stream/pipeline/pipeline.ts

@@ -1,114 +0,0 @@
-import type { Readable, Writable } from 'node:stream'
-import { Transform } from 'node:stream'
-import { pipeline } from 'node:stream/promises'
-import { _last } from '@naturalcycles/js-lib/array/array.util.js'
-import type { DeferredPromise } from '@naturalcycles/js-lib/promise'
-import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js'
-import type { AnyFunction } from '@naturalcycles/js-lib/types'
-import { writablePushToArray } from '../writable/writablePushToArray.js'
-
-type AnyStream = NodeJS.ReadableStream | NodeJS.WritableStream | NodeJS.ReadWriteStream
-
-export interface PipelineOptions {
-  /**
-   * Set to true to allow ERR_STREAM_PREMATURE_CLOSE.
-   * Required to support graceful close when using transformLimit
-   */
-  allowClose?: boolean
-
-  /**
-   * Set to true to allow graceful abort (via AbortSignal).
-   * "Graceful" means it'll swallow the AbortError and let the pipeline resolve normally.
-   *
-   * Default is false - AbortError will be thrown.
-   */
-  allowGracefulAbort?: boolean
-
-  signal?: AbortSignal
-}
-
-/**
- * Promisified `stream.pipeline`.
- *
- * Supports opt.allowClose, which allows transformLimit to work (to actually stop source Readable)
- * without throwing an error (ERR_STREAM_PREMATURE_CLOSE).
- */
-export async function _pipeline(streams: AnyStream[], opt: PipelineOptions = {}): Promise<void> {
-  const first = streams[0] as any
-  const rest = streams.slice(1)
-
-  if (opt.allowClose) {
-    // Do the magic of making the pipeline "abortable"
-    //
-    // How does it work:
-    // It finds `sourceReadable` (basically, it's just first item in the passed array of streams)
-    // Finds last "writable" (last item), patches the `_final` method of it to detect when the whole pipeline is "done",
-    // sets the `streamDone` DeferredPromise that resolves when the pipeline is done.
-    // Scans through all passed items, finds those that are capable of "closing" the stream
-    // (currently its `transformLimit` or `transformMap`)
-    // Patches them by attaching `sourceReadable` and `streamDone`.
-    // These items (transformLimit and transformMap), when they need to "close the stream" - call `pipelineClose`.
-    // `pipelineClose` is the result of 2 sleepless nights of googling and experimentation:)
-    // It does:
-    // 1. Stops the "downstream" by doing `this.push(null)`.
-    // 2. Pauses the `sourceReadable` by calling sourceReadable.unpipe()
-    // 3. Waits for `streamDone` to ensure that downstream chunks are fully processed (e.g written to disk).
-    // 4. Calls `sourceReadable.destroy()`, which emits ERR_STREAM_PREMATURE_CLOSE
-    // 5. _pipeline (this function) catches that specific error and suppresses it (because it's expected and
-    // inevitable in this flow). Know a better way to close the stream? Tell me!
-    const streamDone = pDefer()
-    const sourceReadable = first as Readable
-    const last = _last(streams) as Writable
-    const lastFinal = last._final?.bind(last) || ((cb: AnyFunction) => cb())
-    last._final = cb => {
-      lastFinal(() => {
-        cb()
-        streamDone.resolve()
-      })
-    }
-
-    rest.forEach(s => {
-      // console.log(s)
-      if (s instanceof AbortableTransform || s.constructor.name === 'AbortableTransform') {
-        // console.log(`found ${s.constructor.name}, setting props`)
-        ;(s as AbortableTransform).sourceReadable = sourceReadable
-        ;(s as AbortableTransform).streamDone = streamDone
-      }
-    })
-  }
-
-  try {
-    return await pipeline([first, ...rest], opt)
-  } catch (err) {
-    if (opt.allowClose && (err as any)?.code === 'ERR_STREAM_PREMATURE_CLOSE') {
-      console.log('_pipeline closed (as expected)')
-      return
-    }
-
-    if (opt.allowGracefulAbort && (err as any)?.name === 'AbortError') {
-      console.log('_pipeline closed via AbortSignal (as expected)')
-      return
-    }
-
-    // console.log(`_pipeline error`, err)
-    throw err
-  }
-}
-
-/**
- * Convenience function to make _pipeline collect all items at the end of the stream (should be Transform, not Writeable!)
- * and return.
- */
-export async function _pipelineToArray<T>(
-  streams: AnyStream[],
-  opt: PipelineOptions = {},
-): Promise<T[]> {
-  const a: T[] = []
-  await _pipeline([...streams, writablePushToArray(a)], opt)
-  return a
-}
-
-export class AbortableTransform extends Transform {
-  sourceReadable?: Readable
-  streamDone?: DeferredPromise
-}

package/src/stream/readable/readableForEach.ts

@@ -1,42 +0,0 @@
-import type { AbortableAsyncMapper, IndexedMapper } from '@naturalcycles/js-lib/types'
-import { _passNothingPredicate } from '@naturalcycles/js-lib/types'
-import { _pipeline } from '../pipeline/pipeline.js'
-import type { ReadableTyped } from '../stream.model.js'
-import type { TransformMapOptions } from '../transform/transformMap.js'
-import { transformMap } from '../transform/transformMap.js'
-
-/**
- * Convenience function to do `.forEach` over a Readable.
- * Typed! (unlike default Readable).
- *
- * Try native readable.forEach() instead!
- *
- * @experimental
- */
-export async function readableForEach<T>(
-  readable: ReadableTyped<T>,
-  mapper: AbortableAsyncMapper<T, void>,
-  opt: TransformMapOptions<T, void> = {},
-): Promise<void> {
-  await _pipeline([
-    readable,
-    transformMap<T, void>(mapper, { ...opt, predicate: _passNothingPredicate }),
-  ])
-}
-
-/**
- * Convenience function to do `.forEach` over a Readable.
- * Typed! (unlike default Readable).
- *
- * @experimental
- */
-export async function readableForEachSync<T>(
-  readable: ReadableTyped<T>,
-  mapper: IndexedMapper<T, void>,
-): Promise<void> {
-  // async iteration
-  let index = 0
-  for await (const item of readable) {
-    mapper(item, index++)
-  }
-}