@naturalcycles/nodejs-lib 15.22.0 → 15.23.0

package/src/stream/transform/transformChunk.ts

@@ -1,22 +1,19 @@
 import { Transform } from 'node:stream'
+import type { PositiveInteger } from '@naturalcycles/js-lib/types'
 import type { TransformOptions, TransformTyped } from '../stream.model.js'
 
-export interface TransformChunkOptions extends TransformOptions {
-  /**
-   * How many items to include in each chunk.
-   * Last chunk will contain the remaining items, possibly less than chunkSize.
-   */
-  chunkSize: number
-}
-
 /**
  * Similar to RxJS bufferCount(),
  * allows to "chunk" the input stream into chunks of `opt.chunkSize` size.
  * Last chunk will contain the remaining items, possibly less than chunkSize.
+ *
+ * `chunkSize` indicates how many items to include in each chunk.
+ * Last chunk will contain the remaining items, possibly less than chunkSize.
  */
-export function transformChunk<IN = any>(opt: TransformChunkOptions): TransformTyped<IN, IN[]> {
-  const { chunkSize } = opt
-
+export function transformChunk<IN = any>(
+  chunkSize: PositiveInteger,
+  opt?: TransformOptions,
+): TransformTyped<IN, IN[]> {
   let buf: IN[] = []
 
   return new Transform({

package/src/stream/transform/transformFlatten.ts

@@ -5,13 +5,25 @@ export function transformFlatten<T>(): TransformTyped<T[], T> {
   return new Transform({
     objectMode: true,
     transform(chunk: T[], _, cb) {
-      if (!Array.isArray(chunk)) {
-        // As a safety precaution, to not crash the pipeline - push as is
-        this.push(chunk)
-      } else {
+      for (const item of chunk) {
+        this.push(item)
+      }
+      cb() // acknowledge
+    },
+  })
+}
+
+export function transformFlattenIfNeeded<T>(): TransformTyped<T[], T> {
+  return new Transform({
+    objectMode: true,
+    transform(chunk: T[], _, cb) {
+      if (Array.isArray(chunk)) {
         for (const item of chunk) {
           this.push(item)
         }
+      } else {
+        // As a safety precaution, to not crash the pipeline - push as is
+        this.push(chunk)
       }
       cb() // acknowledge
     },

package/src/stream/transform/transformLimit.ts

@@ -1,8 +1,7 @@
-import type { Readable } from 'node:stream'
-import type { CommonLogger } from '@naturalcycles/js-lib/log'
-import { AbortableTransform } from '../pipeline/pipeline.js'
+import { Transform } from 'node:stream'
+import type { AbortableSignal } from '@naturalcycles/js-lib'
 import type { TransformOptions, TransformTyped } from '../stream.model.js'
-import { pipelineClose } from '../stream.util.js'
+import { PIPELINE_GRACEFUL_ABORT } from '../stream.util.js'
 import { transformNoOp } from './transformNoOp.js'
 
 export interface TransformLimitOptions extends TransformOptions {
@@ -12,72 +11,42 @@ export interface TransformLimitOptions extends TransformOptions {
   limit?: number
 
   /**
-   * If provided (recommended!) - it will call readable.destroy() on limit.
-   * Without it - it will only stop the downstream consumers, but won't stop
-   * the Readable ("source" of the stream).
-   * It is almost always crucial to stop the Source too, so, please provide the Readable here!
+   * Allows to abort (gracefully stop) the stream from inside the Transform.
    */
-  sourceReadable?: Readable
-
-  /**
-   * Please provide it (a Promise that resolves when the Stream is done, e.g finished consuming things)
-   * to be able to wait for Consumers before calling `readable.destroy`.
-   * Has no effect if `readable` is not provided.
-   */
-  streamDone?: Promise<void>
-
-  logger?: CommonLogger
-
-  /**
-   * Set to true to enable additional debug messages, e.g it'll log
-   * when readable still emits values after the limit is reached.
-   */
-  debug?: boolean
+  signal: AbortableSignal
 }
 
-/**
- * Class only exists to be able to do `instanceof TransformLimit`
- * and to set sourceReadable+streamDone to it in `_pipeline`.
- */
-export class TransformLimit extends AbortableTransform {}
-
 export function transformLimit<IN>(opt: TransformLimitOptions): TransformTyped<IN, IN> {
-  const { logger = console, limit, debug } = opt
+  const { limit, signal } = opt
 
   if (!limit) {
-    // No limit - returning pass-through transform
     return transformNoOp()
   }
 
   let i = 0 // so we start first chunk with 1
   let ended = false
-  return new TransformLimit({
+  return new Transform({
     objectMode: true,
     ...opt,
-    transform(this: TransformLimit, chunk, _, cb) {
+    transform(chunk, _, cb) {
+      if (ended) {
+        return
+      }
+
       i++
 
       if (i === limit) {
         ended = true
-        logger.log(`transformLimit of ${limit} reached`)
         this.push(chunk)
-
-        pipelineClose(
-          'transformLimit',
-          this,
-          opt.sourceReadable || this.sourceReadable,
-          opt.streamDone || this.streamDone,
-          logger,
-        )
-
-        cb() // after pause
-      } else if (!ended) {
-        cb(null, chunk)
-      } else {
-        if (debug) logger.log(`transformLimit.transform after limit`, i)
-        // If we ever HANG (don't call cb) - Node will do process.exit(0) to us
-        cb() // ended, don't emit anything
+        this.push(null) // tell downstream that we're done
+        cb()
+        queueMicrotask(() => {
+          signal.abort(new Error(PIPELINE_GRACEFUL_ABORT))
+        })
+        return
       }
+
+      cb(null, chunk)
     },
   })
 }

package/src/stream/transform/transformMap.ts

@@ -1,6 +1,6 @@
-import { _hc } from '@naturalcycles/js-lib'
+import { _hc, type AbortableSignal } from '@naturalcycles/js-lib'
 import { _since } from '@naturalcycles/js-lib/datetime/time.util.js'
-import { _anyToError, ErrorMode } from '@naturalcycles/js-lib/error'
+import { _anyToError, _assert, ErrorMode } from '@naturalcycles/js-lib/error'
 import type { CommonLogger } from '@naturalcycles/js-lib/log'
 import { _stringify } from '@naturalcycles/js-lib/string/stringify.js'
 import {
@@ -15,9 +15,8 @@ import {
 } from '@naturalcycles/js-lib/types'
 import through2Concurrent from 'through2-concurrent'
 import { yellow } from '../../colors/colors.js'
-import type { AbortableTransform } from '../pipeline/pipeline.js'
 import type { TransformTyped } from '../stream.model.js'
-import { pipelineClose } from '../stream.util.js'
+import { PIPELINE_GRACEFUL_ABORT } from '../stream.util.js'
 
 export interface TransformMapOptions<IN = any, OUT = IN> {
   /**
@@ -81,6 +80,11 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
   metric?: string
 
   logger?: CommonLogger
+
+  /**
+   * Allows to abort (gracefully stop) the stream from inside the Transform.
+   */
+  signal?: AbortableSignal
 }
 
 export interface TransformMapStats {
@@ -140,12 +144,14 @@ export function transformMap<IN = any, OUT = IN>(
     onDone,
     metric = 'stream',
     logger = console,
+    signal,
   } = opt
 
   const started = Date.now() as UnixTimestampMillis
   let index = -1
   let countOut = 0
   let isSettled = false
+  let ok = true
   let errors = 0
   const collectedErrors: Error[] = [] // only used if errorMode == THROW_AGGREGATED
 
@@ -185,7 +191,7 @@ export function transformMap<IN = any, OUT = IN>(
 
           try {
             await onDone?.({
-              ok: true,
+              ok,
               collectedErrors,
               countErrors: errors,
               countIn: index + 1,
@@ -200,7 +206,7 @@ export function transformMap<IN = any, OUT = IN>(
         }
       },
     },
-    async function transformMapFn(this: AbortableTransform, chunk: IN, _, cb) {
+    async function transformMapFn(chunk: IN, _, cb) {
       // Stop processing if isSettled (either THROW_IMMEDIATELY was fired or END received)
       if (isSettled) return cb()
 
@@ -214,7 +220,8 @@ export function transformMap<IN = any, OUT = IN>(
         if (res === END) {
           isSettled = true
           logger.log(`transformMap END received at index ${currentIndex}`)
-          pipelineClose('transformMap', this, this.sourceReadable, this.streamDone, logger)
+          _assert(signal, 'signal is required when using END')
+          signal.abort(new Error(PIPELINE_GRACEFUL_ABORT))
           return cb()
         }
 
@@ -243,19 +250,23 @@ export function transformMap<IN = any, OUT = IN>(
 
         if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
           isSettled = true
-
-          try {
-            await onDone?.({
-              ok: false,
-              collectedErrors,
-              countErrors: errors,
-              countIn: index + 1,
-              countOut,
-              started,
-            })
-          } catch (err) {
-            logger.error(err)
-          }
+          ok = false
+
+          // Tests show that onDone is still called at `final` (second time),
+          // so, we no longer call it here
+
+          // try {
+          //   await onDone?.({
+          //     ok: false,
+          //     collectedErrors,
+          //     countErrors: errors,
+          //     countIn: index + 1,
+          //     countOut,
+          //     started,
+          //   })
+          // } catch (err) {
+          //   logger.error(err)
+          // }
 
           return cb(err) // Emit error immediately
         }

package/src/stream/transform/transformMapSync.ts

@@ -1,11 +1,12 @@
-import { _anyToError, ErrorMode } from '@naturalcycles/js-lib/error'
+import { Transform } from 'node:stream'
+import type { AbortableSignal } from '@naturalcycles/js-lib'
+import { _anyToError, _assert, ErrorMode } from '@naturalcycles/js-lib/error'
 import type { CommonLogger } from '@naturalcycles/js-lib/log'
 import type { IndexedMapper, Predicate, UnixTimestampMillis } from '@naturalcycles/js-lib/types'
 import { END, SKIP } from '@naturalcycles/js-lib/types'
 import { yellow } from '../../colors/colors.js'
-import { AbortableTransform } from '../pipeline/pipeline.js'
 import type { TransformTyped } from '../stream.model.js'
-import { pipelineClose } from '../stream.util.js'
+import { PIPELINE_GRACEFUL_ABORT } from '../stream.util.js'
 import type { TransformMapStats } from './transformMap.js'
 
 export interface TransformMapSyncOptions<IN = any, OUT = IN> {
@@ -54,9 +55,12 @@ export interface TransformMapSyncOptions<IN = any, OUT = IN> {
   metric?: string
 
   logger?: CommonLogger
-}
 
-export class TransformMapSync extends AbortableTransform {}
+  /**
+   * Allows to abort (gracefully stop) the stream from inside the Transform.
+   */
+  signal?: AbortableSignal
+}
 
 /**
  * Sync (not async) version of transformMap.
@@ -74,6 +78,7 @@ export function transformMapSync<IN = any, OUT = IN>(
     metric = 'stream',
     objectMode = true,
     logger = console,
+    signal,
   } = opt
 
   const started = Date.now() as UnixTimestampMillis
@@ -83,10 +88,10 @@ export function transformMapSync<IN = any, OUT = IN>(
   let errors = 0
   const collectedErrors: Error[] = [] // only used if errorMode == THROW_AGGREGATED
 
-  return new TransformMapSync({
+  return new Transform({
     objectMode,
     ...opt,
-    transform(this: AbortableTransform, chunk: IN, _, cb) {
+    transform(chunk: IN, _, cb) {
       // Stop processing if isSettled
       if (isSettled) return cb()
 
@@ -99,7 +104,8 @@ export function transformMapSync<IN = any, OUT = IN>(
         if (v === END) {
           isSettled = true // will be checked later
           logger.log(`transformMapSync END received at index ${currentIndex}`)
-          pipelineClose('transformMapSync', this, this.sourceReadable, this.streamDone, logger)
+          _assert(signal, 'signal is required when using END')
+          signal.abort(new Error(PIPELINE_GRACEFUL_ABORT))
           return cb()
         }
 

package/src/stream/transform/transformTee.ts

@@ -1,5 +1,5 @@
 import { Transform } from 'node:stream'
-import { _pipeline } from '../pipeline/pipeline.js'
+import { pipeline } from 'node:stream/promises'
 import { readableCreate } from '../readable/readableCreate.js'
 import type { TransformTyped } from '../stream.model.js'
 
@@ -17,11 +17,14 @@ type AnyStream = NodeJS.WritableStream | NodeJS.ReadWriteStream
 export function transformTee<T>(streams: AnyStream[]): TransformTyped<T, T> {
   const readable = readableCreate<T>()
 
-  const secondPipelinePromise = _pipeline([readable, ...streams])
+  const secondPipelinePromise = pipeline([readable, ...streams])
 
   return new Transform({
     objectMode: true,
     transform(chunk: T, _, cb) {
+      // todo: it's possible to start respecting backpressure,
+      // if we start to listen to the boolean output of .push()
+
       // pass to the "secondary" pipeline
       readable.push(chunk)
 

package/src/stream/writable/writableForEach.ts

@@ -2,7 +2,7 @@ import type { AsyncIndexedMapper, IndexedMapper } from '@naturalcycles/js-lib/ty
 import { _passNothingPredicate } from '@naturalcycles/js-lib/types'
 import type { WritableTyped } from '../stream.model.js'
 import { transformMap, type TransformMapOptions } from '../transform/transformMap.js'
-import { transformMapSync } from '../transform/transformMapSync.js'
+import { transformMapSync, type TransformMapSyncOptions } from '../transform/transformMapSync.js'
 
 /**
  * Just an alias to transformMap that declares OUT as void.
@@ -19,7 +19,7 @@ export function writableForEach<IN = any>(
  */
 export function writableForEachSync<IN = any>(
   mapper: IndexedMapper<IN, void>,
-  opt: TransformMapOptions<IN, void> = {},
+  opt: TransformMapSyncOptions<IN, void> = {},
 ): WritableTyped<IN> {
   return transformMapSync<IN, void>(mapper, { ...opt, predicate: _passNothingPredicate })
 }

package/src/stream/writable/writableFork.ts

@@ -1,5 +1,5 @@
 import { Writable } from 'node:stream'
-import { _pipeline } from '../pipeline/pipeline.js'
+import { pipeline } from 'node:stream/promises'
 import { readableCreate } from '../readable/readableCreate.js'
 import type { ReadableTyped, TransformOptions, WritableTyped } from '../stream.model.js'
 
@@ -22,7 +22,7 @@ export function writableFork<T>(
       const readable = readableCreate<T>()
       readables.push(readable)
 
-      return await _pipeline([readable, ...chain])
+      return await pipeline([readable, ...chain])
     }),
   ).catch(err => {
     console.error(err) // ensure the error is logged

package/dist/stream/pipeline/pipeline.d.ts

@@ -1,36 +0,0 @@
-import type { Readable } from 'node:stream';
-import { Transform } from 'node:stream';
-import type { DeferredPromise } from '@naturalcycles/js-lib/promise';
-type AnyStream = NodeJS.ReadableStream | NodeJS.WritableStream | NodeJS.ReadWriteStream;
-export interface PipelineOptions {
-    /**
-     * Set to true to allow ERR_STREAM_PREMATURE_CLOSE.
-     * Required to support graceful close when using transformLimit
-     */
-    allowClose?: boolean;
-    /**
-     * Set to true to allow graceful abort (via AbortSignal).
-     * "Graceful" means it'll swallow the AbortError and let the pipeline resolve normally.
-     *
-     * Default is false - AbortError will be thrown.
-     */
-    allowGracefulAbort?: boolean;
-    signal?: AbortSignal;
-}
-/**
- * Promisified `stream.pipeline`.
- *
- * Supports opt.allowClose, which allows transformLimit to work (to actually stop source Readable)
- * without throwing an error (ERR_STREAM_PREMATURE_CLOSE).
- */
-export declare function _pipeline(streams: AnyStream[], opt?: PipelineOptions): Promise<void>;
-/**
- * Convenience function to make _pipeline collect all items at the end of the stream (should be Transform, not Writeable!)
- * and return.
- */
-export declare function _pipelineToArray<T>(streams: AnyStream[], opt?: PipelineOptions): Promise<T[]>;
-export declare class AbortableTransform extends Transform {
-    sourceReadable?: Readable;
-    streamDone?: DeferredPromise;
-}
-export {};

package/dist/stream/pipeline/pipeline.js

@@ -1,82 +0,0 @@
-import { Transform } from 'node:stream';
-import { pipeline } from 'node:stream/promises';
-import { _last } from '@naturalcycles/js-lib/array/array.util.js';
-import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
-import { writablePushToArray } from '../writable/writablePushToArray.js';
-/**
- * Promisified `stream.pipeline`.
- *
- * Supports opt.allowClose, which allows transformLimit to work (to actually stop source Readable)
- * without throwing an error (ERR_STREAM_PREMATURE_CLOSE).
- */
-export async function _pipeline(streams, opt = {}) {
-    const first = streams[0];
-    const rest = streams.slice(1);
-    if (opt.allowClose) {
-        // Do the magic of making the pipeline "abortable"
-        //
-        // How does it work:
-        // It finds `sourceReadable` (basically, it's just first item in the passed array of streams)
-        // Finds last "writable" (last item), patches the `_final` method of it to detect when the whole pipeline is "done",
-        // sets the `streamDone` DeferredPromise that resolves when the pipeline is done.
-        // Scans through all passed items, finds those that are capable of "closing" the stream
-        // (currently its `transformLimit` or `transformMap`)
-        // Patches them by attaching `sourceReadable` and `streamDone`.
-        // These items (transformLimit and transformMap), when they need to "close the stream" - call `pipelineClose`.
-        // `pipelineClose` is the result of 2 sleepless nights of googling and experimentation:)
-        // It does:
-        // 1. Stops the "downstream" by doing `this.push(null)`.
-        // 2. Pauses the `sourceReadable` by calling sourceReadable.unpipe()
-        // 3. Waits for `streamDone` to ensure that downstream chunks are fully processed (e.g written to disk).
-        // 4. Calls `sourceReadable.destroy()`, which emits ERR_STREAM_PREMATURE_CLOSE
-        // 5. _pipeline (this function) catches that specific error and suppresses it (because it's expected and
-        // inevitable in this flow). Know a better way to close the stream? Tell me!
-        const streamDone = pDefer();
-        const sourceReadable = first;
-        const last = _last(streams);
-        const lastFinal = last._final?.bind(last) || ((cb) => cb());
-        last._final = cb => {
-            lastFinal(() => {
-                cb();
-                streamDone.resolve();
-            });
-        };
-        rest.forEach(s => {
-            // console.log(s)
-            if (s instanceof AbortableTransform || s.constructor.name === 'AbortableTransform') {
-                // console.log(`found ${s.constructor.name}, setting props`)
-                ;
-                s.sourceReadable = sourceReadable;
-                s.streamDone = streamDone;
-            }
-        });
-    }
-    try {
-        return await pipeline([first, ...rest], opt);
-    }
-    catch (err) {
-        if (opt.allowClose && err?.code === 'ERR_STREAM_PREMATURE_CLOSE') {
-            console.log('_pipeline closed (as expected)');
-            return;
-        }
-        if (opt.allowGracefulAbort && err?.name === 'AbortError') {
-            console.log('_pipeline closed via AbortSignal (as expected)');
-            return;
-        }
-        // console.log(`_pipeline error`, err)
-        throw err;
-    }
-}
-/**
- * Convenience function to make _pipeline collect all items at the end of the stream (should be Transform, not Writeable!)
- * and return.
- */
-export async function _pipelineToArray(streams, opt = {}) {
-    const a = [];
-    await _pipeline([...streams, writablePushToArray(a)], opt);
-    return a;
-}
-export class AbortableTransform extends Transform {
-    sourceReadable;
-    streamDone;
-}

package/dist/stream/readable/readableForEach.d.ts

@@ -1,19 +0,0 @@
-import type { AbortableAsyncMapper, IndexedMapper } from '@naturalcycles/js-lib/types';
-import type { ReadableTyped } from '../stream.model.js';
-import type { TransformMapOptions } from '../transform/transformMap.js';
-/**
- * Convenience function to do `.forEach` over a Readable.
- * Typed! (unlike default Readable).
- *
- * Try native readable.forEach() instead!
- *
- * @experimental
- */
-export declare function readableForEach<T>(readable: ReadableTyped<T>, mapper: AbortableAsyncMapper<T, void>, opt?: TransformMapOptions<T, void>): Promise<void>;
-/**
- * Convenience function to do `.forEach` over a Readable.
- * Typed! (unlike default Readable).
- *
- * @experimental
- */
-export declare function readableForEachSync<T>(readable: ReadableTyped<T>, mapper: IndexedMapper<T, void>): Promise<void>;

package/dist/stream/readable/readableForEach.js

@@ -1,30 +0,0 @@
-import { _passNothingPredicate } from '@naturalcycles/js-lib/types';
-import { _pipeline } from '../pipeline/pipeline.js';
-import { transformMap } from '../transform/transformMap.js';
-/**
- * Convenience function to do `.forEach` over a Readable.
- * Typed! (unlike default Readable).
- *
- * Try native readable.forEach() instead!
- *
- * @experimental
- */
-export async function readableForEach(readable, mapper, opt = {}) {
-    await _pipeline([
-        readable,
-        transformMap(mapper, { ...opt, predicate: _passNothingPredicate }),
-    ]);
-}
-/**
- * Convenience function to do `.forEach` over a Readable.
- * Typed! (unlike default Readable).
- *
- * @experimental
- */
-export async function readableForEachSync(readable, mapper) {
-    // async iteration
-    let index = 0;
-    for await (const item of readable) {
-        mapper(item, index++);
-    }
-}