@naturalcycles/nodejs-lib 15.22.0 → 15.24.0

package/src/stream/transform/transformMapSync.ts

@@ -1,11 +1,12 @@
-import { _anyToError, ErrorMode } from '@naturalcycles/js-lib/error'
+import { Transform } from 'node:stream'
+import type { AbortableSignal } from '@naturalcycles/js-lib'
+import { _anyToError, _assert, ErrorMode } from '@naturalcycles/js-lib/error'
 import type { CommonLogger } from '@naturalcycles/js-lib/log'
 import type { IndexedMapper, Predicate, UnixTimestampMillis } from '@naturalcycles/js-lib/types'
 import { END, SKIP } from '@naturalcycles/js-lib/types'
 import { yellow } from '../../colors/colors.js'
-import { AbortableTransform } from '../pipeline/pipeline.js'
 import type { TransformTyped } from '../stream.model.js'
-import { pipelineClose } from '../stream.util.js'
+import { PIPELINE_GRACEFUL_ABORT } from '../stream.util.js'
 import type { TransformMapStats } from './transformMap.js'
 
 export interface TransformMapSyncOptions<IN = any, OUT = IN> {
@@ -54,9 +55,12 @@ export interface TransformMapSyncOptions<IN = any, OUT = IN> {
   metric?: string
 
   logger?: CommonLogger
-}
 
-export class TransformMapSync extends AbortableTransform {}
+  /**
+   * Allows to abort (gracefully stop) the stream from inside the Transform.
+   */
+  signal?: AbortableSignal
+}
 
 /**
  * Sync (not async) version of transformMap.
@@ -74,6 +78,7 @@ export function transformMapSync<IN = any, OUT = IN>(
     metric = 'stream',
     objectMode = true,
     logger = console,
+    signal,
   } = opt
 
   const started = Date.now() as UnixTimestampMillis
@@ -83,10 +88,10 @@ export function transformMapSync<IN = any, OUT = IN>(
   let errors = 0
   const collectedErrors: Error[] = [] // only used if errorMode == THROW_AGGREGATED
 
-  return new TransformMapSync({
+  return new Transform({
     objectMode,
     ...opt,
-    transform(this: AbortableTransform, chunk: IN, _, cb) {
+    transform(chunk: IN, _, cb) {
       // Stop processing if isSettled
       if (isSettled) return cb()
 
@@ -99,7 +104,8 @@ export function transformMapSync<IN = any, OUT = IN>(
         if (v === END) {
           isSettled = true // will be checked later
           logger.log(`transformMapSync END received at index ${currentIndex}`)
-          pipelineClose('transformMapSync', this, this.sourceReadable, this.streamDone, logger)
+          _assert(signal, 'signal is required when using END')
+          signal.abort(new Error(PIPELINE_GRACEFUL_ABORT))
           return cb()
         }
 

package/src/stream/transform/transformSplit.ts

@@ -15,8 +15,9 @@ export function transformSplitOnNewline(): TransformTyped<Buffer, Buffer> {
   let buffered: Buffer | undefined
 
   return new Transform({
-    readableObjectMode: true,
+    writableObjectMode: false,
     writableHighWaterMark: 64 * 1024,
+    readableObjectMode: true,
 
     transform(buf: Buffer, _enc, done) {
       let offset = 0

package/src/stream/transform/transformTee.ts

@@ -1,5 +1,5 @@
 import { Transform } from 'node:stream'
-import { _pipeline } from '../pipeline/pipeline.js'
+import { pipeline } from 'node:stream/promises'
 import { readableCreate } from '../readable/readableCreate.js'
 import type { TransformTyped } from '../stream.model.js'
 
@@ -17,11 +17,14 @@ type AnyStream = NodeJS.WritableStream | NodeJS.ReadWriteStream
 export function transformTee<T>(streams: AnyStream[]): TransformTyped<T, T> {
   const readable = readableCreate<T>()
 
-  const secondPipelinePromise = _pipeline([readable, ...streams])
+  const secondPipelinePromise = pipeline([readable, ...streams])
 
   return new Transform({
     objectMode: true,
     transform(chunk: T, _, cb) {
+      // todo: it's possible to start respecting backpressure,
+      // if we start to listen to the boolean output of .push()
+
       // pass to the "secondary" pipeline
       readable.push(chunk)
 

package/src/stream/writable/writableForEach.ts

@@ -2,7 +2,7 @@ import type { AsyncIndexedMapper, IndexedMapper } from '@naturalcycles/js-lib/ty
 import { _passNothingPredicate } from '@naturalcycles/js-lib/types'
 import type { WritableTyped } from '../stream.model.js'
 import { transformMap, type TransformMapOptions } from '../transform/transformMap.js'
-import { transformMapSync } from '../transform/transformMapSync.js'
+import { transformMapSync, type TransformMapSyncOptions } from '../transform/transformMapSync.js'
 
 /**
  * Just an alias to transformMap that declares OUT as void.
@@ -19,7 +19,7 @@ export function writableForEach<IN = any>(
  */
 export function writableForEachSync<IN = any>(
   mapper: IndexedMapper<IN, void>,
-  opt: TransformMapOptions<IN, void> = {},
+  opt: TransformMapSyncOptions<IN, void> = {},
 ): WritableTyped<IN> {
   return transformMapSync<IN, void>(mapper, { ...opt, predicate: _passNothingPredicate })
 }

package/src/stream/writable/writableFork.ts

@@ -1,5 +1,5 @@
 import { Writable } from 'node:stream'
-import { _pipeline } from '../pipeline/pipeline.js'
+import { pipeline } from 'node:stream/promises'
 import { readableCreate } from '../readable/readableCreate.js'
 import type { ReadableTyped, TransformOptions, WritableTyped } from '../stream.model.js'
 
@@ -22,7 +22,7 @@ export function writableFork<T>(
       const readable = readableCreate<T>()
       readables.push(readable)
 
-      return await _pipeline([readable, ...chain])
+      return await pipeline([readable, ...chain])
     }),
   ).catch(err => {
     console.error(err) // ensure the error is logged

package/dist/stream/pipeline/pipeline.d.ts

@@ -1,36 +0,0 @@
-import type { Readable } from 'node:stream';
-import { Transform } from 'node:stream';
-import type { DeferredPromise } from '@naturalcycles/js-lib/promise';
-type AnyStream = NodeJS.ReadableStream | NodeJS.WritableStream | NodeJS.ReadWriteStream;
-export interface PipelineOptions {
-    /**
-     * Set to true to allow ERR_STREAM_PREMATURE_CLOSE.
-     * Required to support graceful close when using transformLimit
-     */
-    allowClose?: boolean;
-    /**
-     * Set to true to allow graceful abort (via AbortSignal).
-     * "Graceful" means it'll swallow the AbortError and let the pipeline resolve normally.
-     *
-     * Default is false - AbortError will be thrown.
-     */
-    allowGracefulAbort?: boolean;
-    signal?: AbortSignal;
-}
-/**
- * Promisified `stream.pipeline`.
- *
- * Supports opt.allowClose, which allows transformLimit to work (to actually stop source Readable)
- * without throwing an error (ERR_STREAM_PREMATURE_CLOSE).
- */
-export declare function _pipeline(streams: AnyStream[], opt?: PipelineOptions): Promise<void>;
-/**
- * Convenience function to make _pipeline collect all items at the end of the stream (should be Transform, not Writeable!)
- * and return.
- */
-export declare function _pipelineToArray<T>(streams: AnyStream[], opt?: PipelineOptions): Promise<T[]>;
-export declare class AbortableTransform extends Transform {
-    sourceReadable?: Readable;
-    streamDone?: DeferredPromise;
-}
-export {};

package/dist/stream/pipeline/pipeline.js

@@ -1,82 +0,0 @@
-import { Transform } from 'node:stream';
-import { pipeline } from 'node:stream/promises';
-import { _last } from '@naturalcycles/js-lib/array/array.util.js';
-import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js';
-import { writablePushToArray } from '../writable/writablePushToArray.js';
-/**
- * Promisified `stream.pipeline`.
- *
- * Supports opt.allowClose, which allows transformLimit to work (to actually stop source Readable)
- * without throwing an error (ERR_STREAM_PREMATURE_CLOSE).
- */
-export async function _pipeline(streams, opt = {}) {
-    const first = streams[0];
-    const rest = streams.slice(1);
-    if (opt.allowClose) {
-        // Do the magic of making the pipeline "abortable"
-        //
-        // How does it work:
-        // It finds `sourceReadable` (basically, it's just first item in the passed array of streams)
-        // Finds last "writable" (last item), patches the `_final` method of it to detect when the whole pipeline is "done",
-        // sets the `streamDone` DeferredPromise that resolves when the pipeline is done.
-        // Scans through all passed items, finds those that are capable of "closing" the stream
-        // (currently its `transformLimit` or `transformMap`)
-        // Patches them by attaching `sourceReadable` and `streamDone`.
-        // These items (transformLimit and transformMap), when they need to "close the stream" - call `pipelineClose`.
-        // `pipelineClose` is the result of 2 sleepless nights of googling and experimentation:)
-        // It does:
-        // 1. Stops the "downstream" by doing `this.push(null)`.
-        // 2. Pauses the `sourceReadable` by calling sourceReadable.unpipe()
-        // 3. Waits for `streamDone` to ensure that downstream chunks are fully processed (e.g written to disk).
-        // 4. Calls `sourceReadable.destroy()`, which emits ERR_STREAM_PREMATURE_CLOSE
-        // 5. _pipeline (this function) catches that specific error and suppresses it (because it's expected and
-        // inevitable in this flow). Know a better way to close the stream? Tell me!
-        const streamDone = pDefer();
-        const sourceReadable = first;
-        const last = _last(streams);
-        const lastFinal = last._final?.bind(last) || ((cb) => cb());
-        last._final = cb => {
-            lastFinal(() => {
-                cb();
-                streamDone.resolve();
-            });
-        };
-        rest.forEach(s => {
-            // console.log(s)
-            if (s instanceof AbortableTransform || s.constructor.name === 'AbortableTransform') {
-                // console.log(`found ${s.constructor.name}, setting props`)
-                ;
-                s.sourceReadable = sourceReadable;
-                s.streamDone = streamDone;
-            }
-        });
-    }
-    try {
-        return await pipeline([first, ...rest], opt);
-    }
-    catch (err) {
-        if (opt.allowClose && err?.code === 'ERR_STREAM_PREMATURE_CLOSE') {
-            console.log('_pipeline closed (as expected)');
-            return;
-        }
-        if (opt.allowGracefulAbort && err?.name === 'AbortError') {
-            console.log('_pipeline closed via AbortSignal (as expected)');
-            return;
-        }
-        // console.log(`_pipeline error`, err)
-        throw err;
-    }
-}
-/**
- * Convenience function to make _pipeline collect all items at the end of the stream (should be Transform, not Writeable!)
- * and return.
- */
-export async function _pipelineToArray(streams, opt = {}) {
-    const a = [];
-    await _pipeline([...streams, writablePushToArray(a)], opt);
-    return a;
-}
-export class AbortableTransform extends Transform {
-    sourceReadable;
-    streamDone;
-}

package/dist/stream/readable/readableForEach.d.ts

@@ -1,19 +0,0 @@
-import type { AbortableAsyncMapper, IndexedMapper } from '@naturalcycles/js-lib/types';
-import type { ReadableTyped } from '../stream.model.js';
-import type { TransformMapOptions } from '../transform/transformMap.js';
-/**
- * Convenience function to do `.forEach` over a Readable.
- * Typed! (unlike default Readable).
- *
- * Try native readable.forEach() instead!
- *
- * @experimental
- */
-export declare function readableForEach<T>(readable: ReadableTyped<T>, mapper: AbortableAsyncMapper<T, void>, opt?: TransformMapOptions<T, void>): Promise<void>;
-/**
- * Convenience function to do `.forEach` over a Readable.
- * Typed! (unlike default Readable).
- *
- * @experimental
- */
-export declare function readableForEachSync<T>(readable: ReadableTyped<T>, mapper: IndexedMapper<T, void>): Promise<void>;

package/dist/stream/readable/readableForEach.js

@@ -1,30 +0,0 @@
-import { _passNothingPredicate } from '@naturalcycles/js-lib/types';
-import { _pipeline } from '../pipeline/pipeline.js';
-import { transformMap } from '../transform/transformMap.js';
-/**
- * Convenience function to do `.forEach` over a Readable.
- * Typed! (unlike default Readable).
- *
- * Try native readable.forEach() instead!
- *
- * @experimental
- */
-export async function readableForEach(readable, mapper, opt = {}) {
-    await _pipeline([
-        readable,
-        transformMap(mapper, { ...opt, predicate: _passNothingPredicate }),
-    ]);
-}
-/**
- * Convenience function to do `.forEach` over a Readable.
- * Typed! (unlike default Readable).
- *
- * @experimental
- */
-export async function readableForEachSync(readable, mapper) {
-    // async iteration
-    let index = 0;
-    for await (const item of readable) {
-        mapper(item, index++);
-    }
-}

package/src/stream/pipeline/pipeline.ts

@@ -1,114 +0,0 @@
-import type { Readable, Writable } from 'node:stream'
-import { Transform } from 'node:stream'
-import { pipeline } from 'node:stream/promises'
-import { _last } from '@naturalcycles/js-lib/array/array.util.js'
-import type { DeferredPromise } from '@naturalcycles/js-lib/promise'
-import { pDefer } from '@naturalcycles/js-lib/promise/pDefer.js'
-import type { AnyFunction } from '@naturalcycles/js-lib/types'
-import { writablePushToArray } from '../writable/writablePushToArray.js'
-
-type AnyStream = NodeJS.ReadableStream | NodeJS.WritableStream | NodeJS.ReadWriteStream
-
-export interface PipelineOptions {
-  /**
-   * Set to true to allow ERR_STREAM_PREMATURE_CLOSE.
-   * Required to support graceful close when using transformLimit
-   */
-  allowClose?: boolean
-
-  /**
-   * Set to true to allow graceful abort (via AbortSignal).
-   * "Graceful" means it'll swallow the AbortError and let the pipeline resolve normally.
-   *
-   * Default is false - AbortError will be thrown.
-   */
-  allowGracefulAbort?: boolean
-
-  signal?: AbortSignal
-}
-
-/**
- * Promisified `stream.pipeline`.
- *
- * Supports opt.allowClose, which allows transformLimit to work (to actually stop source Readable)
- * without throwing an error (ERR_STREAM_PREMATURE_CLOSE).
- */
-export async function _pipeline(streams: AnyStream[], opt: PipelineOptions = {}): Promise<void> {
-  const first = streams[0] as any
-  const rest = streams.slice(1)
-
-  if (opt.allowClose) {
-    // Do the magic of making the pipeline "abortable"
-    //
-    // How does it work:
-    // It finds `sourceReadable` (basically, it's just first item in the passed array of streams)
-    // Finds last "writable" (last item), patches the `_final` method of it to detect when the whole pipeline is "done",
-    // sets the `streamDone` DeferredPromise that resolves when the pipeline is done.
-    // Scans through all passed items, finds those that are capable of "closing" the stream
-    // (currently its `transformLimit` or `transformMap`)
-    // Patches them by attaching `sourceReadable` and `streamDone`.
-    // These items (transformLimit and transformMap), when they need to "close the stream" - call `pipelineClose`.
-    // `pipelineClose` is the result of 2 sleepless nights of googling and experimentation:)
-    // It does:
-    // 1. Stops the "downstream" by doing `this.push(null)`.
-    // 2. Pauses the `sourceReadable` by calling sourceReadable.unpipe()
-    // 3. Waits for `streamDone` to ensure that downstream chunks are fully processed (e.g written to disk).
-    // 4. Calls `sourceReadable.destroy()`, which emits ERR_STREAM_PREMATURE_CLOSE
-    // 5. _pipeline (this function) catches that specific error and suppresses it (because it's expected and
-    // inevitable in this flow). Know a better way to close the stream? Tell me!
-    const streamDone = pDefer()
-    const sourceReadable = first as Readable
-    const last = _last(streams) as Writable
-    const lastFinal = last._final?.bind(last) || ((cb: AnyFunction) => cb())
-    last._final = cb => {
-      lastFinal(() => {
-        cb()
-        streamDone.resolve()
-      })
-    }
-
-    rest.forEach(s => {
-      // console.log(s)
-      if (s instanceof AbortableTransform || s.constructor.name === 'AbortableTransform') {
-        // console.log(`found ${s.constructor.name}, setting props`)
-        ;(s as AbortableTransform).sourceReadable = sourceReadable
-        ;(s as AbortableTransform).streamDone = streamDone
-      }
-    })
-  }
-
-  try {
-    return await pipeline([first, ...rest], opt)
-  } catch (err) {
-    if (opt.allowClose && (err as any)?.code === 'ERR_STREAM_PREMATURE_CLOSE') {
-      console.log('_pipeline closed (as expected)')
-      return
-    }
-
-    if (opt.allowGracefulAbort && (err as any)?.name === 'AbortError') {
-      console.log('_pipeline closed via AbortSignal (as expected)')
-      return
-    }
-
-    // console.log(`_pipeline error`, err)
-    throw err
-  }
-}
-
-/**
- * Convenience function to make _pipeline collect all items at the end of the stream (should be Transform, not Writeable!)
- * and return.
- */
-export async function _pipelineToArray<T>(
-  streams: AnyStream[],
-  opt: PipelineOptions = {},
-): Promise<T[]> {
-  const a: T[] = []
-  await _pipeline([...streams, writablePushToArray(a)], opt)
-  return a
-}
-
-export class AbortableTransform extends Transform {
-  sourceReadable?: Readable
-  streamDone?: DeferredPromise
-}

package/src/stream/readable/readableForEach.ts

@@ -1,42 +0,0 @@
-import type { AbortableAsyncMapper, IndexedMapper } from '@naturalcycles/js-lib/types'
-import { _passNothingPredicate } from '@naturalcycles/js-lib/types'
-import { _pipeline } from '../pipeline/pipeline.js'
-import type { ReadableTyped } from '../stream.model.js'
-import type { TransformMapOptions } from '../transform/transformMap.js'
-import { transformMap } from '../transform/transformMap.js'
-
-/**
- * Convenience function to do `.forEach` over a Readable.
- * Typed! (unlike default Readable).
- *
- * Try native readable.forEach() instead!
- *
- * @experimental
- */
-export async function readableForEach<T>(
-  readable: ReadableTyped<T>,
-  mapper: AbortableAsyncMapper<T, void>,
-  opt: TransformMapOptions<T, void> = {},
-): Promise<void> {
-  await _pipeline([
-    readable,
-    transformMap<T, void>(mapper, { ...opt, predicate: _passNothingPredicate }),
-  ])
-}
-
-/**
- * Convenience function to do `.forEach` over a Readable.
- * Typed! (unlike default Readable).
- *
- * @experimental
- */
-export async function readableForEachSync<T>(
-  readable: ReadableTyped<T>,
-  mapper: IndexedMapper<T, void>,
-): Promise<void> {
-  // async iteration
-  let index = 0
-  for await (const item of readable) {
-    mapper(item, index++)
-  }
-}