@naturalcycles/nodejs-lib 15.22.0 → 15.24.0

package/src/stream/pipeline.ts

@@ -0,0 +1,351 @@
+import { Readable, type Transform } from 'node:stream'
+import { pipeline } from 'node:stream/promises'
+import type { ReadableStream as WebReadableStream } from 'node:stream/web'
+import { createUnzip, type ZlibOptions } from 'node:zlib'
+import { createGzip } from 'node:zlib'
+import { createAbortableSignal } from '@naturalcycles/js-lib'
+import type {
+  AbortableAsyncMapper,
+  AsyncIndexedMapper,
+  AsyncPredicate,
+  END,
+  IndexedMapper,
+  Integer,
+  NonNegativeInteger,
+  PositiveInteger,
+  Predicate,
+  SKIP,
+} from '@naturalcycles/js-lib/types'
+import { fs2 } from '../fs/fs2.js'
+import { createReadStreamAsNDJSON } from './ndjson/createReadStreamAsNDJSON.js'
+import { transformJsonParse } from './ndjson/transformJsonParse.js'
+import { transformToNDJson } from './ndjson/transformToNDJson.js'
+import type {
+  ReadableTyped,
+  TransformOptions,
+  TransformTyped,
+  WritableTyped,
+} from './stream.model.js'
+import { PIPELINE_GRACEFUL_ABORT } from './stream.util.js'
+import { transformChunk } from './transform/transformChunk.js'
+import { transformFilterSync } from './transform/transformFilter.js'
+import { transformFlatten, transformFlattenIfNeeded } from './transform/transformFlatten.js'
+import { transformLimit } from './transform/transformLimit.js'
+import {
+  transformLogProgress,
+  type TransformLogProgressOptions,
+} from './transform/transformLogProgress.js'
+import { transformMap, type TransformMapOptions } from './transform/transformMap.js'
+import {
+  transformMapSimple,
+  type TransformMapSimpleOptions,
+} from './transform/transformMapSimple.js'
+import { transformMapSync, type TransformMapSyncOptions } from './transform/transformMapSync.js'
+import { transformOffset, type TransformOffsetOptions } from './transform/transformOffset.js'
+import { transformSplitOnNewline } from './transform/transformSplit.js'
+import { transformTap, type TransformTapOptions } from './transform/transformTap.js'
+import { transformThrottle, type TransformThrottleOptions } from './transform/transformThrottle.js'
+import { writablePushToArray } from './writable/writablePushToArray.js'
+import { writableVoid } from './writable/writableVoid.js'
+
+export class Pipeline<T> {
+  // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
+  private readonly source: Readable
+  private transforms: NodeJS.ReadWriteStream[] = []
+  private destination?: NodeJS.WritableStream
+  private readableLimit?: Integer
+  // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
+  private objectMode: boolean
+  private abortableSignal = createAbortableSignal()
+
+  private constructor(source: ReadableTyped<T>, objectMode = true) {
+    this.source = source
+    this.objectMode = objectMode
+  }
+
+  static from<T>(source: ReadableTyped<T>): Pipeline<T> {
+    return new Pipeline(source)
+  }
+
+  static fromWeb<T>(webReadableStream: WebReadableStream<T>): Pipeline<T> {
+    return new Pipeline(Readable.fromWeb(webReadableStream))
+  }
+
+  /**
+   * Technically same as `fromIterable` (since Array is Iterable),
+   * but named a bit friendlier.
+   */
+  static fromArray<T>(input: T[]): Pipeline<T> {
+    return new Pipeline(Readable.from(input))
+  }
+
+  static fromIterable<T>(input: Iterable<T> | AsyncIterable<T>): Pipeline<T> {
+    return new Pipeline(Readable.from(input))
+  }
+
+  static fromFile(sourceFilePath: string): Pipeline<Uint8Array> {
+    return new Pipeline(fs2.createReadStream(sourceFilePath), false)
+  }
+
+  static fromNDJsonFile<T>(sourceFilePath: string): Pipeline<T> {
+    return new Pipeline(createReadStreamAsNDJSON<T>(sourceFilePath))
+  }
+
+  /**
+   * Limits the source Readable, but using `.take(limit)` on it.
+   * This is THE preferred way of limiting the source.
+   */
+  limitSource(limit: NonNegativeInteger | undefined): this {
+    this.readableLimit = limit
+    return this
+  }
+
+  /**
+   * If possible - STRONGLY PREFER applying `.take(limit)` on the source Readable,
+   * as it's a clean graceful way of limiting the Readable. Example:
+   *
+   * Pipeline.from(myReadable.take(10))
+   *
+   * or
+   *
+   * Pipeline
+   *   .from(myReadable)
+   *   .limitSource(10)
+   *
+   * If applying `take` on Readable is not possible - use this method at your own risk.
+   * Why warning?
+   * The limit works by aborting the stream, and then catching the error - certainly
+   * less clean than `.take()` on the source.
+   */
+  limit(limit: NonNegativeInteger | undefined): this {
+    this.transforms.push(
+      transformLimit({
+        limit,
+        signal: this.abortableSignal,
+      }),
+    )
+    return this
+  }
+
+  chunk(chunkSize: PositiveInteger, opt?: TransformOptions): Pipeline<T[]> {
+    this.transforms.push(transformChunk(chunkSize, opt))
+    return this as any
+  }
+
+  flatten<TO>(this: Pipeline<readonly TO[]>): Pipeline<TO> {
+    this.transforms.push(transformFlatten())
+    return this as any
+  }
+
+  flattenIfNeeded(): Pipeline<T extends readonly (infer TO)[] ? TO : T> {
+    this.transforms.push(transformFlattenIfNeeded())
+    return this as any
+  }
+
+  // TransformLogProgressOptions intentionally doesn't have <T> passed, as it's inconvenient in many cases
+  logProgress(opt?: TransformLogProgressOptions): this {
+    this.transforms.push(transformLogProgress(opt))
+    return this
+  }
+
+  map<TO>(
+    mapper: AbortableAsyncMapper<T, TO | typeof SKIP | typeof END>,
+    opt?: TransformMapOptions<T, TO>,
+  ): Pipeline<TO> {
+    this.transforms.push(
+      transformMap(mapper, {
+        ...opt,
+        signal: this.abortableSignal,
+      }),
+    )
+    return this as any
+  }
+
+  mapSync<TO>(
+    mapper: IndexedMapper<T, TO | typeof SKIP | typeof END>,
+    opt?: TransformMapSyncOptions,
+  ): Pipeline<TO> {
+    this.transforms.push(
+      transformMapSync(mapper, {
+        ...opt,
+        signal: this.abortableSignal,
+      }),
+    )
+    return this as any
+  }
+
+  mapSimple<TO>(mapper: IndexedMapper<T, TO>, opt?: TransformMapSimpleOptions): Pipeline<TO> {
+    this.transforms.push(transformMapSimple(mapper, opt))
+    return this as any
+  }
+
+  filter(predicate: AsyncPredicate<T>, opt?: TransformMapOptions): this {
+    this.transforms.push(
+      transformMap(v => v, {
+        predicate,
+        ...opt,
+        signal: this.abortableSignal,
+      }),
+    )
+    return this
+  }
+
+  filterSync(predicate: Predicate<T>, opt?: TransformOptions): this {
+    this.transforms.push(transformFilterSync(predicate, opt))
+    return this
+  }
+
+  offset(opt: TransformOffsetOptions): this {
+    this.transforms.push(transformOffset(opt))
+    return this
+  }
+
+  tap(fn: AsyncIndexedMapper<T, any>, opt?: TransformTapOptions): this {
+    this.transforms.push(transformTap(fn, opt))
+    return this
+  }
+
+  throttle(opt: TransformThrottleOptions): this {
+    this.transforms.push(transformThrottle(opt))
+    return this
+  }
+
+  // todo: tee/fork
+
+  transform<TO>(transform: TransformTyped<T, TO>): Pipeline<TO> {
+    this.transforms.push(transform)
+    return this as any
+  }
+
+  /**
+   * Helper method to add multiple transforms at once.
+   * Not type safe! Prefer using singular `transform()` multiple times for type safety.
+   */
+  transformMany<TO>(transforms: Transform[]): Pipeline<TO> {
+    this.transforms.push(...transforms)
+    return this as any
+  }
+
+  /**
+   * Utility method just to conveniently type-cast the current Pipeline type.
+   * No runtime effect.
+   */
+  typeCastAs<TO>(): Pipeline<TO> {
+    return this as any
+  }
+
+  setObjectMode(objectMode: boolean): this {
+    this.objectMode = objectMode
+    return this
+  }
+
+  /**
+   * Transform the stream of Objects into a stream of JSON lines.
+   * Technically, it goes into objectMode=false, so it's a binary stream at the end.
+   */
+  toNDJson(): Pipeline<Uint8Array> {
+    this.transforms.push(transformToNDJson())
+    this.objectMode = false
+    return this as any
+  }
+
+  parseNDJson<TO = unknown>(this: Pipeline<Uint8Array>): Pipeline<TO> {
+    // It was said that transformJsonParse() separately is 10% or more slower than .map(line => JSON.parse(line))
+    // So, we can investigate a speedup
+    this.transforms.push(transformSplitOnNewline(), transformJsonParse())
+    this.objectMode = true
+    return this as any
+  }
+
+  gzip(this: Pipeline<Uint8Array>, opt?: ZlibOptions): Pipeline<Uint8Array> {
+    this.transforms.push(createGzip(opt))
+    this.objectMode = false
+    return this as any
+  }
+
+  gunzip(this: Pipeline<Uint8Array>, opt?: ZlibOptions): Pipeline<Uint8Array> {
+    this.transforms.push(createUnzip(opt))
+    this.objectMode = false
+    return this as any
+  }
+
+  async toArray(opt?: TransformOptions): Promise<T[]> {
+    const arr: T[] = []
+    this.destination = writablePushToArray(arr, opt)
+    await this.run()
+    return arr
+  }
+
+  async toFile(outputFilePath: string): Promise<void> {
+    fs2.ensureFile(outputFilePath)
+    this.destination = fs2.createWriteStream(outputFilePath)
+    await this.run()
+  }
+
+  async toNDJsonFile(outputFilePath: string): Promise<void> {
+    fs2.ensureFile(outputFilePath)
+    this.transforms.push(transformToNDJson())
+    if (outputFilePath.endsWith('.gz')) {
+      this.transforms.push(
+        createGzip({
+          // chunkSize: 64 * 1024, // no observed speedup
+        }),
+      )
+    }
+    this.destination = fs2.createWriteStream(outputFilePath, {
+      // highWaterMark: 64 * 1024, // no observed speedup
+    })
+    await this.run()
+  }
+
+  async to(destination: WritableTyped<T>): Promise<void> {
+    this.destination = destination
+    await this.run()
+  }
+
+  async forEach(
+    fn: AsyncIndexedMapper<T, void>,
+    opt?: TransformMapOptions<T, void>,
+  ): Promise<void> {
+    this.transforms.push(
+      transformMap(fn, {
+        ...opt,
+        signal: this.abortableSignal,
+      }),
+    )
+    await this.run()
+  }
+
+  async forEachSync(
+    fn: IndexedMapper<T, void>,
+    opt?: TransformMapSyncOptions<T, void>,
+  ): Promise<void> {
+    this.transforms.push(
+      transformMapSync(fn, {
+        ...opt,
+        signal: this.abortableSignal,
+      }),
+    )
+    await this.run()
+  }
+
+  async run(): Promise<void> {
+    this.destination ||= writableVoid()
+    let { source } = this
+    if (this.readableLimit) {
+      source = source.take(this.readableLimit)
+    }
+
+    try {
+      await pipeline([source, ...this.transforms, this.destination], {
+        signal: this.abortableSignal,
+      })
+    } catch (err) {
+      if (err instanceof Error && (err.cause as any)?.message === PIPELINE_GRACEFUL_ABORT) {
+        console.log('pipeline gracefully aborted') // todo: this message may be removed later
+        return
+      }
+      throw err
+    }
+  }
+}

package/src/stream/stream.util.ts

@@ -1,29 +1 @@
-import type { Readable } from 'node:stream'
-import type { CommonLogger } from '@naturalcycles/js-lib/log'
-
-export function pipelineClose(
-  name: string,
-  readableDownstream: Readable,
-  sourceReadable: Readable | undefined,
-  streamDone: Promise<void> | undefined,
-  logger: CommonLogger,
-): void {
-  readableDownstream.push(null) // this closes the stream, so downstream Readable will receive `end` and won't write anything
-
-  if (!sourceReadable) {
-    logger.warn(`${name} sourceReadable is not provided, readable stream will not be stopped`)
-  } else {
-    logger.log(`${name} is calling readable.unpipe() to pause the stream`)
-    sourceReadable.unpipe() // it is expected to pause the stream
-
-    if (!streamDone) {
-      logger.log(`${name} streamDone is not provided, will do readable.destroy right away`)
-      sourceReadable.destroy()
-    } else {
-      void streamDone.then(() => {
-        logger.log(`${name} streamDone, calling readable.destroy()`)
-        sourceReadable.destroy() // this throws ERR_STREAM_PREMATURE_CLOSE
-      })
-    }
-  }
-}
+export const PIPELINE_GRACEFUL_ABORT = 'PIPELINE_GRACEFUL_ABORT'

package/src/stream/transform/transformChunk.ts

@@ -1,22 +1,19 @@
 import { Transform } from 'node:stream'
+import type { PositiveInteger } from '@naturalcycles/js-lib/types'
 import type { TransformOptions, TransformTyped } from '../stream.model.js'
 
-export interface TransformChunkOptions extends TransformOptions {
-  /**
-   * How many items to include in each chunk.
-   * Last chunk will contain the remaining items, possibly less than chunkSize.
-   */
-  chunkSize: number
-}
-
 /**
  * Similar to RxJS bufferCount(),
  * allows to "chunk" the input stream into chunks of `opt.chunkSize` size.
  * Last chunk will contain the remaining items, possibly less than chunkSize.
+ *
+ * `chunkSize` indicates how many items to include in each chunk.
+ * Last chunk will contain the remaining items, possibly less than chunkSize.
  */
-export function transformChunk<IN = any>(opt: TransformChunkOptions): TransformTyped<IN, IN[]> {
-  const { chunkSize } = opt
-
+export function transformChunk<IN = any>(
+  chunkSize: PositiveInteger,
+  opt?: TransformOptions,
+): TransformTyped<IN, IN[]> {
   let buf: IN[] = []
 
   return new Transform({

package/src/stream/transform/transformFlatten.ts

@@ -5,13 +5,25 @@ export function transformFlatten<T>(): TransformTyped<T[], T> {
   return new Transform({
     objectMode: true,
     transform(chunk: T[], _, cb) {
-      if (!Array.isArray(chunk)) {
-        // As a safety precaution, to not crash the pipeline - push as is
-        this.push(chunk)
-      } else {
+      for (const item of chunk) {
+        this.push(item)
+      }
+      cb() // acknowledge
+    },
+  })
+}
+
+export function transformFlattenIfNeeded<T>(): TransformTyped<T[], T> {
+  return new Transform({
+    objectMode: true,
+    transform(chunk: T[], _, cb) {
+      if (Array.isArray(chunk)) {
         for (const item of chunk) {
           this.push(item)
         }
+      } else {
+        // As a safety precaution, to not crash the pipeline - push as is
+        this.push(chunk)
       }
       cb() // acknowledge
     },

package/src/stream/transform/transformLimit.ts

@@ -1,8 +1,7 @@
-import type { Readable } from 'node:stream'
-import type { CommonLogger } from '@naturalcycles/js-lib/log'
-import { AbortableTransform } from '../pipeline/pipeline.js'
+import { Transform } from 'node:stream'
+import type { AbortableSignal } from '@naturalcycles/js-lib'
 import type { TransformOptions, TransformTyped } from '../stream.model.js'
-import { pipelineClose } from '../stream.util.js'
+import { PIPELINE_GRACEFUL_ABORT } from '../stream.util.js'
 import { transformNoOp } from './transformNoOp.js'
 
 export interface TransformLimitOptions extends TransformOptions {
@@ -12,72 +11,42 @@ export interface TransformLimitOptions extends TransformOptions {
   limit?: number
 
   /**
-   * If provided (recommended!) - it will call readable.destroy() on limit.
-   * Without it - it will only stop the downstream consumers, but won't stop
-   * the Readable ("source" of the stream).
-   * It is almost always crucial to stop the Source too, so, please provide the Readable here!
+   * Allows to abort (gracefully stop) the stream from inside the Transform.
    */
-  sourceReadable?: Readable
-
-  /**
-   * Please provide it (a Promise that resolves when the Stream is done, e.g finished consuming things)
-   * to be able to wait for Consumers before calling `readable.destroy`.
-   * Has no effect if `readable` is not provided.
-   */
-  streamDone?: Promise<void>
-
-  logger?: CommonLogger
-
-  /**
-   * Set to true to enable additional debug messages, e.g it'll log
-   * when readable still emits values after the limit is reached.
-   */
-  debug?: boolean
+  signal: AbortableSignal
 }
 
-/**
- * Class only exists to be able to do `instanceof TransformLimit`
- * and to set sourceReadable+streamDone to it in `_pipeline`.
- */
-export class TransformLimit extends AbortableTransform {}
-
 export function transformLimit<IN>(opt: TransformLimitOptions): TransformTyped<IN, IN> {
-  const { logger = console, limit, debug } = opt
+  const { limit, signal } = opt
 
   if (!limit) {
-    // No limit - returning pass-through transform
     return transformNoOp()
   }
 
   let i = 0 // so we start first chunk with 1
   let ended = false
-  return new TransformLimit({
+  return new Transform({
     objectMode: true,
     ...opt,
-    transform(this: TransformLimit, chunk, _, cb) {
+    transform(chunk, _, cb) {
+      if (ended) {
+        return
+      }
+
       i++
 
       if (i === limit) {
         ended = true
-        logger.log(`transformLimit of ${limit} reached`)
         this.push(chunk)
-
-        pipelineClose(
-          'transformLimit',
-          this,
-          opt.sourceReadable || this.sourceReadable,
-          opt.streamDone || this.streamDone,
-          logger,
-        )
-
-        cb() // after pause
-      } else if (!ended) {
-        cb(null, chunk)
-      } else {
-        if (debug) logger.log(`transformLimit.transform after limit`, i)
-        // If we ever HANG (don't call cb) - Node will do process.exit(0) to us
-        cb() // ended, don't emit anything
+        this.push(null) // tell downstream that we're done
+        cb()
+        queueMicrotask(() => {
+          signal.abort(new Error(PIPELINE_GRACEFUL_ABORT))
+        })
+        return
       }
+
+      cb(null, chunk)
     },
   })
 }

package/src/stream/transform/transformMap.ts

@@ -1,6 +1,6 @@
-import { _hc } from '@naturalcycles/js-lib'
+import { _hc, type AbortableSignal } from '@naturalcycles/js-lib'
 import { _since } from '@naturalcycles/js-lib/datetime/time.util.js'
-import { _anyToError, ErrorMode } from '@naturalcycles/js-lib/error'
+import { _anyToError, _assert, ErrorMode } from '@naturalcycles/js-lib/error'
 import type { CommonLogger } from '@naturalcycles/js-lib/log'
 import { _stringify } from '@naturalcycles/js-lib/string/stringify.js'
 import {
@@ -15,9 +15,8 @@ import {
 } from '@naturalcycles/js-lib/types'
 import through2Concurrent from 'through2-concurrent'
 import { yellow } from '../../colors/colors.js'
-import type { AbortableTransform } from '../pipeline/pipeline.js'
 import type { TransformTyped } from '../stream.model.js'
-import { pipelineClose } from '../stream.util.js'
+import { PIPELINE_GRACEFUL_ABORT } from '../stream.util.js'
 
 export interface TransformMapOptions<IN = any, OUT = IN> {
   /**
@@ -81,6 +80,11 @@ export interface TransformMapOptions<IN = any, OUT = IN> {
   metric?: string
 
   logger?: CommonLogger
+
+  /**
+   * Allows to abort (gracefully stop) the stream from inside the Transform.
+   */
+  signal?: AbortableSignal
 }
 
 export interface TransformMapStats {
@@ -140,12 +144,14 @@ export function transformMap<IN = any, OUT = IN>(
     onDone,
     metric = 'stream',
     logger = console,
+    signal,
   } = opt
 
   const started = Date.now() as UnixTimestampMillis
   let index = -1
   let countOut = 0
   let isSettled = false
+  let ok = true
   let errors = 0
   const collectedErrors: Error[] = [] // only used if errorMode == THROW_AGGREGATED
 
@@ -185,7 +191,7 @@ export function transformMap<IN = any, OUT = IN>(
 
           try {
             await onDone?.({
-              ok: true,
+              ok,
               collectedErrors,
               countErrors: errors,
               countIn: index + 1,
@@ -200,7 +206,7 @@ export function transformMap<IN = any, OUT = IN>(
         }
       },
     },
-    async function transformMapFn(this: AbortableTransform, chunk: IN, _, cb) {
+    async function transformMapFn(chunk: IN, _, cb) {
       // Stop processing if isSettled (either THROW_IMMEDIATELY was fired or END received)
       if (isSettled) return cb()
 
@@ -214,7 +220,8 @@ export function transformMap<IN = any, OUT = IN>(
         if (res === END) {
           isSettled = true
           logger.log(`transformMap END received at index ${currentIndex}`)
-          pipelineClose('transformMap', this, this.sourceReadable, this.streamDone, logger)
+          _assert(signal, 'signal is required when using END')
+          signal.abort(new Error(PIPELINE_GRACEFUL_ABORT))
           return cb()
         }
 
@@ -243,19 +250,23 @@ export function transformMap<IN = any, OUT = IN>(
 
         if (errorMode === ErrorMode.THROW_IMMEDIATELY) {
           isSettled = true
-
-          try {
-            await onDone?.({
-              ok: false,
-              collectedErrors,
-              countErrors: errors,
-              countIn: index + 1,
-              countOut,
-              started,
-            })
-          } catch (err) {
-            logger.error(err)
-          }
+          ok = false
+
+          // Tests show that onDone is still called at `final` (second time),
+          // so, we no longer call it here
+
+          // try {
+          //   await onDone?.({
+          //     ok: false,
+          //     collectedErrors,
+          //     countErrors: errors,
+          //     countIn: index + 1,
+          //     countOut,
+          //     started,
+          //   })
+          // } catch (err) {
+          //   logger.error(err)
+          // }
 
           return cb(err) // Emit error immediately
         }