@naturalcycles/nodejs-lib 15.21.0 → 15.23.0

package/src/stream/pipeline.ts

@@ -0,0 +1,301 @@
+import { Readable, type Transform } from 'node:stream'
+import { pipeline } from 'node:stream/promises'
+import { createGzip } from 'node:zlib'
+import { createAbortableSignal } from '@naturalcycles/js-lib'
+import type {
+  AbortableAsyncMapper,
+  AsyncIndexedMapper,
+  AsyncPredicate,
+  END,
+  IndexedMapper,
+  Integer,
+  NonNegativeInteger,
+  PositiveInteger,
+  Predicate,
+  SKIP,
+} from '@naturalcycles/js-lib/types'
+import { fs2 } from '../fs/fs2.js'
+import { createReadStreamAsNDJSON } from './ndjson/createReadStreamAsNDJSON.js'
+import { transformToNDJson } from './ndjson/transformToNDJson.js'
+import type {
+  ReadableTyped,
+  TransformOptions,
+  TransformTyped,
+  WritableTyped,
+} from './stream.model.js'
+import { PIPELINE_GRACEFUL_ABORT } from './stream.util.js'
+import { transformChunk } from './transform/transformChunk.js'
+import { transformFilterSync } from './transform/transformFilter.js'
+import { transformFlatten, transformFlattenIfNeeded } from './transform/transformFlatten.js'
+import { transformLimit } from './transform/transformLimit.js'
+import {
+  transformLogProgress,
+  type TransformLogProgressOptions,
+} from './transform/transformLogProgress.js'
+import { transformMap, type TransformMapOptions } from './transform/transformMap.js'
+import {
+  transformMapSimple,
+  type TransformMapSimpleOptions,
+} from './transform/transformMapSimple.js'
+import { transformMapSync, type TransformMapSyncOptions } from './transform/transformMapSync.js'
+import { transformOffset, type TransformOffsetOptions } from './transform/transformOffset.js'
+import { transformTap, type TransformTapOptions } from './transform/transformTap.js'
+import { transformThrottle, type TransformThrottleOptions } from './transform/transformThrottle.js'
+import { writablePushToArray } from './writable/writablePushToArray.js'
+import { writableVoid } from './writable/writableVoid.js'
+
+export class Pipeline<T> {
+  // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
+  private readonly source: Readable
+  private transforms: NodeJS.ReadWriteStream[] = []
+  private destination?: NodeJS.WritableStream
+  private readableLimit?: Integer
+  private abortableSignal = createAbortableSignal()
+
+  private constructor(source: ReadableTyped<T>) {
+    this.source = source
+  }
+
+  static from<T>(source: ReadableTyped<T>): Pipeline<T> {
+    return new Pipeline(source)
+  }
+
+  /**
+   * Technically same as `fromIterable` (since Array is Iterable),
+   * but named a bit friendlier.
+   */
+  static fromArray<T>(input: T[]): Pipeline<T> {
+    return new Pipeline(Readable.from(input))
+  }
+
+  static fromIterable<T>(input: Iterable<T> | AsyncIterable<T>): Pipeline<T> {
+    return new Pipeline(Readable.from(input))
+  }
+
+  static fromNDJsonFile<T>(sourceFilePath: string): Pipeline<T> {
+    return new Pipeline(createReadStreamAsNDJSON<T>(sourceFilePath))
+  }
+
+  /**
+   * Limits the source Readable, but using `.take(limit)` on it.
+   * This is THE preferred way of limiting the source.
+   */
+  limitSource(limit: NonNegativeInteger | undefined): this {
+    this.readableLimit = limit
+    return this
+  }
+
+  /**
+   * If possible - STRONGLY PREFER applying `.take(limit)` on the source Readable,
+   * as it's a clean graceful way of limiting the Readable. Example:
+   *
+   * Pipeline.from(myReadable.take(10))
+   *
+   * or
+   *
+   * Pipeline
+   *   .from(myReadable)
+   *   .limitSource(10)
+   *
+   * If applying `take` on Readable is not possible - use this method at your own risk.
+   * Why warning?
+   * The limit works by aborting the stream, and then catching the error - certainly
+   * less clean than `.take()` on the source.
+   */
+  limit(limit: NonNegativeInteger | undefined): this {
+    this.transforms.push(
+      transformLimit({
+        limit,
+        signal: this.abortableSignal,
+      }),
+    )
+    return this
+  }
+
+  chunk(chunkSize: PositiveInteger, opt?: TransformOptions): Pipeline<T[]> {
+    this.transforms.push(transformChunk(chunkSize, opt))
+    return this as any
+  }
+
+  flatten<TO>(this: Pipeline<readonly TO[]>): Pipeline<TO> {
+    this.transforms.push(transformFlatten())
+    return this as any
+  }
+
+  flattenIfNeeded(): Pipeline<T extends readonly (infer TO)[] ? TO : T> {
+    this.transforms.push(transformFlattenIfNeeded())
+    return this as any
+  }
+
+  // TransformLogProgressOptions intentionally doesn't have <T> passed, as it's inconvenient in many cases
+  logProgress(opt?: TransformLogProgressOptions): this {
+    this.transforms.push(transformLogProgress(opt))
+    return this
+  }
+
+  map<TO>(
+    mapper: AbortableAsyncMapper<T, TO | typeof SKIP | typeof END>,
+    opt?: TransformMapOptions<T, TO>,
+  ): Pipeline<TO> {
+    this.transforms.push(
+      transformMap(mapper, {
+        ...opt,
+        signal: this.abortableSignal,
+      }),
+    )
+    return this as any
+  }
+
+  mapSync<TO>(
+    mapper: IndexedMapper<T, TO | typeof SKIP | typeof END>,
+    opt?: TransformMapSyncOptions,
+  ): Pipeline<TO> {
+    this.transforms.push(
+      transformMapSync(mapper, {
+        ...opt,
+        signal: this.abortableSignal,
+      }),
+    )
+    return this as any
+  }
+
+  mapSimple<TO>(mapper: IndexedMapper<T, TO>, opt?: TransformMapSimpleOptions): Pipeline<TO> {
+    this.transforms.push(transformMapSimple(mapper, opt))
+    return this as any
+  }
+
+  filter(predicate: AsyncPredicate<T>, opt?: TransformMapOptions): this {
+    this.transforms.push(
+      transformMap(v => v, {
+        predicate,
+        ...opt,
+        signal: this.abortableSignal,
+      }),
+    )
+    return this
+  }
+
+  filterSync(predicate: Predicate<T>, opt?: TransformOptions): this {
+    this.transforms.push(transformFilterSync(predicate, opt))
+    return this
+  }
+
+  offset(opt: TransformOffsetOptions): this {
+    this.transforms.push(transformOffset(opt))
+    return this
+  }
+
+  tap(fn: AsyncIndexedMapper<T, any>, opt?: TransformTapOptions): this {
+    this.transforms.push(transformTap(fn, opt))
+    return this
+  }
+
+  throttle(opt: TransformThrottleOptions): this {
+    this.transforms.push(transformThrottle(opt))
+    return this
+  }
+
+  // todo: tee/fork
+
+  transform<TO>(transform: TransformTyped<T, TO>): Pipeline<TO> {
+    this.transforms.push(transform)
+    return this as any
+  }
+
+  /**
+   * Helper method to add multiple transforms at once.
+   * Not type safe! Prefer using singular `transform()` multiple times for type safety.
+   */
+  transformMany<TO>(transforms: Transform[]): Pipeline<TO> {
+    this.transforms.push(...transforms)
+    return this as any
+  }
+
+  /**
+   * Utility method just to conveniently type-cast the current Pipeline type.
+   * No runtime effect.
+   */
+  typeCastAs<TO>(): Pipeline<TO> {
+    return this as any
+  }
+
+  async toArray(opt?: TransformOptions): Promise<T[]> {
+    const arr: T[] = []
+    this.destination = writablePushToArray(arr, opt)
+    await this.run()
+    return arr
+  }
+
+  async toFile(outputFilePath: string): Promise<void> {
+    fs2.ensureFile(outputFilePath)
+    this.destination = fs2.createWriteStream(outputFilePath)
+    await this.run()
+  }
+
+  async toNDJsonFile(outputFilePath: string): Promise<void> {
+    fs2.ensureFile(outputFilePath)
+    this.transforms.push(transformToNDJson())
+    if (outputFilePath.endsWith('.gz')) {
+      this.transforms.push(
+        createGzip({
+          // chunkSize: 64 * 1024, // no observed speedup
+        }),
+      )
+    }
+    this.destination = fs2.createWriteStream(outputFilePath, {
+      // highWaterMark: 64 * 1024, // no observed speedup
+    })
+    await this.run()
+  }
+
+  async to(destination: WritableTyped<T>): Promise<void> {
+    this.destination = destination
+    await this.run()
+  }
+
+  async forEach(
+    fn: AsyncIndexedMapper<T, void>,
+    opt?: TransformMapOptions<T, void>,
+  ): Promise<void> {
+    this.transforms.push(
+      transformMap(fn, {
+        ...opt,
+        signal: this.abortableSignal,
+      }),
+    )
+    await this.run()
+  }
+
+  async forEachSync(
+    fn: IndexedMapper<T, void>,
+    opt?: TransformMapSyncOptions<T, void>,
+  ): Promise<void> {
+    this.transforms.push(
+      transformMapSync(fn, {
+        ...opt,
+        signal: this.abortableSignal,
+      }),
+    )
+    await this.run()
+  }
+
+  async run(): Promise<void> {
+    this.destination ||= writableVoid()
+    let { source } = this
+    if (this.readableLimit) {
+      source = source.take(this.readableLimit)
+    }
+
+    try {
+      await pipeline([source, ...this.transforms, this.destination], {
+        signal: this.abortableSignal,
+      })
+    } catch (err) {
+      if (err instanceof Error && (err.cause as any)?.message === PIPELINE_GRACEFUL_ABORT) {
+        console.log('pipeline gracefully aborted') // todo: this message may be removed later
+        return
+      }
+      throw err
+    }
+  }
+}

package/src/stream/readable/readableCombined.ts

@@ -0,0 +1,87 @@
+import { Readable } from 'node:stream'
+import { type DeferredPromise, pDefer } from '@naturalcycles/js-lib/promise/pDefer.js'
+import { pMap } from '@naturalcycles/js-lib/promise/pMap.js'
+import type { ReadableTyped } from '@naturalcycles/nodejs-lib/stream'
+
+/**
+ * Allows to combine multiple Readables into 1 Readable.
+ * As soon as any of the input Readables emit - the output Readable emits
+ * (passes through).
+ * Order is not preserved in any way, first come first served!
+ *
+ * Readable completes when all input Readables complete.
+ *
+ * @experimental
+ */
+export class ReadableCombined<T> extends Readable implements ReadableTyped<T> {
+  static create<T>(inputs: Readable[]): ReadableCombined<T> {
+    return new ReadableCombined<T>(inputs)
+  }
+
+  private constructor(public inputs: Readable[]) {
+    super({ objectMode: true })
+    void this.start()
+  }
+
+  /**
+   * If defined - we are in Paused mode
+   * and should await the lock to be resolved before proceeding.
+   *
+   * If not defined - we are in Flowing mode, no limits in data flow.
+   */
+  private lock?: DeferredPromise
+
+  // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
+  private countIn = 0
+  // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
+  private countOut = 0
+  // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
+  private countReads = 0
+
+  private async start(): Promise<void> {
+    await pMap(this.inputs, async (input, i) => {
+      for await (const item of input) {
+        this.countIn++
+        this.logStats()
+        if (this.lock) {
+          await this.lock
+          // lock is undefined at this point
+        }
+
+        const shouldContinue = this.push(item)
+        this.countOut++
+        if (!shouldContinue && !this.lock) {
+          this.lock = pDefer()
+          console.log(`ReadableCombined.push #${i} returned false, pausing the flow!`)
+        }
+      }
+
+      console.log(`ReadableCombined: input #${i} done`)
+    })
+
+    console.log(`ReadableCombined: all inputs done!`)
+    this.push(null)
+  }
+
+  override _read(): void {
+    this.countReads++
+
+    if (this.lock) {
+      console.log(`ReadableCombined._read: resuming the flow!`)
+      // calling it in this order is important!
+      // this.lock should be undefined BEFORE we call lock.resolve()
+      const { lock } = this
+      this.lock = undefined
+      lock.resolve()
+    }
+  }
+
+  private logStats(): void {
+    const { countIn, countOut, countReads } = this
+    console.log({
+      countIn,
+      countOut,
+      countReads,
+    })
+  }
+}

package/src/stream/stream.util.ts

@@ -1,29 +1 @@
-import type { Readable } from 'node:stream'
-import type { CommonLogger } from '@naturalcycles/js-lib/log'
-
-export function pipelineClose(
-  name: string,
-  readableDownstream: Readable,
-  sourceReadable: Readable | undefined,
-  streamDone: Promise<void> | undefined,
-  logger: CommonLogger,
-): void {
-  readableDownstream.push(null) // this closes the stream, so downstream Readable will receive `end` and won't write anything
-
-  if (!sourceReadable) {
-    logger.warn(`${name} sourceReadable is not provided, readable stream will not be stopped`)
-  } else {
-    logger.log(`${name} is calling readable.unpipe() to pause the stream`)
-    sourceReadable.unpipe() // it is expected to pause the stream
-
-    if (!streamDone) {
-      logger.log(`${name} streamDone is not provided, will do readable.destroy right away`)
-      sourceReadable.destroy()
-    } else {
-      void streamDone.then(() => {
-        logger.log(`${name} streamDone, calling readable.destroy()`)
-        sourceReadable.destroy() // this throws ERR_STREAM_PREMATURE_CLOSE
-      })
-    }
-  }
-}
+export const PIPELINE_GRACEFUL_ABORT = 'PIPELINE_GRACEFUL_ABORT'

package/src/stream/transform/transformChunk.ts

@@ -1,22 +1,19 @@
 import { Transform } from 'node:stream'
+import type { PositiveInteger } from '@naturalcycles/js-lib/types'
 import type { TransformOptions, TransformTyped } from '../stream.model.js'
 
-export interface TransformChunkOptions extends TransformOptions {
-  /**
-   * How many items to include in each chunk.
-   * Last chunk will contain the remaining items, possibly less than chunkSize.
-   */
-  chunkSize: number
-}
-
 /**
  * Similar to RxJS bufferCount(),
  * allows to "chunk" the input stream into chunks of `opt.chunkSize` size.
  * Last chunk will contain the remaining items, possibly less than chunkSize.
+ *
+ * `chunkSize` indicates how many items to include in each chunk.
+ * Last chunk will contain the remaining items, possibly less than chunkSize.
  */
-export function transformChunk<IN = any>(opt: TransformChunkOptions): TransformTyped<IN, IN[]> {
-  const { chunkSize } = opt
-
+export function transformChunk<IN = any>(
+  chunkSize: PositiveInteger,
+  opt?: TransformOptions,
+): TransformTyped<IN, IN[]> {
   let buf: IN[] = []
 
   return new Transform({

package/src/stream/transform/transformFlatten.ts

@@ -5,13 +5,25 @@ export function transformFlatten<T>(): TransformTyped<T[], T> {
   return new Transform({
     objectMode: true,
     transform(chunk: T[], _, cb) {
-      if (!Array.isArray(chunk)) {
-        // As a safety precaution, to not crash the pipeline - push as is
-        this.push(chunk)
-      } else {
+      for (const item of chunk) {
+        this.push(item)
+      }
+      cb() // acknowledge
+    },
+  })
+}
+
+export function transformFlattenIfNeeded<T>(): TransformTyped<T[], T> {
+  return new Transform({
+    objectMode: true,
+    transform(chunk: T[], _, cb) {
+      if (Array.isArray(chunk)) {
         for (const item of chunk) {
           this.push(item)
         }
+      } else {
+        // As a safety precaution, to not crash the pipeline - push as is
+        this.push(chunk)
       }
       cb() // acknowledge
     },

package/src/stream/transform/transformLimit.ts

@@ -1,8 +1,7 @@
-import type { Readable } from 'node:stream'
-import type { CommonLogger } from '@naturalcycles/js-lib/log'
-import { AbortableTransform } from '../pipeline/pipeline.js'
+import { Transform } from 'node:stream'
+import type { AbortableSignal } from '@naturalcycles/js-lib'
 import type { TransformOptions, TransformTyped } from '../stream.model.js'
-import { pipelineClose } from '../stream.util.js'
+import { PIPELINE_GRACEFUL_ABORT } from '../stream.util.js'
 import { transformNoOp } from './transformNoOp.js'
 
 export interface TransformLimitOptions extends TransformOptions {
@@ -12,72 +11,42 @@ export interface TransformLimitOptions extends TransformOptions {
   limit?: number
 
   /**
-   * If provided (recommended!) - it will call readable.destroy() on limit.
-   * Without it - it will only stop the downstream consumers, but won't stop
-   * the Readable ("source" of the stream).
-   * It is almost always crucial to stop the Source too, so, please provide the Readable here!
+   * Allows to abort (gracefully stop) the stream from inside the Transform.
    */
-  sourceReadable?: Readable
-
-  /**
-   * Please provide it (a Promise that resolves when the Stream is done, e.g finished consuming things)
-   * to be able to wait for Consumers before calling `readable.destroy`.
-   * Has no effect if `readable` is not provided.
-   */
-  streamDone?: Promise<void>
-
-  logger?: CommonLogger
-
-  /**
-   * Set to true to enable additional debug messages, e.g it'll log
-   * when readable still emits values after the limit is reached.
-   */
-  debug?: boolean
+  signal: AbortableSignal
 }
 
-/**
- * Class only exists to be able to do `instanceof TransformLimit`
- * and to set sourceReadable+streamDone to it in `_pipeline`.
- */
-export class TransformLimit extends AbortableTransform {}
-
 export function transformLimit<IN>(opt: TransformLimitOptions): TransformTyped<IN, IN> {
-  const { logger = console, limit, debug } = opt
+  const { limit, signal } = opt
 
   if (!limit) {
-    // No limit - returning pass-through transform
     return transformNoOp()
   }
 
   let i = 0 // so we start first chunk with 1
   let ended = false
-  return new TransformLimit({
+  return new Transform({
     objectMode: true,
     ...opt,
-    transform(this: TransformLimit, chunk, _, cb) {
+    transform(chunk, _, cb) {
+      if (ended) {
+        return
+      }
+
       i++
 
       if (i === limit) {
         ended = true
-        logger.log(`transformLimit of ${limit} reached`)
         this.push(chunk)
-
-        pipelineClose(
-          'transformLimit',
-          this,
-          opt.sourceReadable || this.sourceReadable,
-          opt.streamDone || this.streamDone,
-          logger,
-        )
-
-        cb() // after pause
-      } else if (!ended) {
-        cb(null, chunk)
-      } else {
-        if (debug) logger.log(`transformLimit.transform after limit`, i)
-        // If we ever HANG (don't call cb) - Node will do process.exit(0) to us
-        cb() // ended, don't emit anything
+        this.push(null) // tell downstream that we're done
+        cb()
+        queueMicrotask(() => {
+          signal.abort(new Error(PIPELINE_GRACEFUL_ABORT))
+        })
+        return
       }
+
+      cb(null, chunk)
     },
   })
 }