@naturalcycles/nodejs-lib 15.25.0 → 15.27.0

package/dist/stream/writable/writableLimit.js

@@ -1,25 +0,0 @@
-import { Writable } from 'node:stream';
-/**
- * Allows to stop the Readable stream after the pipeline has processed X number of rows.
- * It counts OUTPUT rows (not input), because this Writable is always at the end of the Pipeline.
- * It ensures that everything has been processed before issuing a STOP on the readable.
- */
-export function writableLimit(readable, limit) {
-    let i = 0;
-    return new Writable({
-        objectMode: true,
-        write(_chunk, _, cb) {
-            if (limit === 0)
-                return cb(); // no limit, just passthrough
-            i++;
-            if (i === limit) {
-                console.log(`writableLimit of ${limit} reached`);
-                readable.destroy();
-                cb(); // do we need it?
-            }
-            else {
-                cb(); // passthrough
-            }
-        },
-    });
-}

package/src/stream/ndjson/createWriteStreamAsNDJSON.ts

@@ -1,30 +0,0 @@
-import { createGzip } from 'node:zlib'
-import { _isTruthy } from '@naturalcycles/js-lib'
-import { fs2 } from '../../fs/fs2.js'
-import type { TransformTyped } from '../stream.model.js'
-import { transformToNDJson } from './transformToNDJson.js'
-
-/**
-  Returns an array of Transforms, so that you can ...destructure them at
-  the end of the _pipeline.
- 
-  Replaces a list of operations:
-  - transformToNDJson
-  - createGzip (only if path ends with '.gz')
-  - fs.createWriteStream
- */
-export function createWriteStreamAsNDJSON(outputPath: string): TransformTyped<any, any>[] {
-  fs2.ensureFile(outputPath)
-
-  return [
-    transformToNDJson(),
-    outputPath.endsWith('.gz')
-      ? createGzip({
-          // chunkSize: 64 * 1024, // no observed speedup
-        })
-      : undefined,
-    fs2.createWriteStream(outputPath, {
-      // highWaterMark: 64 * 1024, // no observed speedup
-    }),
-  ].filter(_isTruthy) as TransformTyped<any, any>[]
-}

package/src/stream/ndjson/ndjsonStreamForEach.ts

@@ -1,28 +0,0 @@
-import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js'
-import type { AbortableAsyncMapper } from '@naturalcycles/js-lib/types'
-import { Pipeline } from '../pipeline.js'
-import type { TransformLogProgressOptions } from '../transform/transformLogProgress.js'
-import type { TransformMapOptions } from '../transform/transformMap.js'
-
-export interface NDJSONStreamForEachOptions<IN = any>
-  extends TransformMapOptions<IN, void>,
-    TransformLogProgressOptions<IN> {
-  inputFilePath: string
-}
-
-/**
- * Convenience function to `forEach` through an ndjson file.
- */
-export async function ndjsonStreamForEach<T>(
-  mapper: AbortableAsyncMapper<T, void>,
-  opt: NDJSONStreamForEachOptions<T>,
-): Promise<void> {
-  await Pipeline.fromNDJsonFile<T>(opt.inputFilePath)
-    .map(mapper, {
-      errorMode: ErrorMode.THROW_AGGREGATED,
-      ...opt,
-      predicate: () => true, // to log progress properly
-    })
-    .logProgress(opt)
-    .run()
-}

package/src/stream/readable/readableToArray.ts

@@ -1,19 +0,0 @@
-import type { ReadableTyped } from '../stream.model.js'
-
-/**
- * Convenience function to read the whole Readable stream into Array (in-memory)
- * and return that array.
- *
- * Native `await readable.toArray()` can be used instead.
- * This helper is kept for type-safery support.
- */
-export async function readableToArray<T>(readable: ReadableTyped<T>): Promise<T[]> {
-  return await readable.toArray()
-  // const a: T[] = []
-  //
-  // for await (const item of readable) {
-  //   a.push(item)
-  // }
-  //
-  // return a
-}

package/src/stream/transform/transformTee.ts

@@ -1,48 +0,0 @@
-import { Transform } from 'node:stream'
-import { pipeline } from 'node:stream/promises'
-import { readableCreate } from '../readable/readableCreate.js'
-import type { TransformTyped } from '../stream.model.js'
-
-type AnyStream = NodeJS.WritableStream | NodeJS.ReadWriteStream
-
-/**
- * Allows to "tee"/"fork" away from the "main pipeline" into the "secondary pipeline".
- *
- * Important, that the main pipeline works "as normal", keeps backpressure, etc.
- * Secondary pipeline DOES NOT keep backpressure.
- * Therefor, the "slowest" pipeline should be made Primary (to keep backpressure).
- *
- * @experimental
- */
-export function transformTee<T>(streams: AnyStream[]): TransformTyped<T, T> {
-  const readable = readableCreate<T>()
-
-  const secondPipelinePromise = pipeline([readable, ...streams])
-
-  return new Transform({
-    objectMode: true,
-    transform(chunk: T, _, cb) {
-      // todo: it's possible to start respecting backpressure,
-      // if we start to listen to the boolean output of .push()
-
-      // pass to the "secondary" pipeline
-      readable.push(chunk)
-
-      // pass through to the "main" pipeline
-      cb(null, chunk)
-    },
-    async final(cb) {
-      console.log('transformTee final')
-
-      // Pushing null "closes"/ends the secondary pipeline correctly
-      readable.push(null)
-
-      // Second pipeline is expected to finish now, let's await it
-      await secondPipelinePromise
-      console.log('transformTee final secondPipeline done')
-
-      // Because second pipeline is done - now we can signal main pipeline to be done as well
-      cb()
-    },
-  })
-}

package/src/stream/transform/transformToArray.ts

@@ -1,23 +0,0 @@
-import { Transform } from 'node:stream'
-import type { TransformOptions, TransformTyped } from '../stream.model.js'
-
-/**
- * Will collect all stream results in the array (keeping it in memory) and emit in the end as one result.
- */
-export function transformToArray<IN>(opt: TransformOptions = {}): TransformTyped<IN, IN[]> {
-  const res: IN[] = []
-
-  return new Transform({
-    objectMode: true,
-    ...opt,
-    transform(chunk: IN, _, cb) {
-      res.push(chunk)
-      // callback to signal that we processed input, but not emitting any output
-      cb()
-    },
-    final(this: Transform, cb) {
-      this.push(res)
-      cb()
-    },
-  })
-}

package/src/stream/writable/writableForEach.ts

@@ -1,25 +0,0 @@
-import type { AsyncIndexedMapper, IndexedMapper } from '@naturalcycles/js-lib/types'
-import { _passNothingPredicate } from '@naturalcycles/js-lib/types'
-import type { WritableTyped } from '../stream.model.js'
-import { transformMap, type TransformMapOptions } from '../transform/transformMap.js'
-import { transformMapSync, type TransformMapSyncOptions } from '../transform/transformMapSync.js'
-
-/**
- * Just an alias to transformMap that declares OUT as void.
- */
-export function writableForEach<IN = any>(
-  mapper: AsyncIndexedMapper<IN, void>,
-  opt: TransformMapOptions<IN, void> = {},
-): WritableTyped<IN> {
-  return transformMap<IN, void>(mapper, { ...opt, predicate: _passNothingPredicate })
-}
-
-/**
- * Just an alias to transformMap that declares OUT as void.
- */
-export function writableForEachSync<IN = any>(
-  mapper: IndexedMapper<IN, void>,
-  opt: TransformMapSyncOptions<IN, void> = {},
-): WritableTyped<IN> {
-  return transformMapSync<IN, void>(mapper, { ...opt, predicate: _passNothingPredicate })
-}

package/src/stream/writable/writableFork.ts

@@ -1,56 +0,0 @@
-import { Writable } from 'node:stream'
-import { pipeline } from 'node:stream/promises'
-import { readableCreate } from '../readable/readableCreate.js'
-import type { ReadableTyped, TransformOptions, WritableTyped } from '../stream.model.js'
-
-/**
- * Allows "forking" a stream inside pipeline into a number of pipeline chains (2 or more).
- * Currently does NOT (!) maintain backpressure.
- * Error in the forked pipeline will propagate up to the main pipeline (and log error, to be sure).
- * Will wait until all forked pipelines are completed before completing the stream.
- *
- * @experimental
- */
-export function writableFork<T>(
-  chains: NodeJS.WritableStream[][],
-  opt?: TransformOptions,
-): WritableTyped<T> {
-  const readables: ReadableTyped<T>[] = []
-
-  const allChainsDone = Promise.all(
-    chains.map(async chain => {
-      const readable = readableCreate<T>()
-      readables.push(readable)
-
-      return await pipeline([readable, ...chain])
-    }),
-  ).catch(err => {
-    console.error(err) // ensure the error is logged
-    throw err
-  })
-
-  return new Writable({
-    objectMode: true,
-    ...opt,
-    write(chunk: T, _, cb) {
-      // Push/fork to all sub-streams
-      // No backpressure is ensured here, it'll push regardless of the
-      readables.forEach(readable => readable.push(chunk))
-
-      cb()
-    },
-    async final(cb) {
-      try {
-        // Push null (complete) to all sub-streams
-        readables.forEach(readable => readable.push(null))
-
-        console.log(`writableFork.final is waiting for all chains to be done`)
-        await allChainsDone
-        console.log(`writableFork.final all chains done`)
-        cb()
-      } catch (err) {
-        cb(err as Error)
-      }
-    },
-  })
-}

package/src/stream/writable/writableLimit.ts

@@ -1,29 +0,0 @@
-import type { Readable } from 'node:stream'
-import { Writable } from 'node:stream'
-import type { WritableTyped } from '../stream.model.js'
-
-/**
- * Allows to stop the Readable stream after the pipeline has processed X number of rows.
- * It counts OUTPUT rows (not input), because this Writable is always at the end of the Pipeline.
- * It ensures that everything has been processed before issuing a STOP on the readable.
- */
-export function writableLimit<T>(readable: Readable, limit: number): WritableTyped<T> {
-  let i = 0
-
-  return new Writable({
-    objectMode: true,
-    write(_chunk, _, cb) {
-      if (limit === 0) return cb() // no limit, just passthrough
-
-      i++
-
-      if (i === limit) {
-        console.log(`writableLimit of ${limit} reached`)
-        readable.destroy()
-        cb() // do we need it?
-      } else {
-        cb() // passthrough
-      }
-    },
-  })
-}