@naturalcycles/nodejs-lib 15.24.0 → 15.26.0

package/dist/stream/index.d.ts

@@ -1,8 +1,5 @@
-export * from './ndjson/createReadStreamAsNDJSON.js';
-export * from './ndjson/createWriteStreamAsNDJSON.js';
 export * from './ndjson/ndjson.model.js';
 export * from './ndjson/ndjsonMap.js';
-export * from './ndjson/ndjsonStreamForEach.js';
 export * from './ndjson/transformJsonParse.js';
 export * from './ndjson/transformToNDJson.js';
 export * from './pipeline.js';
@@ -10,7 +7,6 @@ export * from './progressLogger.js';
 export * from './readable/readableCombined.js';
 export * from './readable/readableCreate.js';
 export * from './readable/readableFromArray.js';
-export * from './readable/readableToArray.js';
 export * from './stream.model.js';
 export * from './transform/transformChunk.js';
 export * from './transform/transformFilter.js';
@@ -30,7 +26,6 @@ export * from './transform/transformToArray.js';
 export * from './transform/worker/baseWorkerClass.js';
 export * from './transform/worker/transformMultiThreaded.js';
 export * from './transform/worker/transformMultiThreaded.model.js';
-export * from './writable/writableForEach.js';
 export * from './writable/writableFork.js';
 export * from './writable/writablePushToArray.js';
 export * from './writable/writableVoid.js';

package/dist/stream/index.js

@@ -1,8 +1,5 @@
-export * from './ndjson/createReadStreamAsNDJSON.js';
-export * from './ndjson/createWriteStreamAsNDJSON.js';
 export * from './ndjson/ndjson.model.js';
 export * from './ndjson/ndjsonMap.js';
-export * from './ndjson/ndjsonStreamForEach.js';
 export * from './ndjson/transformJsonParse.js';
 export * from './ndjson/transformToNDJson.js';
 export * from './pipeline.js';
@@ -10,7 +7,6 @@ export * from './progressLogger.js';
 export * from './readable/readableCombined.js';
 export * from './readable/readableCreate.js';
 export * from './readable/readableFromArray.js';
-export * from './readable/readableToArray.js';
 export * from './stream.model.js';
 export * from './transform/transformChunk.js';
 export * from './transform/transformFilter.js';
@@ -30,7 +26,6 @@ export * from './transform/transformToArray.js';
 export * from './transform/worker/baseWorkerClass.js';
 export * from './transform/worker/transformMultiThreaded.js';
 export * from './transform/worker/transformMultiThreaded.model.js';
-export * from './writable/writableForEach.js';
 export * from './writable/writableFork.js';
 export * from './writable/writablePushToArray.js';
 export * from './writable/writableVoid.js';

package/dist/stream/ndjson/ndjsonMap.d.ts

@@ -13,5 +13,7 @@ export interface NDJSONMapOptions<IN = any, OUT = IN> extends TransformMapOption
 /**
  * Unzips input file automatically, if it ends with `.gz`.
  * Zips output file automatically, if it ends with `.gz`.
+ *
+ * @deprecated use Pipeline directly
  */
 export declare function ndjsonMap<IN = any, OUT = any>(mapper: AbortableAsyncMapper<IN, OUT>, opt: NDJSONMapOptions<IN, OUT>): Promise<void>;

package/dist/stream/ndjson/ndjsonMap.js

@@ -3,6 +3,8 @@ import { Pipeline } from '../pipeline.js';
 /**
  * Unzips input file automatically, if it ends with `.gz`.
  * Zips output file automatically, if it ends with `.gz`.
+ *
+ * @deprecated use Pipeline directly
  */
 export async function ndjsonMap(mapper, opt) {
     const { inputFilePath, outputFilePath, logEveryOutput = 100_000, limitInput, limitOutput } = opt;

package/dist/stream/pipeline.d.ts

@@ -26,8 +26,8 @@ export declare class Pipeline<T> {
      */
     static fromArray<T>(input: T[]): Pipeline<T>;
     static fromIterable<T>(input: Iterable<T> | AsyncIterable<T>): Pipeline<T>;
-    static fromFile(sourceFilePath: string): Pipeline<Uint8Array>;
     static fromNDJsonFile<T>(sourceFilePath: string): Pipeline<T>;
+    static fromFile(sourceFilePath: string): Pipeline<Uint8Array>;
     /**
      * Limits the source Readable, but using `.take(limit)` on it.
      * This is THE preferred way of limiting the source.
@@ -81,6 +81,8 @@ export declare class Pipeline<T> {
      */
     toNDJson(): Pipeline<Uint8Array>;
     parseNDJson<TO = unknown>(this: Pipeline<Uint8Array>): Pipeline<TO>;
+    splitOnNewline(this: Pipeline<Uint8Array>): Pipeline<Buffer>;
+    parseJson<TO = unknown>(this: Pipeline<Buffer> | Pipeline<Uint8Array> | Pipeline<string>): Pipeline<TO>;
     gzip(this: Pipeline<Uint8Array>, opt?: ZlibOptions): Pipeline<Uint8Array>;
     gunzip(this: Pipeline<Uint8Array>, opt?: ZlibOptions): Pipeline<Uint8Array>;
     toArray(opt?: TransformOptions): Promise<T[]>;

package/dist/stream/pipeline.js

@@ -4,7 +4,6 @@ import { createUnzip } from 'node:zlib';
 import { createGzip } from 'node:zlib';
 import { createAbortableSignal } from '@naturalcycles/js-lib';
 import { fs2 } from '../fs/fs2.js';
-import { createReadStreamAsNDJSON } from './ndjson/createReadStreamAsNDJSON.js';
 import { transformJsonParse } from './ndjson/transformJsonParse.js';
 import { transformToNDJson } from './ndjson/transformToNDJson.js';
 import { PIPELINE_GRACEFUL_ABORT } from './stream.util.js';
@@ -51,11 +50,21 @@ export class Pipeline {
     static fromIterable(input) {
         return new Pipeline(Readable.from(input));
     }
-    static fromFile(sourceFilePath) {
-        return new Pipeline(fs2.createReadStream(sourceFilePath), false);
-    }
     static fromNDJsonFile(sourceFilePath) {
-        return new Pipeline(createReadStreamAsNDJSON(sourceFilePath));
+        fs2.requireFileToExist(sourceFilePath);
+        const p = Pipeline.fromFile(sourceFilePath);
+        if (sourceFilePath.endsWith('.gz')) {
+            p.gunzip();
+        }
+        return p.parseJson();
+        // return stream.pipe(transformSplitOnNewline()).map(line => JSON.parse(line))
+        // For some crazy reason .map is much faster than transformJsonParse!
+        // ~5000 vs ~4000 rps !!!
+    }
+    static fromFile(sourceFilePath) {
+        return new Pipeline(fs2.createReadStream(sourceFilePath, {
+            highWaterMark: 64 * 1024, // no observed speedup
+        }), false);
     }
     /**
      * Limits the source Readable, but using `.take(limit)` on it.
@@ -188,13 +197,33 @@ export class Pipeline {
         this.objectMode = true;
         return this;
     }
+    splitOnNewline() {
+        // Input: objectMode=false - binary stream
+        // Output: objectMode=true - stream of Buffer objects (which are also strings?)
+        this.transforms.push(transformSplitOnNewline());
+        this.objectMode = true;
+        return this;
+    }
+    parseJson() {
+        // Input: objectMode=false - takes a stream of strings one by one
+        // Output: objectMode=true - stream of json-parsed Objects
+        this.transforms.push(transformJsonParse());
+        this.objectMode = true;
+        return this;
+    }
     gzip(opt) {
-        this.transforms.push(createGzip(opt));
+        this.transforms.push(createGzip({
+            // chunkSize: 64 * 1024, // no observed speedup
+            ...opt,
+        }));
         this.objectMode = false;
         return this;
     }
     gunzip(opt) {
-        this.transforms.push(createUnzip(opt));
+        this.transforms.push(createUnzip({
+            chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
+            ...opt,
+        }));
         this.objectMode = false;
         return this;
     }

package/dist/stream/writable/writableVoid.d.ts

@@ -1,15 +1,8 @@
 import { Writable } from 'node:stream';
-import type { DeferredPromise } from '@naturalcycles/js-lib/promise';
 import type { TransformOptions } from '../stream.model.js';
-export interface WritableVoidOptions extends TransformOptions {
-    /**
-     * If set - it will be Resolved when the Stream is done (after final.cb)
-     */
-    streamDone?: DeferredPromise;
-}
 /**
  * Use as a "null-terminator" of stream.pipeline.
  * It consumes the stream as quickly as possible without doing anything.
  * Put it in the end of your pipeline in case it ends with Transform that needs a consumer.
  */
-export declare function writableVoid(opt?: WritableVoidOptions): Writable;
+export declare function writableVoid(opt?: TransformOptions): Writable;

package/dist/stream/writable/writableVoid.js

@@ -13,7 +13,6 @@ export function writableVoid(opt = {}) {
         },
         final(cb) {
             cb();
-            opt.streamDone?.resolve();
         },
     });
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.24.0",
+  "version": "15.26.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@types/js-yaml": "^4",

package/src/stream/index.ts

@@ -1,8 +1,5 @@
-export * from './ndjson/createReadStreamAsNDJSON.js'
-export * from './ndjson/createWriteStreamAsNDJSON.js'
 export * from './ndjson/ndjson.model.js'
 export * from './ndjson/ndjsonMap.js'
-export * from './ndjson/ndjsonStreamForEach.js'
 export * from './ndjson/transformJsonParse.js'
 export * from './ndjson/transformToNDJson.js'
 export * from './pipeline.js'
@@ -10,7 +7,6 @@ export * from './progressLogger.js'
 export * from './readable/readableCombined.js'
 export * from './readable/readableCreate.js'
 export * from './readable/readableFromArray.js'
-export * from './readable/readableToArray.js'
 export * from './stream.model.js'
 export * from './transform/transformChunk.js'
 export * from './transform/transformFilter.js'
@@ -30,7 +26,6 @@ export * from './transform/transformToArray.js'
 export * from './transform/worker/baseWorkerClass.js'
 export * from './transform/worker/transformMultiThreaded.js'
 export * from './transform/worker/transformMultiThreaded.model.js'
-export * from './writable/writableForEach.js'
 export * from './writable/writableFork.js'
 export * from './writable/writablePushToArray.js'
 export * from './writable/writableVoid.js'

package/src/stream/ndjson/ndjsonMap.ts

@@ -21,6 +21,8 @@ export interface NDJSONMapOptions<IN = any, OUT = IN>
 /**
  * Unzips input file automatically, if it ends with `.gz`.
  * Zips output file automatically, if it ends with `.gz`.
+ *
+ * @deprecated use Pipeline directly
  */
 export async function ndjsonMap<IN = any, OUT = any>(
   mapper: AbortableAsyncMapper<IN, OUT>,

package/src/stream/pipeline.ts

@@ -17,7 +17,6 @@ import type {
   SKIP,
 } from '@naturalcycles/js-lib/types'
 import { fs2 } from '../fs/fs2.js'
-import { createReadStreamAsNDJSON } from './ndjson/createReadStreamAsNDJSON.js'
 import { transformJsonParse } from './ndjson/transformJsonParse.js'
 import { transformToNDJson } from './ndjson/transformToNDJson.js'
 import type {
@@ -83,12 +82,26 @@ export class Pipeline<T> {
     return new Pipeline(Readable.from(input))
   }
 
-  static fromFile(sourceFilePath: string): Pipeline<Uint8Array> {
-    return new Pipeline(fs2.createReadStream(sourceFilePath), false)
+  static fromNDJsonFile<T>(sourceFilePath: string): Pipeline<T> {
+    fs2.requireFileToExist(sourceFilePath)
+
+    const p = Pipeline.fromFile(sourceFilePath)
+    if (sourceFilePath.endsWith('.gz')) {
+      p.gunzip()
+    }
+    return p.parseJson()
+    // return stream.pipe(transformSplitOnNewline()).map(line => JSON.parse(line))
+    // For some crazy reason .map is much faster than transformJsonParse!
+    // ~5000 vs ~4000 rps !!!
   }
 
-  static fromNDJsonFile<T>(sourceFilePath: string): Pipeline<T> {
-    return new Pipeline(createReadStreamAsNDJSON<T>(sourceFilePath))
+  static fromFile(sourceFilePath: string): Pipeline<Uint8Array> {
+    return new Pipeline(
+      fs2.createReadStream(sourceFilePath, {
+        highWaterMark: 64 * 1024, // no observed speedup
+      }),
+      false,
+    )
   }
 
   /**
@@ -257,14 +270,42 @@ export class Pipeline<T> {
     return this as any
   }
 
+  splitOnNewline(this: Pipeline<Uint8Array>): Pipeline<Buffer> {
+    // Input: objectMode=false - binary stream
+    // Output: objectMode=true - stream of Buffer objects (which are also strings?)
+    this.transforms.push(transformSplitOnNewline())
+    this.objectMode = true
+    return this as any
+  }
+
+  parseJson<TO = unknown>(
+    this: Pipeline<Buffer> | Pipeline<Uint8Array> | Pipeline<string>,
+  ): Pipeline<TO> {
+    // Input: objectMode=false - takes a stream of strings one by one
+    // Output: objectMode=true - stream of json-parsed Objects
+    this.transforms.push(transformJsonParse())
+    this.objectMode = true
+    return this as any
+  }
+
   gzip(this: Pipeline<Uint8Array>, opt?: ZlibOptions): Pipeline<Uint8Array> {
-    this.transforms.push(createGzip(opt))
+    this.transforms.push(
+      createGzip({
+        // chunkSize: 64 * 1024, // no observed speedup
+        ...opt,
+      }),
+    )
     this.objectMode = false
     return this as any
   }
 
   gunzip(this: Pipeline<Uint8Array>, opt?: ZlibOptions): Pipeline<Uint8Array> {
-    this.transforms.push(createUnzip(opt))
+    this.transforms.push(
+      createUnzip({
+        chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
+        ...opt,
+      }),
+    )
     this.objectMode = false
     return this as any
   }

package/src/stream/writable/writableVoid.ts

@@ -1,20 +1,12 @@
 import { Writable } from 'node:stream'
-import type { DeferredPromise } from '@naturalcycles/js-lib/promise'
 import type { TransformOptions } from '../stream.model.js'
 
-export interface WritableVoidOptions extends TransformOptions {
-  /**
-   * If set - it will be Resolved when the Stream is done (after final.cb)
-   */
-  streamDone?: DeferredPromise
-}
-
 /**
  * Use as a "null-terminator" of stream.pipeline.
  * It consumes the stream as quickly as possible without doing anything.
  * Put it in the end of your pipeline in case it ends with Transform that needs a consumer.
  */
-export function writableVoid(opt: WritableVoidOptions = {}): Writable {
+export function writableVoid(opt: TransformOptions = {}): Writable {
   return new Writable({
     objectMode: true,
     ...opt,
@@ -23,7 +15,6 @@ export function writableVoid(opt: WritableVoidOptions = {}): Writable {
     },
     final(cb) {
       cb()
-      opt.streamDone?.resolve()
     },
   })
 }

package/dist/stream/ndjson/createReadStreamAsNDJSON.d.ts

@@ -1,19 +0,0 @@
-import type { ReadableTyped } from '../stream.model.js';
-/**
-  Returns a Readable of [already parsed] NDJSON objects.
- 
-  Replaces a list of operations:
-  - requireFileToExist(inputPath)
-  - fs.createReadStream
-  - createUnzip (only if path ends with '.gz')
-  - transformSplitOnNewline
-  - transformJsonParse
- 
-  To add a Limit or Offset: just add .take() or .drop(), example:
- 
-  _pipeline([
-    fs2.createReadStreamAsNDJSON().take(100),
-    transformX(),
-  ])
- */
-export declare function createReadStreamAsNDJSON<ROW = any>(inputPath: string): ReadableTyped<ROW>;

package/dist/stream/ndjson/createReadStreamAsNDJSON.js

@@ -1,38 +0,0 @@
-import { createUnzip } from 'node:zlib';
-import { fs2 } from '../../fs/fs2.js';
-import { transformSplitOnNewline } from '../transform/transformSplit.js';
-/**
-  Returns a Readable of [already parsed] NDJSON objects.
- 
-  Replaces a list of operations:
-  - requireFileToExist(inputPath)
-  - fs.createReadStream
-  - createUnzip (only if path ends with '.gz')
-  - transformSplitOnNewline
-  - transformJsonParse
- 
-  To add a Limit or Offset: just add .take() or .drop(), example:
- 
-  _pipeline([
-    fs2.createReadStreamAsNDJSON().take(100),
-    transformX(),
-  ])
- */
-export function createReadStreamAsNDJSON(inputPath) {
-    fs2.requireFileToExist(inputPath);
-    let stream = fs2
-        .createReadStream(inputPath, {
-        highWaterMark: 64 * 1024, // no observed speedup
-    })
-        .on('error', err => stream.emit('error', err));
-    if (inputPath.endsWith('.gz')) {
-        stream = stream.pipe(createUnzip({
-            chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
-        }));
-    }
-    return stream.pipe(transformSplitOnNewline()).map(line => JSON.parse(line));
-    // For some crazy reason .map is much faster than transformJsonParse!
-    // ~5000 vs ~4000 rps !!!
-    // .on('error', err => stream.emit('error', err))
-    // .pipe(transformJsonParse<ROW>())
-}

package/dist/stream/ndjson/createWriteStreamAsNDJSON.d.ts

@@ -1,11 +0,0 @@
-import type { TransformTyped } from '../stream.model.js';
-/**
-  Returns an array of Transforms, so that you can ...destructure them at
-  the end of the _pipeline.
- 
-  Replaces a list of operations:
-  - transformToNDJson
-  - createGzip (only if path ends with '.gz')
-  - fs.createWriteStream
- */
-export declare function createWriteStreamAsNDJSON(outputPath: string): TransformTyped<any, any>[];

package/dist/stream/ndjson/createWriteStreamAsNDJSON.js

@@ -1,27 +0,0 @@
-import { createGzip } from 'node:zlib';
-import { _isTruthy } from '@naturalcycles/js-lib';
-import { fs2 } from '../../fs/fs2.js';
-import { transformToNDJson } from './transformToNDJson.js';
-/**
-  Returns an array of Transforms, so that you can ...destructure them at
-  the end of the _pipeline.
- 
-  Replaces a list of operations:
-  - transformToNDJson
-  - createGzip (only if path ends with '.gz')
-  - fs.createWriteStream
- */
-export function createWriteStreamAsNDJSON(outputPath) {
-    fs2.ensureFile(outputPath);
-    return [
-        transformToNDJson(),
-        outputPath.endsWith('.gz')
-            ? createGzip({
-            // chunkSize: 64 * 1024, // no observed speedup
-            })
-            : undefined,
-        fs2.createWriteStream(outputPath, {
-        // highWaterMark: 64 * 1024, // no observed speedup
-        }),
-    ].filter(_isTruthy);
-}

package/dist/stream/ndjson/ndjsonStreamForEach.d.ts

@@ -1,10 +0,0 @@
-import type { AbortableAsyncMapper } from '@naturalcycles/js-lib/types';
-import type { TransformLogProgressOptions } from '../transform/transformLogProgress.js';
-import type { TransformMapOptions } from '../transform/transformMap.js';
-export interface NDJSONStreamForEachOptions<IN = any> extends TransformMapOptions<IN, void>, TransformLogProgressOptions<IN> {
-    inputFilePath: string;
-}
-/**
- * Convenience function to `forEach` through an ndjson file.
- */
-export declare function ndjsonStreamForEach<T>(mapper: AbortableAsyncMapper<T, void>, opt: NDJSONStreamForEachOptions<T>): Promise<void>;

package/dist/stream/ndjson/ndjsonStreamForEach.js

@@ -1,15 +0,0 @@
-import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js';
-import { Pipeline } from '../pipeline.js';
-/**
- * Convenience function to `forEach` through an ndjson file.
- */
-export async function ndjsonStreamForEach(mapper, opt) {
-    await Pipeline.fromNDJsonFile(opt.inputFilePath)
-        .map(mapper, {
-        errorMode: ErrorMode.THROW_AGGREGATED,
-        ...opt,
-        predicate: () => true, // to log progress properly
-    })
-        .logProgress(opt)
-        .run();
-}

package/dist/stream/readable/readableToArray.d.ts

@@ -1,9 +0,0 @@
-import type { ReadableTyped } from '../stream.model.js';
-/**
- * Convenience function to read the whole Readable stream into Array (in-memory)
- * and return that array.
- *
- * Native `await readable.toArray()` can be used instead.
- * This helper is kept for type-safery support.
- */
-export declare function readableToArray<T>(readable: ReadableTyped<T>): Promise<T[]>;

package/dist/stream/readable/readableToArray.js

@@ -1,17 +0,0 @@
-/**
- * Convenience function to read the whole Readable stream into Array (in-memory)
- * and return that array.
- *
- * Native `await readable.toArray()` can be used instead.
- * This helper is kept for type-safery support.
- */
-export async function readableToArray(readable) {
-    return await readable.toArray();
-    // const a: T[] = []
-    //
-    // for await (const item of readable) {
-    //   a.push(item)
-    // }
-    //
-    // return a
-}

package/dist/stream/writable/writableForEach.d.ts

@@ -1,12 +0,0 @@
-import type { AsyncIndexedMapper, IndexedMapper } from '@naturalcycles/js-lib/types';
-import type { WritableTyped } from '../stream.model.js';
-import { type TransformMapOptions } from '../transform/transformMap.js';
-import { type TransformMapSyncOptions } from '../transform/transformMapSync.js';
-/**
- * Just an alias to transformMap that declares OUT as void.
- */
-export declare function writableForEach<IN = any>(mapper: AsyncIndexedMapper<IN, void>, opt?: TransformMapOptions<IN, void>): WritableTyped<IN>;
-/**
- * Just an alias to transformMap that declares OUT as void.
- */
-export declare function writableForEachSync<IN = any>(mapper: IndexedMapper<IN, void>, opt?: TransformMapSyncOptions<IN, void>): WritableTyped<IN>;

package/dist/stream/writable/writableForEach.js

@@ -1,15 +0,0 @@
-import { _passNothingPredicate } from '@naturalcycles/js-lib/types';
-import { transformMap } from '../transform/transformMap.js';
-import { transformMapSync } from '../transform/transformMapSync.js';
-/**
- * Just an alias to transformMap that declares OUT as void.
- */
-export function writableForEach(mapper, opt = {}) {
-    return transformMap(mapper, { ...opt, predicate: _passNothingPredicate });
-}
-/**
- * Just an alias to transformMap that declares OUT as void.
- */
-export function writableForEachSync(mapper, opt = {}) {
-    return transformMapSync(mapper, { ...opt, predicate: _passNothingPredicate });
-}

package/dist/stream/writable/writableLimit.d.ts

@@ -1,8 +0,0 @@
-import type { Readable } from 'node:stream';
-import type { WritableTyped } from '../stream.model.js';
-/**
- * Allows to stop the Readable stream after the pipeline has processed X number of rows.
- * It counts OUTPUT rows (not input), because this Writable is always at the end of the Pipeline.
- * It ensures that everything has been processed before issuing a STOP on the readable.
- */
-export declare function writableLimit<T>(readable: Readable, limit: number): WritableTyped<T>;

package/dist/stream/writable/writableLimit.js

@@ -1,25 +0,0 @@
-import { Writable } from 'node:stream';
-/**
- * Allows to stop the Readable stream after the pipeline has processed X number of rows.
- * It counts OUTPUT rows (not input), because this Writable is always at the end of the Pipeline.
- * It ensures that everything has been processed before issuing a STOP on the readable.
- */
-export function writableLimit(readable, limit) {
-    let i = 0;
-    return new Writable({
-        objectMode: true,
-        write(_chunk, _, cb) {
-            if (limit === 0)
-                return cb(); // no limit, just passthrough
-            i++;
-            if (i === limit) {
-                console.log(`writableLimit of ${limit} reached`);
-                readable.destroy();
-                cb(); // do we need it?
-            }
-            else {
-                cb(); // passthrough
-            }
-        },
-    });
-}

package/src/stream/ndjson/createReadStreamAsNDJSON.ts

@@ -1,46 +0,0 @@
-import { createUnzip } from 'node:zlib'
-import { fs2 } from '../../fs/fs2.js'
-import type { ReadableTyped } from '../stream.model.js'
-import { transformSplitOnNewline } from '../transform/transformSplit.js'
-
-/**
-  Returns a Readable of [already parsed] NDJSON objects.
- 
-  Replaces a list of operations:
-  - requireFileToExist(inputPath)
-  - fs.createReadStream
-  - createUnzip (only if path ends with '.gz')
-  - transformSplitOnNewline
-  - transformJsonParse
- 
-  To add a Limit or Offset: just add .take() or .drop(), example:
- 
-  _pipeline([
-    fs2.createReadStreamAsNDJSON().take(100),
-    transformX(),
-  ])
- */
-
-export function createReadStreamAsNDJSON<ROW = any>(inputPath: string): ReadableTyped<ROW> {
-  fs2.requireFileToExist(inputPath)
-
-  let stream: ReadableTyped<ROW> = fs2
-    .createReadStream(inputPath, {
-      highWaterMark: 64 * 1024, // no observed speedup
-    })
-    .on('error', err => stream.emit('error', err))
-
-  if (inputPath.endsWith('.gz')) {
-    stream = stream.pipe(
-      createUnzip({
-        chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
-      }),
-    )
-  }
-
-  return stream.pipe(transformSplitOnNewline()).map(line => JSON.parse(line))
-  // For some crazy reason .map is much faster than transformJsonParse!
-  // ~5000 vs ~4000 rps !!!
-  // .on('error', err => stream.emit('error', err))
-  // .pipe(transformJsonParse<ROW>())
-}

package/src/stream/ndjson/createWriteStreamAsNDJSON.ts

@@ -1,30 +0,0 @@
-import { createGzip } from 'node:zlib'
-import { _isTruthy } from '@naturalcycles/js-lib'
-import { fs2 } from '../../fs/fs2.js'
-import type { TransformTyped } from '../stream.model.js'
-import { transformToNDJson } from './transformToNDJson.js'
-
-/**
-  Returns an array of Transforms, so that you can ...destructure them at
-  the end of the _pipeline.
- 
-  Replaces a list of operations:
-  - transformToNDJson
-  - createGzip (only if path ends with '.gz')
-  - fs.createWriteStream
- */
-export function createWriteStreamAsNDJSON(outputPath: string): TransformTyped<any, any>[] {
-  fs2.ensureFile(outputPath)
-
-  return [
-    transformToNDJson(),
-    outputPath.endsWith('.gz')
-      ? createGzip({
-          // chunkSize: 64 * 1024, // no observed speedup
-        })
-      : undefined,
-    fs2.createWriteStream(outputPath, {
-      // highWaterMark: 64 * 1024, // no observed speedup
-    }),
-  ].filter(_isTruthy) as TransformTyped<any, any>[]
-}

package/src/stream/ndjson/ndjsonStreamForEach.ts

@@ -1,28 +0,0 @@
-import { ErrorMode } from '@naturalcycles/js-lib/error/errorMode.js'
-import type { AbortableAsyncMapper } from '@naturalcycles/js-lib/types'
-import { Pipeline } from '../pipeline.js'
-import type { TransformLogProgressOptions } from '../transform/transformLogProgress.js'
-import type { TransformMapOptions } from '../transform/transformMap.js'
-
-export interface NDJSONStreamForEachOptions<IN = any>
-  extends TransformMapOptions<IN, void>,
-    TransformLogProgressOptions<IN> {
-  inputFilePath: string
-}
-
-/**
- * Convenience function to `forEach` through an ndjson file.
- */
-export async function ndjsonStreamForEach<T>(
-  mapper: AbortableAsyncMapper<T, void>,
-  opt: NDJSONStreamForEachOptions<T>,
-): Promise<void> {
-  await Pipeline.fromNDJsonFile<T>(opt.inputFilePath)
-    .map(mapper, {
-      errorMode: ErrorMode.THROW_AGGREGATED,
-      ...opt,
-      predicate: () => true, // to log progress properly
-    })
-    .logProgress(opt)
-    .run()
-}

package/src/stream/readable/readableToArray.ts

@@ -1,19 +0,0 @@
-import type { ReadableTyped } from '../stream.model.js'
-
-/**
- * Convenience function to read the whole Readable stream into Array (in-memory)
- * and return that array.
- *
- * Native `await readable.toArray()` can be used instead.
- * This helper is kept for type-safery support.
- */
-export async function readableToArray<T>(readable: ReadableTyped<T>): Promise<T[]> {
-  return await readable.toArray()
-  // const a: T[] = []
-  //
-  // for await (const item of readable) {
-  //   a.push(item)
-  // }
-  //
-  // return a
-}

package/src/stream/writable/writableForEach.ts

@@ -1,25 +0,0 @@
-import type { AsyncIndexedMapper, IndexedMapper } from '@naturalcycles/js-lib/types'
-import { _passNothingPredicate } from '@naturalcycles/js-lib/types'
-import type { WritableTyped } from '../stream.model.js'
-import { transformMap, type TransformMapOptions } from '../transform/transformMap.js'
-import { transformMapSync, type TransformMapSyncOptions } from '../transform/transformMapSync.js'
-
-/**
- * Just an alias to transformMap that declares OUT as void.
- */
-export function writableForEach<IN = any>(
-  mapper: AsyncIndexedMapper<IN, void>,
-  opt: TransformMapOptions<IN, void> = {},
-): WritableTyped<IN> {
-  return transformMap<IN, void>(mapper, { ...opt, predicate: _passNothingPredicate })
-}
-
-/**
- * Just an alias to transformMap that declares OUT as void.
- */
-export function writableForEachSync<IN = any>(
-  mapper: IndexedMapper<IN, void>,
-  opt: TransformMapSyncOptions<IN, void> = {},
-): WritableTyped<IN> {
-  return transformMapSync<IN, void>(mapper, { ...opt, predicate: _passNothingPredicate })
-}

package/src/stream/writable/writableLimit.ts

@@ -1,29 +0,0 @@
-import type { Readable } from 'node:stream'
-import { Writable } from 'node:stream'
-import type { WritableTyped } from '../stream.model.js'
-
-/**
- * Allows to stop the Readable stream after the pipeline has processed X number of rows.
- * It counts OUTPUT rows (not input), because this Writable is always at the end of the Pipeline.
- * It ensures that everything has been processed before issuing a STOP on the readable.
- */
-export function writableLimit<T>(readable: Readable, limit: number): WritableTyped<T> {
-  let i = 0
-
-  return new Writable({
-    objectMode: true,
-    write(_chunk, _, cb) {
-      if (limit === 0) return cb() // no limit, just passthrough
-
-      i++
-
-      if (i === limit) {
-        console.log(`writableLimit of ${limit} reached`)
-        readable.destroy()
-        cb() // do we need it?
-      } else {
-        cb() // passthrough
-      }
-    },
-  })
-}