@naturalcycles/nodejs-lib 15.23.0 → 15.25.0

package/dist/stream/pipeline.d.ts

@@ -1,4 +1,6 @@
 import { type Transform } from 'node:stream';
+import type { ReadableStream as WebReadableStream } from 'node:stream/web';
+import { type ZlibOptions } from 'node:zlib';
 import type { AbortableAsyncMapper, AsyncIndexedMapper, AsyncPredicate, END, IndexedMapper, NonNegativeInteger, PositiveInteger, Predicate, SKIP } from '@naturalcycles/js-lib/types';
 import type { ReadableTyped, TransformOptions, TransformTyped, WritableTyped } from './stream.model.js';
 import { type TransformLogProgressOptions } from './transform/transformLogProgress.js';
@@ -13,15 +15,18 @@ export declare class Pipeline<T> {
     private transforms;
     private destination?;
     private readableLimit?;
+    private objectMode;
     private abortableSignal;
     private constructor();
     static from<T>(source: ReadableTyped<T>): Pipeline<T>;
+    static fromWeb<T>(webReadableStream: WebReadableStream<T>): Pipeline<T>;
     /**
      * Technically same as `fromIterable` (since Array is Iterable),
      * but named a bit friendlier.
      */
     static fromArray<T>(input: T[]): Pipeline<T>;
     static fromIterable<T>(input: Iterable<T> | AsyncIterable<T>): Pipeline<T>;
+    static fromFile(sourceFilePath: string): Pipeline<Uint8Array>;
     static fromNDJsonFile<T>(sourceFilePath: string): Pipeline<T>;
     /**
      * Limits the source Readable, but using `.take(limit)` on it.
@@ -69,6 +74,17 @@ export declare class Pipeline<T> {
      * No runtime effect.
      */
     typeCastAs<TO>(): Pipeline<TO>;
+    setObjectMode(objectMode: boolean): this;
+    /**
+     * Transform the stream of Objects into a stream of JSON lines.
+     * Technically, it goes into objectMode=false, so it's a binary stream at the end.
+     */
+    toNDJson(): Pipeline<Uint8Array>;
+    parseNDJson<TO = unknown>(this: Pipeline<Uint8Array>): Pipeline<TO>;
+    splitOnNewline(this: Pipeline<Uint8Array>): Pipeline<Buffer>;
+    parseJson<TO = unknown>(this: Pipeline<Buffer> | Pipeline<Uint8Array> | Pipeline<string>): Pipeline<TO>;
+    gzip(this: Pipeline<Uint8Array>, opt?: ZlibOptions): Pipeline<Uint8Array>;
+    gunzip(this: Pipeline<Uint8Array>, opt?: ZlibOptions): Pipeline<Uint8Array>;
     toArray(opt?: TransformOptions): Promise<T[]>;
     toFile(outputFilePath: string): Promise<void>;
     toNDJsonFile(outputFilePath: string): Promise<void>;

package/dist/stream/pipeline.js

@@ -1,9 +1,11 @@
 import { Readable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
+import { createUnzip } from 'node:zlib';
 import { createGzip } from 'node:zlib';
 import { createAbortableSignal } from '@naturalcycles/js-lib';
 import { fs2 } from '../fs/fs2.js';
 import { createReadStreamAsNDJSON } from './ndjson/createReadStreamAsNDJSON.js';
+import { transformJsonParse } from './ndjson/transformJsonParse.js';
 import { transformToNDJson } from './ndjson/transformToNDJson.js';
 import { PIPELINE_GRACEFUL_ABORT } from './stream.util.js';
 import { transformChunk } from './transform/transformChunk.js';
@@ -15,6 +17,7 @@ import { transformMap } from './transform/transformMap.js';
 import { transformMapSimple, } from './transform/transformMapSimple.js';
 import { transformMapSync } from './transform/transformMapSync.js';
 import { transformOffset } from './transform/transformOffset.js';
+import { transformSplitOnNewline } from './transform/transformSplit.js';
 import { transformTap } from './transform/transformTap.js';
 import { transformThrottle } from './transform/transformThrottle.js';
 import { writablePushToArray } from './writable/writablePushToArray.js';
@@ -25,13 +28,19 @@ export class Pipeline {
     transforms = [];
     destination;
     readableLimit;
+    // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
+    objectMode;
     abortableSignal = createAbortableSignal();
-    constructor(source) {
+    constructor(source, objectMode = true) {
         this.source = source;
+        this.objectMode = objectMode;
     }
     static from(source) {
         return new Pipeline(source);
     }
+    static fromWeb(webReadableStream) {
+        return new Pipeline(Readable.fromWeb(webReadableStream));
+    }
     /**
      * Technically same as `fromIterable` (since Array is Iterable),
      * but named a bit friendlier.
@@ -42,6 +51,9 @@ export class Pipeline {
     static fromIterable(input) {
         return new Pipeline(Readable.from(input));
     }
+    static fromFile(sourceFilePath) {
+        return new Pipeline(fs2.createReadStream(sourceFilePath), false);
+    }
     static fromNDJsonFile(sourceFilePath) {
         return new Pipeline(createReadStreamAsNDJSON(sourceFilePath));
     }
@@ -156,6 +168,50 @@ export class Pipeline {
     typeCastAs() {
         return this;
     }
+    setObjectMode(objectMode) {
+        this.objectMode = objectMode;
+        return this;
+    }
+    /**
+     * Transform the stream of Objects into a stream of JSON lines.
+     * Technically, it goes into objectMode=false, so it's a binary stream at the end.
+     */
+    toNDJson() {
+        this.transforms.push(transformToNDJson());
+        this.objectMode = false;
+        return this;
+    }
+    parseNDJson() {
+        // It was said that transformJsonParse() separately is 10% or more slower than .map(line => JSON.parse(line))
+        // So, we can investigate a speedup
+        this.transforms.push(transformSplitOnNewline(), transformJsonParse());
+        this.objectMode = true;
+        return this;
+    }
+    splitOnNewline() {
+        // Input: objectMode=false - binary stream
+        // Output: objectMode=true - stream of Buffer objects (which are also strings?)
+        this.transforms.push(transformSplitOnNewline());
+        this.objectMode = true;
+        return this;
+    }
+    parseJson() {
+        // Input: objectMode=false - takes a stream of strings one by one
+        // Output: objectMode=true - stream of json-parsed Objects
+        this.transforms.push(transformJsonParse());
+        this.objectMode = true;
+        return this;
+    }
+    gzip(opt) {
+        this.transforms.push(createGzip(opt));
+        this.objectMode = false;
+        return this;
+    }
+    gunzip(opt) {
+        this.transforms.push(createUnzip(opt));
+        this.objectMode = false;
+        return this;
+    }
     async toArray(opt) {
         const arr = [];
         this.destination = writablePushToArray(arr, opt);

package/dist/stream/transform/transformSplit.js

@@ -11,8 +11,9 @@ import { Transform } from 'node:stream';
 export function transformSplitOnNewline() {
     let buffered;
     return new Transform({
-        readableObjectMode: true,
+        writableObjectMode: false,
         writableHighWaterMark: 64 * 1024,
+        readableObjectMode: true,
         transform(buf, _enc, done) {
             let offset = 0;
             let lastMatch = 0;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.23.0",
+  "version": "15.25.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@types/js-yaml": "^4",

package/src/stream/pipeline.ts

@@ -1,5 +1,7 @@
 import { Readable, type Transform } from 'node:stream'
 import { pipeline } from 'node:stream/promises'
+import type { ReadableStream as WebReadableStream } from 'node:stream/web'
+import { createUnzip, type ZlibOptions } from 'node:zlib'
 import { createGzip } from 'node:zlib'
 import { createAbortableSignal } from '@naturalcycles/js-lib'
 import type {
@@ -16,6 +18,7 @@ import type {
 } from '@naturalcycles/js-lib/types'
 import { fs2 } from '../fs/fs2.js'
 import { createReadStreamAsNDJSON } from './ndjson/createReadStreamAsNDJSON.js'
+import { transformJsonParse } from './ndjson/transformJsonParse.js'
 import { transformToNDJson } from './ndjson/transformToNDJson.js'
 import type {
   ReadableTyped,
@@ -39,6 +42,7 @@ import {
 } from './transform/transformMapSimple.js'
 import { transformMapSync, type TransformMapSyncOptions } from './transform/transformMapSync.js'
 import { transformOffset, type TransformOffsetOptions } from './transform/transformOffset.js'
+import { transformSplitOnNewline } from './transform/transformSplit.js'
 import { transformTap, type TransformTapOptions } from './transform/transformTap.js'
 import { transformThrottle, type TransformThrottleOptions } from './transform/transformThrottle.js'
 import { writablePushToArray } from './writable/writablePushToArray.js'
@@ -50,16 +54,23 @@ export class Pipeline<T> {
   private transforms: NodeJS.ReadWriteStream[] = []
   private destination?: NodeJS.WritableStream
   private readableLimit?: Integer
+  // biome-ignore lint/correctness/noUnusedPrivateClassMembers: ok
+  private objectMode: boolean
   private abortableSignal = createAbortableSignal()
 
-  private constructor(source: ReadableTyped<T>) {
+  private constructor(source: ReadableTyped<T>, objectMode = true) {
     this.source = source
+    this.objectMode = objectMode
   }
 
   static from<T>(source: ReadableTyped<T>): Pipeline<T> {
     return new Pipeline(source)
   }
 
+  static fromWeb<T>(webReadableStream: WebReadableStream<T>): Pipeline<T> {
+    return new Pipeline(Readable.fromWeb(webReadableStream))
+  }
+
   /**
    * Technically same as `fromIterable` (since Array is Iterable),
    * but named a bit friendlier.
@@ -72,6 +83,10 @@ export class Pipeline<T> {
     return new Pipeline(Readable.from(input))
   }
 
+  static fromFile(sourceFilePath: string): Pipeline<Uint8Array> {
+    return new Pipeline(fs2.createReadStream(sourceFilePath), false)
+  }
+
   static fromNDJsonFile<T>(sourceFilePath: string): Pipeline<T> {
     return new Pipeline(createReadStreamAsNDJSON<T>(sourceFilePath))
   }
@@ -219,6 +234,59 @@ export class Pipeline<T> {
     return this as any
   }
 
+  setObjectMode(objectMode: boolean): this {
+    this.objectMode = objectMode
+    return this
+  }
+
+  /**
+   * Transform the stream of Objects into a stream of JSON lines.
+   * Technically, it goes into objectMode=false, so it's a binary stream at the end.
+   */
+  toNDJson(): Pipeline<Uint8Array> {
+    this.transforms.push(transformToNDJson())
+    this.objectMode = false
+    return this as any
+  }
+
+  parseNDJson<TO = unknown>(this: Pipeline<Uint8Array>): Pipeline<TO> {
+    // It was said that transformJsonParse() separately is 10% or more slower than .map(line => JSON.parse(line))
+    // So, we can investigate a speedup
+    this.transforms.push(transformSplitOnNewline(), transformJsonParse())
+    this.objectMode = true
+    return this as any
+  }
+
+  splitOnNewline(this: Pipeline<Uint8Array>): Pipeline<Buffer> {
+    // Input: objectMode=false - binary stream
+    // Output: objectMode=true - stream of Buffer objects (which are also strings?)
+    this.transforms.push(transformSplitOnNewline())
+    this.objectMode = true
+    return this as any
+  }
+
+  parseJson<TO = unknown>(
+    this: Pipeline<Buffer> | Pipeline<Uint8Array> | Pipeline<string>,
+  ): Pipeline<TO> {
+    // Input: objectMode=false - takes a stream of strings one by one
+    // Output: objectMode=true - stream of json-parsed Objects
+    this.transforms.push(transformJsonParse())
+    this.objectMode = true
+    return this as any
+  }
+
+  gzip(this: Pipeline<Uint8Array>, opt?: ZlibOptions): Pipeline<Uint8Array> {
+    this.transforms.push(createGzip(opt))
+    this.objectMode = false
+    return this as any
+  }
+
+  gunzip(this: Pipeline<Uint8Array>, opt?: ZlibOptions): Pipeline<Uint8Array> {
+    this.transforms.push(createUnzip(opt))
+    this.objectMode = false
+    return this as any
+  }
+
   async toArray(opt?: TransformOptions): Promise<T[]> {
     const arr: T[] = []
     this.destination = writablePushToArray(arr, opt)

package/src/stream/transform/transformSplit.ts

@@ -15,8 +15,9 @@ export function transformSplitOnNewline(): TransformTyped<Buffer, Buffer> {
   let buffered: Buffer | undefined
 
   return new Transform({
-    readableObjectMode: true,
+    writableObjectMode: false,
     writableHighWaterMark: 64 * 1024,
+    readableObjectMode: true,
 
     transform(buf: Buffer, _enc, done) {
       let offset = 0