@naturalcycles/nodejs-lib 15.125.0 → 15.125.2

package/dist/stream/ndjson/createReadStreamAsNDJson.js

@@ -1,6 +1,9 @@
+import { pipeline } from 'node:stream';
 import { createUnzip, createZstdDecompress } from 'node:zlib';
+import { _noop } from '@naturalcycles/js-lib/types';
 import { fs2 } from '../../fs/fs2.js';
 import { transformSplitOnNewline } from '../transform/transformSplit.js';
+import { transformJsonParse } from './transformJsonParse.js';
 /**
  Returns a Readable of [already parsed] NDJSON objects.
 
@@ -17,24 +20,30 @@ import { transformSplitOnNewline } from '../transform/transformSplit.js';
  */
 export function createReadStreamAsNDJson(inputPath) {
     fs2.requireFileToExist(inputPath);
-    let stream = fs2
-        .createReadStream(inputPath, {
-        highWaterMark: 64 * 1024, // no observed speedup
-    })
-        .on('error', err => stream.destroy(err));
+    const streams = [
+        fs2.createReadStream(inputPath, {
+            highWaterMark: 256 * 1024, // 64KB had no observed speedup over the default; 256KB is +6-12% end-to-end on ~9KB rows, neutral on ~180B rows, 1MB brings nothing more
+        }),
+    ];
     if (inputPath.endsWith('.gz')) {
-        stream = stream.pipe(createUnzip({
-            chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
+        streams.push(createUnzip({
+            chunkSize: 256 * 1024, // 64KB: speedup from ~3200 to 3800 rps! 256KB: +6-8% more on ~9KB rows, neutral on ~180B rows
         }));
     }
     else if (inputPath.endsWith('.zst')) {
-        stream = stream.pipe(createZstdDecompress({
-            chunkSize: 64 * 1024, // todo: test it
+        streams.push(createZstdDecompress({
+            chunkSize: 256 * 1024, // tested: 256KB is +10-12% over 64KB on ~9KB rows, neutral on ~180B rows
         }));
     }
-    return stream.pipe(transformSplitOnNewline()).map(line => JSON.parse(line));
-    // For some crazy reason .map is much faster than transformJsonParse!
-    // ~5000 vs ~4000 rps !!!
-    // .on('error', err => stream.emit('error', err))
-    // .pipe(transformJsonParse<ROW>())
+    const output = transformJsonParse();
+    streams.push(transformSplitOnNewline(), output);
+    // `pipeline` (unlike `.pipe`) forwards errors: when any stream fails (e.g. a corrupt .gz file),
+    // all of them are destroyed with that error, so the consumer of `output` gets a rejection,
+    // instead of the process crashing with an unhandled 'error' event.
+    // The error is also passed to the callback, where there's nothing more to do with it.
+    pipeline(streams, _noop);
+    return output;
+    // It used to be `.map(line => JSON.parse(line))`, because "for some crazy reason .map is much faster than transformJsonParse!
+    // ~5000 vs ~4000 rps !!!". Measured again on Node 24, it's the opposite: Readable.map costs ~1µs per row in promise machinery,
+    // transformJsonParse() is 1.5x (consumed via pipeline) to 1.75x (consumed via for-await) faster on ~180B rows.
 }

package/dist/stream/pipeline.js

@@ -73,7 +73,7 @@ export class Pipeline {
     }
     static fromFile(sourceFilePath) {
         return new Pipeline(fs2.createReadStream(sourceFilePath, {
-            highWaterMark: 64 * 1024, // no observed speedup
+            highWaterMark: 256 * 1024, // 64KB had no observed speedup over the default; 256KB is +6-12% end-to-end on ~9KB rows, neutral on ~180B rows, 1MB brings nothing more
         }), false);
     }
     /**
@@ -225,7 +225,7 @@ export class Pipeline {
     }
     parseNDJson() {
         // It was said that transformJsonParse() separately is 10% or more slower than .map(line => JSON.parse(line))
-        // So, we can investigate a speedup
+        // Measured again on Node 24: the opposite, transformJsonParse() is 1.5x faster (Readable.map costs ~1µs per row), see createReadStreamAsNDJson
         this.transforms.push(transformSplitOnNewline(), transformJsonParse());
         this.objectMode = true;
         return this;
@@ -254,7 +254,7 @@ export class Pipeline {
     }
     gunzip(opt) {
         this.transforms.push(createUnzip({
-            chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
+            chunkSize: 256 * 1024, // 64KB: speedup from ~3200 to 3800 rps! 256KB: +6-8% more on ~9KB rows, neutral on ~180B rows
             ...opt,
         }));
         this.objectMode = false;
@@ -268,7 +268,7 @@ export class Pipeline {
     }
     zstdDecompress(opt) {
         this.transforms.push(createZstdDecompress({
-            chunkSize: 64 * 1024, // todo: test it
+            chunkSize: 256 * 1024, // tested: 256KB is +10-12% over 64KB on ~9KB rows, neutral on ~180B rows
             ...opt,
         }));
         this.objectMode = false;

package/dist/stream/transform/transformSplit.js

@@ -30,61 +30,91 @@ export function transformSplit(separator = '\n') {
     const needle = matcherLength === 1 ? matcher[0] : matcher;
     // a multi-byte separator may straddle a chunk boundary, so we keep that many trailing bytes around
     const overlap = matcherLength - 1;
-    // Unterminated data is collected as a list of chunks and concatenated only once a separator arrives,
-    // so a long line spanning many chunks is copied once rather than on every chunk (linear, not quadratic)
+    // Unterminated data is collected as a list of chunks and concatenated only once its separator arrives,
+    // so a line spanning many chunks is copied once rather than on every chunk (linear, not quadratic).
+    // Only that one line is copied: the rest of the chunk is scanned in place and emitted as views into it.
     let pending = [];
     let pendingLength = 0;
     let edge; // last `overlap` bytes of pending data, only tracked when overlap > 0
-    return new Transform({
+    const transform = new Transform({
         writableObjectMode: false,
         writableHighWaterMark: 64 * 1024,
         readableObjectMode: true,
         transform(chunk, _enc, cb) {
-            let buf = chunk;
-            let offset = 0;
+            // Deferred by a microtask, so that an async upstream (zlib, fs) can dispatch its next chunk to its thread
+            // before we do the (synchronous) splitting + whatever the consumer does with the lines:
+            // decompression of chunk N+1 then overlaps with processing of chunk N, measured +5-15% on gzip/zstd input.
+            // Without it all of that runs inside the upstream's push, and the two never overlap.
+            queueMicrotask(() => {
+                // Whatever is thrown here (by a consumer most likely, as push() runs 'data' listeners synchronously)
+                // must become a stream error, like it did when this ran inside transform() itself:
+                // thrown from a microtask it would be an uncaught exception and crash the process.
+                try {
+                    processChunk(chunk);
+                }
+                catch (err) {
+                    cb(err);
+                    return;
+                }
+                cb();
+            });
+        },
+        flush(cb) {
             if (pendingLength > 0) {
-                const straddles = overlap > 0 && Buffer.concat([edge, chunk.subarray(0, overlap)]).includes(matcher);
-                const idxInChunk = straddles ? -1 : chunk.indexOf(needle);
-                if (!straddles && idxInChunk === -1) {
+                this.push(Buffer.concat(pending, pendingLength));
+            }
+            cb();
+        },
+    });
+    function processChunk(chunk) {
+        let start = 0;
+        if (pendingLength > 0) {
+            // A multi-byte separator may straddle the chunk boundary: look for it in the last `overlap` bytes
+            // of pending data followed by the first `overlap` bytes of this chunk
+            const straddleIdx = overlap > 0 ? Buffer.concat([edge, chunk.subarray(0, overlap)]).indexOf(matcher) : -1;
+            // Position of the first separator in chunk coordinates, negative when it started inside the pending data
+            let idx;
+            if (straddleIdx === -1) {
+                idx = chunk.indexOf(needle);
+                if (idx === -1) {
                     // No separator yet: keep accumulating without copying
                     pending.push(chunk);
                     pendingLength += chunk.length;
                     if (overlap > 0) {
                         edge = Buffer.concat([edge, chunk.subarray(-overlap)]).subarray(-overlap);
                     }
-                    cb();
                     return;
                 }
-                pending.push(chunk);
-                buf = Buffer.concat(pending, pendingLength + chunk.length);
-                // Start scanning where the first separator can possibly be
-                offset = straddles ? Math.max(0, pendingLength - overlap) : pendingLength + idxInChunk;
-                pending = [];
-                pendingLength = 0;
             }
-            let start = 0;
-            let idx = buf.indexOf(needle, offset);
-            while (idx !== -1) {
-                if (idx > start) {
-                    this.push(buf.subarray(start, idx));
-                }
-                start = idx + matcherLength;
-                idx = buf.indexOf(needle, start);
+            else {
+                idx = straddleIdx - edge.length;
             }
-            if (start < buf.length) {
-                const rest = buf.subarray(start);
-                pending.push(rest);
-                pendingLength = rest.length;
-                if (overlap > 0)
-                    edge = rest.subarray(-overlap);
+            // Complete the pending line with the head of this chunk
+            if (idx > 0)
+                pending.push(chunk.subarray(0, idx));
+            const lineLength = pendingLength + idx;
+            if (lineLength > 0) {
+                transform.push(Buffer.concat(pending, lineLength));
             }
-            cb();
-        },
-        flush(cb) {
-            if (pendingLength > 0) {
-                this.push(Buffer.concat(pending, pendingLength));
+            pending = [];
+            pendingLength = 0;
+            start = idx + matcherLength;
+        }
+        let idx = chunk.indexOf(needle, start);
+        while (idx !== -1) {
+            if (idx > start) {
+                transform.push(chunk.subarray(start, idx));
             }
-            cb();
-        },
-    });
+            start = idx + matcherLength;
+            idx = chunk.indexOf(needle, start);
+        }
+        if (start < chunk.length) {
+            const rest = chunk.subarray(start);
+            pending.push(rest);
+            pendingLength = rest.length;
+            if (overlap > 0)
+                edge = rest.subarray(-overlap);
+        }
+    }
+    return transform;
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.125.0",
+  "version": "15.125.2",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@standard-schema/spec": "^1",

package/src/stream/ndjson/createReadStreamAsNDJson.ts

@@ -1,7 +1,11 @@
+import type { Readable } from 'node:stream'
+import { pipeline } from 'node:stream'
 import { createUnzip, createZstdDecompress } from 'node:zlib'
+import { _noop } from '@naturalcycles/js-lib/types'
 import { fs2 } from '../../fs/fs2.js'
 import type { ReadableTyped } from '../stream.model.js'
 import { transformSplitOnNewline } from '../transform/transformSplit.js'
+import { transformJsonParse } from './transformJsonParse.js'
 
 /**
  Returns a Readable of [already parsed] NDJSON objects.
@@ -21,29 +25,37 @@ import { transformSplitOnNewline } from '../transform/transformSplit.js'
 export function createReadStreamAsNDJson<ROW = any>(inputPath: string): ReadableTyped<ROW> {
   fs2.requireFileToExist(inputPath)
 
-  let stream: ReadableTyped<ROW> = fs2
-    .createReadStream(inputPath, {
-      highWaterMark: 64 * 1024, // no observed speedup
-    })
-    .on('error', err => stream.destroy(err))
+  const streams: Readable[] = [
+    fs2.createReadStream(inputPath, {
+      highWaterMark: 256 * 1024, // 64KB had no observed speedup over the default; 256KB is +6-12% end-to-end on ~9KB rows, neutral on ~180B rows, 1MB brings nothing more
+    }),
+  ]
 
   if (inputPath.endsWith('.gz')) {
-    stream = stream.pipe(
+    streams.push(
       createUnzip({
-        chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
+        chunkSize: 256 * 1024, // 64KB: speedup from ~3200 to 3800 rps! 256KB: +6-8% more on ~9KB rows, neutral on ~180B rows
       }),
     )
   } else if (inputPath.endsWith('.zst')) {
-    stream = stream.pipe(
+    streams.push(
       createZstdDecompress({
-        chunkSize: 64 * 1024, // todo: test it
+        chunkSize: 256 * 1024, // tested: 256KB is +10-12% over 64KB on ~9KB rows, neutral on ~180B rows
       }),
     )
   }
 
-  return stream.pipe(transformSplitOnNewline()).map(line => JSON.parse(line))
-  // For some crazy reason .map is much faster than transformJsonParse!
-  // ~5000 vs ~4000 rps !!!
-  // .on('error', err => stream.emit('error', err))
-  // .pipe(transformJsonParse<ROW>())
+  const output = transformJsonParse<ROW>()
+  streams.push(transformSplitOnNewline(), output)
+
+  // `pipeline` (unlike `.pipe`) forwards errors: when any stream fails (e.g. a corrupt .gz file),
+  // all of them are destroyed with that error, so the consumer of `output` gets a rejection,
+  // instead of the process crashing with an unhandled 'error' event.
+  // The error is also passed to the callback, where there's nothing more to do with it.
+  pipeline(streams, _noop)
+
+  return output
+  // It used to be `.map(line => JSON.parse(line))`, because "for some crazy reason .map is much faster than transformJsonParse!
+  // ~5000 vs ~4000 rps !!!". Measured again on Node 24, it's the opposite: Readable.map costs ~1µs per row in promise machinery,
+  // transformJsonParse() is 1.5x (consumed via pipeline) to 1.75x (consumed via for-await) faster on ~180B rows.
 }

package/src/stream/pipeline.ts

@@ -114,7 +114,7 @@ export class Pipeline<T = unknown> {
   static fromFile(sourceFilePath: string): Pipeline<Uint8Array> {
     return new Pipeline(
       fs2.createReadStream(sourceFilePath, {
-        highWaterMark: 64 * 1024, // no observed speedup
+        highWaterMark: 256 * 1024, // 64KB had no observed speedup over the default; 256KB is +6-12% end-to-end on ~9KB rows, neutral on ~180B rows, 1MB brings nothing more
       }),
       false,
     )
@@ -309,7 +309,7 @@ export class Pipeline<T = unknown> {
 
   parseNDJson<TO = unknown>(this: Pipeline<Uint8Array>): Pipeline<TO> {
     // It was said that transformJsonParse() separately is 10% or more slower than .map(line => JSON.parse(line))
-    // So, we can investigate a speedup
+    // Measured again on Node 24: the opposite, transformJsonParse() is 1.5x faster (Readable.map costs ~1µs per row), see createReadStreamAsNDJson
     this.transforms.push(transformSplitOnNewline(), transformJsonParse())
     this.objectMode = true
     return this as any
@@ -347,7 +347,7 @@ export class Pipeline<T = unknown> {
   gunzip(this: Pipeline<Uint8Array>, opt?: ZlibOptions): Pipeline<Uint8Array> {
     this.transforms.push(
       createUnzip({
-        chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
+        chunkSize: 256 * 1024, // 64KB: speedup from ~3200 to 3800 rps! 256KB: +6-8% more on ~9KB rows, neutral on ~180B rows
         ...opt,
       }),
     )
@@ -368,7 +368,7 @@ export class Pipeline<T = unknown> {
   zstdDecompress(this: Pipeline<Uint8Array>, opt?: ZstdOptions): Pipeline<Uint8Array> {
     this.transforms.push(
       createZstdDecompress({
-        chunkSize: 64 * 1024, // todo: test it
+        chunkSize: 256 * 1024, // tested: 256KB is +10-12% over 64KB on ~9KB rows, neutral on ~180B rows
         ...opt,
       }),
     )

package/src/stream/transform/transformSplit.ts

@@ -35,68 +35,98 @@ export function transformSplit(separator = '\n'): TransformTyped<Buffer, Buffer>
   // a multi-byte separator may straddle a chunk boundary, so we keep that many trailing bytes around
   const overlap = matcherLength - 1
 
-  // Unterminated data is collected as a list of chunks and concatenated only once a separator arrives,
-  // so a long line spanning many chunks is copied once rather than on every chunk (linear, not quadratic)
+  // Unterminated data is collected as a list of chunks and concatenated only once its separator arrives,
+  // so a line spanning many chunks is copied once rather than on every chunk (linear, not quadratic).
+  // Only that one line is copied: the rest of the chunk is scanned in place and emitted as views into it.
   let pending: Buffer[] = []
   let pendingLength = 0
   let edge: Buffer | undefined // last `overlap` bytes of pending data, only tracked when overlap > 0
 
-  return new Transform({
+  const transform = new Transform({
     writableObjectMode: false,
     writableHighWaterMark: 64 * 1024,
     readableObjectMode: true,
 
     transform(chunk: Buffer, _enc, cb) {
-      let buf = chunk
-      let offset = 0
+      // Deferred by a microtask, so that an async upstream (zlib, fs) can dispatch its next chunk to its thread
+      // before we do the (synchronous) splitting + whatever the consumer does with the lines:
+      // decompression of chunk N+1 then overlaps with processing of chunk N, measured +5-15% on gzip/zstd input.
+      // Without it all of that runs inside the upstream's push, and the two never overlap.
+      queueMicrotask(() => {
+        // Whatever is thrown here (by a consumer most likely, as push() runs 'data' listeners synchronously)
+        // must become a stream error, like it did when this ran inside transform() itself:
+        // thrown from a microtask it would be an uncaught exception and crash the process.
+        try {
+          processChunk(chunk)
+        } catch (err) {
+          cb(err as Error)
+          return
+        }
+        cb()
+      })
+    },
 
+    flush(cb) {
       if (pendingLength > 0) {
-        const straddles =
-          overlap > 0 && Buffer.concat([edge!, chunk.subarray(0, overlap)]).includes(matcher)
-        const idxInChunk = straddles ? -1 : chunk.indexOf(needle)
-        if (!straddles && idxInChunk === -1) {
+        this.push(Buffer.concat(pending, pendingLength))
+      }
+      cb()
+    },
+  })
+
+  function processChunk(chunk: Buffer): void {
+    let start = 0
+
+    if (pendingLength > 0) {
+      // A multi-byte separator may straddle the chunk boundary: look for it in the last `overlap` bytes
+      // of pending data followed by the first `overlap` bytes of this chunk
+      const straddleIdx =
+        overlap > 0 ? Buffer.concat([edge!, chunk.subarray(0, overlap)]).indexOf(matcher) : -1
+
+      // Position of the first separator in chunk coordinates, negative when it started inside the pending data
+      let idx: number
+      if (straddleIdx === -1) {
+        idx = chunk.indexOf(needle)
+        if (idx === -1) {
           // No separator yet: keep accumulating without copying
           pending.push(chunk)
           pendingLength += chunk.length
           if (overlap > 0) {
             edge = Buffer.concat([edge!, chunk.subarray(-overlap)]).subarray(-overlap)
           }
-          cb()
           return
         }
-        pending.push(chunk)
-        buf = Buffer.concat(pending, pendingLength + chunk.length)
-        // Start scanning where the first separator can possibly be
-        offset = straddles ? Math.max(0, pendingLength - overlap) : pendingLength + idxInChunk
-        pending = []
-        pendingLength = 0
+      } else {
+        idx = straddleIdx - edge!.length
       }
 
-      let start = 0
-      let idx = buf.indexOf(needle, offset)
-      while (idx !== -1) {
-        if (idx > start) {
-          this.push(buf.subarray(start, idx))
-        }
-        start = idx + matcherLength
-        idx = buf.indexOf(needle, start)
+      // Complete the pending line with the head of this chunk
+      if (idx > 0) pending.push(chunk.subarray(0, idx))
+      const lineLength = pendingLength + idx
+      if (lineLength > 0) {
+        transform.push(Buffer.concat(pending, lineLength))
       }
+      pending = []
+      pendingLength = 0
+      start = idx + matcherLength
+    }
 
-      if (start < buf.length) {
-        const rest = buf.subarray(start)
-        pending.push(rest)
-        pendingLength = rest.length
-        if (overlap > 0) edge = rest.subarray(-overlap)
+    let idx = chunk.indexOf(needle, start)
+    while (idx !== -1) {
+      if (idx > start) {
+        transform.push(chunk.subarray(start, idx))
       }
+      start = idx + matcherLength
+      idx = chunk.indexOf(needle, start)
+    }
 
-      cb()
-    },
+    if (start < chunk.length) {
+      const rest = chunk.subarray(start)
+      pending.push(rest)
+      pendingLength = rest.length
+      if (overlap > 0) edge = rest.subarray(-overlap)
+    }
+  }
 
-    flush(cb) {
-      if (pendingLength > 0) {
-        this.push(Buffer.concat(pending, pendingLength))
-      }
-      cb()
-    },
-  })
+  return transform
 }