@naturalcycles/nodejs-lib 15.125.1 → 15.125.2

package/dist/stream/ndjson/createReadStreamAsNDJson.js

@@ -1,6 +1,9 @@
+import { pipeline } from 'node:stream';
 import { createUnzip, createZstdDecompress } from 'node:zlib';
+import { _noop } from '@naturalcycles/js-lib/types';
 import { fs2 } from '../../fs/fs2.js';
 import { transformSplitOnNewline } from '../transform/transformSplit.js';
+import { transformJsonParse } from './transformJsonParse.js';
 /**
  Returns a Readable of [already parsed] NDJSON objects.
 
@@ -17,24 +20,30 @@ import { transformSplitOnNewline } from '../transform/transformSplit.js';
  */
 export function createReadStreamAsNDJson(inputPath) {
     fs2.requireFileToExist(inputPath);
-    let stream = fs2
-        .createReadStream(inputPath, {
-        highWaterMark: 256 * 1024, // 64KB had no observed speedup over the default; 256KB is +6-12% end-to-end on ~9KB rows, neutral on ~180B rows, 1MB brings nothing more
-    })
-        .on('error', err => stream.destroy(err));
+    const streams = [
+        fs2.createReadStream(inputPath, {
+            highWaterMark: 256 * 1024, // 64KB had no observed speedup over the default; 256KB is +6-12% end-to-end on ~9KB rows, neutral on ~180B rows, 1MB brings nothing more
+        }),
+    ];
     if (inputPath.endsWith('.gz')) {
-        stream = stream.pipe(createUnzip({
+        streams.push(createUnzip({
             chunkSize: 256 * 1024, // 64KB: speedup from ~3200 to 3800 rps! 256KB: +6-8% more on ~9KB rows, neutral on ~180B rows
         }));
     }
     else if (inputPath.endsWith('.zst')) {
-        stream = stream.pipe(createZstdDecompress({
+        streams.push(createZstdDecompress({
             chunkSize: 256 * 1024, // tested: 256KB is +10-12% over 64KB on ~9KB rows, neutral on ~180B rows
         }));
     }
-    return stream.pipe(transformSplitOnNewline()).map(line => JSON.parse(line));
-    // For some crazy reason .map is much faster than transformJsonParse!
-    // ~5000 vs ~4000 rps !!!
-    // .on('error', err => stream.emit('error', err))
-    // .pipe(transformJsonParse<ROW>())
+    const output = transformJsonParse();
+    streams.push(transformSplitOnNewline(), output);
+    // `pipeline` (unlike `.pipe`) forwards errors: when any stream fails (e.g. a corrupt .gz file),
+    // all of them are destroyed with that error, so the consumer of `output` gets a rejection,
+    // instead of the process crashing with an unhandled 'error' event.
+    // The error is also passed to the callback, where there's nothing more to do with it.
+    pipeline(streams, _noop);
+    return output;
+    // It used to be `.map(line => JSON.parse(line))`, because "for some crazy reason .map is much faster than transformJsonParse!
+    // ~5000 vs ~4000 rps !!!". Measured again on Node 24, it's the opposite: Readable.map costs ~1µs per row in promise machinery,
+    // transformJsonParse() is 1.5x (consumed via pipeline) to 1.75x (consumed via for-await) faster on ~180B rows.
 }

package/dist/stream/pipeline.js

@@ -225,7 +225,7 @@ export class Pipeline {
     }
     parseNDJson() {
         // It was said that transformJsonParse() separately is 10% or more slower than .map(line => JSON.parse(line))
-        // So, we can investigate a speedup
+        // Measured again on Node 24: the opposite, transformJsonParse() is 1.5x faster (Readable.map costs ~1µs per row), see createReadStreamAsNDJson
         this.transforms.push(transformSplitOnNewline(), transformJsonParse());
         this.objectMode = true;
         return this;

package/dist/stream/transform/transformSplit.js

@@ -36,61 +36,28 @@ export function transformSplit(separator = '\n') {
     let pending = [];
     let pendingLength = 0;
     let edge; // last `overlap` bytes of pending data, only tracked when overlap > 0
-    return new Transform({
+    const transform = new Transform({
         writableObjectMode: false,
         writableHighWaterMark: 64 * 1024,
         readableObjectMode: true,
         transform(chunk, _enc, cb) {
-            let start = 0;
-            if (pendingLength > 0) {
-                // A multi-byte separator may straddle the chunk boundary: look for it in the last `overlap` bytes
-                // of pending data followed by the first `overlap` bytes of this chunk
-                const straddleIdx = overlap > 0 ? Buffer.concat([edge, chunk.subarray(0, overlap)]).indexOf(matcher) : -1;
-                // Position of the first separator in chunk coordinates, negative when it started inside the pending data
-                let idx;
-                if (straddleIdx === -1) {
-                    idx = chunk.indexOf(needle);
-                    if (idx === -1) {
-                        // No separator yet: keep accumulating without copying
-                        pending.push(chunk);
-                        pendingLength += chunk.length;
-                        if (overlap > 0) {
-                            edge = Buffer.concat([edge, chunk.subarray(-overlap)]).subarray(-overlap);
-                        }
-                        cb();
-                        return;
-                    }
-                }
-                else {
-                    idx = straddleIdx - edge.length;
-                }
-                // Complete the pending line with the head of this chunk
-                if (idx > 0)
-                    pending.push(chunk.subarray(0, idx));
-                const lineLength = pendingLength + idx;
-                if (lineLength > 0) {
-                    this.push(Buffer.concat(pending, lineLength));
+            // Deferred by a microtask, so that an async upstream (zlib, fs) can dispatch its next chunk to its thread
+            // before we do the (synchronous) splitting + whatever the consumer does with the lines:
+            // decompression of chunk N+1 then overlaps with processing of chunk N, measured +5-15% on gzip/zstd input.
+            // Without it all of that runs inside the upstream's push, and the two never overlap.
+            queueMicrotask(() => {
+                // Whatever is thrown here (by a consumer most likely, as push() runs 'data' listeners synchronously)
+                // must become a stream error, like it did when this ran inside transform() itself:
+                // thrown from a microtask it would be an uncaught exception and crash the process.
+                try {
+                    processChunk(chunk);
                 }
-                pending = [];
-                pendingLength = 0;
-                start = idx + matcherLength;
-            }
-            let idx = chunk.indexOf(needle, start);
-            while (idx !== -1) {
-                if (idx > start) {
-                    this.push(chunk.subarray(start, idx));
+                catch (err) {
+                    cb(err);
+                    return;
                 }
-                start = idx + matcherLength;
-                idx = chunk.indexOf(needle, start);
-            }
-            if (start < chunk.length) {
-                const rest = chunk.subarray(start);
-                pending.push(rest);
-                pendingLength = rest.length;
-                if (overlap > 0)
-                    edge = rest.subarray(-overlap);
-            }
-            cb();
+                cb();
+            });
         },
         flush(cb) {
             if (pendingLength > 0) {
@@ -99,4 +66,55 @@ export function transformSplit(separator = '\n') {
             cb();
         },
     });
+    function processChunk(chunk) {
+        let start = 0;
+        if (pendingLength > 0) {
+            // A multi-byte separator may straddle the chunk boundary: look for it in the last `overlap` bytes
+            // of pending data followed by the first `overlap` bytes of this chunk
+            const straddleIdx = overlap > 0 ? Buffer.concat([edge, chunk.subarray(0, overlap)]).indexOf(matcher) : -1;
+            // Position of the first separator in chunk coordinates, negative when it started inside the pending data
+            let idx;
+            if (straddleIdx === -1) {
+                idx = chunk.indexOf(needle);
+                if (idx === -1) {
+                    // No separator yet: keep accumulating without copying
+                    pending.push(chunk);
+                    pendingLength += chunk.length;
+                    if (overlap > 0) {
+                        edge = Buffer.concat([edge, chunk.subarray(-overlap)]).subarray(-overlap);
+                    }
+                    return;
+                }
+            }
+            else {
+                idx = straddleIdx - edge.length;
+            }
+            // Complete the pending line with the head of this chunk
+            if (idx > 0)
+                pending.push(chunk.subarray(0, idx));
+            const lineLength = pendingLength + idx;
+            if (lineLength > 0) {
+                transform.push(Buffer.concat(pending, lineLength));
+            }
+            pending = [];
+            pendingLength = 0;
+            start = idx + matcherLength;
+        }
+        let idx = chunk.indexOf(needle, start);
+        while (idx !== -1) {
+            if (idx > start) {
+                transform.push(chunk.subarray(start, idx));
+            }
+            start = idx + matcherLength;
+            idx = chunk.indexOf(needle, start);
+        }
+        if (start < chunk.length) {
+            const rest = chunk.subarray(start);
+            pending.push(rest);
+            pendingLength = rest.length;
+            if (overlap > 0)
+                edge = rest.subarray(-overlap);
+        }
+    }
+    return transform;
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.125.1",
+  "version": "15.125.2",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@standard-schema/spec": "^1",

package/src/stream/ndjson/createReadStreamAsNDJson.ts

@@ -1,7 +1,11 @@
+import type { Readable } from 'node:stream'
+import { pipeline } from 'node:stream'
 import { createUnzip, createZstdDecompress } from 'node:zlib'
+import { _noop } from '@naturalcycles/js-lib/types'
 import { fs2 } from '../../fs/fs2.js'
 import type { ReadableTyped } from '../stream.model.js'
 import { transformSplitOnNewline } from '../transform/transformSplit.js'
+import { transformJsonParse } from './transformJsonParse.js'
 
 /**
  Returns a Readable of [already parsed] NDJSON objects.
@@ -21,29 +25,37 @@ import { transformSplitOnNewline } from '../transform/transformSplit.js'
 export function createReadStreamAsNDJson<ROW = any>(inputPath: string): ReadableTyped<ROW> {
   fs2.requireFileToExist(inputPath)
 
-  let stream: ReadableTyped<ROW> = fs2
-    .createReadStream(inputPath, {
+  const streams: Readable[] = [
+    fs2.createReadStream(inputPath, {
       highWaterMark: 256 * 1024, // 64KB had no observed speedup over the default; 256KB is +6-12% end-to-end on ~9KB rows, neutral on ~180B rows, 1MB brings nothing more
-    })
-    .on('error', err => stream.destroy(err))
+    }),
+  ]
 
   if (inputPath.endsWith('.gz')) {
-    stream = stream.pipe(
+    streams.push(
       createUnzip({
         chunkSize: 256 * 1024, // 64KB: speedup from ~3200 to 3800 rps! 256KB: +6-8% more on ~9KB rows, neutral on ~180B rows
       }),
     )
   } else if (inputPath.endsWith('.zst')) {
-    stream = stream.pipe(
+    streams.push(
       createZstdDecompress({
         chunkSize: 256 * 1024, // tested: 256KB is +10-12% over 64KB on ~9KB rows, neutral on ~180B rows
       }),
     )
   }
 
-  return stream.pipe(transformSplitOnNewline()).map(line => JSON.parse(line))
-  // For some crazy reason .map is much faster than transformJsonParse!
-  // ~5000 vs ~4000 rps !!!
-  // .on('error', err => stream.emit('error', err))
-  // .pipe(transformJsonParse<ROW>())
+  const output = transformJsonParse<ROW>()
+  streams.push(transformSplitOnNewline(), output)
+
+  // `pipeline` (unlike `.pipe`) forwards errors: when any stream fails (e.g. a corrupt .gz file),
+  // all of them are destroyed with that error, so the consumer of `output` gets a rejection,
+  // instead of the process crashing with an unhandled 'error' event.
+  // The error is also passed to the callback, where there's nothing more to do with it.
+  pipeline(streams, _noop)
+
+  return output
+  // It used to be `.map(line => JSON.parse(line))`, because "for some crazy reason .map is much faster than transformJsonParse!
+  // ~5000 vs ~4000 rps !!!". Measured again on Node 24, it's the opposite: Readable.map costs ~1µs per row in promise machinery,
+  // transformJsonParse() is 1.5x (consumed via pipeline) to 1.75x (consumed via for-await) faster on ~180B rows.
 }

package/src/stream/pipeline.ts

@@ -309,7 +309,7 @@ export class Pipeline<T = unknown> {
 
   parseNDJson<TO = unknown>(this: Pipeline<Uint8Array>): Pipeline<TO> {
     // It was said that transformJsonParse() separately is 10% or more slower than .map(line => JSON.parse(line))
-    // So, we can investigate a speedup
+    // Measured again on Node 24: the opposite, transformJsonParse() is 1.5x faster (Readable.map costs ~1µs per row), see createReadStreamAsNDJson
     this.transforms.push(transformSplitOnNewline(), transformJsonParse())
     this.objectMode = true
     return this as any

package/src/stream/transform/transformSplit.ts

@@ -42,73 +42,91 @@ export function transformSplit(separator = '\n'): TransformTyped<Buffer, Buffer>
   let pendingLength = 0
   let edge: Buffer | undefined // last `overlap` bytes of pending data, only tracked when overlap > 0
 
-  return new Transform({
+  const transform = new Transform({
     writableObjectMode: false,
     writableHighWaterMark: 64 * 1024,
     readableObjectMode: true,
 
     transform(chunk: Buffer, _enc, cb) {
-      let start = 0
+      // Deferred by a microtask, so that an async upstream (zlib, fs) can dispatch its next chunk to its thread
+      // before we do the (synchronous) splitting + whatever the consumer does with the lines:
+      // decompression of chunk N+1 then overlaps with processing of chunk N, measured +5-15% on gzip/zstd input.
+      // Without it all of that runs inside the upstream's push, and the two never overlap.
+      queueMicrotask(() => {
+        // Whatever is thrown here (by a consumer most likely, as push() runs 'data' listeners synchronously)
+        // must become a stream error, like it did when this ran inside transform() itself:
+        // thrown from a microtask it would be an uncaught exception and crash the process.
+        try {
+          processChunk(chunk)
+        } catch (err) {
+          cb(err as Error)
+          return
+        }
+        cb()
+      })
+    },
 
+    flush(cb) {
       if (pendingLength > 0) {
-        // A multi-byte separator may straddle the chunk boundary: look for it in the last `overlap` bytes
-        // of pending data followed by the first `overlap` bytes of this chunk
-        const straddleIdx =
-          overlap > 0 ? Buffer.concat([edge!, chunk.subarray(0, overlap)]).indexOf(matcher) : -1
+        this.push(Buffer.concat(pending, pendingLength))
+      }
+      cb()
+    },
+  })
 
-        // Position of the first separator in chunk coordinates, negative when it started inside the pending data
-        let idx: number
-        if (straddleIdx === -1) {
-          idx = chunk.indexOf(needle)
-          if (idx === -1) {
-            // No separator yet: keep accumulating without copying
-            pending.push(chunk)
-            pendingLength += chunk.length
-            if (overlap > 0) {
-              edge = Buffer.concat([edge!, chunk.subarray(-overlap)]).subarray(-overlap)
-            }
-            cb()
-            return
-          }
-        } else {
-          idx = straddleIdx - edge!.length
-        }
+  function processChunk(chunk: Buffer): void {
+    let start = 0
+
+    if (pendingLength > 0) {
+      // A multi-byte separator may straddle the chunk boundary: look for it in the last `overlap` bytes
+      // of pending data followed by the first `overlap` bytes of this chunk
+      const straddleIdx =
+        overlap > 0 ? Buffer.concat([edge!, chunk.subarray(0, overlap)]).indexOf(matcher) : -1
 
-        // Complete the pending line with the head of this chunk
-        if (idx > 0) pending.push(chunk.subarray(0, idx))
-        const lineLength = pendingLength + idx
-        if (lineLength > 0) {
-          this.push(Buffer.concat(pending, lineLength))
+      // Position of the first separator in chunk coordinates, negative when it started inside the pending data
+      let idx: number
+      if (straddleIdx === -1) {
+        idx = chunk.indexOf(needle)
+        if (idx === -1) {
+          // No separator yet: keep accumulating without copying
+          pending.push(chunk)
+          pendingLength += chunk.length
+          if (overlap > 0) {
+            edge = Buffer.concat([edge!, chunk.subarray(-overlap)]).subarray(-overlap)
+          }
+          return
         }
-        pending = []
-        pendingLength = 0
-        start = idx + matcherLength
+      } else {
+        idx = straddleIdx - edge!.length
       }
 
-      let idx = chunk.indexOf(needle, start)
-      while (idx !== -1) {
-        if (idx > start) {
-          this.push(chunk.subarray(start, idx))
-        }
-        start = idx + matcherLength
-        idx = chunk.indexOf(needle, start)
+      // Complete the pending line with the head of this chunk
+      if (idx > 0) pending.push(chunk.subarray(0, idx))
+      const lineLength = pendingLength + idx
+      if (lineLength > 0) {
+        transform.push(Buffer.concat(pending, lineLength))
       }
+      pending = []
+      pendingLength = 0
+      start = idx + matcherLength
+    }
 
-      if (start < chunk.length) {
-        const rest = chunk.subarray(start)
-        pending.push(rest)
-        pendingLength = rest.length
-        if (overlap > 0) edge = rest.subarray(-overlap)
+    let idx = chunk.indexOf(needle, start)
+    while (idx !== -1) {
+      if (idx > start) {
+        transform.push(chunk.subarray(start, idx))
       }
+      start = idx + matcherLength
+      idx = chunk.indexOf(needle, start)
+    }
 
-      cb()
-    },
+    if (start < chunk.length) {
+      const rest = chunk.subarray(start)
+      pending.push(rest)
+      pendingLength = rest.length
+      if (overlap > 0) edge = rest.subarray(-overlap)
+    }
+  }
 
-    flush(cb) {
-      if (pendingLength > 0) {
-        this.push(Buffer.concat(pending, pendingLength))
-      }
-      cb()
-    },
-  })
+  return transform
 }