@naturalcycles/nodejs-lib 15.125.0 → 15.125.1

package/dist/stream/ndjson/createReadStreamAsNDJson.js

@@ -19,17 +19,17 @@ export function createReadStreamAsNDJson(inputPath) {
     fs2.requireFileToExist(inputPath);
     let stream = fs2
         .createReadStream(inputPath, {
-        highWaterMark: 64 * 1024, // no observed speedup
+        highWaterMark: 256 * 1024, // 64KB had no observed speedup over the default; 256KB is +6-12% end-to-end on ~9KB rows, neutral on ~180B rows, 1MB brings nothing more
     })
         .on('error', err => stream.destroy(err));
     if (inputPath.endsWith('.gz')) {
         stream = stream.pipe(createUnzip({
-            chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
+            chunkSize: 256 * 1024, // 64KB: speedup from ~3200 to 3800 rps! 256KB: +6-8% more on ~9KB rows, neutral on ~180B rows
         }));
     }
     else if (inputPath.endsWith('.zst')) {
         stream = stream.pipe(createZstdDecompress({
-            chunkSize: 64 * 1024, // todo: test it
+            chunkSize: 256 * 1024, // tested: 256KB is +10-12% over 64KB on ~9KB rows, neutral on ~180B rows
         }));
     }
     return stream.pipe(transformSplitOnNewline()).map(line => JSON.parse(line));

package/dist/stream/pipeline.js

@@ -73,7 +73,7 @@ export class Pipeline {
     }
     static fromFile(sourceFilePath) {
         return new Pipeline(fs2.createReadStream(sourceFilePath, {
-            highWaterMark: 64 * 1024, // no observed speedup
+            highWaterMark: 256 * 1024, // 64KB had no observed speedup over the default; 256KB is +6-12% end-to-end on ~9KB rows, neutral on ~180B rows, 1MB brings nothing more
         }), false);
     }
     /**
@@ -254,7 +254,7 @@ export class Pipeline {
     }
     gunzip(opt) {
         this.transforms.push(createUnzip({
-            chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
+            chunkSize: 256 * 1024, // 64KB: speedup from ~3200 to 3800 rps! 256KB: +6-8% more on ~9KB rows, neutral on ~180B rows
             ...opt,
         }));
         this.objectMode = false;
@@ -268,7 +268,7 @@ export class Pipeline {
     }
     zstdDecompress(opt) {
         this.transforms.push(createZstdDecompress({
-            chunkSize: 64 * 1024, // todo: test it
+            chunkSize: 256 * 1024, // tested: 256KB is +10-12% over 64KB on ~9KB rows, neutral on ~180B rows
             ...opt,
         }));
         this.objectMode = false;

package/dist/stream/transform/transformSplit.js

@@ -30,8 +30,9 @@ export function transformSplit(separator = '\n') {
     const needle = matcherLength === 1 ? matcher[0] : matcher;
     // a multi-byte separator may straddle a chunk boundary, so we keep that many trailing bytes around
     const overlap = matcherLength - 1;
-    // Unterminated data is collected as a list of chunks and concatenated only once a separator arrives,
-    // so a long line spanning many chunks is copied once rather than on every chunk (linear, not quadratic)
+    // Unterminated data is collected as a list of chunks and concatenated only once its separator arrives,
+    // so a line spanning many chunks is copied once rather than on every chunk (linear, not quadratic).
+    // Only that one line is copied: the rest of the chunk is scanned in place and emitted as views into it.
     let pending = [];
     let pendingLength = 0;
     let edge; // last `overlap` bytes of pending data, only tracked when overlap > 0
@@ -40,39 +41,50 @@ export function transformSplit(separator = '\n') {
         writableHighWaterMark: 64 * 1024,
         readableObjectMode: true,
         transform(chunk, _enc, cb) {
-            let buf = chunk;
-            let offset = 0;
+            let start = 0;
             if (pendingLength > 0) {
-                const straddles = overlap > 0 && Buffer.concat([edge, chunk.subarray(0, overlap)]).includes(matcher);
-                const idxInChunk = straddles ? -1 : chunk.indexOf(needle);
-                if (!straddles && idxInChunk === -1) {
-                    // No separator yet: keep accumulating without copying
-                    pending.push(chunk);
-                    pendingLength += chunk.length;
-                    if (overlap > 0) {
-                        edge = Buffer.concat([edge, chunk.subarray(-overlap)]).subarray(-overlap);
+                // A multi-byte separator may straddle the chunk boundary: look for it in the last `overlap` bytes
+                // of pending data followed by the first `overlap` bytes of this chunk
+                const straddleIdx = overlap > 0 ? Buffer.concat([edge, chunk.subarray(0, overlap)]).indexOf(matcher) : -1;
+                // Position of the first separator in chunk coordinates, negative when it started inside the pending data
+                let idx;
+                if (straddleIdx === -1) {
+                    idx = chunk.indexOf(needle);
+                    if (idx === -1) {
+                        // No separator yet: keep accumulating without copying
+                        pending.push(chunk);
+                        pendingLength += chunk.length;
+                        if (overlap > 0) {
+                            edge = Buffer.concat([edge, chunk.subarray(-overlap)]).subarray(-overlap);
+                        }
+                        cb();
+                        return;
                     }
-                    cb();
-                    return;
                 }
-                pending.push(chunk);
-                buf = Buffer.concat(pending, pendingLength + chunk.length);
-                // Start scanning where the first separator can possibly be
-                offset = straddles ? Math.max(0, pendingLength - overlap) : pendingLength + idxInChunk;
+                else {
+                    idx = straddleIdx - edge.length;
+                }
+                // Complete the pending line with the head of this chunk
+                if (idx > 0)
+                    pending.push(chunk.subarray(0, idx));
+                const lineLength = pendingLength + idx;
+                if (lineLength > 0) {
+                    this.push(Buffer.concat(pending, lineLength));
+                }
                 pending = [];
                 pendingLength = 0;
+                start = idx + matcherLength;
             }
-            let start = 0;
-            let idx = buf.indexOf(needle, offset);
+            let idx = chunk.indexOf(needle, start);
             while (idx !== -1) {
                 if (idx > start) {
-                    this.push(buf.subarray(start, idx));
+                    this.push(chunk.subarray(start, idx));
                 }
                 start = idx + matcherLength;
-                idx = buf.indexOf(needle, start);
+                idx = chunk.indexOf(needle, start);
             }
-            if (start < buf.length) {
-                const rest = buf.subarray(start);
+            if (start < chunk.length) {
+                const rest = chunk.subarray(start);
                 pending.push(rest);
                 pendingLength = rest.length;
                 if (overlap > 0)

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.125.0",
+  "version": "15.125.1",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@standard-schema/spec": "^1",

package/src/stream/ndjson/createReadStreamAsNDJson.ts

@@ -23,20 +23,20 @@ export function createReadStreamAsNDJson<ROW = any>(inputPath: string): Readable
 
   let stream: ReadableTyped<ROW> = fs2
     .createReadStream(inputPath, {
-      highWaterMark: 64 * 1024, // no observed speedup
+      highWaterMark: 256 * 1024, // 64KB had no observed speedup over the default; 256KB is +6-12% end-to-end on ~9KB rows, neutral on ~180B rows, 1MB brings nothing more
     })
     .on('error', err => stream.destroy(err))
 
   if (inputPath.endsWith('.gz')) {
     stream = stream.pipe(
       createUnzip({
-        chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
+        chunkSize: 256 * 1024, // 64KB: speedup from ~3200 to 3800 rps! 256KB: +6-8% more on ~9KB rows, neutral on ~180B rows
       }),
     )
   } else if (inputPath.endsWith('.zst')) {
     stream = stream.pipe(
       createZstdDecompress({
-        chunkSize: 64 * 1024, // todo: test it
+        chunkSize: 256 * 1024, // tested: 256KB is +10-12% over 64KB on ~9KB rows, neutral on ~180B rows
       }),
     )
   }

package/src/stream/pipeline.ts

@@ -114,7 +114,7 @@ export class Pipeline<T = unknown> {
   static fromFile(sourceFilePath: string): Pipeline<Uint8Array> {
     return new Pipeline(
       fs2.createReadStream(sourceFilePath, {
-        highWaterMark: 64 * 1024, // no observed speedup
+        highWaterMark: 256 * 1024, // 64KB had no observed speedup over the default; 256KB is +6-12% end-to-end on ~9KB rows, neutral on ~180B rows, 1MB brings nothing more
       }),
       false,
     )
@@ -347,7 +347,7 @@ export class Pipeline<T = unknown> {
   gunzip(this: Pipeline<Uint8Array>, opt?: ZlibOptions): Pipeline<Uint8Array> {
     this.transforms.push(
       createUnzip({
-        chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
+        chunkSize: 256 * 1024, // 64KB: speedup from ~3200 to 3800 rps! 256KB: +6-8% more on ~9KB rows, neutral on ~180B rows
         ...opt,
       }),
     )
@@ -368,7 +368,7 @@ export class Pipeline<T = unknown> {
   zstdDecompress(this: Pipeline<Uint8Array>, opt?: ZstdOptions): Pipeline<Uint8Array> {
     this.transforms.push(
       createZstdDecompress({
-        chunkSize: 64 * 1024, // todo: test it
+        chunkSize: 256 * 1024, // tested: 256KB is +10-12% over 64KB on ~9KB rows, neutral on ~180B rows
         ...opt,
       }),
     )

package/src/stream/transform/transformSplit.ts

@@ -35,8 +35,9 @@ export function transformSplit(separator = '\n'): TransformTyped<Buffer, Buffer>
   // a multi-byte separator may straddle a chunk boundary, so we keep that many trailing bytes around
   const overlap = matcherLength - 1
 
-  // Unterminated data is collected as a list of chunks and concatenated only once a separator arrives,
-  // so a long line spanning many chunks is copied once rather than on every chunk (linear, not quadratic)
+  // Unterminated data is collected as a list of chunks and concatenated only once its separator arrives,
+  // so a line spanning many chunks is copied once rather than on every chunk (linear, not quadratic).
+  // Only that one line is copied: the rest of the chunk is scanned in place and emitted as views into it.
   let pending: Buffer[] = []
   let pendingLength = 0
   let edge: Buffer | undefined // last `overlap` bytes of pending data, only tracked when overlap > 0
@@ -47,43 +48,54 @@ export function transformSplit(separator = '\n'): TransformTyped<Buffer, Buffer>
     readableObjectMode: true,
 
     transform(chunk: Buffer, _enc, cb) {
-      let buf = chunk
-      let offset = 0
+      let start = 0
 
       if (pendingLength > 0) {
-        const straddles =
-          overlap > 0 && Buffer.concat([edge!, chunk.subarray(0, overlap)]).includes(matcher)
-        const idxInChunk = straddles ? -1 : chunk.indexOf(needle)
-        if (!straddles && idxInChunk === -1) {
-          // No separator yet: keep accumulating without copying
-          pending.push(chunk)
-          pendingLength += chunk.length
-          if (overlap > 0) {
-            edge = Buffer.concat([edge!, chunk.subarray(-overlap)]).subarray(-overlap)
+        // A multi-byte separator may straddle the chunk boundary: look for it in the last `overlap` bytes
+        // of pending data followed by the first `overlap` bytes of this chunk
+        const straddleIdx =
+          overlap > 0 ? Buffer.concat([edge!, chunk.subarray(0, overlap)]).indexOf(matcher) : -1
+
+        // Position of the first separator in chunk coordinates, negative when it started inside the pending data
+        let idx: number
+        if (straddleIdx === -1) {
+          idx = chunk.indexOf(needle)
+          if (idx === -1) {
+            // No separator yet: keep accumulating without copying
+            pending.push(chunk)
+            pendingLength += chunk.length
+            if (overlap > 0) {
+              edge = Buffer.concat([edge!, chunk.subarray(-overlap)]).subarray(-overlap)
+            }
+            cb()
+            return
           }
-          cb()
-          return
+        } else {
+          idx = straddleIdx - edge!.length
+        }
+
+        // Complete the pending line with the head of this chunk
+        if (idx > 0) pending.push(chunk.subarray(0, idx))
+        const lineLength = pendingLength + idx
+        if (lineLength > 0) {
+          this.push(Buffer.concat(pending, lineLength))
         }
-        pending.push(chunk)
-        buf = Buffer.concat(pending, pendingLength + chunk.length)
-        // Start scanning where the first separator can possibly be
-        offset = straddles ? Math.max(0, pendingLength - overlap) : pendingLength + idxInChunk
         pending = []
         pendingLength = 0
+        start = idx + matcherLength
       }
 
-      let start = 0
-      let idx = buf.indexOf(needle, offset)
+      let idx = chunk.indexOf(needle, start)
       while (idx !== -1) {
         if (idx > start) {
-          this.push(buf.subarray(start, idx))
+          this.push(chunk.subarray(start, idx))
         }
         start = idx + matcherLength
-        idx = buf.indexOf(needle, start)
+        idx = chunk.indexOf(needle, start)
       }
 
-      if (start < buf.length) {
-        const rest = buf.subarray(start)
+      if (start < chunk.length) {
+        const rest = chunk.subarray(start)
         pending.push(rest)
         pendingLength = rest.length
         if (overlap > 0) edge = rest.subarray(-overlap)