@naturalcycles/nodejs-lib 15.124.0 → 15.125.1

package/dist/stream/ndjson/createReadStreamAsNDJson.js

@@ -19,17 +19,17 @@ export function createReadStreamAsNDJson(inputPath) {
     fs2.requireFileToExist(inputPath);
     let stream = fs2
         .createReadStream(inputPath, {
-        highWaterMark: 64 * 1024, // no observed speedup
+        highWaterMark: 256 * 1024, // 64KB had no observed speedup over the default; 256KB is +6-12% end-to-end on ~9KB rows, neutral on ~180B rows, 1MB brings nothing more
     })
         .on('error', err => stream.destroy(err));
     if (inputPath.endsWith('.gz')) {
         stream = stream.pipe(createUnzip({
-            chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
+            chunkSize: 256 * 1024, // 64KB: speedup from ~3200 to 3800 rps! 256KB: +6-8% more on ~9KB rows, neutral on ~180B rows
         }));
     }
     else if (inputPath.endsWith('.zst')) {
         stream = stream.pipe(createZstdDecompress({
-            chunkSize: 64 * 1024, // todo: test it
+            chunkSize: 256 * 1024, // tested: 256KB is +10-12% over 64KB on ~9KB rows, neutral on ~180B rows
         }));
     }
     return stream.pipe(transformSplitOnNewline()).map(line => JSON.parse(line));

package/dist/stream/pipeline.js

@@ -73,7 +73,7 @@ export class Pipeline {
     }
     static fromFile(sourceFilePath) {
         return new Pipeline(fs2.createReadStream(sourceFilePath, {
-            highWaterMark: 64 * 1024, // no observed speedup
+            highWaterMark: 256 * 1024, // 64KB had no observed speedup over the default; 256KB is +6-12% end-to-end on ~9KB rows, neutral on ~180B rows, 1MB brings nothing more
         }), false);
     }
     /**
@@ -254,7 +254,7 @@ export class Pipeline {
     }
     gunzip(opt) {
         this.transforms.push(createUnzip({
-            chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
+            chunkSize: 256 * 1024, // 64KB: speedup from ~3200 to 3800 rps! 256KB: +6-8% more on ~9KB rows, neutral on ~180B rows
             ...opt,
         }));
         this.objectMode = false;
@@ -268,7 +268,7 @@ export class Pipeline {
     }
     zstdDecompress(opt) {
         this.transforms.push(createZstdDecompress({
-            chunkSize: 64 * 1024, // todo: test it
+            chunkSize: 256 * 1024, // tested: 256KB is +10-12% over 64KB on ~9KB rows, neutral on ~180B rows
             ...opt,
         }));
         this.objectMode = false;

package/dist/stream/transform/transformSplit.d.ts

@@ -4,15 +4,16 @@ import type { TransformTyped } from '../stream.model.js';
  *
  * Useful for reading NDJSON files from fs.
  *
- * Same as binarySplit, but optimized (hard-coded) to split on NEWLINE (aka `\n`).
- * (+5-10% _pipeline speedup measured, compared to generic `binarySplit` on variable length delimiter)
+ * Same as `transformSplit('\n')`.
+ * (It used to be a separate hard-coded implementation, +5-10% faster than the generic one,
+ * but since the generic one searches 1-byte separators by byte value there's no difference anymore)
  */
 export declare function transformSplitOnNewline(): TransformTyped<Buffer, Buffer>;
 /**
  * Input: stream (objectMode=false) of arbitrary string|Buffer chunks, like when read from fs
  * Output: stream (objectMode=true) or string|Buffer chunks split by `separator` (@default to `\n`)
  *
- * Please use slightly more optimized `transformSplitOnNewline` for NDJSON file parsing.
- * (+5-10% _pipeline speedup measured!)
+ * Empty lines (consecutive separators) are skipped.
+ * Trailing data without a separator is emitted on flush.
  */
 export declare function transformSplit(separator?: string): TransformTyped<Buffer, Buffer>;

package/dist/stream/transform/transformSplit.js

@@ -1,133 +1,102 @@
 import { Transform } from 'node:stream';
 // The code below is carefully adopted from: https://github.com/max-mapper/binary-split
+// (including the v2.0.0 performance improvements: native Buffer#indexOf and linear-time long lines)
 /**
  * Transforms input Buffer/string stream into Buffer chunks (objectMode: true) split by newLine.
  *
  * Useful for reading NDJSON files from fs.
  *
- * Same as binarySplit, but optimized (hard-coded) to split on NEWLINE (aka `\n`).
- * (+5-10% _pipeline speedup measured, compared to generic `binarySplit` on variable length delimiter)
+ * Same as `transformSplit('\n')`.
+ * (It used to be a separate hard-coded implementation, +5-10% faster than the generic one,
+ * but since the generic one searches 1-byte separators by byte value there's no difference anymore)
  */
 export function transformSplitOnNewline() {
-    let buffered;
-    return new Transform({
-        writableObjectMode: false,
-        writableHighWaterMark: 64 * 1024,
-        readableObjectMode: true,
-        transform(buf, _enc, cb) {
-            let offset = 0;
-            let lastMatch = 0;
-            if (buffered) {
-                buf = Buffer.concat([buffered, buf]);
-                offset = buffered.length;
-                buffered = undefined;
-            }
-            while (true) {
-                const idx = firstNewlineMatch(buf, offset);
-                if (idx !== -1 && idx < buf.length) {
-                    if (lastMatch !== idx) {
-                        this.push(buf.slice(lastMatch, idx));
-                    }
-                    offset = idx + 1;
-                    lastMatch = offset;
-                }
-                else {
-                    buffered = buf.slice(lastMatch);
-                    break;
-                }
-            }
-            cb();
-        },
-        flush(done) {
-            if (buffered && buffered.length > 0)
-                this.push(buffered);
-            done();
-        },
-    });
+    return transformSplit('\n');
 }
 /**
  * Input: stream (objectMode=false) of arbitrary string|Buffer chunks, like when read from fs
  * Output: stream (objectMode=true) or string|Buffer chunks split by `separator` (@default to `\n`)
  *
- * Please use slightly more optimized `transformSplitOnNewline` for NDJSON file parsing.
- * (+5-10% _pipeline speedup measured!)
+ * Empty lines (consecutive separators) are skipped.
+ * Trailing data without a separator is emitted on flush.
  */
 export function transformSplit(separator = '\n') {
     const matcher = Buffer.from(separator);
-    let buffered;
+    if (matcher.length === 0) {
+        throw new Error('transformSplit: separator must not be empty');
+    }
+    const matcherLength = matcher.length;
+    // indexOf with a byte value is much faster than with a 1-byte Buffer, which matters for short lines
+    const needle = matcherLength === 1 ? matcher[0] : matcher;
+    // a multi-byte separator may straddle a chunk boundary, so we keep that many trailing bytes around
+    const overlap = matcherLength - 1;
+    // Unterminated data is collected as a list of chunks and concatenated only once its separator arrives,
+    // so a line spanning many chunks is copied once rather than on every chunk (linear, not quadratic).
+    // Only that one line is copied: the rest of the chunk is scanned in place and emitted as views into it.
+    let pending = [];
+    let pendingLength = 0;
+    let edge; // last `overlap` bytes of pending data, only tracked when overlap > 0
     return new Transform({
-        readableObjectMode: true,
+        writableObjectMode: false,
         writableHighWaterMark: 64 * 1024,
-        transform(buf, _enc, done) {
-            let offset = 0;
-            let lastMatch = 0;
-            if (buffered) {
-                buf = Buffer.concat([buffered, buf]);
-                offset = buffered.length;
-                buffered = undefined;
-            }
-            while (true) {
-                const idx = firstMatch(buf, offset - matcher.length + 1, matcher);
-                if (idx !== -1 && idx < buf.length) {
-                    if (lastMatch !== idx) {
-                        this.push(buf.slice(lastMatch, idx));
+        readableObjectMode: true,
+        transform(chunk, _enc, cb) {
+            let start = 0;
+            if (pendingLength > 0) {
+                // A multi-byte separator may straddle the chunk boundary: look for it in the last `overlap` bytes
+                // of pending data followed by the first `overlap` bytes of this chunk
+                const straddleIdx = overlap > 0 ? Buffer.concat([edge, chunk.subarray(0, overlap)]).indexOf(matcher) : -1;
+                // Position of the first separator in chunk coordinates, negative when it started inside the pending data
+                let idx;
+                if (straddleIdx === -1) {
+                    idx = chunk.indexOf(needle);
+                    if (idx === -1) {
+                        // No separator yet: keep accumulating without copying
+                        pending.push(chunk);
+                        pendingLength += chunk.length;
+                        if (overlap > 0) {
+                            edge = Buffer.concat([edge, chunk.subarray(-overlap)]).subarray(-overlap);
+                        }
+                        cb();
+                        return;
                     }
-                    offset = idx + matcher.length;
-                    lastMatch = offset;
                 }
                 else {
-                    buffered = buf.slice(lastMatch);
-                    break;
+                    idx = straddleIdx - edge.length;
+                }
+                // Complete the pending line with the head of this chunk
+                if (idx > 0)
+                    pending.push(chunk.subarray(0, idx));
+                const lineLength = pendingLength + idx;
+                if (lineLength > 0) {
+                    this.push(Buffer.concat(pending, lineLength));
                 }
+                pending = [];
+                pendingLength = 0;
+                start = idx + matcherLength;
             }
-            done();
-        },
-        flush(done) {
-            if (buffered && buffered.length > 0)
-                this.push(buffered);
-            done();
-        },
-    });
-}
-// const NEWLINE = Buffer.from('\n')
-// const NEWLINE_CODE = NEWLINE[0]! // it is `10`
-const NEWLINE_CODE = 10;
-/**
- * Same as firstMatch, but optimized (hard-coded) to find NEWLINE (aka `\n`).
- */
-function firstNewlineMatch(buf, offset) {
-    const bufLength = buf.length;
-    if (offset >= bufLength)
-        return -1;
-    for (let i = offset; i < bufLength; i++) {
-        if (buf[i] === NEWLINE_CODE) {
-            return i;
-        }
-    }
-    return -1; // this code is unreachable, because i is guaranteed to be found in the loop above
-}
-function firstMatch(buf, offset, matcher) {
-    if (offset >= buf.length)
-        return -1;
-    let i;
-    for (i = offset; i < buf.length; i++) {
-        if (buf[i] === matcher[0]) {
-            if (matcher.length > 1) {
-                let fullMatch = true;
-                let j = i;
-                for (let k = 0; j < i + matcher.length; j++, k++) {
-                    if (buf[j] !== matcher[k]) {
-                        fullMatch = false;
-                        break;
-                    }
+            let idx = chunk.indexOf(needle, start);
+            while (idx !== -1) {
+                if (idx > start) {
+                    this.push(chunk.subarray(start, idx));
                 }
-                if (fullMatch)
-                    return j - matcher.length;
+                start = idx + matcherLength;
+                idx = chunk.indexOf(needle, start);
             }
-            else {
-                break;
+            if (start < chunk.length) {
+                const rest = chunk.subarray(start);
+                pending.push(rest);
+                pendingLength = rest.length;
+                if (overlap > 0)
+                    edge = rest.subarray(-overlap);
             }
-        }
-    }
-    return i + matcher.length - 1;
+            cb();
+        },
+        flush(cb) {
+            if (pendingLength > 0) {
+                this.push(Buffer.concat(pending, pendingLength));
+            }
+            cb();
+        },
+    });
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.124.0",
+  "version": "15.125.1",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@standard-schema/spec": "^1",
@@ -14,7 +14,7 @@
     "yaml": "^2"
   },
   "devDependencies": {
-    "@naturalcycles/dev-lib": "20.64.0",
+    "@naturalcycles/dev-lib": "0.0.0",
     "typescript": "^7"
   },
   "exports": {

package/src/stream/ndjson/createReadStreamAsNDJson.ts

@@ -23,20 +23,20 @@ export function createReadStreamAsNDJson<ROW = any>(inputPath: string): Readable
 
   let stream: ReadableTyped<ROW> = fs2
     .createReadStream(inputPath, {
-      highWaterMark: 64 * 1024, // no observed speedup
+      highWaterMark: 256 * 1024, // 64KB had no observed speedup over the default; 256KB is +6-12% end-to-end on ~9KB rows, neutral on ~180B rows, 1MB brings nothing more
     })
     .on('error', err => stream.destroy(err))
 
   if (inputPath.endsWith('.gz')) {
     stream = stream.pipe(
       createUnzip({
-        chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
+        chunkSize: 256 * 1024, // 64KB: speedup from ~3200 to 3800 rps! 256KB: +6-8% more on ~9KB rows, neutral on ~180B rows
       }),
     )
   } else if (inputPath.endsWith('.zst')) {
     stream = stream.pipe(
       createZstdDecompress({
-        chunkSize: 64 * 1024, // todo: test it
+        chunkSize: 256 * 1024, // tested: 256KB is +10-12% over 64KB on ~9KB rows, neutral on ~180B rows
       }),
     )
   }

package/src/stream/pipeline.ts

@@ -114,7 +114,7 @@ export class Pipeline<T = unknown> {
   static fromFile(sourceFilePath: string): Pipeline<Uint8Array> {
     return new Pipeline(
       fs2.createReadStream(sourceFilePath, {
-        highWaterMark: 64 * 1024, // no observed speedup
+        highWaterMark: 256 * 1024, // 64KB had no observed speedup over the default; 256KB is +6-12% end-to-end on ~9KB rows, neutral on ~180B rows, 1MB brings nothing more
       }),
       false,
     )
@@ -347,7 +347,7 @@ export class Pipeline<T = unknown> {
   gunzip(this: Pipeline<Uint8Array>, opt?: ZlibOptions): Pipeline<Uint8Array> {
     this.transforms.push(
       createUnzip({
-        chunkSize: 64 * 1024, // speedup from ~3200 to 3800 rps!
+        chunkSize: 256 * 1024, // 64KB: speedup from ~3200 to 3800 rps! 256KB: +6-8% more on ~9KB rows, neutral on ~180B rows
         ...opt,
       }),
     )
@@ -368,7 +368,7 @@ export class Pipeline<T = unknown> {
   zstdDecompress(this: Pipeline<Uint8Array>, opt?: ZstdOptions): Pipeline<Uint8Array> {
     this.transforms.push(
       createZstdDecompress({
-        chunkSize: 64 * 1024, // todo: test it
+        chunkSize: 256 * 1024, // tested: 256KB is +10-12% over 64KB on ~9KB rows, neutral on ~180B rows
         ...opt,
       }),
     )

package/src/stream/transform/transformSplit.ts

@@ -2,142 +2,113 @@ import { Transform } from 'node:stream'
 import type { TransformTyped } from '../stream.model.js'
 
 // The code below is carefully adopted from: https://github.com/max-mapper/binary-split
+// (including the v2.0.0 performance improvements: native Buffer#indexOf and linear-time long lines)
 
 /**
  * Transforms input Buffer/string stream into Buffer chunks (objectMode: true) split by newLine.
  *
  * Useful for reading NDJSON files from fs.
  *
- * Same as binarySplit, but optimized (hard-coded) to split on NEWLINE (aka `\n`).
- * (+5-10% _pipeline speedup measured, compared to generic `binarySplit` on variable length delimiter)
+ * Same as `transformSplit('\n')`.
+ * (It used to be a separate hard-coded implementation, +5-10% faster than the generic one,
+ * but since the generic one searches 1-byte separators by byte value there's no difference anymore)
  */
 export function transformSplitOnNewline(): TransformTyped<Buffer, Buffer> {
-  let buffered: Buffer | undefined
-
-  return new Transform({
-    writableObjectMode: false,
-    writableHighWaterMark: 64 * 1024,
-    readableObjectMode: true,
-
-    transform(buf: Buffer, _enc, cb) {
-      let offset = 0
-      let lastMatch = 0
-      if (buffered) {
-        buf = Buffer.concat([buffered, buf])
-        offset = buffered.length
-        buffered = undefined
-      }
-
-      while (true) {
-        const idx = firstNewlineMatch(buf, offset)
-        if (idx !== -1 && idx < buf.length) {
-          if (lastMatch !== idx) {
-            this.push(buf.slice(lastMatch, idx))
-          }
-          offset = idx + 1
-          lastMatch = offset
-        } else {
-          buffered = buf.slice(lastMatch)
-          break
-        }
-      }
-
-      cb()
-    },
-
-    flush(done) {
-      if (buffered && buffered.length > 0) this.push(buffered)
-      done()
-    },
-  })
+  return transformSplit('\n')
 }
 
 /**
  * Input: stream (objectMode=false) of arbitrary string|Buffer chunks, like when read from fs
  * Output: stream (objectMode=true) or string|Buffer chunks split by `separator` (@default to `\n`)
  *
- * Please use slightly more optimized `transformSplitOnNewline` for NDJSON file parsing.
- * (+5-10% _pipeline speedup measured!)
+ * Empty lines (consecutive separators) are skipped.
+ * Trailing data without a separator is emitted on flush.
  */
 export function transformSplit(separator = '\n'): TransformTyped<Buffer, Buffer> {
   const matcher = Buffer.from(separator)
-  let buffered: Buffer | undefined
+  if (matcher.length === 0) {
+    throw new Error('transformSplit: separator must not be empty')
+  }
+  const matcherLength = matcher.length
+  // indexOf with a byte value is much faster than with a 1-byte Buffer, which matters for short lines
+  const needle = matcherLength === 1 ? matcher[0]! : matcher
+  // a multi-byte separator may straddle a chunk boundary, so we keep that many trailing bytes around
+  const overlap = matcherLength - 1
+
+  // Unterminated data is collected as a list of chunks and concatenated only once its separator arrives,
+  // so a line spanning many chunks is copied once rather than on every chunk (linear, not quadratic).
+  // Only that one line is copied: the rest of the chunk is scanned in place and emitted as views into it.
+  let pending: Buffer[] = []
+  let pendingLength = 0
+  let edge: Buffer | undefined // last `overlap` bytes of pending data, only tracked when overlap > 0
 
   return new Transform({
-    readableObjectMode: true,
+    writableObjectMode: false,
     writableHighWaterMark: 64 * 1024,
+    readableObjectMode: true,
 
-    transform(buf: Buffer, _enc, done) {
-      let offset = 0
-      let lastMatch = 0
-      if (buffered) {
-        buf = Buffer.concat([buffered, buf])
-        offset = buffered.length
-        buffered = undefined
-      }
-
-      while (true) {
-        const idx = firstMatch(buf, offset - matcher.length + 1, matcher)
-        if (idx !== -1 && idx < buf.length) {
-          if (lastMatch !== idx) {
-            this.push(buf.slice(lastMatch, idx))
+    transform(chunk: Buffer, _enc, cb) {
+      let start = 0
+
+      if (pendingLength > 0) {
+        // A multi-byte separator may straddle the chunk boundary: look for it in the last `overlap` bytes
+        // of pending data followed by the first `overlap` bytes of this chunk
+        const straddleIdx =
+          overlap > 0 ? Buffer.concat([edge!, chunk.subarray(0, overlap)]).indexOf(matcher) : -1
+
+        // Position of the first separator in chunk coordinates, negative when it started inside the pending data
+        let idx: number
+        if (straddleIdx === -1) {
+          idx = chunk.indexOf(needle)
+          if (idx === -1) {
+            // No separator yet: keep accumulating without copying
+            pending.push(chunk)
+            pendingLength += chunk.length
+            if (overlap > 0) {
+              edge = Buffer.concat([edge!, chunk.subarray(-overlap)]).subarray(-overlap)
+            }
+            cb()
+            return
           }
-          offset = idx + matcher.length
-          lastMatch = offset
         } else {
-          buffered = buf.slice(lastMatch)
-          break
+          idx = straddleIdx - edge!.length
         }
-      }
 
-      done()
-    },
+        // Complete the pending line with the head of this chunk
+        if (idx > 0) pending.push(chunk.subarray(0, idx))
+        const lineLength = pendingLength + idx
+        if (lineLength > 0) {
+          this.push(Buffer.concat(pending, lineLength))
+        }
+        pending = []
+        pendingLength = 0
+        start = idx + matcherLength
+      }
 
-    flush(done) {
-      if (buffered && buffered.length > 0) this.push(buffered)
-      done()
-    },
-  })
-}
+      let idx = chunk.indexOf(needle, start)
+      while (idx !== -1) {
+        if (idx > start) {
+          this.push(chunk.subarray(start, idx))
+        }
+        start = idx + matcherLength
+        idx = chunk.indexOf(needle, start)
+      }
 
-// const NEWLINE = Buffer.from('\n')
-// const NEWLINE_CODE = NEWLINE[0]! // it is `10`
-const NEWLINE_CODE = 10
+      if (start < chunk.length) {
+        const rest = chunk.subarray(start)
+        pending.push(rest)
+        pendingLength = rest.length
+        if (overlap > 0) edge = rest.subarray(-overlap)
+      }
 
-/**
- * Same as firstMatch, but optimized (hard-coded) to find NEWLINE (aka `\n`).
- */
-function firstNewlineMatch(buf: Buffer, offset: number): number {
-  const bufLength = buf.length
-  if (offset >= bufLength) return -1
-  for (let i = offset; i < bufLength; i++) {
-    if (buf[i] === NEWLINE_CODE) {
-      return i
-    }
-  }
-  return -1 // this code is unreachable, because i is guaranteed to be found in the loop above
-}
+      cb()
+    },
 
-function firstMatch(buf: Buffer, offset: number, matcher: Buffer): number {
-  if (offset >= buf.length) return -1
-  let i: number
-  for (i = offset; i < buf.length; i++) {
-    if (buf[i] === matcher[0]) {
-      if (matcher.length > 1) {
-        let fullMatch = true
-        let j = i
-        for (let k = 0; j < i + matcher.length; j++, k++) {
-          if (buf[j] !== matcher[k]) {
-            fullMatch = false
-            break
-          }
-        }
-        if (fullMatch) return j - matcher.length
-      } else {
-        break
+    flush(cb) {
+      if (pendingLength > 0) {
+        this.push(Buffer.concat(pending, pendingLength))
       }
-    }
-  }
-
-  return i + matcher.length - 1
+      cb()
+    },
+  })
 }