@naturalcycles/nodejs-lib 15.124.0 → 15.125.0

package/dist/stream/transform/transformSplit.d.ts

@@ -4,15 +4,16 @@ import type { TransformTyped } from '../stream.model.js';
  *
  * Useful for reading NDJSON files from fs.
  *
- * Same as binarySplit, but optimized (hard-coded) to split on NEWLINE (aka `\n`).
- * (+5-10% _pipeline speedup measured, compared to generic `binarySplit` on variable length delimiter)
+ * Same as `transformSplit('\n')`.
+ * (It used to be a separate hard-coded implementation, +5-10% faster than the generic one,
+ * but since the generic one searches 1-byte separators by byte value there's no difference anymore)
  */
 export declare function transformSplitOnNewline(): TransformTyped<Buffer, Buffer>;
 /**
  * Input: stream (objectMode=false) of arbitrary string|Buffer chunks, like when read from fs
  * Output: stream (objectMode=true) or string|Buffer chunks split by `separator` (@default to `\n`)
  *
- * Please use slightly more optimized `transformSplitOnNewline` for NDJSON file parsing.
- * (+5-10% _pipeline speedup measured!)
+ * Empty lines (consecutive separators) are skipped.
+ * Trailing data without a separator is emitted on flush.
  */
 export declare function transformSplit(separator?: string): TransformTyped<Buffer, Buffer>;

package/dist/stream/transform/transformSplit.js

@@ -1,133 +1,90 @@
 import { Transform } from 'node:stream';
 // The code below is carefully adopted from: https://github.com/max-mapper/binary-split
+// (including the v2.0.0 performance improvements: native Buffer#indexOf and linear-time long lines)
 /**
  * Transforms input Buffer/string stream into Buffer chunks (objectMode: true) split by newLine.
  *
  * Useful for reading NDJSON files from fs.
  *
- * Same as binarySplit, but optimized (hard-coded) to split on NEWLINE (aka `\n`).
- * (+5-10% _pipeline speedup measured, compared to generic `binarySplit` on variable length delimiter)
+ * Same as `transformSplit('\n')`.
+ * (It used to be a separate hard-coded implementation, +5-10% faster than the generic one,
+ * but since the generic one searches 1-byte separators by byte value there's no difference anymore)
  */
 export function transformSplitOnNewline() {
-    let buffered;
-    return new Transform({
-        writableObjectMode: false,
-        writableHighWaterMark: 64 * 1024,
-        readableObjectMode: true,
-        transform(buf, _enc, cb) {
-            let offset = 0;
-            let lastMatch = 0;
-            if (buffered) {
-                buf = Buffer.concat([buffered, buf]);
-                offset = buffered.length;
-                buffered = undefined;
-            }
-            while (true) {
-                const idx = firstNewlineMatch(buf, offset);
-                if (idx !== -1 && idx < buf.length) {
-                    if (lastMatch !== idx) {
-                        this.push(buf.slice(lastMatch, idx));
-                    }
-                    offset = idx + 1;
-                    lastMatch = offset;
-                }
-                else {
-                    buffered = buf.slice(lastMatch);
-                    break;
-                }
-            }
-            cb();
-        },
-        flush(done) {
-            if (buffered && buffered.length > 0)
-                this.push(buffered);
-            done();
-        },
-    });
+    return transformSplit('\n');
 }
 /**
  * Input: stream (objectMode=false) of arbitrary string|Buffer chunks, like when read from fs
  * Output: stream (objectMode=true) or string|Buffer chunks split by `separator` (@default to `\n`)
  *
- * Please use slightly more optimized `transformSplitOnNewline` for NDJSON file parsing.
- * (+5-10% _pipeline speedup measured!)
+ * Empty lines (consecutive separators) are skipped.
+ * Trailing data without a separator is emitted on flush.
  */
 export function transformSplit(separator = '\n') {
     const matcher = Buffer.from(separator);
-    let buffered;
+    if (matcher.length === 0) {
+        throw new Error('transformSplit: separator must not be empty');
+    }
+    const matcherLength = matcher.length;
+    // indexOf with a byte value is much faster than with a 1-byte Buffer, which matters for short lines
+    const needle = matcherLength === 1 ? matcher[0] : matcher;
+    // a multi-byte separator may straddle a chunk boundary, so we keep that many trailing bytes around
+    const overlap = matcherLength - 1;
+    // Unterminated data is collected as a list of chunks and concatenated only once a separator arrives,
+    // so a long line spanning many chunks is copied once rather than on every chunk (linear, not quadratic)
+    let pending = [];
+    let pendingLength = 0;
+    let edge; // last `overlap` bytes of pending data, only tracked when overlap > 0
     return new Transform({
-        readableObjectMode: true,
+        writableObjectMode: false,
         writableHighWaterMark: 64 * 1024,
-        transform(buf, _enc, done) {
+        readableObjectMode: true,
+        transform(chunk, _enc, cb) {
+            let buf = chunk;
             let offset = 0;
-            let lastMatch = 0;
-            if (buffered) {
-                buf = Buffer.concat([buffered, buf]);
-                offset = buffered.length;
-                buffered = undefined;
-            }
-            while (true) {
-                const idx = firstMatch(buf, offset - matcher.length + 1, matcher);
-                if (idx !== -1 && idx < buf.length) {
-                    if (lastMatch !== idx) {
-                        this.push(buf.slice(lastMatch, idx));
+            if (pendingLength > 0) {
+                const straddles = overlap > 0 && Buffer.concat([edge, chunk.subarray(0, overlap)]).includes(matcher);
+                const idxInChunk = straddles ? -1 : chunk.indexOf(needle);
+                if (!straddles && idxInChunk === -1) {
+                    // No separator yet: keep accumulating without copying
+                    pending.push(chunk);
+                    pendingLength += chunk.length;
+                    if (overlap > 0) {
+                        edge = Buffer.concat([edge, chunk.subarray(-overlap)]).subarray(-overlap);
                     }
-                    offset = idx + matcher.length;
-                    lastMatch = offset;
+                    cb();
+                    return;
                 }
-                else {
-                    buffered = buf.slice(lastMatch);
-                    break;
+                pending.push(chunk);
+                buf = Buffer.concat(pending, pendingLength + chunk.length);
+                // Start scanning where the first separator can possibly be
+                offset = straddles ? Math.max(0, pendingLength - overlap) : pendingLength + idxInChunk;
+                pending = [];
+                pendingLength = 0;
+            }
+            let start = 0;
+            let idx = buf.indexOf(needle, offset);
+            while (idx !== -1) {
+                if (idx > start) {
+                    this.push(buf.subarray(start, idx));
                 }
+                start = idx + matcherLength;
+                idx = buf.indexOf(needle, start);
             }
-            done();
+            if (start < buf.length) {
+                const rest = buf.subarray(start);
+                pending.push(rest);
+                pendingLength = rest.length;
+                if (overlap > 0)
+                    edge = rest.subarray(-overlap);
+            }
+            cb();
         },
-        flush(done) {
-            if (buffered && buffered.length > 0)
-                this.push(buffered);
-            done();
+        flush(cb) {
+            if (pendingLength > 0) {
+                this.push(Buffer.concat(pending, pendingLength));
+            }
+            cb();
         },
     });
 }
-// const NEWLINE = Buffer.from('\n')
-// const NEWLINE_CODE = NEWLINE[0]! // it is `10`
-const NEWLINE_CODE = 10;
-/**
- * Same as firstMatch, but optimized (hard-coded) to find NEWLINE (aka `\n`).
- */
-function firstNewlineMatch(buf, offset) {
-    const bufLength = buf.length;
-    if (offset >= bufLength)
-        return -1;
-    for (let i = offset; i < bufLength; i++) {
-        if (buf[i] === NEWLINE_CODE) {
-            return i;
-        }
-    }
-    return -1; // this code is unreachable, because i is guaranteed to be found in the loop above
-}
-function firstMatch(buf, offset, matcher) {
-    if (offset >= buf.length)
-        return -1;
-    let i;
-    for (i = offset; i < buf.length; i++) {
-        if (buf[i] === matcher[0]) {
-            if (matcher.length > 1) {
-                let fullMatch = true;
-                let j = i;
-                for (let k = 0; j < i + matcher.length; j++, k++) {
-                    if (buf[j] !== matcher[k]) {
-                        fullMatch = false;
-                        break;
-                    }
-                }
-                if (fullMatch)
-                    return j - matcher.length;
-            }
-            else {
-                break;
-            }
-        }
-    }
-    return i + matcher.length - 1;
-}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.124.0",
+  "version": "15.125.0",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@standard-schema/spec": "^1",
@@ -14,7 +14,7 @@
     "yaml": "^2"
   },
   "devDependencies": {
-    "@naturalcycles/dev-lib": "20.64.0",
+    "@naturalcycles/dev-lib": "0.0.0",
     "typescript": "^7"
   },
   "exports": {

package/src/stream/transform/transformSplit.ts

@@ -2,142 +2,101 @@ import { Transform } from 'node:stream'
 import type { TransformTyped } from '../stream.model.js'
 
 // The code below is carefully adopted from: https://github.com/max-mapper/binary-split
+// (including the v2.0.0 performance improvements: native Buffer#indexOf and linear-time long lines)
 
 /**
  * Transforms input Buffer/string stream into Buffer chunks (objectMode: true) split by newLine.
  *
  * Useful for reading NDJSON files from fs.
  *
- * Same as binarySplit, but optimized (hard-coded) to split on NEWLINE (aka `\n`).
- * (+5-10% _pipeline speedup measured, compared to generic `binarySplit` on variable length delimiter)
+ * Same as `transformSplit('\n')`.
+ * (It used to be a separate hard-coded implementation, +5-10% faster than the generic one,
+ * but since the generic one searches 1-byte separators by byte value there's no difference anymore)
  */
 export function transformSplitOnNewline(): TransformTyped<Buffer, Buffer> {
-  let buffered: Buffer | undefined
-
-  return new Transform({
-    writableObjectMode: false,
-    writableHighWaterMark: 64 * 1024,
-    readableObjectMode: true,
-
-    transform(buf: Buffer, _enc, cb) {
-      let offset = 0
-      let lastMatch = 0
-      if (buffered) {
-        buf = Buffer.concat([buffered, buf])
-        offset = buffered.length
-        buffered = undefined
-      }
-
-      while (true) {
-        const idx = firstNewlineMatch(buf, offset)
-        if (idx !== -1 && idx < buf.length) {
-          if (lastMatch !== idx) {
-            this.push(buf.slice(lastMatch, idx))
-          }
-          offset = idx + 1
-          lastMatch = offset
-        } else {
-          buffered = buf.slice(lastMatch)
-          break
-        }
-      }
-
-      cb()
-    },
-
-    flush(done) {
-      if (buffered && buffered.length > 0) this.push(buffered)
-      done()
-    },
-  })
+  return transformSplit('\n')
 }
 
 /**
  * Input: stream (objectMode=false) of arbitrary string|Buffer chunks, like when read from fs
  * Output: stream (objectMode=true) or string|Buffer chunks split by `separator` (@default to `\n`)
  *
- * Please use slightly more optimized `transformSplitOnNewline` for NDJSON file parsing.
- * (+5-10% _pipeline speedup measured!)
+ * Empty lines (consecutive separators) are skipped.
+ * Trailing data without a separator is emitted on flush.
  */
 export function transformSplit(separator = '\n'): TransformTyped<Buffer, Buffer> {
   const matcher = Buffer.from(separator)
-  let buffered: Buffer | undefined
+  if (matcher.length === 0) {
+    throw new Error('transformSplit: separator must not be empty')
+  }
+  const matcherLength = matcher.length
+  // indexOf with a byte value is much faster than with a 1-byte Buffer, which matters for short lines
+  const needle = matcherLength === 1 ? matcher[0]! : matcher
+  // a multi-byte separator may straddle a chunk boundary, so we keep that many trailing bytes around
+  const overlap = matcherLength - 1
+
+  // Unterminated data is collected as a list of chunks and concatenated only once a separator arrives,
+  // so a long line spanning many chunks is copied once rather than on every chunk (linear, not quadratic)
+  let pending: Buffer[] = []
+  let pendingLength = 0
+  let edge: Buffer | undefined // last `overlap` bytes of pending data, only tracked when overlap > 0
 
   return new Transform({
-    readableObjectMode: true,
+    writableObjectMode: false,
     writableHighWaterMark: 64 * 1024,
+    readableObjectMode: true,
 
-    transform(buf: Buffer, _enc, done) {
+    transform(chunk: Buffer, _enc, cb) {
+      let buf = chunk
       let offset = 0
-      let lastMatch = 0
-      if (buffered) {
-        buf = Buffer.concat([buffered, buf])
-        offset = buffered.length
-        buffered = undefined
-      }
 
-      while (true) {
-        const idx = firstMatch(buf, offset - matcher.length + 1, matcher)
-        if (idx !== -1 && idx < buf.length) {
-          if (lastMatch !== idx) {
-            this.push(buf.slice(lastMatch, idx))
+      if (pendingLength > 0) {
+        const straddles =
+          overlap > 0 && Buffer.concat([edge!, chunk.subarray(0, overlap)]).includes(matcher)
+        const idxInChunk = straddles ? -1 : chunk.indexOf(needle)
+        if (!straddles && idxInChunk === -1) {
+          // No separator yet: keep accumulating without copying
+          pending.push(chunk)
+          pendingLength += chunk.length
+          if (overlap > 0) {
+            edge = Buffer.concat([edge!, chunk.subarray(-overlap)]).subarray(-overlap)
           }
-          offset = idx + matcher.length
-          lastMatch = offset
-        } else {
-          buffered = buf.slice(lastMatch)
-          break
+          cb()
+          return
         }
+        pending.push(chunk)
+        buf = Buffer.concat(pending, pendingLength + chunk.length)
+        // Start scanning where the first separator can possibly be
+        offset = straddles ? Math.max(0, pendingLength - overlap) : pendingLength + idxInChunk
+        pending = []
+        pendingLength = 0
       }
 
-      done()
-    },
-
-    flush(done) {
-      if (buffered && buffered.length > 0) this.push(buffered)
-      done()
-    },
-  })
-}
+      let start = 0
+      let idx = buf.indexOf(needle, offset)
+      while (idx !== -1) {
+        if (idx > start) {
+          this.push(buf.subarray(start, idx))
+        }
+        start = idx + matcherLength
+        idx = buf.indexOf(needle, start)
+      }
 
-// const NEWLINE = Buffer.from('\n')
-// const NEWLINE_CODE = NEWLINE[0]! // it is `10`
-const NEWLINE_CODE = 10
+      if (start < buf.length) {
+        const rest = buf.subarray(start)
+        pending.push(rest)
+        pendingLength = rest.length
+        if (overlap > 0) edge = rest.subarray(-overlap)
+      }
 
-/**
- * Same as firstMatch, but optimized (hard-coded) to find NEWLINE (aka `\n`).
- */
-function firstNewlineMatch(buf: Buffer, offset: number): number {
-  const bufLength = buf.length
-  if (offset >= bufLength) return -1
-  for (let i = offset; i < bufLength; i++) {
-    if (buf[i] === NEWLINE_CODE) {
-      return i
-    }
-  }
-  return -1 // this code is unreachable, because i is guaranteed to be found in the loop above
-}
+      cb()
+    },
 
-function firstMatch(buf: Buffer, offset: number, matcher: Buffer): number {
-  if (offset >= buf.length) return -1
-  let i: number
-  for (i = offset; i < buf.length; i++) {
-    if (buf[i] === matcher[0]) {
-      if (matcher.length > 1) {
-        let fullMatch = true
-        let j = i
-        for (let k = 0; j < i + matcher.length; j++, k++) {
-          if (buf[j] !== matcher[k]) {
-            fullMatch = false
-            break
-          }
-        }
-        if (fullMatch) return j - matcher.length
-      } else {
-        break
+    flush(cb) {
+      if (pendingLength > 0) {
+        this.push(Buffer.concat(pending, pendingLength))
       }
-    }
-  }
-
-  return i + matcher.length - 1
+      cb()
+    },
+  })
 }