@zakkster/lite-bake-stream 1.0.0

package/src/PreserveTokenizer.js

@@ -0,0 +1,172 @@
+// @zakkster/lite-bake-stream / PreserveTokenizer
+// NDJSON record-boundary scanner for preserve-mode. Copyright (c) 2026 Zahary Shinikchiev. MIT.
+//
+// The schema-mode Tokenizer walks the full JSON AST (numbers, strings, keys,
+// object/array structure) so the Writer can pack values into typed lanes.
+// Preserve-mode doesn't crack open records — it just needs to find where one
+// record ends and the next begins, so the record's original bytes can be
+// shoved into a shard intact.
+//
+// Contract:
+//   - Input:  NDJSON bytes. One JSON value per line, delimited by 0x0A.
+//   - Output: sink.onRecord(bytes, from, to) for each complete record.
+//             The byte range excludes the trailing newline. CRLF endings are
+//             stripped (trailing \r trimmed). Empty and whitespace-only lines
+//             are skipped.
+//   - Chunk safety: records that span multiple feed() calls are buffered
+//             internally and emitted only when the closing newline is seen.
+//   - Depth aware: the scanner tracks brace/bracket depth AND whether it's
+//             inside a string (with backslash-escape awareness), so a literal
+//             0x0A only counts as a boundary when it's outside any string and
+//             at depth 0. A well-formed NDJSON stream never has a raw 0x0A
+//             inside a string (must be escaped as \n = 0x5C 0x6E), but the
+//             scanner handles adversarial input safely.
+//
+// Errors (stable E_* codes):
+//   E_TRUNCATED         - end of input reached mid-record (depth != 0 or in-string)
+//   E_RECORD_TOO_LARGE  - single record exceeds maxRecordBytes (default: none)
+
+export const VERSION = '1.0.0';
+
+export class PreserveTokenizerError extends Error {
+  constructor(code, msg) { super(msg); this.code = code; this.name = 'PreserveTokenizerError'; }
+}
+
+// Byte values used by the scanner (named so the hot loop reads cleanly).
+const BYTE_LF     = 0x0A;
+const BYTE_CR     = 0x0D;
+const BYTE_QUOTE  = 0x22;
+const BYTE_BSLASH = 0x5C;
+const BYTE_LBRACE = 0x7B;
+const BYTE_RBRACE = 0x7D;
+const BYTE_LBRACK = 0x5B;
+const BYTE_RBRACK = 0x5D;
+const BYTE_SPACE  = 0x20;
+const BYTE_TAB    = 0x09;
+
+const INITIAL_BUF = 1 << 16;   // 64 KiB — grows on demand for larger records
+
+export class PreserveTokenizer {
+  constructor(sink, opts) {
+    if (!sink || typeof sink.onRecord !== 'function') {
+      throw new TypeError('PreserveTokenizer: sink must implement onRecord(bytes, from, to)');
+    }
+    opts = opts || {};
+    if (opts.framing && opts.framing !== 'ndjson') {
+      // 'array' framing (top-level `[a, b, c]`) is a future addition.
+      throw new PreserveTokenizerError('E_UNSUPPORTED_FRAMING',
+        'preserve-mode currently supports NDJSON only; got framing=' + opts.framing);
+    }
+    this._sink = sink;
+    this._maxRecordBytes = typeof opts.maxRecordBytes === 'number' ? opts.maxRecordBytes : 0;   // 0 = unlimited
+    // Accumulator: bytes seen since the last emitted record's terminating LF.
+    this._buf = new Uint8Array(INITIAL_BUF);
+    this._bufLen = 0;
+    this._recordStart = 0;
+    // Scanner state
+    this._depth = 0;
+    this._inString = false;
+    this._escape = false;
+    this._absOffset = 0;    // for error messages
+  }
+
+  feed(chunk) {
+    if (!(chunk instanceof Uint8Array)) {
+      throw new TypeError('PreserveTokenizer.feed: expected Uint8Array');
+    }
+    if (chunk.length === 0) return;
+    this._ensureCapacity(this._bufLen + chunk.length);
+    this._buf.set(chunk, this._bufLen);
+    const scanStart = this._bufLen;
+    this._bufLen += chunk.length;
+    this._scan(scanStart);
+    this._compact();
+  }
+
+  end() {
+    if (this._bufLen > this._recordStart) {
+      // There's pending content that never terminated.
+      if (this._depth !== 0 || this._inString || this._escape) {
+        throw new PreserveTokenizerError('E_TRUNCATED',
+          'input ended mid-record at byte ' + (this._absOffset + this._bufLen) +
+          ' (depth=' + this._depth + ', inString=' + this._inString + ')');
+      }
+      // Depth is balanced and we're not inside a string -- treat the tail as
+      // the last record even without a terminating LF.
+      this._emitRecord(this._recordStart, this._bufLen);
+      this._recordStart = this._bufLen;
+    }
+  }
+
+  _scan(from) {
+    const buf = this._buf;
+    const to = this._bufLen;
+    let depth = this._depth;
+    let inString = this._inString;
+    let escape = this._escape;
+    for (let i = from; i < to; i++) {
+      const b = buf[i];
+      if (escape) { escape = false; continue; }
+      if (inString) {
+        if (b === BYTE_BSLASH)     escape = true;
+        else if (b === BYTE_QUOTE) inString = false;
+      } else {
+        if (b === BYTE_QUOTE) {
+          inString = true;
+        } else if (b === BYTE_LBRACE || b === BYTE_LBRACK) {
+          depth++;
+        } else if (b === BYTE_RBRACE || b === BYTE_RBRACK) {
+          depth--;
+          if (depth < 0) {
+            this._depth = depth; this._inString = inString; this._escape = escape;
+            throw new PreserveTokenizerError('E_UNBALANCED',
+              'closing bracket without matching open at byte ' + (this._absOffset + i));
+          }
+        } else if (b === BYTE_LF && depth === 0) {
+          // Record boundary. Flush [recordStart, i).
+          this._depth = depth; this._inString = inString; this._escape = escape;
+          this._emitRecord(this._recordStart, i);
+          this._recordStart = i + 1;
+        }
+      }
+      if (this._maxRecordBytes > 0 && (i - this._recordStart) > this._maxRecordBytes) {
+        throw new PreserveTokenizerError('E_RECORD_TOO_LARGE',
+          'record starting at byte ' + (this._absOffset + this._recordStart) +
+          ' exceeded maxRecordBytes=' + this._maxRecordBytes);
+      }
+    }
+    this._depth = depth;
+    this._inString = inString;
+    this._escape = escape;
+  }
+
+  _emitRecord(from, to) {
+    // Trim trailing CR (CRLF ending).
+    while (to > from && this._buf[to - 1] === BYTE_CR) to--;
+    if (to === from) return;
+    // Skip whitespace-only records (blank lines between records).
+    let firstNonWs = from;
+    while (firstNonWs < to && (this._buf[firstNonWs] === BYTE_SPACE || this._buf[firstNonWs] === BYTE_TAB)) firstNonWs++;
+    if (firstNonWs === to) return;
+    this._sink.onRecord(this._buf, from, to);
+  }
+
+  _compact() {
+    if (this._recordStart === 0) return;
+    // Move [recordStart, bufLen) to the front. copyWithin is in-place.
+    const remaining = this._bufLen - this._recordStart;
+    if (remaining > 0) this._buf.copyWithin(0, this._recordStart, this._bufLen);
+    this._absOffset += this._recordStart;
+    this._bufLen = remaining;
+    this._recordStart = 0;
+  }
+
+  _ensureCapacity(need) {
+    if (this._buf.length >= need) return;
+    let cap = this._buf.length;
+    while (cap < need) cap *= 2;
+    const next = new Uint8Array(cap);
+    next.set(this._buf.subarray(0, this._bufLen));
+    this._buf = next;
+  }
+}

package/src/PreserveWriter.js

@@ -0,0 +1,218 @@
+// @zakkster/lite-bake-stream / PreserveWriter
+// Copyright (c) 2026 Zahary Shinikchiev. MIT.
+//
+// Preserve-mode sink: opaque record blobs packed into shards, with a trailing
+// u32 offset table per shard. No schema, no lanes, no string table, no zone
+// maps. Bytes in, same bytes out.
+//
+// Shard payload layout:
+//     [ blob0 | blob1 | ... | blob(N-1) | u32 offsets[N] ]
+//     ^payload_off                       ^offsetTableStart = payload_off + payload_len - N*4
+//
+// Container header uses the same 48-byte layout as schema-mode, but with:
+//   - flags byte at offset 7 has bit 0 set (preserve mode)
+//   - schema_block_off = 0
+//   - metadata_off = 0
+// Reader dispatch happens via the flag bit; a schema-mode Reader refuses this
+// container with R_WRONG_MODE.
+
+export const VERSION = '1.0.0';
+
+export class PreserveWriterError extends Error {
+  constructor(code, msg) { super(msg); this.code = code; this.name = 'PreserveWriterError'; }
+}
+
+const CONTAINER_HEADER_BYTES = 48;
+const SHARD_ENTRY_BYTES = 40;
+const FOOTER_BYTES = 16;
+const DEFAULT_TARGET_SHARD_BYTES = 8 * 1024 * 1024;   // 8 MiB — smaller than schema mode
+const INITIAL_OFFSETS_CAP = 4096;
+
+export class PreserveWriter {
+  constructor(opts) {
+    opts = opts || {};
+    this._targetShardBytes = opts.targetShardBytes || DEFAULT_TARGET_SHARD_BYTES;
+    this._shards = [];
+    this._totalRows = 0;
+    this._finalized = false;
+    this._allocateShard();
+  }
+
+  _allocateShard() {
+    // Pre-allocate up to targetShardBytes for zero-GC record writes.
+    this._currentBlobs = new Uint8Array(this._targetShardBytes);
+    this._currentOffsets = new Uint32Array(INITIAL_OFFSETS_CAP);
+    this._currentBlobBytes = 0;
+    this._currentRowCount = 0;
+  }
+
+  // Sink protocol for PreserveTokenizer. Bytes are copied immediately into
+  // the shard buffer, so the caller may reuse `bytes` on return.
+  onRecord(bytes, from, to) {
+    if (this._finalized) throw new PreserveWriterError('W_FINALIZED', 'writer already finalized');
+    const len = to - from;
+    if (len === 0) return;
+
+    // Would this record push us over the soft cap? Roll to a new shard first,
+    // unless the current shard is still empty (i.e. this is an oversized
+    // single record; keep it in its own shard).
+    if (this._currentBlobBytes + len > this._targetShardBytes && this._currentRowCount > 0) {
+      this._finalizeCurrentShard();
+      this._allocateShard();
+    }
+    // Grow shard buffer if this single record exceeds the pre-allocated size.
+    if (this._currentBlobBytes + len > this._currentBlobs.length) {
+      this._growBlobBuffer(this._currentBlobBytes + len);
+    }
+    // Grow offsets array if we've hit the pre-allocated ceiling.
+    if (this._currentRowCount >= this._currentOffsets.length) {
+      this._growOffsetsArray();
+    }
+
+    // Copy record bytes into the shard buffer at the current cursor.
+    // Uint8Array.set is the fastest byte copy V8 offers; subarray on the RHS
+    // creates a small view (young-gen, collected trivially).
+    this._currentOffsets[this._currentRowCount] = this._currentBlobBytes;
+    if (from === 0 && to === bytes.length) {
+      this._currentBlobs.set(bytes, this._currentBlobBytes);
+    } else {
+      this._currentBlobs.set(bytes.subarray(from, to), this._currentBlobBytes);
+    }
+    this._currentBlobBytes += len;
+    this._currentRowCount++;
+    this._totalRows++;
+  }
+
+  // Public API for callers that want to feed complete records directly.
+  writeRecord(bytes) {
+    if (!(bytes instanceof Uint8Array)) {
+      throw new TypeError('PreserveWriter.writeRecord: expected Uint8Array');
+    }
+    this.onRecord(bytes, 0, bytes.length);
+  }
+
+  _growBlobBuffer(needed) {
+    let cap = this._currentBlobs.length;
+    while (cap < needed) cap *= 2;
+    const next = new Uint8Array(cap);
+    next.set(this._currentBlobs.subarray(0, this._currentBlobBytes));
+    this._currentBlobs = next;
+  }
+
+  _growOffsetsArray() {
+    const next = new Uint32Array(this._currentOffsets.length * 2);
+    next.set(this._currentOffsets);
+    this._currentOffsets = next;
+  }
+
+  _finalizeCurrentShard() {
+    if (this._currentRowCount === 0) return;
+    const rowCount = this._currentRowCount;
+    const blobLen = this._currentBlobBytes;
+    const offsetTableBytes = rowCount * 4;
+    const shardBytesLen = blobLen + offsetTableBytes;
+
+    const shardBytes = new Uint8Array(shardBytesLen);
+    shardBytes.set(this._currentBlobs.subarray(0, blobLen), 0);
+    // Trailing offset table: u32 LE per record.
+    const dv = new DataView(shardBytes.buffer, shardBytes.byteOffset, shardBytes.byteLength);
+    for (let i = 0; i < rowCount; i++) {
+      dv.setUint32(blobLen + i * 4, this._currentOffsets[i], true);
+    }
+    this._shards.push({
+      bytes: shardBytes,
+      rowCount,
+      blobLen,
+    });
+    // Zero out the working buffers so a stale ref can't leak old bytes into
+    // the next shard by accident. The buffers are reused via _allocateShard.
+    this._currentBlobs = null;
+    this._currentOffsets = null;
+    this._currentBlobBytes = 0;
+    this._currentRowCount = 0;
+  }
+
+  finalize() {
+    if (this._finalized) throw new PreserveWriterError('W_FINALIZED', 'writer already finalized');
+    if (this._currentRowCount > 0) this._finalizeCurrentShard();
+    if (this._shards.length === 0) {
+      // Emit an empty container? Or error? Match schema-mode: throw.
+      throw new PreserveWriterError('W_EMPTY_INPUT', 'no records written; nothing to finalize');
+    }
+    this._finalized = true;
+    return this._assembleContainer();
+  }
+
+  get totalRows()  { return this._totalRows; }
+  get shardCount() { return this._shards.length; }
+
+  _assembleContainer() {
+    const shardCount = this._shards.length;
+    const shardDirBytes = shardCount * SHARD_ENTRY_BYTES;
+
+    // Layout: header (48) | shard directory | shard payloads | footer (16)
+    // No schema block, no zone maps in preserve mode; schema_block_off = 0.
+    const shardDirOff = CONTAINER_HEADER_BYTES;
+    let cursor = shardDirOff + shardDirBytes;
+    const shardPayloadOffsets = new Array(shardCount);
+    for (let i = 0; i < shardCount; i++) {
+      shardPayloadOffsets[i] = cursor;
+      cursor += this._shards[i].bytes.length;
+    }
+    const totalBytes = cursor + FOOTER_BYTES;
+
+    const container = new ArrayBuffer(totalBytes);
+    const dv = new DataView(container);
+    const bytes = new Uint8Array(container);
+
+    // Header
+    bytes[0] = 0x4C; bytes[1] = 0x42; bytes[2] = 0x4B; bytes[3] = 0x31;   // 'LBK1'
+    dv.setUint16(4, 1, true);            // format_version
+    bytes[6] = 1;                         // endian LE
+    bytes[7] = 0x01;                      // flags: bit 0 = preserve mode
+    dv.setBigUint64(8,  0n, true);        // schema_block_off = 0 (no schema)
+    dv.setBigUint64(16, 0n, true);        // metadata_off = 0 (no zone maps)
+    dv.setBigUint64(24, BigInt(shardDirOff), true);
+    dv.setUint32(32, shardCount, true);
+    dv.setUint32(36, 0, true);            // reserved1
+    dv.setBigUint64(40, BigInt(this._totalRows), true);
+
+    // Shard directory (40 bytes per entry, u64 payload_off, u32 payload_len,
+    // u32 row_count, u16 min_reader_version, u16 flags, u32 reserved,
+    // u64 local_string_off=0, u64 local_string_len=0)
+    for (let i = 0; i < shardCount; i++) {
+      const entryOff = shardDirOff + i * SHARD_ENTRY_BYTES;
+      const s = this._shards[i];
+      dv.setBigUint64(entryOff + 0,  BigInt(shardPayloadOffsets[i]), true);
+      dv.setUint32(entryOff + 8,  s.bytes.length, true);
+      dv.setUint32(entryOff + 12, s.rowCount, true);
+      dv.setUint16(entryOff + 16, 1, true);   // min_reader_version
+      dv.setUint16(entryOff + 18, 0, true);   // shard flags
+      dv.setUint32(entryOff + 20, 0, true);   // reserved
+      dv.setBigUint64(entryOff + 24, 0n, true);   // no string table
+      dv.setBigUint64(entryOff + 32, 0n, true);
+    }
+
+    // Shard payloads
+    for (let i = 0; i < shardCount; i++) {
+      bytes.set(this._shards[i].bytes, shardPayloadOffsets[i]);
+    }
+
+    // Footer
+    const footerOff = totalBytes - FOOTER_BYTES;
+    dv.setUint32(footerOff + 0, 0xFFFFFFFF, true);   // CRC absent
+    dv.setUint32(footerOff + 4, 0, true);
+    bytes[footerOff + 8]  = 0x31;   // '1'
+    bytes[footerOff + 9]  = 0x4B;   // 'K'
+    bytes[footerOff + 10] = 0x42;   // 'B'
+    bytes[footerOff + 11] = 0x4C;   // 'L'
+    dv.setUint32(footerOff + 12, FOOTER_BYTES, true);
+
+    return {
+      buffer: container,
+      totalRows: this._totalRows,
+      shardCount,
+      mode: 'preserve',
+    };
+  }
+}