@zakkster/lite-bake-stream 1.0.0

package/package.json

@@ -0,0 +1,117 @@
+{
+  "name": "@zakkster/lite-bake-stream",
+  "version": "1.0.0",
+  "description": "Streaming byte-level JSON to lite-bake binary compiler. Zero-GC, tree-shakeable, gigabyte-scale.",
+  "type": "module",
+  "sideEffects": false,
+  "exports": {
+    ".": {
+      "types": "./types/index.d.ts",
+      "import": "./src/index.js",
+      "default": "./src/index.js"
+    },
+    "./tokenizer": {
+      "types": "./types/Tokenizer.d.ts",
+      "import": "./src/Tokenizer.js",
+      "default": "./src/Tokenizer.js"
+    },
+    "./writer": {
+      "types": "./types/Writer.d.ts",
+      "import": "./src/Writer.js",
+      "default": "./src/Writer.js"
+    },
+    "./reader": {
+      "types": "./types/Reader.d.ts",
+      "import": "./src/Reader.js",
+      "default": "./src/Reader.js"
+    },
+    "./string-table": {
+      "types": "./types/StringTable.d.ts",
+      "import": "./src/StringTable.js",
+      "default": "./src/StringTable.js"
+    },
+    "./file-ingest": {
+      "types": "./types/FileIngest.d.ts",
+      "import": "./src/FileIngest.js",
+      "default": "./src/FileIngest.js"
+    },
+    "./range-reader": {
+      "types": "./types/RangeReader.d.ts",
+      "import": "./src/RangeReader.js",
+      "default": "./src/RangeReader.js"
+    },
+    "./multi-reader": {
+      "types": "./types/MultiReader.d.ts",
+      "import": "./src/MultiReader.js",
+      "default": "./src/MultiReader.js"
+    },
+    "./split": {
+      "types": "./types/Split.d.ts",
+      "import": "./src/Split.js",
+      "default": "./src/Split.js"
+    },
+    "./preserve-tokenizer": {
+      "types": "./types/PreserveTokenizer.d.ts",
+      "import": "./src/PreserveTokenizer.js",
+      "default": "./src/PreserveTokenizer.js"
+    },
+    "./preserve-writer": {
+      "types": "./types/PreserveWriter.d.ts",
+      "import": "./src/PreserveWriter.js",
+      "default": "./src/PreserveWriter.js"
+    },
+    "./preserve-reader": {
+      "types": "./types/PreserveReader.d.ts",
+      "import": "./src/PreserveReader.js",
+      "default": "./src/PreserveReader.js"
+    }
+  },
+  "files": [
+    "src",
+    "types",
+    "SPEC.md",
+    "CHANGELOG.md",
+    "llms.txt",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "test": "node --test test/Tokenizer.test.js test/RoundTrip.test.js test/StringRoundTrip.test.js test/DataPreservation.test.js test/NumericBoundary.test.js test/RangeReader.test.js test/FileIngest.test.js test/ZoneMaps.test.js test/M4.test.js test/M6.test.js test/Conformance.test.js test/Fuzz.test.js test/OffsetOverflow.test.js test/Preserve.test.js",
+    "bench": "node --expose-gc bench/bench-tokenizer.js",
+    "torture": "node --expose-gc bench/torture.js --tier=fast",
+    "torture:full": "node --expose-gc bench/torture.js --tier=full",
+    "soak": "node --expose-gc bench/soak.js --mb=100 --verify",
+    "soak:500": "node --expose-gc bench/soak.js --mb=500 --verify",
+    "soak:1gb": "node --expose-gc bench/soak.js --gb=1 --verify",
+    "soak:overnight": "node --expose-gc bench/soak.js --gb=8 --verify",
+    "demo": "npx serve ."
+  },
+  "keywords": [
+    "json",
+    "streaming",
+    "sax",
+    "zero-gc",
+    "binary",
+    "lite-bake",
+    "ingestion"
+  ],
+  "author": "Zahary Shinikchiev",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/PeshoVurtoleta/lite-bake-stream.git"
+  },
+  "bugs": {
+    "url": "https://github.com/PeshoVurtoleta/lite-bake-stream/issues"
+  },
+  "homepage": "https://github.com/PeshoVurtoleta/lite-bake-stream#readme",
+  "engines": {
+    "node": ">=18"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "devDependencies": {
+    "@zakkster/lite-gc-profiler": "^1.0.0"
+  }
+}

package/src/FileIngest.js

@@ -0,0 +1,104 @@
+// @zakkster/lite-bake-stream / FileIngest
+// Browser-side helper: File.stream() -> Tokenizer -> Writer -> Reader.
+// Copyright (c) 2026 Zahary Shinikchiev. MIT.
+//
+// The natural browser ingest path is:
+//
+//   const file = fileInputEl.files[0];
+//   const reader = await ingestStream(file.stream(), file.size, {
+//     framing: 'ndjson',
+//     onProgress: ({bytesIngested, totalBytes, rowsWritten, shardsCommitted}) => {...}
+//   });
+//   // reader is a fully-materialized in-memory LBK1 Reader.
+//
+// The helper is stream-agnostic (works with any ReadableStream<Uint8Array>),
+// so it's usable for fetch() bodies, WebSocket-fed inputs, and Node's
+// ReadableStream too. Progress fires every chunk plus a final "done" event
+// with the completed Reader.
+//
+// For containers larger than the browser's ArrayBuffer capacity (~2 GB in
+// most engines), use server-side pre-compilation and RangeReader instead.
+
+import { Tokenizer } from './Tokenizer.js';
+import { Writer } from './Writer.js';
+import { Reader } from './Reader.js';
+import { PreserveTokenizer } from './PreserveTokenizer.js';
+import { PreserveWriter } from './PreserveWriter.js';
+import { PreserveReader } from './PreserveReader.js';
+
+export const VERSION = '1.0.0';
+
+// Ingest a ReadableStream<Uint8Array> and return a Reader over the resulting
+// LBK1 container. Options:
+//   preserve:           when true, use preserve mode (opaque byte blobs, any
+//                       JSON shape allowed). Default false = schema mode.
+//   framing:            'ndjson' | 'array' | 'auto' (default 'ndjson').
+//                       Preserve mode is NDJSON-only.
+//   writer:             Writer options. Schema: { schema, targetShardBytes,
+//                       sampleBytes }. Preserve: { targetShardBytes, maxRecordBytes }.
+//   onProgress(state):  called after each chunk with { bytesIngested,
+//                       totalBytes, rowsWritten, shardsCommitted, chunkBytes,
+//                       chunkCount, elapsedMs }
+//   totalBytes:         optional total for progress percentage. Pass file.size
+//                       when ingesting a File; omit for indeterminate streams.
+export async function ingestStream(readableStream, opts) {
+  opts = opts || {};
+  const preserve = opts.preserve === true;
+  const framing  = opts.framing || 'ndjson';
+  const writerOpts = opts.writer || {};
+  const onProgress = typeof opts.onProgress === 'function' ? opts.onProgress : null;
+  const totalBytes = typeof opts.totalBytes === 'number' ? opts.totalBytes : -1;
+
+  const writer = preserve ? new PreserveWriter(writerOpts) : new Writer(writerOpts);
+  const tokenizer = preserve
+    ? new PreserveTokenizer(writer, { framing, maxRecordBytes: writerOpts.maxRecordBytes || 0 })
+    : new Tokenizer(writer, { framing });
+
+  const reader = readableStream.getReader();
+  const t0 = (globalThis.performance && performance.now) ? performance.now() : Date.now();
+  let bytesIngested = 0;
+  let chunkCount = 0;
+
+  try {
+    while (true) {
+      const { value, done } = await reader.read();
+      if (done) break;
+      if (!(value instanceof Uint8Array)) {
+        throw new TypeError('ingestStream: expected Uint8Array chunks, got ' + typeof value);
+      }
+      tokenizer.feed(value);
+      bytesIngested += value.length;
+      chunkCount++;
+      if (onProgress) {
+        const now = (globalThis.performance && performance.now) ? performance.now() : Date.now();
+        onProgress({
+          bytesIngested, totalBytes,
+          rowsWritten: writer.totalRows,
+          shardsCommitted: writer.shardCount,
+          chunkBytes: value.length,
+          chunkCount,
+          elapsedMs: now - t0,
+        });
+      }
+    }
+  } finally {
+    reader.releaseLock();
+  }
+
+  tokenizer.end();
+  const { buffer } = writer.finalize();
+  return preserve ? new PreserveReader(buffer) : new Reader(buffer);
+}
+
+// Convenience for File / Blob inputs -- pulls the stream and size from the
+// blob. Rejects if the runtime doesn't support Blob.stream().
+export async function ingestFile(file, opts) {
+  if (!file || typeof file.stream !== 'function') {
+    throw new TypeError('ingestFile: expected a Blob/File with .stream()');
+  }
+  const merged = Object.assign({}, opts || {});
+  if (typeof file.size === 'number' && merged.totalBytes === undefined) {
+    merged.totalBytes = file.size;
+  }
+  return ingestStream(file.stream(), merged);
+}

package/src/MultiReader.js

@@ -0,0 +1,160 @@
+// @zakkster/lite-bake-stream / MultiReader
+// Union view over N LBK1 Readers as one logical stream.
+// Copyright (c) 2026 Zahary Shinikchiev. MIT.
+//
+// Motivation:
+//   - Worker-parallel compilation produces one container per worker; a query
+//     layer wants to see them as a single dataset.
+//   - Checkpoint/resume ingest: append a new container per session, browse
+//     the whole history.
+//   - Split-and-merge (M6): the merge step is optional if a MultiReader can
+//     serve queries directly over N separate LBK1 containers.
+//
+// Requirements:
+//   - Schemas MUST match across all readers (field names, lane kinds, order).
+//     Enforced at construction; mismatches raise M_SCHEMA_MISMATCH.
+//   - Row indices are cumulative in reader order: rows 0..N_0 are in reader
+//     0, rows N_0..N_0+N_1 in reader 1, etc.
+//   - Zone maps merge naturally: findShards returns global shard indices
+//     spanning all sub-readers.
+//
+// Error codes:
+//   M_EMPTY               - no readers passed
+//   M_SCHEMA_MISMATCH     - schemas differ across readers
+//   M_UNKNOWN_FIELD       - .get() / .fieldIndex() called with unknown name
+//   M_ROW_OUT_OF_RANGE    - rowIdx >= totalRows
+
+export const VERSION = '1.0.0';
+
+export class MultiReaderError extends Error {
+  constructor(code, msg) { super(msg); this.code = code; this.name = 'MultiReaderError'; }
+}
+
+function schemasEqual(a, b) {
+  if (a.fields.length !== b.fields.length) return false;
+  if (a.rowStride !== b.rowStride) return false;
+  for (let i = 0; i < a.fields.length; i++) {
+    const fa = a.fields[i], fb = b.fields[i];
+    if (fa.name !== fb.name) return false;
+    if (fa.laneKind !== fb.laneKind) return false;
+    if (fa.offsetInRow !== fb.offsetInRow) return false;
+  }
+  return true;
+}
+
+export class MultiReader {
+  constructor(readers) {
+    if (!Array.isArray(readers) || readers.length === 0) {
+      throw new MultiReaderError('M_EMPTY', 'MultiReader requires a non-empty array of Readers');
+    }
+    this._readers = readers.slice();
+    // All readers must share the same schema.
+    const schema0 = readers[0].schema;
+    for (let i = 1; i < readers.length; i++) {
+      if (!schemasEqual(schema0, readers[i].schema)) {
+        throw new MultiReaderError('M_SCHEMA_MISMATCH',
+          'reader ' + i + ' schema differs from reader 0');
+      }
+    }
+    this._schema = schema0;
+
+    // Cumulative row offsets and shard offsets. Row rowIdx belongs to reader
+    // r iff _rowOffsets[r] <= rowIdx < _rowOffsets[r+1].
+    let cumRows = 0, cumShards = 0;
+    this._rowOffsets   = new Uint32Array(readers.length + 1);
+    this._shardOffsets = new Uint32Array(readers.length + 1);
+    for (let i = 0; i < readers.length; i++) {
+      this._rowOffsets[i]   = cumRows;
+      this._shardOffsets[i] = cumShards;
+      cumRows   += readers[i].totalRows;
+      cumShards += readers[i].shardCount;
+    }
+    this._rowOffsets[readers.length]   = cumRows;
+    this._shardOffsets[readers.length] = cumShards;
+    this._totalRows = cumRows;
+    this._shardCount = cumShards;
+
+    // Field-name -> index lookup
+    this._fieldIndex = new Map();
+    for (let i = 0; i < this._schema.fields.length; i++) {
+      this._fieldIndex.set(this._schema.fields[i].name, i);
+    }
+  }
+
+  get schema()      { return this._schema; }
+  get totalRows()   { return this._totalRows; }
+  get shardCount()  { return this._shardCount; }
+  get readerCount() { return this._readers.length; }
+
+  // Which reader contains global row rowIdx? Returns { readerIdx, localRow }.
+  _locateRow(rowIdx) {
+    if (rowIdx < 0 || rowIdx >= this._totalRows) {
+      throw new MultiReaderError('M_ROW_OUT_OF_RANGE',
+        'rowIdx ' + rowIdx + ' out of range [0, ' + this._totalRows + ')');
+    }
+    // Linear-scan the cumulative offsets. Reader counts are small (usually
+    // <= a few dozen); binary search's overhead isn't worth it.
+    for (let r = 0; r < this._readers.length; r++) {
+      if (rowIdx < this._rowOffsets[r + 1]) {
+        return { readerIdx: r, localRow: rowIdx - this._rowOffsets[r] };
+      }
+    }
+    return null;  // unreachable given the range check above
+  }
+
+  // Which reader owns global shard globalShardIdx? Returns { readerIdx, localShard }.
+  _locateShard(globalShardIdx) {
+    for (let r = 0; r < this._readers.length; r++) {
+      if (globalShardIdx < this._shardOffsets[r + 1]) {
+        return { readerIdx: r, localShard: globalShardIdx - this._shardOffsets[r] };
+      }
+    }
+    return null;
+  }
+
+  fieldIndex(name) {
+    const i = this._fieldIndex.get(name);
+    if (i === undefined) throw new MultiReaderError('M_UNKNOWN_FIELD', 'no field named ' + JSON.stringify(name));
+    return i;
+  }
+
+  // Get value at global row rowIdx.
+  get(rowIdx, fieldName) {
+    const loc = this._locateRow(rowIdx);
+    return this._readers[loc.readerIdx].get(loc.localRow, fieldName);
+  }
+
+  // Zone-maps query: returns GLOBAL shard indices whose bounds overlap [min, max].
+  // For sub-readers that have zone maps, the shard-level pruning happens per
+  // reader; results are concatenated and re-numbered into the global shard
+  // index space.
+  findShards(fieldName, opts) {
+    const out = [];
+    for (let r = 0; r < this._readers.length; r++) {
+      const localShards = this._readers[r].findShards(fieldName, opts);
+      const shift = this._shardOffsets[r];
+      for (const s of localShards) out.push(shift + s);
+    }
+    return out;
+  }
+
+  shardBounds(globalShardIdx, fieldName) {
+    const loc = this._locateShard(globalShardIdx);
+    if (!loc) return null;
+    return this._readers[loc.readerIdx].shardBounds(loc.localShard, fieldName);
+  }
+
+  // Access the underlying sub-Reader and its local shard offset for advanced
+  // consumers (e.g. building a MultiRangeReader or worker RPC).
+  reader(readerIdx)         { return this._readers[readerIdx]; }
+  readerForShard(globalShardIdx) { return this._locateShard(globalShardIdx); }
+  readerForRow(rowIdx)      { return this._locateRow(rowIdx); }
+
+  get hasZoneMaps() {
+    // MultiReader reports zone maps only if EVERY sub-reader has them.
+    // Mixed availability would silently fall through to full-scan in
+    // findShards for the sub-readers without zone maps; better to be honest.
+    for (const r of this._readers) if (!r.hasZoneMaps) return false;
+    return true;
+  }
+}

package/src/PreserveReader.js

@@ -0,0 +1,180 @@
+// @zakkster/lite-bake-stream / PreserveReader
+// Copyright (c) 2026 Zahary Shinikchiev. MIT.
+//
+// Random-access reader for preserve-mode LBK1 containers. Same 48-byte header
+// and 40-byte ShardEntry as schema mode, but interpreted differently:
+//   - flags byte at offset 7 has bit 0 set (validated at open)
+//   - schema_block_off is 0 (no schema block to parse)
+//   - metadata_off is 0 (no zone maps)
+//   - each shard payload ends with a u32 offset table sized rowCount * 4
+//
+// Public tri-API:
+//   getBytes(rowIdx)  -> Uint8Array (view into container, zero-alloc)
+//   getString(rowIdx) -> string (allocates one string via TextDecoder)
+//   getJSON(rowIdx)   -> parsed value (allocates via getString + JSON.parse)
+//
+// The Uint8Array returned by getBytes is a live subarray view of the container
+// buffer. It is valid for as long as the Reader instance holds the container.
+// Do NOT mutate it, and do NOT retain it past the Reader's lifetime.
+
+export const VERSION = '1.0.0';
+
+const CONTAINER_HEADER_BYTES = 48;
+const SHARD_ENTRY_BYTES = 40;
+
+export class PreserveReaderError extends Error {
+  constructor(code, msg) { super(msg); this.code = code; this.name = 'PreserveReaderError'; }
+}
+
+export class PreserveReader {
+  static fromBuffer(input) {
+    if (input instanceof ArrayBuffer) return new PreserveReader(input);
+    if (input instanceof Uint8Array) {
+      if (input.byteOffset === 0 && input.byteLength === input.buffer.byteLength) {
+        return new PreserveReader(input.buffer);
+      }
+      const copy = new Uint8Array(input.byteLength);
+      copy.set(input);
+      return new PreserveReader(copy.buffer);
+    }
+    throw new TypeError('PreserveReader.fromBuffer: expected ArrayBuffer or Uint8Array');
+  }
+
+  constructor(buffer) {
+    if (!(buffer instanceof ArrayBuffer)) {
+      throw new TypeError('PreserveReader: expected ArrayBuffer');
+    }
+    this._buffer = buffer;
+    this._dv = new DataView(buffer);
+    this._bytes = new Uint8Array(buffer);
+    this._decoder = new TextDecoder('utf-8', { fatal: false });
+    this._parseHeader();
+    this._parseShardDirectory();
+  }
+
+  _parseHeader() {
+    if (this._buffer.byteLength < CONTAINER_HEADER_BYTES) {
+      throw new PreserveReaderError('R_TRUNCATED', 'container smaller than 48-byte header');
+    }
+    const b = this._bytes;
+    if (b[0] !== 0x4C || b[1] !== 0x42 || b[2] !== 0x4B || b[3] !== 0x31) {
+      throw new PreserveReaderError('R_BAD_MAGIC', 'not an LBK1 container');
+    }
+    const version = this._dv.getUint16(4, true);
+    if (version > 1) throw new PreserveReaderError('R_UNSUPPORTED_VERSION', 'format_version=' + version);
+    const endian = b[6];
+    if (endian !== 1) throw new PreserveReaderError('R_UNSUPPORTED_ENDIAN', 'BE payloads not implemented in v1');
+    const flags = b[7];
+    if (!(flags & 0x01)) {
+      throw new PreserveReaderError('R_WRONG_MODE',
+        'container is schema-mode; use Reader (from @zakkster/lite-bake-stream/reader) or deserialize() which auto-dispatches');
+    }
+    if (flags & ~0x01) {
+      throw new PreserveReaderError('R_BAD_FLAGS', 'unknown flag bits set: 0x' + flags.toString(16));
+    }
+
+    this._schemaBlockOff = Number(this._dv.getBigUint64(8, true));
+    this._metadataOff    = Number(this._dv.getBigUint64(16, true));
+    this._shardDirOff    = Number(this._dv.getBigUint64(24, true));
+    this._shardCount     = this._dv.getUint32(32, true);
+    // 4 bytes reserved at 36
+    this._totalRows      = Number(this._dv.getBigUint64(40, true));
+
+    if (this._schemaBlockOff !== 0) {
+      throw new PreserveReaderError('R_INVALID', 'preserve container has non-zero schema_block_off');
+    }
+  }
+
+  _parseShardDirectory() {
+    const need = this._shardCount * SHARD_ENTRY_BYTES;
+    if (this._shardDirOff + need > this._buffer.byteLength) {
+      throw new PreserveReaderError('R_TRUNCATED', 'shard directory truncated');
+    }
+    const shards = new Array(this._shardCount);
+    // Also compute cumulative row offsets so _locateShard can binary-search
+    // in O(log N) rather than linear-scanning entries with row bounds.
+    let cumulativeRow = 0;
+    for (let i = 0; i < this._shardCount; i++) {
+      const entryOff = this._shardDirOff + i * SHARD_ENTRY_BYTES;
+      const payloadOff = Number(this._dv.getBigUint64(entryOff + 0, true));
+      const payloadLen = this._dv.getUint32(entryOff + 8, true);
+      const rowCount   = this._dv.getUint32(entryOff + 12, true);
+      const minReaderVer = this._dv.getUint16(entryOff + 16, true);
+      if (minReaderVer > 1) {
+        throw new PreserveReaderError('R_SHARD_VERSION_TOO_NEW',
+          'shard ' + i + ' requires reader version ' + minReaderVer);
+      }
+      // Trailing u32 offset table.
+      const offsetTableOff = payloadOff + payloadLen - rowCount * 4;
+      if (offsetTableOff < payloadOff) {
+        throw new PreserveReaderError('R_INVALID',
+          'shard ' + i + ' payload_len (' + payloadLen + ') too small for offset table of ' + rowCount + ' rows');
+      }
+      shards[i] = {
+        payloadOff,
+        payloadLen,
+        rowCount,
+        blobLen: offsetTableOff - payloadOff,
+        offsetTableOff,
+        firstRow: cumulativeRow,
+        endRow: cumulativeRow + rowCount,
+      };
+      cumulativeRow += rowCount;
+    }
+    if (cumulativeRow !== this._totalRows) {
+      throw new PreserveReaderError('R_INVALID',
+        'shard row counts sum to ' + cumulativeRow + ' but header total_rows = ' + this._totalRows);
+    }
+    this._shards = shards;
+  }
+
+  get totalRows()   { return this._totalRows; }
+  get shardCount()  { return this._shardCount; }
+  get shards()      { return this._shards; }
+  get buffer()      { return this._buffer; }
+  get mode()        { return 'preserve'; }
+  // Preserve mode has nothing to query on. hasZoneMaps=false so query planners
+  // fall back to full-scan gracefully -- same behavior as schema mode without
+  // zone maps.
+  get hasZoneMaps() { return false; }
+
+  _locateShard(rowIdx) {
+    if (rowIdx < 0 || rowIdx >= this._totalRows) {
+      throw new PreserveReaderError('R_ROW_OUT_OF_RANGE',
+        'rowIdx ' + rowIdx + ' out of range [0, ' + this._totalRows + ')');
+    }
+    let lo = 0, hi = this._shards.length;
+    while (lo < hi) {
+      const mid = (lo + hi) >>> 1;
+      if (rowIdx < this._shards[mid].firstRow) hi = mid;
+      else if (rowIdx >= this._shards[mid].endRow) lo = mid + 1;
+      else return this._shards[mid];
+    }
+    return null;   // unreachable given the range check
+  }
+
+  // Zero-alloc: returns a subarray view into the container buffer.
+  // Do NOT mutate. Do NOT retain past the reader's lifetime.
+  getBytes(rowIdx) {
+    const shard = this._locateShard(rowIdx);
+    const localRow = rowIdx - shard.firstRow;
+    // Read record start and end offsets relative to payload_off.
+    const startRel = this._dv.getUint32(shard.offsetTableOff + localRow * 4, true);
+    const endRel = (localRow === shard.rowCount - 1)
+      ? shard.blobLen
+      : this._dv.getUint32(shard.offsetTableOff + (localRow + 1) * 4, true);
+    const absStart = shard.payloadOff + startRel;
+    const absEnd   = shard.payloadOff + endRel;
+    return this._bytes.subarray(absStart, absEnd);
+  }
+
+  // Decodes UTF-8 via TextDecoder (allocates one string).
+  getString(rowIdx) {
+    return this._decoder.decode(this.getBytes(rowIdx));
+  }
+
+  // Full JSON.parse (allocates the parsed value graph).
+  getJSON(rowIdx) {
+    return JSON.parse(this.getString(rowIdx));
+  }
+}