@zakkster/lite-bake-stream 1.0.0

package/src/Reader.js

@@ -0,0 +1,349 @@
+// @zakkster/lite-bake-stream / Reader
+// Parses LBK1 container and exposes typed-array views over shard payloads.
+// Copyright (c) 2026 Zahary Shinikchiev. MIT.
+//
+// v1 lane kinds: F64 (1) and U32 (3, string-table index).
+// Per-shard string tables live immediately after each shard's payload; the
+// shard directory carries their offset+length. Reader parses each on load.
+//
+// Error codes:
+//   R_BAD_MAGIC              - header magic mismatch
+//   R_UNSUPPORTED_VERSION    - format_version > 1
+//   R_UNSUPPORTED_ENDIAN     - endian byte is neither LE nor BE
+//   R_BAD_FIELD_FLAGS        - a FieldDescriptor.flags is non-zero (v2+ reserved)
+//   R_UNSUPPORTED_LANE       - lane_kind is not F64 or U32
+//   R_SHARD_VERSION_TOO_NEW  - a shard's min_reader_version > this reader
+//   R_TRUNCATED              - container ends before an expected structure
+//   R_UNKNOWN_FIELD          - get() called with an unknown field name
+
+import { StringTable } from './StringTable.js';
+
+export const VERSION = '1.0.0';
+
+const CONTAINER_HEADER_BYTES = 48;
+const SHARD_ENTRY_BYTES = 40;
+const FIELD_DESCRIPTOR_BYTES = 24;
+const FOOTER_BYTES = 16;
+
+const LANE_F64 = 1;
+const LANE_U32 = 3;
+const READER_VERSION = 1;
+
+export class ReaderError extends Error {
+  constructor(code, msg) { super(msg); this.code = code; this.name = 'ReaderError'; }
+}
+
+export class Reader {
+  static fromBuffer(bufferOrArrayBuffer) {
+    const buffer = bufferOrArrayBuffer instanceof ArrayBuffer
+      ? bufferOrArrayBuffer
+      : bufferOrArrayBuffer.buffer;
+    return new Reader(buffer);
+  }
+
+  constructor(buffer) {
+    this._buffer = buffer;
+    this._dv = new DataView(buffer);
+    this._bytes = new Uint8Array(buffer);
+    this._parseHeader();
+    this._parseSchema();
+    this._parseShardDirectory();
+    this._parseZoneMaps();      // M7 — no-op if metadata_off is 0
+    this._buildFieldIndex();
+  }
+
+  _parseHeader() {
+    if (this._buffer.byteLength < CONTAINER_HEADER_BYTES) {
+      throw new ReaderError('R_TRUNCATED', 'container smaller than header');
+    }
+    const b = this._bytes;
+    if (b[0] !== 0x4C || b[1] !== 0x42 || b[2] !== 0x4B || b[3] !== 0x31) {
+      throw new ReaderError('R_BAD_MAGIC', 'header magic is not LBK1');
+    }
+    const version = this._dv.getUint16(4, true);
+    if (version > 1) throw new ReaderError('R_UNSUPPORTED_VERSION', 'format_version=' + version);
+    const endian = b[6];
+    if (endian !== 1 && endian !== 2) throw new ReaderError('R_UNSUPPORTED_ENDIAN', 'endian=' + endian);
+    if (endian !== 1) throw new ReaderError('R_UNSUPPORTED_ENDIAN', 'BE payloads not implemented in v1 reader');
+
+    // Header flags byte (offset 7). Bit 0 = preserve mode. This Reader parses
+    // the schema-mode variant only; preserve containers have no schema block
+    // and would return garbage if we tried, so refuse loudly.
+    const flags = this._bytes[7];
+    if (flags & 0x01) {
+      throw new ReaderError('R_WRONG_MODE',
+        'container is preserve-mode; use PreserveReader (from @zakkster/lite-bake-stream/preserve-reader) or deserialize() which auto-dispatches');
+    }
+    if (flags & ~0x01) {
+      throw new ReaderError('R_BAD_FLAGS', 'unknown flag bits set in header byte 7: 0x' + flags.toString(16));
+    }
+    this._flags = flags;
+
+    this._schemaBlockOff  = Number(this._dv.getBigUint64(8, true));
+    this._metadataOff     = Number(this._dv.getBigUint64(16, true));   // 0 = no metadata block; M7+ zone maps
+    this._shardDirOff     = Number(this._dv.getBigUint64(24, true));
+    this._shardCount      = this._dv.getUint32(32, true);
+    // 4 bytes reserved at 36
+    this._totalRows       = Number(this._dv.getBigUint64(40, true));
+    this._formatVersion   = version;
+  }
+
+  _parseSchema() {
+    const off = this._schemaBlockOff;
+    if (off + 8 > this._buffer.byteLength) throw new ReaderError('R_TRUNCATED', 'schema header truncated');
+    const fieldCount  = this._dv.getUint32(off + 0, true);
+    const rowStride   = this._dv.getUint32(off + 4, true);
+    const descriptorsOff = off + 8;
+    const descriptorsBytes = fieldCount * FIELD_DESCRIPTOR_BYTES;
+    if (descriptorsOff + descriptorsBytes + 4 > this._buffer.byteLength) {
+      throw new ReaderError('R_TRUNCATED', 'schema descriptors truncated');
+    }
+
+    const fields = new Array(fieldCount);
+    const nameBlobLenOff = descriptorsOff + descriptorsBytes;
+    const nameBlobLen = this._dv.getUint32(nameBlobLenOff, true);
+    const nameBlobOff = nameBlobLenOff + 4;
+    if (nameBlobOff + nameBlobLen > this._buffer.byteLength) {
+      throw new ReaderError('R_TRUNCATED', 'schema name blob truncated');
+    }
+    const decoder = new TextDecoder('utf-8');
+
+    for (let i = 0; i < fieldCount; i++) {
+      const descOff = descriptorsOff + i * FIELD_DESCRIPTOR_BYTES;
+      const nameLen     = this._dv.getUint16(descOff + 0, true);
+      const offsetInRow = this._dv.getUint16(descOff + 2, true);
+      const laneKind    = this._bytes[descOff + 4];
+      const flags       = this._bytes[descOff + 5];
+      const nameStrOff  = Number(this._dv.getBigUint64(descOff + 8, true));
+
+      if (flags !== 0) throw new ReaderError('R_BAD_FIELD_FLAGS', 'field ' + i + ' has non-zero flags (v2+ reserved)');
+      if (laneKind !== LANE_F64 && laneKind !== LANE_U32)
+        throw new ReaderError('R_UNSUPPORTED_LANE', 'field ' + i + ' lane_kind=' + laneKind);
+
+      const nameBytes = this._bytes.subarray(nameBlobOff + nameStrOff, nameBlobOff + nameStrOff + nameLen);
+      const name = decoder.decode(nameBytes);
+      fields[i] = { name, laneKind, offsetInRow };
+    }
+
+    this._schema = { fields, rowStride };
+  }
+
+  _parseShardDirectory() {
+    const off = this._shardDirOff;
+    const need = this._shardCount * SHARD_ENTRY_BYTES;
+    if (off + need > this._buffer.byteLength) throw new ReaderError('R_TRUNCATED', 'shard directory truncated');
+    const shards = new Array(this._shardCount);
+    for (let i = 0; i < this._shardCount; i++) {
+      const entryOff = off + i * SHARD_ENTRY_BYTES;
+      const payloadOff    = Number(this._dv.getBigUint64(entryOff + 0, true));
+      const payloadLen    = this._dv.getUint32(entryOff + 8, true);
+      const rowCount      = this._dv.getUint32(entryOff + 12, true);
+      const minReaderVer  = this._dv.getUint16(entryOff + 16, true);
+      // 2 bytes flags at 18, 4 bytes reserved at 20
+      const localStrOff   = Number(this._dv.getBigUint64(entryOff + 24, true));
+      const localStrLen   = Number(this._dv.getBigUint64(entryOff + 32, true));
+      if (minReaderVer > READER_VERSION) {
+        throw new ReaderError('R_SHARD_VERSION_TOO_NEW',
+          'shard ' + i + ' requires reader version ' + minReaderVer);
+      }
+      if (payloadOff + payloadLen > this._buffer.byteLength) {
+        throw new ReaderError('R_TRUNCATED', 'shard ' + i + ' payload truncated');
+      }
+      // Byte view over payload (no F64 view; row stride can be non-multiple-of-8 with U32 fields)
+      const payloadBytes = new Uint8Array(this._buffer, payloadOff, payloadLen);
+      const payloadDv = new DataView(this._buffer, payloadOff, payloadLen);
+      // String table view (present iff localStrLen > 0)
+      let stringTable = null;
+      if (localStrLen > 0) {
+        if (localStrOff + localStrLen > this._buffer.byteLength) {
+          throw new ReaderError('R_TRUNCATED', 'shard ' + i + ' string table truncated');
+        }
+        stringTable = StringTable.parse(new Uint8Array(this._buffer, localStrOff, localStrLen), 0);
+      }
+      shards[i] = { payloadOff, payloadLen, rowCount, payloadBytes, payloadDv, stringTable };
+    }
+    this._shards = shards;
+  }
+
+  _buildFieldIndex() {
+    const idx = new Map();
+    for (let i = 0; i < this._schema.fields.length; i++) idx.set(this._schema.fields[i].name, i);
+    this._fieldIndex = idx;
+  }
+
+  // M7: parse the zone maps segment if present. No-op if _metadataOff is 0
+  // or the segment magic doesn't match. Populates:
+  //   _zoneMapsTrackedFields: number[]      -- schema field indices with bounds
+  //   _zoneMapsFieldToPos:    Map<name, t>  -- fast field-name -> tracking pos
+  //   _zoneMapsMins/Maxes:    Float64Array  -- length shardCount * T, row-major
+  _parseZoneMaps() {
+    this._zoneMapsTrackedFields = null;
+    if (this._metadataOff === 0) return;
+    if (this._metadataOff + 16 > this._buffer.byteLength) return;
+    const off = this._metadataOff;
+    // Check 'ZM01' magic
+    if (this._bytes[off]     !== 0x30 || this._bytes[off + 1] !== 0x5A ||
+        this._bytes[off + 2] !== 0x4D || this._bytes[off + 3] !== 0x31) return;
+    const shardCount = this._dv.getUint32(off + 4, true);
+    const T          = this._dv.getUint32(off + 8, true);
+    if (shardCount !== this._shardCount) {
+      throw new ReaderError('R_TRUNCATED', 'zone maps shard_count mismatch');
+    }
+    if (T === 0) return;
+    const fieldTableOff = off + 16;
+    const fieldTablePad = (8 - ((T * 2) & 7)) & 7;
+    const minsOff = fieldTableOff + T * 2 + fieldTablePad;
+    const maxesOff = minsOff + shardCount * T * 8;
+    const requiredEnd = maxesOff + shardCount * T * 8;
+    if (requiredEnd > this._buffer.byteLength) {
+      throw new ReaderError('R_TRUNCATED', 'zone maps segment truncated');
+    }
+    const tracked = new Array(T);
+    const fieldToPos = new Map();
+    for (let t = 0; t < T; t++) {
+      const schemaFieldIdx = this._dv.getUint16(fieldTableOff + t * 2, true);
+      if (schemaFieldIdx >= this._schema.fields.length) {
+        throw new ReaderError('R_TRUNCATED', 'zone maps references field index ' + schemaFieldIdx);
+      }
+      tracked[t] = schemaFieldIdx;
+      fieldToPos.set(this._schema.fields[schemaFieldIdx].name, t);
+    }
+    // Zero-copy views over the container's underlying buffer.
+    this._zoneMapsTrackedFields = tracked;
+    this._zoneMapsFieldToPos = fieldToPos;
+    this._zoneMapsMins  = new Float64Array(this._buffer, minsOff,  shardCount * T);
+    this._zoneMapsMaxes = new Float64Array(this._buffer, maxesOff, shardCount * T);
+    this._zoneMapsT = T;
+  }
+
+  // ---------- public API ----------
+
+  get schema()      { return this._schema; }
+  get totalRows()   { return this._totalRows; }
+  get shardCount()  { return this._shardCount; }
+  get shards()      { return this._shards; }
+  // ---------- internal accessors ----------
+  //
+  // These exist so sibling modules (Split.js) can do container surgery without
+  // reaching into underscore-prefixed fields. They are NOT part of the public
+  // API surface and carry no semver guarantee beyond "Split.js can rely on
+  // them"; consumers should use the documented public API instead.
+
+  /** @internal Byte offset of the shard directory. */
+  get shardDirectoryOffset() { return this._shardDirOff; }
+
+  /**
+   * @internal Raw zone-maps state, or null when the container has none.
+   * Returns { trackedFields: number[], laneCount: number,
+   *           mins: Float64Array, maxes: Float64Array }
+   * where mins/maxes are row-major [shardIdx * laneCount + trackedPos].
+   */
+  zoneMapsRaw() {
+    if (!this._zoneMapsTrackedFields) return null;
+    return {
+      trackedFields: this._zoneMapsTrackedFields,
+      laneCount: this._zoneMapsT,
+      mins: this._zoneMapsMins,
+      maxes: this._zoneMapsMaxes,
+    };
+  }
+
+  get buffer()      { return this._buffer; }
+
+  // Whether the container carries a zone maps segment. Consumers can pivot
+  // to a full-scan strategy when this is false.
+  get hasZoneMaps() { return this._zoneMapsTrackedFields !== null; }
+
+  // {min, max} for the given field within the given shard, or null if:
+  //   - the container has no zone maps, or
+  //   - the field is not tracked (currently: U32/string fields are never tracked in v1).
+  // For empty ranges (a shard where the field defaulted to 0), returns {min:0, max:0}.
+  shardBounds(shardIdx, fieldName) {
+    if (!this._zoneMapsTrackedFields) return null;
+    const t = this._zoneMapsFieldToPos.get(fieldName);
+    if (t === undefined) return null;
+    if (shardIdx < 0 || shardIdx >= this._shardCount) return null;
+    const pos = shardIdx * this._zoneMapsT + t;
+    return { min: this._zoneMapsMins[pos], max: this._zoneMapsMaxes[pos] };
+  }
+
+  // Return the indices of every shard whose bounds overlap [min, max].
+  // When there are no zone maps or the field is not tracked, returns every
+  // shard (i.e. the query planner must fall back to full scan).
+  // Semantics: a shard is INCLUDED iff its [smin, smax] overlaps [min, max].
+  // A pass-through range ({min: -Inf, max: +Inf} or omitted) returns all shards.
+  findShards(fieldName, opts) {
+    const all = () => { const a = new Array(this._shardCount); for (let i = 0; i < this._shardCount; i++) a[i] = i; return a; };
+    if (!this._zoneMapsTrackedFields) return all();
+    const t = this._zoneMapsFieldToPos.get(fieldName);
+    if (t === undefined) return all();
+    const min = (opts && typeof opts.min === 'number') ? opts.min : Number.NEGATIVE_INFINITY;
+    const max = (opts && typeof opts.max === 'number') ? opts.max : Number.POSITIVE_INFINITY;
+    const T = this._zoneMapsT;
+    const out = [];
+    for (let s = 0; s < this._shardCount; s++) {
+      const smin = this._zoneMapsMins[s * T + t];
+      const smax = this._zoneMapsMaxes[s * T + t];
+      // Overlap iff smin <= max AND smax >= min
+      if (smin <= max && smax >= min) out.push(s);
+    }
+    return out;
+  }
+
+  fieldIndex(name) {
+    const i = this._fieldIndex.get(name);
+    if (i === undefined) throw new ReaderError('R_UNKNOWN_FIELD', 'no field named ' + JSON.stringify(name));
+    return i;
+  }
+
+  // Random-access get across all shards. Returns the field's decoded value:
+  //   F64 lanes -> number
+  //   U32 lanes -> JS string (resolved via the shard's local string table)
+  // Not zero-alloc; intended for testing/introspection.
+  get(rowIdx, fieldName) {
+    let remaining = rowIdx;
+    const fieldIdx = this.fieldIndex(fieldName);
+    const field = this._schema.fields[fieldIdx];
+    for (let s = 0; s < this._shards.length; s++) {
+      const shard = this._shards[s];
+      if (remaining < shard.rowCount) {
+        const rowOff = remaining * this._schema.rowStride;
+        if (field.laneKind === LANE_F64) {
+          return shard.payloadDv.getFloat64(rowOff + field.offsetInRow, true);
+        } else {
+          const strIdx = shard.payloadDv.getUint32(rowOff + field.offsetInRow, true);
+          return shard.stringTable ? shard.stringTable.get(strIdx) : undefined;
+        }
+      }
+      remaining -= shard.rowCount;
+    }
+    return undefined;
+  }
+
+  // Hot-loop friendly: shard payload as DataView + stride + per-field byte offset.
+  shardPayload(shardIdx)  { return this._shards[shardIdx].payloadDv; }
+  strideBytes()           { return this._schema.rowStride; }
+  offsetBytes(fieldName)  { return this._schema.fields[this.fieldIndex(fieldName)].offsetInRow; }
+  laneKind(fieldName)     { return this._schema.fields[this.fieldIndex(fieldName)].laneKind; }
+  shardStringTable(shardIdx) { return this._shards[shardIdx].stringTable; }
+
+  // Back-compat with M2: F64-only reader helpers still work when the schema is
+  // all-F64 (rowStride is a multiple of 8). For mixed schemas these throw.
+  shardF64(shardIdx) {
+    const shard = this._shards[shardIdx];
+    if ((this._schema.rowStride & 7) !== 0)
+      throw new ReaderError('R_UNSUPPORTED_LANE', 'shardF64 not usable on mixed-lane schema');
+    return new Float64Array(this._buffer, shard.payloadOff, shard.payloadLen / 8);
+  }
+  strideF64() {
+    if ((this._schema.rowStride & 7) !== 0)
+      throw new ReaderError('R_UNSUPPORTED_LANE', 'strideF64 not usable on mixed-lane schema');
+    return this._schema.rowStride / 8;
+  }
+  offsetF64(fieldName) {
+    const f = this._schema.fields[this.fieldIndex(fieldName)];
+    if (f.laneKind !== LANE_F64) throw new ReaderError('R_UNSUPPORTED_LANE', fieldName + ' is not F64');
+    if ((f.offsetInRow & 7) !== 0) throw new ReaderError('R_UNSUPPORTED_LANE', fieldName + ' offset not F64-aligned');
+    return f.offsetInRow / 8;
+  }
+}