@zakkster/lite-bake-stream 1.0.0

package/src/RangeReader.js

@@ -0,0 +1,470 @@
+// @zakkster/lite-bake-stream / RangeReader
+// LBK1 reader with lazy shard loading via an IO adapter.
+// Copyright (c) 2026 Zahary Shinikchiev. MIT.
+//
+// The base Reader parses a fully-loaded ArrayBuffer synchronously. That's the
+// right choice for small containers and post-ingest local use. For gigabyte-
+// scale containers hosted over HTTP, downloading the whole file to read a few
+// rows is wasteful; SPEC 3.4's shard directory + per-shard payload/string-
+// table offsets exist precisely so a client can fetch only what it needs.
+//
+// RangeReader implements that story:
+//   1. On open, fetch the header (32 bytes) + schema block + shard directory.
+//      That gives us the whole schema and every shard's byte extents without
+//      touching a single row.
+//   2. On get(rowIdx, fieldName), locate the shard containing rowIdx, fetch
+//      its payload + local string table in ONE range request (they are
+//      contiguous by SPEC 3.4), cache it, decode.
+//   3. LRU-evict cached shards when the cache exceeds `maxCachedShards`.
+//
+// IO adapter contract:
+//   interface IOAdapter {
+//     size: number;                                         // container byte length
+//     fetch(byteOffset, byteLength): Promise<Uint8Array>;   // MUST return exactly that many bytes
+//   }
+//
+// Two adapters ship in-box:
+//   HTTPRangeAdapter -- uses fetch() with Range headers.
+//   MockRangeAdapter -- backed by a Uint8Array; used by tests + local demos.
+//
+// Error codes (stable, share prefix with Reader.js where semantics overlap):
+//   R_BAD_MAGIC / R_UNSUPPORTED_VERSION / R_UNSUPPORTED_ENDIAN / R_BAD_FIELD_FLAGS
+//   R_UNSUPPORTED_LANE / R_SHARD_VERSION_TOO_NEW / R_TRUNCATED / R_UNKNOWN_FIELD
+//   R_ADAPTER_SHORT_READ   -- adapter returned fewer bytes than requested
+//   R_ROW_OUT_OF_RANGE     -- rowIdx >= totalRows
+
+import { StringTable } from './StringTable.js';
+
+export const VERSION = '1.0.0';
+
+const CONTAINER_HEADER_BYTES = 48;
+const SHARD_ENTRY_BYTES = 40;
+const FIELD_DESCRIPTOR_BYTES = 24;
+
+const LANE_F64 = 1;
+const LANE_U32 = 3;
+const READER_VERSION = 1;
+
+export class RangeReaderError extends Error {
+  constructor(code, msg) { super(msg); this.code = code; this.name = 'RangeReaderError'; }
+}
+
+// ----- Adapters --------------------------------------------------------------
+
+// Uses fetch() with Range headers. Content-Length is discovered on open() via
+// a HEAD (falls back to a Range: bytes=0-0 GET if HEAD is disallowed by the
+// origin/CDN).
+export class HTTPRangeAdapter {
+  static async open(url, opts) {
+    opts = opts || {};
+    const fetchImpl = opts.fetch || globalThis.fetch;
+    if (typeof fetchImpl !== 'function') {
+      throw new RangeReaderError('R_TRUNCATED', 'no fetch() available in this environment');
+    }
+    let size = -1;
+    // Try HEAD first.
+    try {
+      const head = await fetchImpl(url, { method: 'HEAD' });
+      if (head.ok) {
+        const cl = head.headers.get('content-length');
+        if (cl) size = parseInt(cl, 10);
+      }
+    } catch { /* fall through */ }
+    // Fallback: single-byte range GET.
+    if (size < 0) {
+      const probe = await fetchImpl(url, { headers: { Range: 'bytes=0-0' } });
+      const cr = probe.headers.get('content-range');
+      if (cr) {
+        const m = cr.match(/\/(\d+)$/);
+        if (m) size = parseInt(m[1], 10);
+      }
+      // consume body so the connection doesn't hang
+      await probe.arrayBuffer();
+    }
+    if (size < 0) throw new RangeReaderError('R_TRUNCATED', 'could not determine container size for ' + url);
+    return new HTTPRangeAdapter(url, size, fetchImpl);
+  }
+
+  constructor(url, size, fetchImpl) {
+    this.url = url;
+    this.size = size;
+    this._fetch = fetchImpl;
+    this.stats = { requests: 0, bytesFetched: 0 };
+  }
+
+  async fetch(byteOffset, byteLength) {
+    const rangeEnd = byteOffset + byteLength - 1;
+    const res = await this._fetch(this.url, {
+      headers: { Range: 'bytes=' + byteOffset + '-' + rangeEnd },
+    });
+    if (!res.ok && res.status !== 206) {
+      throw new RangeReaderError('R_TRUNCATED',
+        'range fetch failed: HTTP ' + res.status + ' bytes=' + byteOffset + '-' + rangeEnd);
+    }
+    const buf = new Uint8Array(await res.arrayBuffer());
+    if (buf.length !== byteLength) {
+      throw new RangeReaderError('R_ADAPTER_SHORT_READ',
+        'expected ' + byteLength + ' bytes at offset ' + byteOffset + ', got ' + buf.length);
+    }
+    this.stats.requests++;
+    this.stats.bytesFetched += byteLength;
+    return buf;
+  }
+}
+
+// Backed by an in-memory Uint8Array. Used by tests and the local demo (which
+// wants to prove the RangeReader code path against a container it just built
+// in the browser, without spinning up an HTTP server).
+export class MockRangeAdapter {
+  constructor(bytes) {
+    this.bytes = bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes);
+    this.size = this.bytes.length;
+    this.log = [];    // [[offset, length], ...] for test assertions
+    this.stats = { requests: 0, bytesFetched: 0 };
+  }
+
+  async fetch(byteOffset, byteLength) {
+    if (byteOffset + byteLength > this.size) {
+      throw new RangeReaderError('R_ADAPTER_SHORT_READ',
+        'mock adapter: request bytes=' + byteOffset + '-' + (byteOffset + byteLength - 1) +
+        ' exceeds size ' + this.size);
+    }
+    this.log.push([byteOffset, byteLength]);
+    this.stats.requests++;
+    this.stats.bytesFetched += byteLength;
+    // Return a COPY so mutations by the caller don't affect the "source of truth".
+    // This matches HTTPRangeAdapter's semantics (each fetch returns a fresh buffer).
+    return this.bytes.slice(byteOffset, byteOffset + byteLength);
+  }
+}
+
+// ----- RangeReader ---------------------------------------------------------
+
+export class RangeReader {
+  static async open(adapter, opts) {
+    const r = new RangeReader(adapter, opts);
+    await r._loadHeaderAndSchema();
+    return r;
+  }
+
+  constructor(adapter, opts) {
+    opts = opts || {};
+    this.adapter = adapter;
+    this.maxCachedShards = opts.maxCachedShards || 8;
+    // shard cache: shardIdx -> { payloadBytes, payloadDv, stringTable, lastAccess }
+    this._shardCache = new Map();
+    this._accessCounter = 0;
+  }
+
+  async _loadHeaderAndSchema() {
+    if (this.adapter.size < CONTAINER_HEADER_BYTES) {
+      throw new RangeReaderError('R_TRUNCATED', 'adapter size smaller than header');
+    }
+
+    // Step 1: header (32 bytes) tells us schema + shard-dir offsets.
+    const headerBytes = await this.adapter.fetch(0, CONTAINER_HEADER_BYTES);
+    const hdrDv = new DataView(headerBytes.buffer, headerBytes.byteOffset, headerBytes.byteLength);
+    if (headerBytes[0] !== 0x4C || headerBytes[1] !== 0x42 || headerBytes[2] !== 0x4B || headerBytes[3] !== 0x31) {
+      throw new RangeReaderError('R_BAD_MAGIC', 'header magic is not LBK1');
+    }
+    const version = hdrDv.getUint16(4, true);
+    if (version > 1) throw new RangeReaderError('R_UNSUPPORTED_VERSION', 'format_version=' + version);
+    const endian = headerBytes[6];
+    if (endian !== 1) throw new RangeReaderError('R_UNSUPPORTED_ENDIAN', 'BE payloads not implemented in v1 reader');
+
+    this._schemaBlockOff = Number(hdrDv.getBigUint64(8, true));
+    this._metadataOff    = Number(hdrDv.getBigUint64(16, true));
+    this._shardDirOff    = Number(hdrDv.getBigUint64(24, true));
+    this._shardCount     = hdrDv.getUint32(32, true);
+    // 4 bytes reserved at 36
+    this._totalRows      = Number(hdrDv.getBigUint64(40, true));
+
+    // Step 2: schema block. We know its start but not its length; compute
+    // upper bound: schemaBlockOff .. shardDirOff.
+    const schemaBlockLen = this._shardDirOff - this._schemaBlockOff;
+    const schemaBytes = await this.adapter.fetch(this._schemaBlockOff, schemaBlockLen);
+    this._parseSchema(schemaBytes);
+
+    // Step 3: shard directory (fixed size = shardCount * 32).
+    const dirBytes = await this.adapter.fetch(this._shardDirOff, this._shardCount * SHARD_ENTRY_BYTES);
+    this._parseShardDirectory(dirBytes);
+
+    // Step 4 (M7): zone maps. Fetched ONCE at open, cached. Enables query
+    // pruning without touching shard payloads. When metadata_off is 0, skip
+    // entirely (older writers or all-U32 schemas).
+    this._zoneMapsTrackedFields = null;
+    if (this._metadataOff !== 0) await this._loadZoneMaps();
+
+    // Field-name -> index map for O(1) get()
+    this._fieldIndex = new Map();
+    for (let i = 0; i < this._schema.fields.length; i++) {
+      this._fieldIndex.set(this._schema.fields[i].name, i);
+    }
+  }
+
+  async _loadZoneMaps() {
+    // Fetch the segment header first (16 bytes) so we can size the rest.
+    const hdr = await this.adapter.fetch(this._metadataOff, 16);
+    if (hdr[0] !== 0x30 || hdr[1] !== 0x5A || hdr[2] !== 0x4D || hdr[3] !== 0x31) return;
+    const hdrDv = new DataView(hdr.buffer, hdr.byteOffset, hdr.byteLength);
+    const shardCount = hdrDv.getUint32(4, true);
+    const T          = hdrDv.getUint32(8, true);
+    if (shardCount !== this._shardCount) {
+      throw new RangeReaderError('R_TRUNCATED', 'zone maps shard_count mismatch');
+    }
+    if (T === 0) return;
+    // Fetch the rest in one range: field table + pad + mins + maxes.
+    const fieldTableLen = T * 2;
+    const fieldTablePad = (8 - (fieldTableLen & 7)) & 7;
+    const restLen = fieldTableLen + fieldTablePad + shardCount * T * 8 * 2;
+    const rest = await this.adapter.fetch(this._metadataOff + 16, restLen);
+    const restDv = new DataView(rest.buffer, rest.byteOffset, rest.byteLength);
+    const tracked = new Array(T);
+    const fieldToPos = new Map();
+    for (let t = 0; t < T; t++) {
+      const schemaFieldIdx = restDv.getUint16(t * 2, true);
+      if (schemaFieldIdx >= this._schema.fields.length) {
+        throw new RangeReaderError('R_TRUNCATED', 'zone maps references field index ' + schemaFieldIdx);
+      }
+      tracked[t] = schemaFieldIdx;
+      fieldToPos.set(this._schema.fields[schemaFieldIdx].name, t);
+    }
+    const minsRel = fieldTableLen + fieldTablePad;
+    // Materialize as Float64Array. Zero-copy over rest.buffer, aligned reads.
+    this._zoneMapsMins  = new Float64Array(
+      rest.buffer, rest.byteOffset + minsRel, shardCount * T);
+    this._zoneMapsMaxes = new Float64Array(
+      rest.buffer, rest.byteOffset + minsRel + shardCount * T * 8, shardCount * T);
+    this._zoneMapsTrackedFields = tracked;
+    this._zoneMapsFieldToPos = fieldToPos;
+    this._zoneMapsT = T;
+  }
+
+  _parseSchema(bytes) {
+    const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    const fieldCount = dv.getUint32(0, true);
+    const rowStride  = dv.getUint32(4, true);
+    const descOff = 8;
+    const descBytes = fieldCount * FIELD_DESCRIPTOR_BYTES;
+    const nameBlobLenOff = descOff + descBytes;
+    const nameBlobLen = dv.getUint32(nameBlobLenOff, true);
+    const nameBlobOff = nameBlobLenOff + 4;
+    const decoder = new TextDecoder('utf-8');
+    const fields = new Array(fieldCount);
+    for (let i = 0; i < fieldCount; i++) {
+      const off = descOff + i * FIELD_DESCRIPTOR_BYTES;
+      const nameLen     = dv.getUint16(off + 0, true);
+      const offsetInRow = dv.getUint16(off + 2, true);
+      const laneKind    = bytes[off + 4];
+      const flags       = bytes[off + 5];
+      const nameStrOff  = Number(dv.getBigUint64(off + 8, true));
+      if (flags !== 0) throw new RangeReaderError('R_BAD_FIELD_FLAGS', 'field ' + i + ' has non-zero flags');
+      if (laneKind !== LANE_F64 && laneKind !== LANE_U32)
+        throw new RangeReaderError('R_UNSUPPORTED_LANE', 'field ' + i + ' lane_kind=' + laneKind);
+      const nb = bytes.subarray(nameBlobOff + nameStrOff, nameBlobOff + nameStrOff + nameLen);
+      fields[i] = { name: decoder.decode(nb), laneKind, offsetInRow };
+    }
+    this._schema = { fields, rowStride };
+  }
+
+  _parseShardDirectory(bytes) {
+    const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    const shards = new Array(this._shardCount);
+    let cumulativeRow = 0;
+    for (let i = 0; i < this._shardCount; i++) {
+      const off = i * SHARD_ENTRY_BYTES;
+      const payloadOff   = Number(dv.getBigUint64(off + 0, true));
+      const payloadLen   = dv.getUint32(off + 8, true);
+      const rowCount     = dv.getUint32(off + 12, true);
+      const minReaderVer = dv.getUint16(off + 16, true);
+      // 2 bytes flags at 18, 4 bytes reserved at 20
+      const localStrOff  = Number(dv.getBigUint64(off + 24, true));
+      const localStrLen  = Number(dv.getBigUint64(off + 32, true));
+      if (minReaderVer > READER_VERSION)
+        throw new RangeReaderError('R_SHARD_VERSION_TOO_NEW',
+          'shard ' + i + ' requires reader version ' + minReaderVer);
+      shards[i] = {
+        payloadOff, payloadLen, rowCount, localStrOff, localStrLen,
+        firstRow: cumulativeRow,
+        endRow: cumulativeRow + rowCount,
+      };
+      cumulativeRow += rowCount;
+    }
+    this._shards = shards;
+  }
+
+  // ---------- public API ----------
+
+  get schema()      { return this._schema; }
+  get totalRows()   { return this._totalRows; }
+  get shardCount()  { return this._shardCount; }
+  get shards()      { return this._shards; }
+
+  fieldIndex(name) {
+    const i = this._fieldIndex.get(name);
+    if (i === undefined) throw new RangeReaderError('R_UNKNOWN_FIELD', 'no field named ' + JSON.stringify(name));
+    return i;
+  }
+
+  // Locate the shard containing rowIdx via binary search.
+  _findShardIndex(rowIdx) {
+    if (rowIdx < 0 || rowIdx >= this._totalRows) {
+      throw new RangeReaderError('R_ROW_OUT_OF_RANGE',
+        'rowIdx ' + rowIdx + ' out of range [0, ' + this._totalRows + ')');
+    }
+    let lo = 0, hi = this._shards.length;
+    while (lo < hi) {
+      const mid = (lo + hi) >>> 1;
+      if (rowIdx < this._shards[mid].firstRow) hi = mid;
+      else if (rowIdx >= this._shards[mid].endRow) lo = mid + 1;
+      else return mid;
+    }
+    return -1;   // unreachable given the range check above
+  }
+
+  // Load a shard's payload + local string table with ONE range request.
+  // They are contiguous in the container by SPEC 3.4, so a single fetch covers
+  // both. Returns the cached shard record.
+  async loadShard(shardIdx) {
+    const cached = this._shardCache.get(shardIdx);
+    if (cached) {
+      cached.lastAccess = ++this._accessCounter;
+      return cached;
+    }
+    const s = this._shards[shardIdx];
+    const combinedLen = s.payloadLen + s.localStrLen;
+    const combinedOff = s.payloadOff;
+    // Sanity: string table (if any) MUST be contiguous with payload.
+    if (s.localStrLen > 0 && s.localStrOff !== combinedOff + s.payloadLen) {
+      throw new RangeReaderError('R_TRUNCATED',
+        'shard ' + shardIdx + ' string table not contiguous with payload');
+    }
+    const combined = await this.adapter.fetch(combinedOff, combinedLen);
+    const payloadBytes = combined.subarray(0, s.payloadLen);
+    // DataView over the payload's underlying ArrayBuffer window.
+    const payloadDv = new DataView(payloadBytes.buffer, payloadBytes.byteOffset, payloadBytes.byteLength);
+    let stringTable = null;
+    if (s.localStrLen > 0) {
+      const stBytes = combined.subarray(s.payloadLen, s.payloadLen + s.localStrLen);
+      stringTable = StringTable.parse(stBytes, 0);
+    }
+    const record = {
+      payloadBytes, payloadDv, stringTable,
+      firstRow: s.firstRow, rowCount: s.rowCount,
+      lastAccess: ++this._accessCounter,
+    };
+    this._shardCache.set(shardIdx, record);
+    this._maybeEvict();
+    return record;
+  }
+
+  _maybeEvict() {
+    if (this._shardCache.size <= this.maxCachedShards) return;
+    // Evict LRU: the one with the smallest lastAccess.
+    let evictKey = -1, evictAccess = Infinity;
+    for (const [key, rec] of this._shardCache) {
+      if (rec.lastAccess < evictAccess) { evictAccess = rec.lastAccess; evictKey = key; }
+    }
+    if (evictKey >= 0) this._shardCache.delete(evictKey);
+  }
+
+  // Random-access get: returns a number (F64 lane) or a string (U32 lane).
+  // Async because the containing shard may need to be fetched.
+  async get(rowIdx, fieldName) {
+    const fieldIdx = this.fieldIndex(fieldName);
+    const field = this._schema.fields[fieldIdx];
+    const shardIdx = this._findShardIndex(rowIdx);
+    const shard = await this.loadShard(shardIdx);
+    const localRow = rowIdx - shard.firstRow;
+    const rowOff = localRow * this._schema.rowStride;
+    if (field.laneKind === LANE_F64) {
+      return shard.payloadDv.getFloat64(rowOff + field.offsetInRow, true);
+    }
+    const strIdx = shard.payloadDv.getUint32(rowOff + field.offsetInRow, true);
+    return shard.stringTable ? shard.stringTable.get(strIdx) : undefined;
+  }
+
+  // Ensure shards covering rows [firstRow, lastRow) are cached. Useful for
+  // prefetching before a viewport re-render, so the UI doesn't await per-row.
+  async prefetchRange(firstRow, lastRow) {
+    if (firstRow >= this._totalRows) return;
+    if (lastRow > this._totalRows) lastRow = this._totalRows;
+    const firstShard = this._findShardIndex(firstRow);
+    const lastShard  = this._findShardIndex(lastRow - 1);
+    const requests = [];
+    for (let s = firstShard; s <= lastShard; s++) {
+      if (!this._shardCache.has(s)) requests.push(this.loadShard(s));
+    }
+    await Promise.all(requests);
+  }
+
+  // Return a synchronous row accessor for a range that is already cached.
+  // Throws if any shard in the range hasn't been prefetched. Intended for
+  // hot render paths: the UI calls prefetchRange, then reads without await.
+  syncRange(firstRow, lastRow) {
+    const firstShard = this._findShardIndex(firstRow);
+    const lastShard  = this._findShardIndex(lastRow - 1);
+    for (let s = firstShard; s <= lastShard; s++) {
+      if (!this._shardCache.has(s)) {
+        throw new RangeReaderError('R_TRUNCATED',
+          'syncRange requires shard ' + s + ' to be prefetched (call prefetchRange first)');
+      }
+    }
+    const stride = this._schema.rowStride;
+    const fields = this._schema.fields;
+    const self = this;
+    return {
+      get(rowIdx, fieldName) {
+        const fieldIdx = self._fieldIndex.get(fieldName);
+        if (fieldIdx === undefined) throw new RangeReaderError('R_UNKNOWN_FIELD', fieldName);
+        const field = fields[fieldIdx];
+        const shardIdx = self._findShardIndex(rowIdx);
+        const shard = self._shardCache.get(shardIdx);
+        const localRow = rowIdx - shard.firstRow;
+        const rowOff = localRow * stride;
+        if (field.laneKind === LANE_F64) {
+          return shard.payloadDv.getFloat64(rowOff + field.offsetInRow, true);
+        }
+        const strIdx = shard.payloadDv.getUint32(rowOff + field.offsetInRow, true);
+        return shard.stringTable ? shard.stringTable.get(strIdx) : undefined;
+      },
+    };
+  }
+
+  get cachedShardCount() { return this._shardCache.size; }
+
+  // Whether the container carries a zone maps segment (fetched once at open).
+  get hasZoneMaps() { return this._zoneMapsTrackedFields !== null; }
+
+  // {min, max} for the given field within the given shard, or null if
+  // the container has no zone maps or the field is not tracked. Synchronous
+  // -- zone maps were fetched during open().
+  shardBounds(shardIdx, fieldName) {
+    if (!this._zoneMapsTrackedFields) return null;
+    const t = this._zoneMapsFieldToPos.get(fieldName);
+    if (t === undefined) return null;
+    if (shardIdx < 0 || shardIdx >= this._shardCount) return null;
+    const pos = shardIdx * this._zoneMapsT + t;
+    return { min: this._zoneMapsMins[pos], max: this._zoneMapsMaxes[pos] };
+  }
+
+  // Return the indices of shards whose bounds overlap [min, max]. Synchronous.
+  // When no zone maps are present or the field is not tracked, returns every
+  // shard (query planner must fall back to full scan).
+  findShards(fieldName, opts) {
+    const all = () => { const a = new Array(this._shardCount); for (let i = 0; i < this._shardCount; i++) a[i] = i; return a; };
+    if (!this._zoneMapsTrackedFields) return all();
+    const t = this._zoneMapsFieldToPos.get(fieldName);
+    if (t === undefined) return all();
+    const min = (opts && typeof opts.min === 'number') ? opts.min : Number.NEGATIVE_INFINITY;
+    const max = (opts && typeof opts.max === 'number') ? opts.max : Number.POSITIVE_INFINITY;
+    const T = this._zoneMapsT;
+    const out = [];
+    for (let s = 0; s < this._shardCount; s++) {
+      const smin = this._zoneMapsMins[s * T + t];
+      const smax = this._zoneMapsMaxes[s * T + t];
+      if (smin <= max && smax >= min) out.push(s);
+    }
+    return out;
+  }
+}