@zakkster/lite-bake-stream 1.0.0

package/src/Writer.js

@@ -0,0 +1,713 @@
+// @zakkster/lite-bake-stream / Writer
+// LBK1 shard emitter. Plugs into Tokenizer as a sink.
+// Copyright (c) 2026 Zahary Shinikchiev. MIT.
+//
+// Two modes:
+//   - Explicit schema: consumer passes { fields: [names...] } or
+//     { fields: [{name, laneKind}...] }; no sampling.
+//   - Sample-and-infer (default): first `sampleBytes` of input buffered in a
+//     columnar staging area, per-field lane kind inferred from observed
+//     value types (number->F64, string->U32-into-string-table). Fields that
+//     saw both raise W_MIXED_LANE_TYPES at freeze.
+//
+// v1 lane kinds: F64 (numeric) and U32 (string-table index). Per-shard string
+// tables ride immediately after each shard's payload; the shard directory
+// carries local_string_off/local_string_len so each shard is self-contained.
+//
+// Post-freeze allocation profile: ZERO per record on the numeric hot path.
+// String fields intern via the per-shard StringTable which allocates only on
+// unique inserts (bounded by cardinality, not row count).
+//
+// Error codes (stable):
+//   W_TOP_LEVEL_NOT_OBJECT     - top-level value is not an object
+//   W_NESTED_UNSUPPORTED       - nested object/array inside a record
+//   W_EMPTY_INPUT              - finalize() with zero records seen
+//   W_ROW_STRIDE_ZERO          - schema has zero fields (nothing to store)
+//   W_MIXED_LANE_TYPES         - a field saw both number and string values
+//   W_LANE_MISMATCH            - post-freeze: value type doesn't match schema lane
+//   W_UNKNOWN_LANE_KIND        - explicit schema declares an unknown laneKind
+
+import { StringTable } from './StringTable.js';
+
+export const VERSION = '1.0.0';
+
+const CONTAINER_HEADER_BYTES = 48;
+const SHARD_ENTRY_BYTES = 40;
+const FIELD_DESCRIPTOR_BYTES = 24;
+const FOOTER_BYTES = 16;
+
+const LANE_F64 = 1;
+const LANE_U32 = 3;
+const FIELD_FLAGS_NONE = 0;
+
+const SHARD_MIN_READER_VERSION = 1;
+const SHARD_FLAGS_NONE = 0;
+
+const FORMAT_VERSION = 1;
+const ENDIAN_LE = 1;
+
+const DEFAULT_SHARD_BYTES = 32 * 1024 * 1024;
+const DEFAULT_SAMPLE_ROWS_CAPACITY = 4096;
+
+// Kind tags for sample-time bookkeeping
+const K_UNKNOWN = 0;
+const K_NUMBER  = 1;
+const K_STRING  = 2;
+const K_MIXED   = 3;
+
+function laneKindBytes(k) { return k === LANE_F64 ? 8 : (k === LANE_U32 ? 4 : 0); }
+function normalizeLaneKindStr(s) {
+  if (typeof s !== 'string') return s;
+  const t = s.toLowerCase();
+  if (t === 'f64') return LANE_F64;
+  if (t === 'u32') return LANE_U32;
+  return -1;
+}
+
+export class WriterError extends Error {
+  constructor(code, msg) {
+    super(msg);
+    this.code = code;
+    this.name = 'WriterError';
+  }
+}
+
+function hashBytes(bytes, from, to) {
+  let h = 0x811c9dc5 | 0;
+  for (let i = from; i < to; i++) {
+    h = (h ^ bytes[i]) | 0;
+    h = Math.imul(h, 0x01000193) | 0;
+  }
+  return h >>> 0;
+}
+
+function bytesEqual(a, aFrom, aTo, b) {
+  const n = aTo - aFrom;
+  if (n !== b.length) return false;
+  for (let i = 0; i < n; i++) if (a[aFrom + i] !== b[i]) return false;
+  return true;
+}
+
+// SampleBuffer: columnar SoA during sample window. Each field independently
+// records observed kind (numeric or string). Numeric values -> F64 column;
+// string values -> U32 column, interned into a shared string table that is
+// handed to shard 0 at drain time.
+class SampleBuffer {
+  constructor(initialCapacity, stringTable) {
+    this._rowCount = 0;
+    this._capacity = initialCapacity;
+    this._fieldOrder = [];              // Array<string>
+    this._fieldMeta  = new Map();       // fieldName -> { kind, columnF64?, columnU32? }
+    this._byteEstimate = 0;
+    this._stringTable = stringTable;
+  }
+
+  setNumber(name, value) {
+    let meta = this._ensureField(name);
+    if (meta.kind === K_UNKNOWN) { meta.kind = K_NUMBER; meta.columnF64 = new Float64Array(this._capacity); }
+    else if (meta.kind === K_STRING) { meta.kind = K_MIXED; return; }
+    else if (meta.kind === K_MIXED)  { return; }
+    meta.columnF64[this._rowCount] = value;
+  }
+
+  setString(name, bytes, from, to) {
+    let meta = this._ensureField(name);
+    if (meta.kind === K_UNKNOWN) { meta.kind = K_STRING; meta.columnU32 = new Uint32Array(this._capacity); }
+    else if (meta.kind === K_NUMBER) { meta.kind = K_MIXED; return; }
+    else if (meta.kind === K_MIXED)  { return; }
+    const idx = this._stringTable.intern(bytes, from, to);
+    meta.columnU32[this._rowCount] = idx;
+  }
+
+  _ensureField(name) {
+    let meta = this._fieldMeta.get(name);
+    if (!meta) {
+      meta = { kind: K_UNKNOWN, columnF64: null, columnU32: null };
+      this._fieldMeta.set(name, meta);
+      this._fieldOrder.push(name);
+    }
+    return meta;
+  }
+
+  commitRow(inputBytesConsumed) {
+    this._rowCount++;
+    this._byteEstimate += inputBytesConsumed;
+    if (this._rowCount >= this._capacity) this._growCapacity();
+  }
+
+  _growCapacity() {
+    const newCap = this._capacity * 2;
+    for (const name of this._fieldOrder) {
+      const meta = this._fieldMeta.get(name);
+      if (meta.columnF64) {
+        const grown = new Float64Array(newCap); grown.set(meta.columnF64); meta.columnF64 = grown;
+      }
+      if (meta.columnU32) {
+        const grown = new Uint32Array(newCap); grown.set(meta.columnU32); meta.columnU32 = grown;
+      }
+    }
+    this._capacity = newCap;
+  }
+
+  get byteEstimate() { return this._byteEstimate; }
+  get rowCount()     { return this._rowCount; }
+  get fieldOrder()   { return this._fieldOrder; }
+  getFieldMeta(name) { return this._fieldMeta.get(name); }
+}
+
+export class Writer {
+  constructor(opts) {
+    opts = opts || {};
+    this._targetShardBytes = opts.targetShardBytes || DEFAULT_SHARD_BYTES;
+    this._sampleBytes = opts.sampleBytes || this._targetShardBytes;
+    this._explicitSchema = opts.schema || null;
+
+    // Frozen schema state
+    this._schema = null;                // { fields: [{name, laneKind, offsetInRow}], rowStride }
+    this._fieldNamesUtf8 = null;
+    this._fieldNameHashes = null;
+    this._fieldLaneKinds = null;        // Uint8Array — per-field LANE_* (fast dispatch)
+    this._fieldOffsets  = null;         // Uint16Array — per-field byte offset in row
+    this._rowValueSlotsF64 = null;      // Float64Array<fieldCount> scratch
+    this._rowValueSlotsU32 = null;      // Uint32Array<fieldCount> scratch
+
+    // Sample-window state
+    this._stringTable = new StringTable();
+    this._sample = new SampleBuffer(DEFAULT_SAMPLE_ROWS_CAPACITY, this._stringTable);
+
+    // Shard state
+    this._currentShardBuffer = null;
+    this._currentShardBytes = null;     // Uint8Array view over buffer
+    this._currentShardRowCount = 0;
+    this._currentShardMaxRows = 0;
+    this._shards = [];                  // { bytes, rowCount, stringTableBytes }
+    this._totalRows = 0;
+
+    // Per-record parse state
+    this._recordDepth = 0;
+    this._currentKeyBytes = null;
+    this._currentKeyFrom = 0;
+    this._currentKeyTo = 0;
+    this._currentFieldIdx = -1;
+    this._recordStartByteOffset = 0;
+    this._absOffset = 0;
+
+    this._finalized = false;
+
+    if (this._explicitSchema) this._freezeSchemaExplicit(this._explicitSchema.fields);
+  }
+
+  // -------- sink protocol --------
+
+  onStartObject() {
+    this._recordDepth++;
+    if (this._recordDepth > 1) throw new WriterError('W_NESTED_UNSUPPORTED', 'nested object inside record');
+    if (this._schema) {
+      // clear scratch slots
+      const f = this._rowValueSlotsF64, u = this._rowValueSlotsU32;
+      for (let i = 0; i < f.length; i++) { f[i] = 0; u[i] = 0; }
+    }
+    this._recordStartByteOffset = this._absOffset;
+  }
+
+  onEndObject() {
+    this._recordDepth--;
+    if (this._recordDepth === 0) this._commitRecord();
+  }
+
+  onStartArray() {
+    if (this._recordDepth === 0) throw new WriterError('W_TOP_LEVEL_NOT_OBJECT', 'top-level array (in-record) not supported');
+    throw new WriterError('W_NESTED_UNSUPPORTED', 'nested array inside record');
+  }
+
+  onEndArray() {}
+
+  onKey(bytes, from, to) {
+    if (this._schema) {
+      // Post-freeze: resolve to field index NOW; the bytes are ephemeral (SPEC 5.3).
+      this._currentFieldIdx = this._lookupFieldIdx(bytes, from, to);
+    } else {
+      // Sample window: decode to a JS string NOW. The tokenizer will reuse this
+      // buffer for the value bytes before onNumber/onString fires, so we cannot
+      // defer decoding. String allocation here is expected — bounded to the
+      // sample window; steady-state post-freeze remains zero-alloc.
+      this._currentKeyName = new TextDecoder().decode(bytes.subarray(from, to));
+    }
+  }
+
+  onNumber(v) {
+    if (this._recordDepth !== 1) return;
+    if (this._schema) {
+      const idx = this._currentFieldIdx;
+      if (idx < 0) return;
+      const lane = this._fieldLaneKinds[idx];
+      if (lane !== LANE_F64) throw new WriterError('W_LANE_MISMATCH',
+        'field ' + this._schema.fields[idx].name + ' is not an F64 lane but got a number');
+      this._rowValueSlotsF64[idx] = v;
+    } else {
+      if (!this._currentKeyName) return;
+      this._sample.setNumber(this._currentKeyName, v);
+    }
+  }
+
+  onTrue()  { this.onNumber(1); }
+  onFalse() { this.onNumber(0); }
+  onNull()  { this.onNumber(0); }
+
+  onString(bytes, from, to) {
+    if (this._recordDepth === 0) return;
+    if (this._schema) {
+      const idx = this._currentFieldIdx;
+      if (idx < 0) return;
+      const lane = this._fieldLaneKinds[idx];
+      if (lane !== LANE_U32) throw new WriterError('W_LANE_MISMATCH',
+        'field ' + this._schema.fields[idx].name + ' is not a U32 lane but got a string');
+      const strIdx = this._currentShardStringTable().intern(bytes, from, to);
+      this._rowValueSlotsU32[idx] = strIdx;
+    } else {
+      if (!this._currentKeyName) return;
+      this._sample.setString(this._currentKeyName, bytes, from, to);
+    }
+  }
+
+  onEnd() {}
+
+  // The current shard's string table. On the first record post-freeze we
+  // hand over the sample's string table (which already contains the drained
+  // records' strings). Subsequent shards get a fresh reset table.
+  _currentShardStringTable() {
+    if (!this._perShardStringTable) this._perShardStringTable = this._stringTable;
+    return this._perShardStringTable;
+  }
+
+  // -------- schema management --------
+
+  _freezeSchemaExplicit(rawFields) {
+    if (!rawFields || rawFields.length === 0) throw new WriterError('W_ROW_STRIDE_ZERO', 'schema has zero fields');
+    const fields = new Array(rawFields.length);
+    let offset = 0;
+    for (let i = 0; i < rawFields.length; i++) {
+      const rf = rawFields[i];
+      let name, laneKind;
+      if (typeof rf === 'string') { name = rf; laneKind = LANE_F64; }
+      else {
+        name = rf.name;
+        const asStr = normalizeLaneKindStr(rf.laneKind);
+        laneKind = typeof rf.laneKind === 'number' ? rf.laneKind : asStr;
+        if (laneKind !== LANE_F64 && laneKind !== LANE_U32)
+          throw new WriterError('W_UNKNOWN_LANE_KIND', 'field ' + name + ' laneKind=' + rf.laneKind);
+      }
+      fields[i] = { name, laneKind, offsetInRow: offset };
+      offset += laneKindBytes(laneKind);
+    }
+    // Pad row stride to 8 (per SPEC 3.5)
+    const rowStride = (offset + 7) & ~7;
+    this._finalizeSchema(fields, rowStride);
+  }
+
+  _freezeSchemaFromSample() {
+    const names = this._sample.fieldOrder;
+    if (names.length === 0) throw new WriterError('W_ROW_STRIDE_ZERO', 'sample discovered no fields');
+    const fields = new Array(names.length);
+    let offset = 0;
+    for (let i = 0; i < names.length; i++) {
+      const name = names[i];
+      const meta = this._sample.getFieldMeta(name);
+      let laneKind;
+      if (meta.kind === K_MIXED) throw new WriterError('W_MIXED_LANE_TYPES',
+        'field ' + name + ' had both numeric and string values in sample');
+      if (meta.kind === K_STRING) laneKind = LANE_U32;
+      else laneKind = LANE_F64; // K_NUMBER or K_UNKNOWN default
+      fields[i] = { name, laneKind, offsetInRow: offset };
+      offset += laneKindBytes(laneKind);
+    }
+    const rowStride = (offset + 7) & ~7;
+    this._finalizeSchema(fields, rowStride);
+  }
+
+  _finalizeSchema(fields, rowStride) {
+    this._schema = { fields, rowStride };
+
+    const enc = new TextEncoder();
+    this._fieldNamesUtf8 = new Array(fields.length);
+    this._fieldNameHashes = new Uint32Array(fields.length);
+    this._fieldLaneKinds = new Uint8Array(fields.length);
+    this._fieldOffsets = new Uint16Array(fields.length);
+    for (let i = 0; i < fields.length; i++) {
+      const b = enc.encode(fields[i].name);
+      this._fieldNamesUtf8[i] = b;
+      this._fieldNameHashes[i] = hashBytes(b, 0, b.length);
+      this._fieldLaneKinds[i] = fields[i].laneKind;
+      this._fieldOffsets[i] = fields[i].offsetInRow;
+    }
+    this._rowValueSlotsF64 = new Float64Array(fields.length);
+    this._rowValueSlotsU32 = new Uint32Array(fields.length);
+    this._currentShardMaxRows = Math.max(1, Math.floor(this._targetShardBytes / rowStride));
+
+    // Zone-map tracking (M7): every F64 field gets a min/max slot per shard.
+    // trackedFieldIndices[t] = schema field index; trackedFieldToPos[schemaFieldIdx]
+    // = t or -1. The row-commit hot path uses trackedFieldToPos for O(1) dispatch.
+    const tracked = [];
+    for (let i = 0; i < fields.length; i++) {
+      if (fields[i].laneKind === LANE_F64) tracked.push(i);
+    }
+    this._trackedFieldIndices = new Uint16Array(tracked);
+    this._trackedFieldToPos = new Int32Array(fields.length);
+    for (let i = 0; i < fields.length; i++) this._trackedFieldToPos[i] = -1;
+    for (let t = 0; t < tracked.length; t++) this._trackedFieldToPos[tracked[t]] = t;
+  }
+
+  _lookupFieldIdx(bytes, from, to) {
+    const h = hashBytes(bytes, from, to);
+    const hashes = this._fieldNameHashes;
+    const names = this._fieldNamesUtf8;
+    for (let i = 0; i < hashes.length; i++) {
+      if (hashes[i] === h && bytesEqual(bytes, from, to, names[i])) return i;
+    }
+    return -1;
+  }
+
+  // -------- record write path --------
+
+  _commitRecord() {
+    if (this._schema) {
+      if (!this._currentShardBuffer) this._allocateCurrentShard();
+      const row = this._currentShardRowCount;
+      const stride = this._schema.rowStride;
+      const rowOff = row * stride;
+      const dv = this._currentShardDv;
+      const kinds = this._fieldLaneKinds;
+      const offs = this._fieldOffsets;
+      const f = this._rowValueSlotsF64;
+      const u = this._rowValueSlotsU32;
+      const trackedIdx = this._trackedFieldToPos;    // Int32Array: schemaFieldIdx -> trackingPos or -1
+      const mins = this._currentShardMins;
+      const maxes = this._currentShardMaxes;
+      for (let i = 0; i < kinds.length; i++) {
+        const fo = rowOff + offs[i];
+        if (kinds[i] === LANE_F64) {
+          const v = f[i];
+          dv.setFloat64(fo, v, true);
+          const t = trackedIdx[i];
+          if (t >= 0) {
+            if (v < mins[t])  mins[t]  = v;
+            if (v > maxes[t]) maxes[t] = v;
+          }
+        } else {
+          dv.setUint32(fo, u[i], true);
+        }
+      }
+      this._currentShardRowCount++;
+      this._totalRows++;
+      if (this._currentShardRowCount >= this._currentShardMaxRows) this._finalizeCurrentShard();
+    } else {
+      const consumed = Math.max(1, this._absOffset - this._recordStartByteOffset);
+      this._sample.commitRow(consumed);
+      if (this._sample.byteEstimate >= this._sampleBytes) {
+        this._freezeSchemaFromSample();
+        this._drainSampleToShards();
+      }
+    }
+  }
+
+  _allocateCurrentShard() {
+    const stride = this._schema.rowStride;
+    const bytes = this._currentShardMaxRows * stride;
+    this._currentShardBuffer = new ArrayBuffer(bytes);
+    this._currentShardBytes = new Uint8Array(this._currentShardBuffer);
+    this._currentShardDv = new DataView(this._currentShardBuffer);
+    this._currentShardRowCount = 0;
+    // Zone-map tracking: per-F64-field min/max for this shard. Initialized to
+    // +Inf / -Inf sentinels; every row's write path updates them. Only F64
+    // fields participate (M7 scope); the trackedFieldIndices map converts a
+    // schema field index -> position in the tracking arrays.
+    const T = this._trackedFieldIndices.length;
+    this._currentShardMins  = new Float64Array(T);
+    this._currentShardMaxes = new Float64Array(T);
+    for (let i = 0; i < T; i++) {
+      this._currentShardMins[i]  = Number.POSITIVE_INFINITY;
+      this._currentShardMaxes[i] = Number.NEGATIVE_INFINITY;
+    }
+  }
+
+  _finalizeCurrentShard() {
+    if (this._currentShardRowCount === 0) return;
+    const usedBytes = this._currentShardRowCount * this._schema.rowStride;
+    const copy = new Uint8Array(usedBytes);
+    copy.set(this._currentShardBytes.subarray(0, usedBytes));
+    // Serialize the current shard's string table (empty for F64-only shards).
+    const st = this._perShardStringTable || this._stringTable;
+    const stSer = st.serialize();
+    // Snapshot zone-map bounds for this shard. Copy so subsequent shards can
+    // reuse _currentShardMins/Maxes without clobbering the pushed record.
+    const T = this._trackedFieldIndices.length;
+    const shardMins  = new Float64Array(T);
+    const shardMaxes = new Float64Array(T);
+    shardMins.set(this._currentShardMins);
+    shardMaxes.set(this._currentShardMaxes);
+    this._shards.push({
+      bytes: copy,
+      rowCount: this._currentShardRowCount,
+      stringTableBytes: stSer.bytes,
+      mins: shardMins,
+      maxes: shardMaxes,
+    });
+    this._currentShardBuffer = null;
+    this._currentShardBytes = null;
+    this._currentShardDv = null;
+    this._currentShardRowCount = 0;
+    // Reset string table for next shard (per-shard independence)
+    st.reset();
+    this._perShardStringTable = st;
+  }
+
+  _drainSampleToShards() {
+    const rowCount = this._sample.rowCount;
+    const fields = this._schema.fields;
+    const cols = new Array(fields.length);
+    for (let i = 0; i < fields.length; i++) {
+      const meta = this._sample.getFieldMeta(fields[i].name);
+      cols[i] = fields[i].laneKind === LANE_F64 ? (meta ? meta.columnF64 : null)
+                                               : (meta ? meta.columnU32 : null);
+    }
+
+    // The sample interned all strings into a SHARED table. When the sample
+    // overflows multiple output shards, each shard needs its own self-contained
+    // string table (SPEC 3.3 per-shard independence for HTTP Range fetches).
+    // Solution: re-intern per shard during drain. Look up the original bytes
+    // in the sample table, intern into a per-shard fresh table, remap the U32
+    // index. This preserves per-shard independence at the cost of a modest
+    // string-table copy for tags that appear across shards.
+    const sampleTable = this._stringTable;
+    this._perShardStringTable = new StringTable();
+
+    let rowsRemaining = rowCount;
+    let srcRow = 0;
+    while (rowsRemaining > 0) {
+      this._allocateCurrentShard();
+      const chunkRows = Math.min(rowsRemaining, this._currentShardMaxRows);
+      const dv = this._currentShardDv;
+      const stride = this._schema.rowStride;
+      const dstTable = this._perShardStringTable;
+      const trackedIdx = this._trackedFieldToPos;
+      const mins = this._currentShardMins;
+      const maxes = this._currentShardMaxes;
+      for (let r = 0; r < chunkRows; r++) {
+        const rowOff = r * stride;
+        for (let f = 0; f < fields.length; f++) {
+          const fo = rowOff + fields[f].offsetInRow;
+          const col = cols[f];
+          if (fields[f].laneKind === LANE_F64) {
+            const v = col ? col[srcRow + r] : 0;
+            dv.setFloat64(fo, v, true);
+            const t = trackedIdx[f];
+            if (t >= 0) {
+              if (v < mins[t])  mins[t]  = v;
+              if (v > maxes[t]) maxes[t] = v;
+            }
+          } else {
+            const sampleIdx = col ? col[srcRow + r] : 0;
+            const strBytes = sampleTable.bytesAt(sampleIdx);
+            const newIdx = strBytes ? dstTable.intern(strBytes, 0, strBytes.length) : 0;
+            dv.setUint32(fo, newIdx, true);
+          }
+        }
+      }
+      this._currentShardRowCount = chunkRows;
+      this._totalRows += chunkRows;
+      this._finalizeCurrentShard();     // serializes dstTable, resets in-place for the next iter
+      srcRow += chunkRows;
+      rowsRemaining -= chunkRows;
+    }
+
+    this._sample = null;
+    // _perShardStringTable now holds a reset (empty) StringTable ready to
+    // receive strings from post-drain records that fill subsequent shards.
+  }
+
+  finalize() {
+    if (this._finalized) return this._container;
+    if (!this._schema) {
+      if (this._sample.rowCount === 0) throw new WriterError('W_EMPTY_INPUT', 'no records to write');
+      this._freezeSchemaFromSample();
+      this._drainSampleToShards();
+    }
+    if (this._currentShardRowCount > 0) this._finalizeCurrentShard();
+    if (this._shards.length === 0) throw new WriterError('W_EMPTY_INPUT', 'no records to write');
+    this._container = this._assembleContainer();
+    this._finalized = true;
+    return this._container;
+  }
+
+  setInputByteOffset(n) { this._absOffset = n; }
+
+  // -------- container assembly --------
+
+  _assembleContainer() {
+    const enc = new TextEncoder();
+    const nameBytes = new Array(this._schema.fields.length);
+    let nameBlobLen = 0;
+    for (let i = 0; i < this._schema.fields.length; i++) {
+      nameBytes[i] = enc.encode(this._schema.fields[i].name);
+      nameBlobLen += nameBytes[i].length;
+    }
+
+    const fieldCount = this._schema.fields.length;
+    const descriptorBytes = fieldCount * FIELD_DESCRIPTOR_BYTES;
+    let schemaBlockBytes = 8 + descriptorBytes + 4 + nameBlobLen;
+    const schemaPad = (8 - (schemaBlockBytes & 7)) & 7;
+    schemaBlockBytes += schemaPad;
+
+    const shardCount = this._shards.length;
+    const shardDirBytes = shardCount * SHARD_ENTRY_BYTES;
+
+    const schemaBlockOff = CONTAINER_HEADER_BYTES;
+    const shardDirOff = schemaBlockOff + schemaBlockBytes;
+
+    // Zone maps segment (M7). Placed between shard directory and first shard
+    // so a Reader loading header/schema/dir/zoneMaps up front does it in one
+    // contiguous byte range.
+    const T = this._trackedFieldIndices.length;
+    const zoneMapsEnabled = T > 0 && shardCount > 0;
+    let zoneMapsOff = 0;
+    let zoneMapsBytes = 0;
+    let zoneMapsFieldTableOff = 0;
+    let zoneMapsMinsOff = 0;
+    let zoneMapsMaxesOff = 0;
+    if (zoneMapsEnabled) {
+      zoneMapsOff = shardDirOff + shardDirBytes;
+      // Layout: 16-byte header + T*u16 field indices, padded to 8, then
+      // shardCount*T*8 mins, then shardCount*T*8 maxes.
+      const headerLen = 16;
+      const fieldTableLen = T * 2;
+      const fieldTablePad = (8 - (fieldTableLen & 7)) & 7;
+      zoneMapsFieldTableOff = zoneMapsOff + headerLen;
+      const minsRel = headerLen + fieldTableLen + fieldTablePad;
+      zoneMapsMinsOff = zoneMapsOff + minsRel;
+      const minsMaxesLen = shardCount * T * 8 * 2;
+      zoneMapsMaxesOff = zoneMapsMinsOff + shardCount * T * 8;
+      zoneMapsBytes = minsRel + minsMaxesLen;
+    }
+    const firstShardOff = shardDirOff + shardDirBytes + zoneMapsBytes;
+
+    // Compute per-shard payload and string-table offsets
+    let cursor = firstShardOff;
+    const shardPayloadOffsets = new Array(shardCount);
+    const shardPayloadPadded = new Array(shardCount);
+    const shardStringTableOffsets = new Array(shardCount);
+    const shardStringTableLens = new Array(shardCount);
+    for (let i = 0; i < shardCount; i++) {
+      const s = this._shards[i];
+      shardPayloadOffsets[i] = cursor;
+      const payloadLen = s.bytes.length;
+      const payloadPad = (8 - (payloadLen & 7)) & 7;
+      shardPayloadPadded[i] = payloadLen + payloadPad;
+      cursor += shardPayloadPadded[i];
+      // String table immediately follows the shard payload
+      shardStringTableOffsets[i] = cursor;
+      const stLen = s.stringTableBytes.length;
+      shardStringTableLens[i] = stLen;
+      cursor += stLen;   // stringTable.serialize() already 8-byte padded
+    }
+
+    const totalBytes = cursor + FOOTER_BYTES;
+    const container = new ArrayBuffer(totalBytes);
+    const dv = new DataView(container);
+    const bytes = new Uint8Array(container);
+
+    // Header
+    bytes[0] = 0x4C; bytes[1] = 0x42; bytes[2] = 0x4B; bytes[3] = 0x31;
+    dv.setUint16(4, FORMAT_VERSION, true);
+    bytes[6] = ENDIAN_LE;
+    bytes[7] = 0;
+    dv.setBigUint64(8,  BigInt(schemaBlockOff), true);
+    dv.setBigUint64(16, BigInt(zoneMapsOff),    true);   // metadata_off — 0 iff no zone maps
+    dv.setBigUint64(24, BigInt(shardDirOff),    true);
+    dv.setUint32(32, shardCount, true);
+    dv.setUint32(36, 0, true);            // reserved1
+    dv.setBigUint64(40, BigInt(this._totalRows), true);
+
+    // Schema block
+    dv.setUint32(schemaBlockOff, fieldCount, true);
+    dv.setUint32(schemaBlockOff + 4, this._schema.rowStride, true);
+    let nameOff = 0;
+    for (let i = 0; i < fieldCount; i++) {
+      const descOff = schemaBlockOff + 8 + i * FIELD_DESCRIPTOR_BYTES;
+      const f = this._schema.fields[i];
+      const nb = nameBytes[i];
+      dv.setUint16(descOff + 0, nb.length, true);
+      dv.setUint16(descOff + 2, f.offsetInRow, true);
+      bytes[descOff + 4] = f.laneKind;
+      bytes[descOff + 5] = FIELD_FLAGS_NONE;
+      dv.setUint16(descOff + 6, 0, true);
+      dv.setBigUint64(descOff + 8, BigInt(nameOff), true);
+      dv.setBigUint64(descOff + 16, 0n, true);
+      nameOff += nb.length;
+    }
+    const nameBlobLenOff = schemaBlockOff + 8 + descriptorBytes;
+    dv.setUint32(nameBlobLenOff, nameBlobLen, true);
+    let namePosOff = nameBlobLenOff + 4;
+    for (let i = 0; i < fieldCount; i++) {
+      bytes.set(nameBytes[i], namePosOff);
+      namePosOff += nameBytes[i].length;
+    }
+
+    // Shard directory
+    for (let i = 0; i < shardCount; i++) {
+      const entryOff = shardDirOff + i * SHARD_ENTRY_BYTES;
+      const s = this._shards[i];
+      dv.setBigUint64(entryOff + 0,  BigInt(shardPayloadOffsets[i]), true);
+      dv.setUint32(entryOff + 8,  s.bytes.length, true);
+      dv.setUint32(entryOff + 12, s.rowCount, true);
+      dv.setUint16(entryOff + 16, SHARD_MIN_READER_VERSION, true);
+      dv.setUint16(entryOff + 18, SHARD_FLAGS_NONE, true);
+      dv.setUint32(entryOff + 20, 0, true);   // reserved
+      dv.setBigUint64(entryOff + 24, BigInt(shardStringTableOffsets[i]), true);
+      dv.setBigUint64(entryOff + 32, BigInt(shardStringTableLens[i]), true);
+    }
+
+    // Zone maps segment (M7)
+    if (zoneMapsEnabled) {
+      // Header: 'ZM01' magic + shard_count + tracked_field_count + reserved0
+      bytes[zoneMapsOff + 0] = 0x30; bytes[zoneMapsOff + 1] = 0x5A;
+      bytes[zoneMapsOff + 2] = 0x4D; bytes[zoneMapsOff + 3] = 0x31;
+      dv.setUint32(zoneMapsOff + 4, shardCount, true);
+      dv.setUint32(zoneMapsOff + 8, T, true);
+      dv.setUint32(zoneMapsOff + 12, 0, true);
+      // Field index table
+      for (let t = 0; t < T; t++) {
+        dv.setUint16(zoneMapsFieldTableOff + t * 2, this._trackedFieldIndices[t], true);
+      }
+      // Mins / maxes: row-major, [shard * T + tracked]
+      for (let s = 0; s < shardCount; s++) {
+        const sh = this._shards[s];
+        for (let t = 0; t < T; t++) {
+          dv.setFloat64(zoneMapsMinsOff  + (s * T + t) * 8, sh.mins[t],  true);
+          dv.setFloat64(zoneMapsMaxesOff + (s * T + t) * 8, sh.maxes[t], true);
+        }
+      }
+    }
+
+    // Payloads + local string tables
+    for (let i = 0; i < shardCount; i++) {
+      bytes.set(this._shards[i].bytes, shardPayloadOffsets[i]);
+      bytes.set(this._shards[i].stringTableBytes, shardStringTableOffsets[i]);
+    }
+
+    // Footer
+    const footerOff = totalBytes - FOOTER_BYTES;
+    dv.setUint32(footerOff + 0, 0xFFFFFFFF, true);
+    dv.setUint32(footerOff + 4, 0, true);
+    bytes[footerOff + 8]  = 0x31;
+    bytes[footerOff + 9]  = 0x4B;
+    bytes[footerOff + 10] = 0x42;
+    bytes[footerOff + 11] = 0x4C;
+    dv.setUint32(footerOff + 12, FOOTER_BYTES, true);
+
+    return { buffer: container, totalRows: this._totalRows, shardCount, schema: this._schema };
+  }
+
+  get schema()      { return this._schema; }
+  get totalRows()   { return this._totalRows; }
+  get shardCount()  { return this._shards.length + (this._currentShardRowCount > 0 ? 1 : 0); }
+}