@zakkster/lite-bake-stream 1.4.1 → 1.6.0

package/src/Writer.js

@@ -23,6 +23,21 @@
 // String fields intern via the per-shard StringTable which allocates only on
 // unique inserts (bounded by cardinality, not row count).
 //
+// Shard-byte budget (D2): a shard rolls when its row count reaches the maxRows
+// ceiling OR when payload_bytes + string_table_bytes reaches targetShardBytes.
+// The byte term is CHUNK-INDEPENDENT: it reads only _currentShardRowCount,
+// rowStride, _targetShardBytes and _stringTableBytes -- the first three are
+// chunk-invariant and _stringTableBytes is a pure function of the unique
+// interned byte ranges in arrival order (= logical record order, which the
+// Tokenizer delivers identically for any input chunking). No chunk-local state
+// (absOffset, buffer boundaries) enters, so re-chunked input yields byte-
+// identical containers (the t0 law). A single row or string that alone exceeds
+// targetShardBytes still gets its own oversized shard (maxRows has a
+// Math.max(1, ...) floor and the budget runs post-write); payload_len stays
+// u32-honest via the targetShardBytes constructor guard. A schema with no U32
+// lane keeps a zero string budget and rolls on rows only -- byte-identical to
+// the pre-D2 baseline.
+//
 // Error codes (stable):
 //   W_TOP_LEVEL_NOT_OBJECT     - top-level value is not an object
 //   W_NESTED_UNSUPPORTED       - nested object/array inside a record
@@ -42,8 +57,10 @@
 
 import { StringTable } from './StringTable.js';
 import { checkOpts } from './Opts.js';
+import { crc32cInit, crc32cUpdate, crc32cFinal, crc32cCombine } from './Crc32c.js';
+import { validateSink, isThenable } from './Views.js';
 
-export const VERSION = '1.4.1';
+export const VERSION = '1.6.0';
 
 const U32_MAX = 4294967295;
 // Post-finalize sentinel for _recordDepth. Chosen = 2 so every post-finalize
@@ -56,7 +73,13 @@ const WRITER_OPTS = {
   schema: { t: 'obj', nullable: true },
   targetShardBytes: { t: 'int', min: 1, max: U32_MAX },
   sampleBytes: { t: 'int', min: 0, max: U32_MAX },
+  crc: { t: 'bool' },
+};
+const FINALIZE_TO_SINK_OPTS = {
+  layout: { t: 'enum', values: ['prefix', 'stream'] },
+  crc: { t: 'bool' },
 };
+const CRC_ABSENT = 0xFFFFFFFF;
 function raiseWriter(code, msg) { throw new WriterError(code, msg); }
 
 const CONTAINER_HEADER_BYTES = 48;
@@ -76,6 +99,10 @@ const SHARD_FLAGS_NONE = 0;
 // U32-lane fields). Read-only; never written into.
 const EMPTY_STRING_TABLE_BYTES = new Uint8Array(0);
 
+// Shared read-only zero span for 8-byte shard-payload alignment padding in the
+// streaming path. Never written into.
+const ZERO_PAD = new Uint8Array(8);
+
 const FORMAT_VERSION = 1;
 const ENDIAN_LE = 1;
 
@@ -105,6 +132,12 @@ export class WriterError extends Error {
   }
 }
 
+// Shared decoder for sample-window key names (BS-27 floor). onKey fires once per
+// key per sample record; a fresh TextDecoder each time is pure garbage. One
+// module-scope instance, reused -- decode() is stateless across calls. The other
+// TextDecoders in the codebase are per-open/lazy and stay as they are.
+const KEY_DECODER = new TextDecoder();
+
 function hashBytes(bytes, from, to) {
   let h = 0x811c9dc5 | 0;
   for (let i = from; i < to; i++) {
@@ -204,6 +237,17 @@ export class Writer {
     this._targetShardBytes = opts.targetShardBytes !== undefined ? opts.targetShardBytes : DEFAULT_SHARD_BYTES;
     this._sampleBytes = opts.sampleBytes !== undefined ? opts.sampleBytes : this._targetShardBytes;
     this._explicitSchema = opts.schema !== undefined ? opts.schema : null;
+    this._crc = opts.crc === true;
+
+    // Streaming-emission state (M6). Null sink => classic buffered writer:
+    // shards accumulate in _shards until finalize()/finalizeToSink drains them.
+    // A bound sink (beginStream) switches _finalizeCurrentShard to emit each
+    // shard as it finalizes and retain only a scalar directory descriptor, so
+    // peak RAM stays O(targetShardBytes + directory) instead of O(container).
+    this._sink = null;
+    this._sinkCrcOn = false;
+    this._sinkPos = 0;        // current absolute write position in the sink
+    this._sinkCrc = 0;        // running int32 CRC of the body suffix [48, pos)
 
     // Frozen schema state
     this._schema = null;                // { fields: [{name, laneKind, offsetInRow}], rowStride }
@@ -226,6 +270,15 @@ export class Writer {
     this._shards = [];                  // { bytes, rowCount, stringTableBytes }
     this._totalRows = 0;
 
+    // D2 shard-byte budget (zero-alloc, chunk-independent). _stringTableBytes is
+    // the serialized byte length the CURRENT shard's string table would emit RIGHT
+    // NOW; _maxInternedIdx is the highest interned index seen this shard, so a new
+    // unique arrival (strIdx > _maxInternedIdx) is detected in one compare. Both
+    // recomputed only in the cold unique-string arm (bounded by cardinality, not
+    // rows) and reset at each shard finalize. Real values are set at freeze.
+    this._stringTableBytes = 0;
+    this._maxInternedIdx = 0;
+
     // Per-record parse state
     this._recordDepth = 0;
     this._currentKeyBytes = null;
@@ -285,7 +338,7 @@ export class Writer {
       // buffer for the value bytes before onNumber/onString fires, so we cannot
       // defer decoding. String allocation here is expected — bounded to the
       // sample window; steady-state post-freeze remains zero-alloc.
-      this._currentKeyName = new TextDecoder().decode(bytes.subarray(from, to));
+      this._currentKeyName = KEY_DECODER.decode(bytes.subarray(from, to));
     }
   }
 
@@ -343,8 +396,17 @@ export class Writer {
       const lane = this._fieldLaneKinds[idx];
       if (lane !== LANE_U32) throw new WriterError('W_LANE_MISMATCH',
         'field ' + this._schema.fields[idx].name + ' is not a U32 lane but got a string');
-      const strIdx = this._currentShardStringTable().intern(bytes, from, to);
+      const st = this._currentShardStringTable();
+      const strIdx = st.intern(bytes, from, to);
       this._rowValueSlotsU32[idx] = strIdx;
+      // D2 (cold arm, unique arrivals only): recompute the serialized string-table
+      // byte length. serialize() emits (8 + 4*(count+1) + blobLen), 8-padded, and
+      // count = strIdx + 1 for a fresh entry, so the padded form is
+      // (16 + 4*strIdx + blobLen) rounded up to 8 -- exact vs StringTable.serialize.
+      if (strIdx > this._maxInternedIdx) {
+        this._maxInternedIdx = strIdx;
+        this._stringTableBytes = (16 + 4 * strIdx + st.blobLen + 7) & ~7;
+      }
     } else {
       if (!this._currentKeyName) return;
       this._sample.setString(this._currentKeyName, bytes, from, to);
@@ -452,6 +514,19 @@ export class Writer {
       this._fieldLaneKinds[i] = fields[i].laneKind;
       this._fieldOffsets[i] = fields[i].offsetInRow;
     }
+    // R7 floor: O(1) field-name resolution. hash -> field index, or an array of
+    // indices when two names collide (Array.isArray in the cold arm). Built once
+    // at freeze; _lookupFieldIdx replaces its O(F) scan with one Map.get plus the
+    // mandatory byte-equal confirm, so a hash collision still resolves correctly.
+    const fieldHashMap = new Map();
+    for (let i = 0; i < fields.length; i++) {
+      const h = this._fieldNameHashes[i];
+      const existing = fieldHashMap.get(h);
+      if (existing === undefined) fieldHashMap.set(h, i);
+      else if (Array.isArray(existing)) existing.push(i);
+      else fieldHashMap.set(h, [existing, i]);
+    }
+    this._fieldHashMap = fieldHashMap;
     this._rowValueSlotsF64 = new Float64Array(fields.length);
     this._rowValueSlotsU32 = new Uint32Array(fields.length);
     this._currentShardMaxRows = Math.max(1, Math.floor(this._targetShardBytes / rowStride));
@@ -467,14 +542,27 @@ export class Writer {
     this._trackedFieldToPos = new Int32Array(fields.length);
     for (let i = 0; i < fields.length; i++) this._trackedFieldToPos[i] = -1;
     for (let t = 0; t < tracked.length; t++) this._trackedFieldToPos[tracked[t]] = t;
+
+    // D2 budget baseline. A fresh string table always carries reserved entry 0,
+    // so a U32-lane shard with no real strings still serializes to 16 bytes; a
+    // schema with no U32 lane emits NO table (0 bytes, EMPTY_STRING_TABLE_BYTES).
+    // Seeding _stringTableBytes to 0 for the F64-only case keeps such containers
+    // byte-identical to the pre-D2 baseline (the roll then fires on rows only via
+    // the maxRows term, which is <= the byte term for a zero string budget).
+    this._maxInternedIdx = 0;
+    this._stringTableBytes = this._hasU32 ? 16 : 0;
   }
 
   _lookupFieldIdx(bytes, from, to) {
     const h = hashBytes(bytes, from, to);
-    const hashes = this._fieldNameHashes;
+    const hit = this._fieldHashMap.get(h);
+    if (hit === undefined) return -1;
     const names = this._fieldNamesUtf8;
-    for (let i = 0; i < hashes.length; i++) {
-      if (hashes[i] === h && bytesEqual(bytes, from, to, names[i])) return i;
+    // Hit path (the common case): a single index. Byte-confirm and return.
+    if (typeof hit === 'number') return bytesEqual(bytes, from, to, names[hit]) ? hit : -1;
+    // Cold: >= 2 field names share this hash; byte-confirm each candidate.
+    for (let k = 0; k < hit.length; k++) {
+      if (bytesEqual(bytes, from, to, names[hit[k]])) return hit[k];
     }
     return -1;
   }
@@ -511,7 +599,15 @@ export class Writer {
       }
       this._currentShardRowCount++;
       this._totalRows++;
-      if (this._currentShardRowCount >= this._currentShardMaxRows) this._finalizeCurrentShard();
+      // D2 roll: fire on the row ceiling OR when payload + string-table bytes
+      // reach the target. The row-ceiling term fires first for any F64-only shard
+      // (string budget 0, maxRows = floor(target/stride) <= target/stride), so
+      // such containers stay byte-identical to baseline; string shards can roll
+      // earlier. Zero allocation -- one add + compare per row.
+      const n = this._currentShardRowCount;
+      if (n >= this._currentShardMaxRows ||
+          n * this._schema.rowStride + this._stringTableBytes >= this._targetShardBytes)
+        this._finalizeCurrentShard();
     } else {
       const endOff = this._source !== null ? this._source.absOffset : this._absOffset;
       const consumed = Math.max(1, endOff - this._recordStartByteOffset);
@@ -560,13 +656,25 @@ export class Writer {
     const shardMaxes = new Float64Array(T);
     shardMins.set(this._currentShardMins);
     shardMaxes.set(this._currentShardMaxes);
-    this._shards.push({
-      bytes: copy,
-      rowCount: this._currentShardRowCount,
-      stringTableBytes: stBytes,
-      mins: shardMins,
-      maxes: shardMaxes,
-    });
+    if (this._sink !== null) {
+      // Streaming mode: emit this shard to the bound sink now and retain only a
+      // scalar directory descriptor (payload/string offsets, row count, zone
+      // bounds). The heavy payload + string-table byte arrays are dropped, so
+      // peak RAM does not grow with shard count.
+      this._streamEmitShard(copy, stBytes, this._currentShardRowCount, shardMins, shardMaxes);
+    } else {
+      this._shards.push({
+        bytes: copy,
+        rowCount: this._currentShardRowCount,
+        stringTableBytes: stBytes,
+        // D2 audit: the incrementally-tracked budget at this finalize. MUST equal
+        // stBytes.length (the emitted directory local_string_len). Cross-checked by
+        // Ceilings.test.js via __stringTableBudgetAudit; not part of the public API.
+        budgetTracked: this._stringTableBytes,
+        mins: shardMins,
+        maxes: shardMaxes,
+      });
+    }
     this._currentShardBuffer = null;
     this._currentShardBytes = null;
     this._currentShardDv = null;
@@ -574,6 +682,9 @@ export class Writer {
     // Reset string table for next shard (per-shard independence)
     st.reset();
     this._perShardStringTable = st;
+    // Reset the D2 budget trackers alongside st.reset() (per-shard independence).
+    this._maxInternedIdx = 0;
+    this._stringTableBytes = this._hasU32 ? 16 : 0;
   }
 
   _drainSampleToShards() {
@@ -607,6 +718,11 @@ export class Writer {
       const trackedIdx = this._trackedFieldToPos;
       const mins = this._currentShardMins;
       const maxes = this._currentShardMaxes;
+      // D2: honour the same string-aware byte budget the post-freeze hot path
+      // uses, so a sample-drained container and a post-freeze one agree on shard
+      // boundaries for identical logical input. writtenRows may finish below
+      // chunkRows when the string table pushes the shard over target.
+      let writtenRows = chunkRows;
       for (let r = 0; r < chunkRows; r++) {
         const rowOff = r * stride;
         for (let f = 0; f < fields.length; f++) {
@@ -625,14 +741,24 @@ export class Writer {
             const strBytes = sampleTable.bytesAt(sampleIdx);
             const newIdx = strBytes ? dstTable.intern(strBytes, 0, strBytes.length) : 0;
             dv.setUint32(fo, newIdx, true);
+            if (newIdx > this._maxInternedIdx) {
+              this._maxInternedIdx = newIdx;
+              this._stringTableBytes = (16 + 4 * newIdx + dstTable.blobLen + 7) & ~7;
+            }
           }
         }
+        const done = r + 1;
+        if (done < chunkRows &&
+            done * stride + this._stringTableBytes >= this._targetShardBytes) {
+          writtenRows = done;
+          break;
+        }
       }
-      this._currentShardRowCount = chunkRows;
-      this._totalRows += chunkRows;
+      this._currentShardRowCount = writtenRows;
+      this._totalRows += writtenRows;
       this._finalizeCurrentShard();     // serializes dstTable, resets in-place for the next iter
-      srcRow += chunkRows;
-      rowsRemaining -= chunkRows;
+      srcRow += writtenRows;
+      rowsRemaining -= writtenRows;
     }
 
     this._sample = null;
@@ -640,7 +766,12 @@ export class Writer {
     // receive strings from post-drain records that fill subsequent shards.
   }
 
-  finalize() {
+  // Shared input-completion prologue for finalize() and finalizeToSink(): freeze
+  // an inferred schema, drain the sample, finalize the trailing partial shard,
+  // and fail closed on empty input. After this returns, _shards holds one entry
+  // per shard (a buffered {bytes,...} in the classic path, or a scalar directory
+  // descriptor when a sink was bound via beginStream).
+  _completeInput() {
     if (this._finalized) throw new WriterError('W_FINALIZED', 'writer already finalized');
     if (!this._schema) {
       if (this._sample.rowCount === 0) throw new WriterError('W_EMPTY_INPUT', 'no records to write');
@@ -649,13 +780,94 @@ export class Writer {
     }
     if (this._currentShardRowCount > 0) this._finalizeCurrentShard();
     if (this._shards.length === 0) throw new WriterError('W_EMPTY_INPUT', 'no records to write');
-    this._container = this._assembleContainer();
+  }
+
+  finalize() {
+    this._completeInput();
+    // finalize() IS layout:'prefix' over an implicit in-memory buffer: it builds
+    // the identical classic container the pre-M6 assembler produced (byte-for-
+    // byte when crc is off) and returns the frozen struct.
+    const bytes = this._buildPrefixBytes(this._crc);
     this._finalized = true;
     // Post-finalize sentinel: route every later sink event into a cold arm.
     this._recordDepth = FINALIZED_DEPTH;
+    this._container = {
+      buffer: bytes.buffer,
+      totalRows: this._totalRows,
+      shardCount: this._shards.length,
+      schema: this._schema,
+    };
     return this._container;
   }
 
+  // PUBLIC. Bind a sink and switch to streaming emission BEFORE the first record
+  // is fed (the two-step bounded-RAM contract): call beginStream(sink, opts),
+  // then feed the tokenizer -- each shard is written to the sink as it finalizes
+  // and its bytes are dropped, so peak RAM is O(targetShardBytes + directory) --
+  // then call finalizeToSink(sink, opts), which appends the schema/directory/
+  // zone-map trailer and footer and performs the single header backpatch. Calling
+  // finalizeToSink alone (without beginStream) is BUFFERED mode: correct, but
+  // peak RAM is O(container). opts.layout must be 'stream'; opts.crc (default:
+  // the constructor crc) toggles CRC-32C.
+  beginStream(sink, opts) {
+    if (this._finalized) throw new WriterError('W_FINALIZED', 'writer already finalized');
+    if (this._sink !== null) throw new WriterError('W_FINALIZED', 'writer is already streaming to a sink');
+    if (this._shards.length > 0 || this._currentShardRowCount > 0)
+      throw new WriterError('W_FINALIZED', 'beginStream must be called before the first record');
+    checkOpts('Writer.beginStream', opts, FINALIZE_TO_SINK_OPTS, raiseWriter);
+    opts = opts || {};
+    if (opts.layout !== undefined && opts.layout !== 'stream')
+      raiseWriter('W_BAD_SINK', "beginStream requires layout:'stream'");
+    validateSink(sink, true, raiseWriter);
+    this._sink = sink;
+    this._sinkCrcOn = opts.crc !== undefined ? opts.crc === true : this._crc;
+    this._sinkPos = 0;
+    this._sinkCrc = crc32cInit();
+    // Reserve the 48-byte header up front; it is backpatched by ONE writeAt at
+    // the end once shardCount and the trailer offsets are known. Excluded from
+    // the running suffix CRC (folded in via crc32cCombine at the end).
+    const placeholder = new Uint8Array(CONTAINER_HEADER_BYTES);
+    this._sinkWrite(placeholder, false);
+    this._sinkPos = CONTAINER_HEADER_BYTES;
+  }
+
+  // Write bytes to the bound sink, optionally folding them into the running body
+  // CRC. A thenable return means an async sink -- the contract is synchronous, so
+  // that is a malformed sink (W_BAD_SINK). NOTE: _sinkPos is advanced by callers.
+  _sinkWrite(bytes, fold) {
+    if (fold && this._sinkCrcOn) this._sinkCrc = crc32cUpdate(this._sinkCrc, bytes, 0, bytes.length);
+    let ret;
+    // A sink that throws mid-emission fails the writer closed and its own error
+    // is rethrown verbatim -- never laundered into a package code (the FileIngest
+    // mid-stream precedent). A retry then hits W_FINALIZED.
+    try { ret = this._sink.write(bytes); }
+    catch (e) { this._finalized = true; throw e; }
+    if (isThenable(ret)) { this._finalized = true; raiseWriter('W_BAD_SINK', 'sink.write returned a thenable; sinks must be synchronous'); }
+  }
+
+  // Streaming emit of one finalized shard: payload, 8-alignment pad, local string
+  // table -- each folded into the body CRC. Retains only a scalar descriptor.
+  _streamEmitShard(payloadBytes, stBytes, rowCount, mins, maxes) {
+    const payloadOff = this._sinkPos;
+    const payloadLen = payloadBytes.length;
+    this._sinkWrite(payloadBytes, true);
+    this._sinkPos += payloadLen;
+    const pad = (8 - (payloadLen & 7)) & 7;
+    if (pad > 0) { this._sinkWrite(ZERO_PAD.subarray(0, pad), true); this._sinkPos += pad; }
+    const stOff = this._sinkPos;
+    const stLen = stBytes.length;
+    if (stLen > 0) { this._sinkWrite(stBytes, true); this._sinkPos += stLen; }
+    this._shards.push({
+      rowCount,
+      payloadOff,
+      payloadLen,
+      stringOff: stLen > 0 ? stOff : 0,
+      stringLen: stLen,
+      mins,
+      maxes,
+    });
+  }
+
   // Optional capability hook (BS-07): the Tokenizer calls this at construction
   // to hand over the live tokenizer. The Writer then reads src.absOffset at
   // record boundaries so the sample window is byte-true. A CALL only; the
@@ -667,100 +879,62 @@ export class Writer {
   // dance is a documented no-op retained for source compatibility until 2.0.
   setInputByteOffset() {}
 
-  // -------- container assembly --------
-
-  _assembleContainer() {
+  // -------- container assembly (placement-free emitters) --------
+  //
+  // Both finalize() (layout:'prefix') and finalizeToSink(layout:'stream') drive
+  // the SAME byte-producing emitters below. The classic prefix layout builds one
+  // contiguous buffer; the streaming layout writes payloads first, accumulates a
+  // scalar directory, and emits the schema/directory/zone-map trailer + footer at
+  // the end with a single header backpatch. The emitters take an explicit target
+  // offset so they are reusable across both a full-container buffer and a small
+  // trailer buffer.
+
+  // Encode field names once and size the schema block (8-byte padded). Shared by
+  // every layout.
+  _computeSchemaBlock() {
     const enc = new TextEncoder();
-    const nameBytes = new Array(this._schema.fields.length);
+    const fields = this._schema.fields;
+    const nameBytes = new Array(fields.length);
     let nameBlobLen = 0;
-    for (let i = 0; i < this._schema.fields.length; i++) {
-      nameBytes[i] = enc.encode(this._schema.fields[i].name);
+    for (let i = 0; i < fields.length; i++) {
+      nameBytes[i] = enc.encode(fields[i].name);
       nameBlobLen += nameBytes[i].length;
     }
-
-    const fieldCount = this._schema.fields.length;
-    const descriptorBytes = fieldCount * FIELD_DESCRIPTOR_BYTES;
+    const descriptorBytes = fields.length * FIELD_DESCRIPTOR_BYTES;
     let schemaBlockBytes = 8 + descriptorBytes + 4 + nameBlobLen;
-    const schemaPad = (8 - (schemaBlockBytes & 7)) & 7;
-    schemaBlockBytes += schemaPad;
-
-    const shardCount = this._shards.length;
-    const shardDirBytes = shardCount * SHARD_ENTRY_BYTES;
-
-    const schemaBlockOff = CONTAINER_HEADER_BYTES;
-    const shardDirOff = schemaBlockOff + schemaBlockBytes;
+    schemaBlockBytes += (8 - (schemaBlockBytes & 7)) & 7;
+    return { nameBytes, nameBlobLen, descriptorBytes, schemaBlockBytes };
+  }
 
-    // Zone maps segment (M7). Placed between shard directory and first shard
-    // so a Reader loading header/schema/dir/zoneMaps up front does it in one
-    // contiguous byte range.
-    const T = this._trackedFieldIndices.length;
-    const zoneMapsEnabled = T > 0 && shardCount > 0;
-    let zoneMapsOff = 0;
-    let zoneMapsBytes = 0;
-    let zoneMapsFieldTableOff = 0;
-    let zoneMapsMinsOff = 0;
-    let zoneMapsMaxesOff = 0;
-    if (zoneMapsEnabled) {
-      zoneMapsOff = shardDirOff + shardDirBytes;
-      // Layout: 16-byte header + T*u16 field indices, padded to 8, then
-      // shardCount*T*8 mins, then shardCount*T*8 maxes.
-      const headerLen = 16;
-      const fieldTableLen = T * 2;
-      const fieldTablePad = (8 - (fieldTableLen & 7)) & 7;
-      zoneMapsFieldTableOff = zoneMapsOff + headerLen;
-      const minsRel = headerLen + fieldTableLen + fieldTablePad;
-      zoneMapsMinsOff = zoneMapsOff + minsRel;
-      const minsMaxesLen = shardCount * T * 8 * 2;
-      zoneMapsMaxesOff = zoneMapsMinsOff + shardCount * T * 8;
-      zoneMapsBytes = minsRel + minsMaxesLen;
-    }
-    const firstShardOff = shardDirOff + shardDirBytes + zoneMapsBytes;
+  _zoneMapsByteLen(T, shardCount) {
+    const fieldTableLen = T * 2;
+    const fieldTablePad = (8 - (fieldTableLen & 7)) & 7;
+    return 16 + fieldTableLen + fieldTablePad + shardCount * T * 8 * 2;
+  }
 
-    // Compute per-shard payload and string-table offsets
-    let cursor = firstShardOff;
-    const shardPayloadOffsets = new Array(shardCount);
-    const shardPayloadPadded = new Array(shardCount);
-    const shardStringTableOffsets = new Array(shardCount);
-    const shardStringTableLens = new Array(shardCount);
-    for (let i = 0; i < shardCount; i++) {
-      const s = this._shards[i];
-      shardPayloadOffsets[i] = cursor;
-      const payloadLen = s.bytes.length;
-      const payloadPad = (8 - (payloadLen & 7)) & 7;
-      shardPayloadPadded[i] = payloadLen + payloadPad;
-      cursor += shardPayloadPadded[i];
-      // String table immediately follows the shard payload
-      shardStringTableOffsets[i] = cursor;
-      const stLen = s.stringTableBytes.length;
-      shardStringTableLens[i] = stLen;
-      cursor += stLen;   // stringTable.serialize() already 8-byte padded
-    }
+  _emitHeaderInto(dv, bytes, off, h) {
+    bytes[off + 0] = 0x4C; bytes[off + 1] = 0x42; bytes[off + 2] = 0x4B; bytes[off + 3] = 0x31;
+    dv.setUint16(off + 4, FORMAT_VERSION, true);
+    bytes[off + 6] = ENDIAN_LE;
+    bytes[off + 7] = 0;
+    dv.setBigUint64(off + 8,  BigInt(h.schemaBlockOff), true);
+    dv.setBigUint64(off + 16, BigInt(h.metadataOff),    true);   // metadata_off -- 0 iff no zone maps
+    dv.setBigUint64(off + 24, BigInt(h.shardDirOff),    true);
+    dv.setUint32(off + 32, h.shardCount, true);
+    dv.setUint32(off + 36, 0, true);            // reserved1
+    dv.setBigUint64(off + 40, BigInt(h.totalRows), true);
+  }
 
-    const totalBytes = cursor + FOOTER_BYTES;
-    const container = new ArrayBuffer(totalBytes);
-    const dv = new DataView(container);
-    const bytes = new Uint8Array(container);
-
-    // Header
-    bytes[0] = 0x4C; bytes[1] = 0x42; bytes[2] = 0x4B; bytes[3] = 0x31;
-    dv.setUint16(4, FORMAT_VERSION, true);
-    bytes[6] = ENDIAN_LE;
-    bytes[7] = 0;
-    dv.setBigUint64(8,  BigInt(schemaBlockOff), true);
-    dv.setBigUint64(16, BigInt(zoneMapsOff),    true);   // metadata_off — 0 iff no zone maps
-    dv.setBigUint64(24, BigInt(shardDirOff),    true);
-    dv.setUint32(32, shardCount, true);
-    dv.setUint32(36, 0, true);            // reserved1
-    dv.setBigUint64(40, BigInt(this._totalRows), true);
-
-    // Schema block
-    dv.setUint32(schemaBlockOff, fieldCount, true);
-    dv.setUint32(schemaBlockOff + 4, this._schema.rowStride, true);
+  _emitSchemaInto(dv, bytes, off, sb) {
+    const fields = this._schema.fields;
+    const fieldCount = fields.length;
+    dv.setUint32(off, fieldCount, true);
+    dv.setUint32(off + 4, this._schema.rowStride, true);
     let nameOff = 0;
     for (let i = 0; i < fieldCount; i++) {
-      const descOff = schemaBlockOff + 8 + i * FIELD_DESCRIPTOR_BYTES;
-      const f = this._schema.fields[i];
-      const nb = nameBytes[i];
+      const descOff = off + 8 + i * FIELD_DESCRIPTOR_BYTES;
+      const f = fields[i];
+      const nb = sb.nameBytes[i];
       dv.setUint16(descOff + 0, nb.length, true);
       dv.setUint16(descOff + 2, f.offsetInRow, true);
       bytes[descOff + 4] = f.laneKind;
@@ -770,70 +944,243 @@ export class Writer {
       dv.setBigUint64(descOff + 16, 0n, true);
       nameOff += nb.length;
     }
-    const nameBlobLenOff = schemaBlockOff + 8 + descriptorBytes;
-    dv.setUint32(nameBlobLenOff, nameBlobLen, true);
+    const nameBlobLenOff = off + 8 + sb.descriptorBytes;
+    dv.setUint32(nameBlobLenOff, sb.nameBlobLen, true);
     let namePosOff = nameBlobLenOff + 4;
     for (let i = 0; i < fieldCount; i++) {
-      bytes.set(nameBytes[i], namePosOff);
-      namePosOff += nameBytes[i].length;
+      bytes.set(sb.nameBytes[i], namePosOff);
+      namePosOff += sb.nameBytes[i].length;
     }
+  }
 
-    // Shard directory
-    for (let i = 0; i < shardCount; i++) {
-      const entryOff = shardDirOff + i * SHARD_ENTRY_BYTES;
-      const s = this._shards[i];
-      dv.setBigUint64(entryOff + 0,  BigInt(shardPayloadOffsets[i]), true);
-      dv.setUint32(entryOff + 8,  s.bytes.length, true);
-      dv.setUint32(entryOff + 12, s.rowCount, true);
-      dv.setUint16(entryOff + 16, SHARD_MIN_READER_VERSION, true);
-      dv.setUint16(entryOff + 18, SHARD_FLAGS_NONE, true);
-      dv.setUint32(entryOff + 20, 0, true);   // reserved
-      dv.setBigUint64(entryOff + 24, BigInt(shardStringTableLens[i] > 0 ? shardStringTableOffsets[i] : 0), true);
-      dv.setBigUint64(entryOff + 32, BigInt(shardStringTableLens[i]), true);
-    }
+  _emitDirEntryInto(dv, entryOff, payloadOff, payloadLen, rowCount, stOff, stLen) {
+    dv.setBigUint64(entryOff + 0,  BigInt(payloadOff), true);
+    dv.setUint32(entryOff + 8,  payloadLen, true);
+    dv.setUint32(entryOff + 12, rowCount, true);
+    dv.setUint16(entryOff + 16, SHARD_MIN_READER_VERSION, true);
+    dv.setUint16(entryOff + 18, SHARD_FLAGS_NONE, true);
+    dv.setUint32(entryOff + 20, 0, true);   // reserved
+    dv.setBigUint64(entryOff + 24, BigInt(stLen > 0 ? stOff : 0), true);
+    dv.setBigUint64(entryOff + 32, BigInt(stLen), true);
+  }
 
-    // Zone maps segment (M7)
-    if (zoneMapsEnabled) {
-      // Header: 'ZM01' magic + shard_count + tracked_field_count + reserved0
-      bytes[zoneMapsOff + 0] = 0x30; bytes[zoneMapsOff + 1] = 0x5A;
-      bytes[zoneMapsOff + 2] = 0x4D; bytes[zoneMapsOff + 3] = 0x31;
-      dv.setUint32(zoneMapsOff + 4, shardCount, true);
-      dv.setUint32(zoneMapsOff + 8, T, true);
-      dv.setUint32(zoneMapsOff + 12, 0, true);
-      // Field index table
+  _emitZoneMapsInto(dv, bytes, off, T, shardCount, src) {
+    bytes[off + 0] = 0x30; bytes[off + 1] = 0x5A; bytes[off + 2] = 0x4D; bytes[off + 3] = 0x31;   // 'ZM01'
+    dv.setUint32(off + 4, shardCount, true);
+    dv.setUint32(off + 8, T, true);
+    dv.setUint32(off + 12, 0, true);
+    const fieldTableOff = off + 16;
+    for (let t = 0; t < T; t++) dv.setUint16(fieldTableOff + t * 2, this._trackedFieldIndices[t], true);
+    const fieldTableLen = T * 2;
+    const fieldTablePad = (8 - (fieldTableLen & 7)) & 7;
+    const minsOff  = fieldTableOff + fieldTableLen + fieldTablePad;
+    const maxesOff = minsOff + shardCount * T * 8;
+    for (let s = 0; s < shardCount; s++) {
+      const sh = src[s];
       for (let t = 0; t < T; t++) {
-        dv.setUint16(zoneMapsFieldTableOff + t * 2, this._trackedFieldIndices[t], true);
+        dv.setFloat64(minsOff  + (s * T + t) * 8, sh.mins[t],  true);
+        dv.setFloat64(maxesOff + (s * T + t) * 8, sh.maxes[t], true);
       }
-      // Mins / maxes: row-major, [shard * T + tracked]
-      for (let s = 0; s < shardCount; s++) {
-        const sh = this._shards[s];
-        for (let t = 0; t < T; t++) {
-          dv.setFloat64(zoneMapsMinsOff  + (s * T + t) * 8, sh.mins[t],  true);
-          dv.setFloat64(zoneMapsMaxesOff + (s * T + t) * 8, sh.maxes[t], true);
-        }
-      }
-    }
-
-    // Payloads + local string tables
-    for (let i = 0; i < shardCount; i++) {
-      bytes.set(this._shards[i].bytes, shardPayloadOffsets[i]);
-      bytes.set(this._shards[i].stringTableBytes, shardStringTableOffsets[i]);
     }
+  }
 
-    // Footer
-    const footerOff = totalBytes - FOOTER_BYTES;
-    dv.setUint32(footerOff + 0, 0xFFFFFFFF, true);
+  _emitFooterInto(dv, bytes, footerOff, crcVal) {
+    dv.setUint32(footerOff + 0, crcVal >>> 0, true);
     dv.setUint32(footerOff + 4, 0, true);
     bytes[footerOff + 8]  = 0x31;
     bytes[footerOff + 9]  = 0x4B;
     bytes[footerOff + 10] = 0x42;
     bytes[footerOff + 11] = 0x4C;
     dv.setUint32(footerOff + 12, FOOTER_BYTES, true);
+  }
 
-    return { buffer: container, totalRows: this._totalRows, shardCount, schema: this._schema };
+  // Classic prefix layout: header | schema | directory | zone maps | payloads |
+  // footer, in ONE contiguous buffer. Byte-for-byte identical to the pre-M6
+  // assembler when crc is off; the optional CRC-32C covers [0, footer_off).
+  _buildPrefixBytes(crcOn) {
+    const sb = this._computeSchemaBlock();
+    const shardCount = this._shards.length;
+    const shardDirBytes = shardCount * SHARD_ENTRY_BYTES;
+    const schemaBlockOff = CONTAINER_HEADER_BYTES;
+    const shardDirOff = schemaBlockOff + sb.schemaBlockBytes;
+    const T = this._trackedFieldIndices.length;
+    const zoneMapsEnabled = T > 0 && shardCount > 0;
+    let zoneMapsOff = 0, zoneMapsBytes = 0;
+    if (zoneMapsEnabled) {
+      zoneMapsOff = shardDirOff + shardDirBytes;
+      zoneMapsBytes = this._zoneMapsByteLen(T, shardCount);
+    }
+    const firstShardOff = shardDirOff + shardDirBytes + zoneMapsBytes;
+    let cursor = firstShardOff;
+    const payloadOffs = new Array(shardCount);
+    const stOffs = new Array(shardCount);
+    for (let i = 0; i < shardCount; i++) {
+      const s = this._shards[i];
+      payloadOffs[i] = cursor;
+      const payloadLen = s.bytes.length;
+      const payloadPad = (8 - (payloadLen & 7)) & 7;
+      cursor += payloadLen + payloadPad;
+      stOffs[i] = cursor;
+      cursor += s.stringTableBytes.length;   // stringTable.serialize() is already 8-padded
+    }
+    const footerOff = cursor;
+    const totalBytes = cursor + FOOTER_BYTES;
+    const buffer = new ArrayBuffer(totalBytes);
+    const dv = new DataView(buffer);
+    const bytes = new Uint8Array(buffer);
+
+    this._emitHeaderInto(dv, bytes, 0, {
+      schemaBlockOff, metadataOff: zoneMapsEnabled ? zoneMapsOff : 0,
+      shardDirOff, shardCount, totalRows: this._totalRows,
+    });
+    this._emitSchemaInto(dv, bytes, schemaBlockOff, sb);
+    for (let i = 0; i < shardCount; i++) {
+      const s = this._shards[i];
+      this._emitDirEntryInto(dv, shardDirOff + i * SHARD_ENTRY_BYTES,
+        payloadOffs[i], s.bytes.length, s.rowCount, stOffs[i], s.stringTableBytes.length);
+    }
+    if (zoneMapsEnabled) this._emitZoneMapsInto(dv, bytes, zoneMapsOff, T, shardCount, this._shards);
+    for (let i = 0; i < shardCount; i++) {
+      bytes.set(this._shards[i].bytes, payloadOffs[i]);
+      bytes.set(this._shards[i].stringTableBytes, stOffs[i]);
+    }
+    let crcVal = CRC_ABSENT;
+    if (crcOn) crcVal = crc32cFinal(crc32cUpdate(crc32cInit(), bytes, 0, footerOff));
+    this._emitFooterInto(dv, bytes, footerOff, crcVal);
+    return bytes;
+  }
+
+  // PUBLIC. Emit the container to a caller sink instead of returning a buffer.
+  //   layout:'prefix' -- the classic layout, written in one sink.write().
+  //   layout:'stream' -- payloads stream first, then the trailer + footer, with
+  //                      one sink.writeAt(header, 0) backpatch (needs writeAt).
+  // Returns { totalRows, shardCount, schema|mode, bytesWritten, layout }. When
+  // the writer was pre-bound to this sink via beginStream (bounded-RAM ingest),
+  // the shards are already emitted and this only writes the trailer.
+  finalizeToSink(sink, opts) {
+    if (this._finalized) throw new WriterError('W_FINALIZED', 'writer already finalized');
+    checkOpts('Writer.finalizeToSink', opts, FINALIZE_TO_SINK_OPTS, raiseWriter);
+    opts = opts || {};
+
+    if (this._sink !== null) {
+      // Pre-bound streaming: shards were emitted during ingest. Finish the input
+      // (streams the trailing shard), then write the trailer.
+      if (sink !== this._sink) raiseWriter('W_BAD_SINK', 'finalizeToSink sink differs from the streaming sink');
+      if (opts.layout !== undefined && opts.layout !== 'stream')
+        raiseWriter('W_BAD_SINK', "a streaming writer must be finalized with layout:'stream'");
+      const crcOn = opts.crc !== undefined ? opts.crc === true : this._sinkCrcOn;
+      this._sinkCrcOn = crcOn;
+      this._completeInput();
+      return this._finishStream(sink, crcOn);
+    }
+
+    const layout = opts.layout !== undefined ? opts.layout : 'stream';
+    const crcOn = opts.crc !== undefined ? opts.crc === true : this._crc;
+    validateSink(sink, layout === 'stream', raiseWriter);
+    this._completeInput();
+
+    if (layout === 'prefix') {
+      const bytes = this._buildPrefixBytes(crcOn);
+      let ret;
+      try { ret = sink.write(bytes); }
+      catch (e) { this._finalized = true; throw e; }
+      if (isThenable(ret)) { this._finalized = true; raiseWriter('W_BAD_SINK', 'sink.write returned a thenable; sinks must be synchronous'); }
+      this._finalized = true;
+      this._recordDepth = FINALIZED_DEPTH;
+      return { totalRows: this._totalRows, shardCount: this._shards.length, schema: this._schema, bytesWritten: bytes.length, layout: 'prefix' };
+    }
+
+    // layout 'stream', buffered shards: stream them out now, dropping each.
+    const buffered = this._shards;
+    this._shards = [];
+    this._sink = sink;
+    this._sinkCrcOn = crcOn;
+    this._sinkPos = 0;
+    this._sinkCrc = crc32cInit();
+    this._sinkWrite(new Uint8Array(CONTAINER_HEADER_BYTES), false);
+    this._sinkPos = CONTAINER_HEADER_BYTES;
+    for (let i = 0; i < buffered.length; i++) {
+      const s = buffered[i];
+      this._streamEmitShard(s.bytes, s.stringTableBytes, s.rowCount, s.mins, s.maxes);
+      buffered[i] = null;
+    }
+    return this._finishStream(sink, crcOn);
+  }
+
+  // Write the schema/directory/zone-map trailer + footer and backpatch the header.
+  // _shards holds one scalar descriptor per emitted shard; _sinkPos/_sinkCrc are
+  // the current sink position and running body CRC over [48, pos).
+  _finishStream(sink, crcOn) {
+    const shardCount = this._shards.length;
+    const sb = this._computeSchemaBlock();
+    const schemaBlockOff = this._sinkPos;   // 8-aligned: every payload region is 8-padded
+    const shardDirBytes = shardCount * SHARD_ENTRY_BYTES;
+    const shardDirOff = schemaBlockOff + sb.schemaBlockBytes;
+    const T = this._trackedFieldIndices.length;
+    const zoneMapsEnabled = T > 0 && shardCount > 0;
+    let zoneMapsOff = 0, zoneMapsBytes = 0;
+    if (zoneMapsEnabled) {
+      zoneMapsOff = shardDirOff + shardDirBytes;
+      zoneMapsBytes = this._zoneMapsByteLen(T, shardCount);
+    }
+    const trailerLen = sb.schemaBlockBytes + shardDirBytes + zoneMapsBytes;
+    const footerOff = schemaBlockOff + trailerLen;
+
+    const trailer = new Uint8Array(trailerLen);
+    const tdv = new DataView(trailer.buffer);
+    this._emitSchemaInto(tdv, trailer, 0, sb);
+    for (let i = 0; i < shardCount; i++) {
+      const d = this._shards[i];
+      this._emitDirEntryInto(tdv, sb.schemaBlockBytes + i * SHARD_ENTRY_BYTES,
+        d.payloadOff, d.payloadLen, d.rowCount, d.stringOff, d.stringLen);
+    }
+    if (zoneMapsEnabled) this._emitZoneMapsInto(tdv, trailer, sb.schemaBlockBytes + shardDirBytes, T, shardCount, this._shards);
+    this._sinkWrite(trailer, true);
+    this._sinkPos += trailerLen;
+
+    const header = new Uint8Array(CONTAINER_HEADER_BYTES);
+    const hdv = new DataView(header.buffer);
+    this._emitHeaderInto(hdv, header, 0, {
+      schemaBlockOff, metadataOff: zoneMapsEnabled ? zoneMapsOff : 0,
+      shardDirOff, shardCount, totalRows: this._totalRows,
+    });
+
+    let crcVal = CRC_ABSENT;
+    if (crcOn) {
+      const headerCrc = crc32cFinal(crc32cUpdate(crc32cInit(), header, 0, CONTAINER_HEADER_BYTES));
+      const suffixCrc = crc32cFinal(this._sinkCrc);
+      crcVal = crc32cCombine(headerCrc, suffixCrc, footerOff - CONTAINER_HEADER_BYTES);
+    }
+    const footer = new Uint8Array(FOOTER_BYTES);
+    const fdv = new DataView(footer.buffer);
+    this._emitFooterInto(fdv, footer, 0, crcVal);
+    this._sinkWrite(footer, false);
+    this._sinkPos += FOOTER_BYTES;
+
+    let ret;
+    try { ret = sink.writeAt(header, 0); }
+    catch (e) { this._finalized = true; throw e; }
+    if (isThenable(ret)) { this._finalized = true; raiseWriter('W_BAD_SINK', 'sink.writeAt returned a thenable; sinks must be synchronous'); }
+
+    this._finalized = true;
+    this._recordDepth = FINALIZED_DEPTH;
+    return { totalRows: this._totalRows, shardCount, schema: this._schema, bytesWritten: this._sinkPos, layout: 'stream' };
   }
 
   get schema()      { return this._schema; }
   get totalRows()   { return this._totalRows; }
   get shardCount()  { return this._shards.length + (this._currentShardRowCount > 0 ? 1 : 0); }
+
+  // @internal TEST-ONLY (D2): per-shard [{ tracked, emitted }] string-table byte
+  // lengths. tracked is the incrementally-maintained budget at each finalize;
+  // emitted is the serialized local_string_len that reached the container. They
+  // MUST be equal at every shard -- Ceilings.test.js proves it, closing the
+  // formula-drift bug class. No semver guarantee; absent from the public API.
+  __stringTableBudgetAudit() {
+    const out = new Array(this._shards.length);
+    for (let i = 0; i < this._shards.length; i++) {
+      out[i] = { tracked: this._shards[i].budgetTracked, emitted: this._shards[i].stringTableBytes.length };
+    }
+    return out;
+  }
 }