@zakkster/lite-bake-stream 1.5.0 → 1.6.0

package/src/Writer.js

@@ -57,8 +57,10 @@
 
 import { StringTable } from './StringTable.js';
 import { checkOpts } from './Opts.js';
+import { crc32cInit, crc32cUpdate, crc32cFinal, crc32cCombine } from './Crc32c.js';
+import { validateSink, isThenable } from './Views.js';
 
-export const VERSION = '1.5.0';
+export const VERSION = '1.6.0';
 
 const U32_MAX = 4294967295;
 // Post-finalize sentinel for _recordDepth. Chosen = 2 so every post-finalize
@@ -71,7 +73,13 @@ const WRITER_OPTS = {
   schema: { t: 'obj', nullable: true },
   targetShardBytes: { t: 'int', min: 1, max: U32_MAX },
   sampleBytes: { t: 'int', min: 0, max: U32_MAX },
+  crc: { t: 'bool' },
 };
+const FINALIZE_TO_SINK_OPTS = {
+  layout: { t: 'enum', values: ['prefix', 'stream'] },
+  crc: { t: 'bool' },
+};
+const CRC_ABSENT = 0xFFFFFFFF;
 function raiseWriter(code, msg) { throw new WriterError(code, msg); }
 
 const CONTAINER_HEADER_BYTES = 48;
@@ -91,6 +99,10 @@ const SHARD_FLAGS_NONE = 0;
 // U32-lane fields). Read-only; never written into.
 const EMPTY_STRING_TABLE_BYTES = new Uint8Array(0);
 
+// Shared read-only zero span for 8-byte shard-payload alignment padding in the
+// streaming path. Never written into.
+const ZERO_PAD = new Uint8Array(8);
+
 const FORMAT_VERSION = 1;
 const ENDIAN_LE = 1;
 
@@ -225,6 +237,17 @@ export class Writer {
     this._targetShardBytes = opts.targetShardBytes !== undefined ? opts.targetShardBytes : DEFAULT_SHARD_BYTES;
     this._sampleBytes = opts.sampleBytes !== undefined ? opts.sampleBytes : this._targetShardBytes;
     this._explicitSchema = opts.schema !== undefined ? opts.schema : null;
+    this._crc = opts.crc === true;
+
+    // Streaming-emission state (M6). Null sink => classic buffered writer:
+    // shards accumulate in _shards until finalize()/finalizeToSink drains them.
+    // A bound sink (beginStream) switches _finalizeCurrentShard to emit each
+    // shard as it finalizes and retain only a scalar directory descriptor, so
+    // peak RAM stays O(targetShardBytes + directory) instead of O(container).
+    this._sink = null;
+    this._sinkCrcOn = false;
+    this._sinkPos = 0;        // current absolute write position in the sink
+    this._sinkCrc = 0;        // running int32 CRC of the body suffix [48, pos)
 
     // Frozen schema state
     this._schema = null;                // { fields: [{name, laneKind, offsetInRow}], rowStride }
@@ -633,17 +656,25 @@ export class Writer {
     const shardMaxes = new Float64Array(T);
     shardMins.set(this._currentShardMins);
     shardMaxes.set(this._currentShardMaxes);
-    this._shards.push({
-      bytes: copy,
-      rowCount: this._currentShardRowCount,
-      stringTableBytes: stBytes,
-      // D2 audit: the incrementally-tracked budget at this finalize. MUST equal
-      // stBytes.length (the emitted directory local_string_len). Cross-checked by
-      // Ceilings.test.js via __stringTableBudgetAudit; not part of the public API.
-      budgetTracked: this._stringTableBytes,
-      mins: shardMins,
-      maxes: shardMaxes,
-    });
+    if (this._sink !== null) {
+      // Streaming mode: emit this shard to the bound sink now and retain only a
+      // scalar directory descriptor (payload/string offsets, row count, zone
+      // bounds). The heavy payload + string-table byte arrays are dropped, so
+      // peak RAM does not grow with shard count.
+      this._streamEmitShard(copy, stBytes, this._currentShardRowCount, shardMins, shardMaxes);
+    } else {
+      this._shards.push({
+        bytes: copy,
+        rowCount: this._currentShardRowCount,
+        stringTableBytes: stBytes,
+        // D2 audit: the incrementally-tracked budget at this finalize. MUST equal
+        // stBytes.length (the emitted directory local_string_len). Cross-checked by
+        // Ceilings.test.js via __stringTableBudgetAudit; not part of the public API.
+        budgetTracked: this._stringTableBytes,
+        mins: shardMins,
+        maxes: shardMaxes,
+      });
+    }
     this._currentShardBuffer = null;
     this._currentShardBytes = null;
     this._currentShardDv = null;
@@ -735,7 +766,12 @@ export class Writer {
     // receive strings from post-drain records that fill subsequent shards.
   }
 
-  finalize() {
+  // Shared input-completion prologue for finalize() and finalizeToSink(): freeze
+  // an inferred schema, drain the sample, finalize the trailing partial shard,
+  // and fail closed on empty input. After this returns, _shards holds one entry
+  // per shard (a buffered {bytes,...} in the classic path, or a scalar directory
+  // descriptor when a sink was bound via beginStream).
+  _completeInput() {
     if (this._finalized) throw new WriterError('W_FINALIZED', 'writer already finalized');
     if (!this._schema) {
       if (this._sample.rowCount === 0) throw new WriterError('W_EMPTY_INPUT', 'no records to write');
@@ -744,13 +780,94 @@ export class Writer {
     }
     if (this._currentShardRowCount > 0) this._finalizeCurrentShard();
     if (this._shards.length === 0) throw new WriterError('W_EMPTY_INPUT', 'no records to write');
-    this._container = this._assembleContainer();
+  }
+
+  finalize() {
+    this._completeInput();
+    // finalize() IS layout:'prefix' over an implicit in-memory buffer: it builds
+    // the identical classic container the pre-M6 assembler produced (byte-for-
+    // byte when crc is off) and returns the frozen struct.
+    const bytes = this._buildPrefixBytes(this._crc);
     this._finalized = true;
     // Post-finalize sentinel: route every later sink event into a cold arm.
     this._recordDepth = FINALIZED_DEPTH;
+    this._container = {
+      buffer: bytes.buffer,
+      totalRows: this._totalRows,
+      shardCount: this._shards.length,
+      schema: this._schema,
+    };
     return this._container;
   }
 
+  // PUBLIC. Bind a sink and switch to streaming emission BEFORE the first record
+  // is fed (the two-step bounded-RAM contract): call beginStream(sink, opts),
+  // then feed the tokenizer -- each shard is written to the sink as it finalizes
+  // and its bytes are dropped, so peak RAM is O(targetShardBytes + directory) --
+  // then call finalizeToSink(sink, opts), which appends the schema/directory/
+  // zone-map trailer and footer and performs the single header backpatch. Calling
+  // finalizeToSink alone (without beginStream) is BUFFERED mode: correct, but
+  // peak RAM is O(container). opts.layout must be 'stream'; opts.crc (default:
+  // the constructor crc) toggles CRC-32C.
+  beginStream(sink, opts) {
+    if (this._finalized) throw new WriterError('W_FINALIZED', 'writer already finalized');
+    if (this._sink !== null) throw new WriterError('W_FINALIZED', 'writer is already streaming to a sink');
+    if (this._shards.length > 0 || this._currentShardRowCount > 0)
+      throw new WriterError('W_FINALIZED', 'beginStream must be called before the first record');
+    checkOpts('Writer.beginStream', opts, FINALIZE_TO_SINK_OPTS, raiseWriter);
+    opts = opts || {};
+    if (opts.layout !== undefined && opts.layout !== 'stream')
+      raiseWriter('W_BAD_SINK', "beginStream requires layout:'stream'");
+    validateSink(sink, true, raiseWriter);
+    this._sink = sink;
+    this._sinkCrcOn = opts.crc !== undefined ? opts.crc === true : this._crc;
+    this._sinkPos = 0;
+    this._sinkCrc = crc32cInit();
+    // Reserve the 48-byte header up front; it is backpatched by ONE writeAt at
+    // the end once shardCount and the trailer offsets are known. Excluded from
+    // the running suffix CRC (folded in via crc32cCombine at the end).
+    const placeholder = new Uint8Array(CONTAINER_HEADER_BYTES);
+    this._sinkWrite(placeholder, false);
+    this._sinkPos = CONTAINER_HEADER_BYTES;
+  }
+
+  // Write bytes to the bound sink, optionally folding them into the running body
+  // CRC. A thenable return means an async sink -- the contract is synchronous, so
+  // that is a malformed sink (W_BAD_SINK). NOTE: _sinkPos is advanced by callers.
+  _sinkWrite(bytes, fold) {
+    if (fold && this._sinkCrcOn) this._sinkCrc = crc32cUpdate(this._sinkCrc, bytes, 0, bytes.length);
+    let ret;
+    // A sink that throws mid-emission fails the writer closed and its own error
+    // is rethrown verbatim -- never laundered into a package code (the FileIngest
+    // mid-stream precedent). A retry then hits W_FINALIZED.
+    try { ret = this._sink.write(bytes); }
+    catch (e) { this._finalized = true; throw e; }
+    if (isThenable(ret)) { this._finalized = true; raiseWriter('W_BAD_SINK', 'sink.write returned a thenable; sinks must be synchronous'); }
+  }
+
+  // Streaming emit of one finalized shard: payload, 8-alignment pad, local string
+  // table -- each folded into the body CRC. Retains only a scalar descriptor.
+  _streamEmitShard(payloadBytes, stBytes, rowCount, mins, maxes) {
+    const payloadOff = this._sinkPos;
+    const payloadLen = payloadBytes.length;
+    this._sinkWrite(payloadBytes, true);
+    this._sinkPos += payloadLen;
+    const pad = (8 - (payloadLen & 7)) & 7;
+    if (pad > 0) { this._sinkWrite(ZERO_PAD.subarray(0, pad), true); this._sinkPos += pad; }
+    const stOff = this._sinkPos;
+    const stLen = stBytes.length;
+    if (stLen > 0) { this._sinkWrite(stBytes, true); this._sinkPos += stLen; }
+    this._shards.push({
+      rowCount,
+      payloadOff,
+      payloadLen,
+      stringOff: stLen > 0 ? stOff : 0,
+      stringLen: stLen,
+      mins,
+      maxes,
+    });
+  }
+
   // Optional capability hook (BS-07): the Tokenizer calls this at construction
   // to hand over the live tokenizer. The Writer then reads src.absOffset at
   // record boundaries so the sample window is byte-true. A CALL only; the
@@ -762,100 +879,62 @@ export class Writer {
   // dance is a documented no-op retained for source compatibility until 2.0.
   setInputByteOffset() {}
 
-  // -------- container assembly --------
-
-  _assembleContainer() {
+  // -------- container assembly (placement-free emitters) --------
+  //
+  // Both finalize() (layout:'prefix') and finalizeToSink(layout:'stream') drive
+  // the SAME byte-producing emitters below. The classic prefix layout builds one
+  // contiguous buffer; the streaming layout writes payloads first, accumulates a
+  // scalar directory, and emits the schema/directory/zone-map trailer + footer at
+  // the end with a single header backpatch. The emitters take an explicit target
+  // offset so they are reusable across both a full-container buffer and a small
+  // trailer buffer.
+
+  // Encode field names once and size the schema block (8-byte padded). Shared by
+  // every layout.
+  _computeSchemaBlock() {
     const enc = new TextEncoder();
-    const nameBytes = new Array(this._schema.fields.length);
+    const fields = this._schema.fields;
+    const nameBytes = new Array(fields.length);
     let nameBlobLen = 0;
-    for (let i = 0; i < this._schema.fields.length; i++) {
-      nameBytes[i] = enc.encode(this._schema.fields[i].name);
+    for (let i = 0; i < fields.length; i++) {
+      nameBytes[i] = enc.encode(fields[i].name);
       nameBlobLen += nameBytes[i].length;
     }
-
-    const fieldCount = this._schema.fields.length;
-    const descriptorBytes = fieldCount * FIELD_DESCRIPTOR_BYTES;
+    const descriptorBytes = fields.length * FIELD_DESCRIPTOR_BYTES;
     let schemaBlockBytes = 8 + descriptorBytes + 4 + nameBlobLen;
-    const schemaPad = (8 - (schemaBlockBytes & 7)) & 7;
-    schemaBlockBytes += schemaPad;
-
-    const shardCount = this._shards.length;
-    const shardDirBytes = shardCount * SHARD_ENTRY_BYTES;
-
-    const schemaBlockOff = CONTAINER_HEADER_BYTES;
-    const shardDirOff = schemaBlockOff + schemaBlockBytes;
+    schemaBlockBytes += (8 - (schemaBlockBytes & 7)) & 7;
+    return { nameBytes, nameBlobLen, descriptorBytes, schemaBlockBytes };
+  }
 
-    // Zone maps segment (M7). Placed between shard directory and first shard
-    // so a Reader loading header/schema/dir/zoneMaps up front does it in one
-    // contiguous byte range.
-    const T = this._trackedFieldIndices.length;
-    const zoneMapsEnabled = T > 0 && shardCount > 0;
-    let zoneMapsOff = 0;
-    let zoneMapsBytes = 0;
-    let zoneMapsFieldTableOff = 0;
-    let zoneMapsMinsOff = 0;
-    let zoneMapsMaxesOff = 0;
-    if (zoneMapsEnabled) {
-      zoneMapsOff = shardDirOff + shardDirBytes;
-      // Layout: 16-byte header + T*u16 field indices, padded to 8, then
-      // shardCount*T*8 mins, then shardCount*T*8 maxes.
-      const headerLen = 16;
-      const fieldTableLen = T * 2;
-      const fieldTablePad = (8 - (fieldTableLen & 7)) & 7;
-      zoneMapsFieldTableOff = zoneMapsOff + headerLen;
-      const minsRel = headerLen + fieldTableLen + fieldTablePad;
-      zoneMapsMinsOff = zoneMapsOff + minsRel;
-      const minsMaxesLen = shardCount * T * 8 * 2;
-      zoneMapsMaxesOff = zoneMapsMinsOff + shardCount * T * 8;
-      zoneMapsBytes = minsRel + minsMaxesLen;
-    }
-    const firstShardOff = shardDirOff + shardDirBytes + zoneMapsBytes;
+  _zoneMapsByteLen(T, shardCount) {
+    const fieldTableLen = T * 2;
+    const fieldTablePad = (8 - (fieldTableLen & 7)) & 7;
+    return 16 + fieldTableLen + fieldTablePad + shardCount * T * 8 * 2;
+  }
 
-    // Compute per-shard payload and string-table offsets
-    let cursor = firstShardOff;
-    const shardPayloadOffsets = new Array(shardCount);
-    const shardPayloadPadded = new Array(shardCount);
-    const shardStringTableOffsets = new Array(shardCount);
-    const shardStringTableLens = new Array(shardCount);
-    for (let i = 0; i < shardCount; i++) {
-      const s = this._shards[i];
-      shardPayloadOffsets[i] = cursor;
-      const payloadLen = s.bytes.length;
-      const payloadPad = (8 - (payloadLen & 7)) & 7;
-      shardPayloadPadded[i] = payloadLen + payloadPad;
-      cursor += shardPayloadPadded[i];
-      // String table immediately follows the shard payload
-      shardStringTableOffsets[i] = cursor;
-      const stLen = s.stringTableBytes.length;
-      shardStringTableLens[i] = stLen;
-      cursor += stLen;   // stringTable.serialize() already 8-byte padded
-    }
+  _emitHeaderInto(dv, bytes, off, h) {
+    bytes[off + 0] = 0x4C; bytes[off + 1] = 0x42; bytes[off + 2] = 0x4B; bytes[off + 3] = 0x31;
+    dv.setUint16(off + 4, FORMAT_VERSION, true);
+    bytes[off + 6] = ENDIAN_LE;
+    bytes[off + 7] = 0;
+    dv.setBigUint64(off + 8,  BigInt(h.schemaBlockOff), true);
+    dv.setBigUint64(off + 16, BigInt(h.metadataOff),    true);   // metadata_off -- 0 iff no zone maps
+    dv.setBigUint64(off + 24, BigInt(h.shardDirOff),    true);
+    dv.setUint32(off + 32, h.shardCount, true);
+    dv.setUint32(off + 36, 0, true);            // reserved1
+    dv.setBigUint64(off + 40, BigInt(h.totalRows), true);
+  }
 
-    const totalBytes = cursor + FOOTER_BYTES;
-    const container = new ArrayBuffer(totalBytes);
-    const dv = new DataView(container);
-    const bytes = new Uint8Array(container);
-
-    // Header
-    bytes[0] = 0x4C; bytes[1] = 0x42; bytes[2] = 0x4B; bytes[3] = 0x31;
-    dv.setUint16(4, FORMAT_VERSION, true);
-    bytes[6] = ENDIAN_LE;
-    bytes[7] = 0;
-    dv.setBigUint64(8,  BigInt(schemaBlockOff), true);
-    dv.setBigUint64(16, BigInt(zoneMapsOff),    true);   // metadata_off — 0 iff no zone maps
-    dv.setBigUint64(24, BigInt(shardDirOff),    true);
-    dv.setUint32(32, shardCount, true);
-    dv.setUint32(36, 0, true);            // reserved1
-    dv.setBigUint64(40, BigInt(this._totalRows), true);
-
-    // Schema block
-    dv.setUint32(schemaBlockOff, fieldCount, true);
-    dv.setUint32(schemaBlockOff + 4, this._schema.rowStride, true);
+  _emitSchemaInto(dv, bytes, off, sb) {
+    const fields = this._schema.fields;
+    const fieldCount = fields.length;
+    dv.setUint32(off, fieldCount, true);
+    dv.setUint32(off + 4, this._schema.rowStride, true);
     let nameOff = 0;
     for (let i = 0; i < fieldCount; i++) {
-      const descOff = schemaBlockOff + 8 + i * FIELD_DESCRIPTOR_BYTES;
-      const f = this._schema.fields[i];
-      const nb = nameBytes[i];
+      const descOff = off + 8 + i * FIELD_DESCRIPTOR_BYTES;
+      const f = fields[i];
+      const nb = sb.nameBytes[i];
       dv.setUint16(descOff + 0, nb.length, true);
       dv.setUint16(descOff + 2, f.offsetInRow, true);
       bytes[descOff + 4] = f.laneKind;
@@ -865,67 +944,227 @@ export class Writer {
       dv.setBigUint64(descOff + 16, 0n, true);
       nameOff += nb.length;
     }
-    const nameBlobLenOff = schemaBlockOff + 8 + descriptorBytes;
-    dv.setUint32(nameBlobLenOff, nameBlobLen, true);
+    const nameBlobLenOff = off + 8 + sb.descriptorBytes;
+    dv.setUint32(nameBlobLenOff, sb.nameBlobLen, true);
     let namePosOff = nameBlobLenOff + 4;
     for (let i = 0; i < fieldCount; i++) {
-      bytes.set(nameBytes[i], namePosOff);
-      namePosOff += nameBytes[i].length;
+      bytes.set(sb.nameBytes[i], namePosOff);
+      namePosOff += sb.nameBytes[i].length;
     }
+  }
 
-    // Shard directory
-    for (let i = 0; i < shardCount; i++) {
-      const entryOff = shardDirOff + i * SHARD_ENTRY_BYTES;
-      const s = this._shards[i];
-      dv.setBigUint64(entryOff + 0,  BigInt(shardPayloadOffsets[i]), true);
-      dv.setUint32(entryOff + 8,  s.bytes.length, true);
-      dv.setUint32(entryOff + 12, s.rowCount, true);
-      dv.setUint16(entryOff + 16, SHARD_MIN_READER_VERSION, true);
-      dv.setUint16(entryOff + 18, SHARD_FLAGS_NONE, true);
-      dv.setUint32(entryOff + 20, 0, true);   // reserved
-      dv.setBigUint64(entryOff + 24, BigInt(shardStringTableLens[i] > 0 ? shardStringTableOffsets[i] : 0), true);
-      dv.setBigUint64(entryOff + 32, BigInt(shardStringTableLens[i]), true);
-    }
+  _emitDirEntryInto(dv, entryOff, payloadOff, payloadLen, rowCount, stOff, stLen) {
+    dv.setBigUint64(entryOff + 0,  BigInt(payloadOff), true);
+    dv.setUint32(entryOff + 8,  payloadLen, true);
+    dv.setUint32(entryOff + 12, rowCount, true);
+    dv.setUint16(entryOff + 16, SHARD_MIN_READER_VERSION, true);
+    dv.setUint16(entryOff + 18, SHARD_FLAGS_NONE, true);
+    dv.setUint32(entryOff + 20, 0, true);   // reserved
+    dv.setBigUint64(entryOff + 24, BigInt(stLen > 0 ? stOff : 0), true);
+    dv.setBigUint64(entryOff + 32, BigInt(stLen), true);
+  }
 
-    // Zone maps segment (M7)
-    if (zoneMapsEnabled) {
-      // Header: 'ZM01' magic + shard_count + tracked_field_count + reserved0
-      bytes[zoneMapsOff + 0] = 0x30; bytes[zoneMapsOff + 1] = 0x5A;
-      bytes[zoneMapsOff + 2] = 0x4D; bytes[zoneMapsOff + 3] = 0x31;
-      dv.setUint32(zoneMapsOff + 4, shardCount, true);
-      dv.setUint32(zoneMapsOff + 8, T, true);
-      dv.setUint32(zoneMapsOff + 12, 0, true);
-      // Field index table
+  _emitZoneMapsInto(dv, bytes, off, T, shardCount, src) {
+    bytes[off + 0] = 0x30; bytes[off + 1] = 0x5A; bytes[off + 2] = 0x4D; bytes[off + 3] = 0x31;   // 'ZM01'
+    dv.setUint32(off + 4, shardCount, true);
+    dv.setUint32(off + 8, T, true);
+    dv.setUint32(off + 12, 0, true);
+    const fieldTableOff = off + 16;
+    for (let t = 0; t < T; t++) dv.setUint16(fieldTableOff + t * 2, this._trackedFieldIndices[t], true);
+    const fieldTableLen = T * 2;
+    const fieldTablePad = (8 - (fieldTableLen & 7)) & 7;
+    const minsOff  = fieldTableOff + fieldTableLen + fieldTablePad;
+    const maxesOff = minsOff + shardCount * T * 8;
+    for (let s = 0; s < shardCount; s++) {
+      const sh = src[s];
       for (let t = 0; t < T; t++) {
-        dv.setUint16(zoneMapsFieldTableOff + t * 2, this._trackedFieldIndices[t], true);
-      }
-      // Mins / maxes: row-major, [shard * T + tracked]
-      for (let s = 0; s < shardCount; s++) {
-        const sh = this._shards[s];
-        for (let t = 0; t < T; t++) {
-          dv.setFloat64(zoneMapsMinsOff  + (s * T + t) * 8, sh.mins[t],  true);
-          dv.setFloat64(zoneMapsMaxesOff + (s * T + t) * 8, sh.maxes[t], true);
-        }
+        dv.setFloat64(minsOff  + (s * T + t) * 8, sh.mins[t],  true);
+        dv.setFloat64(maxesOff + (s * T + t) * 8, sh.maxes[t], true);
       }
     }
+  }
 
-    // Payloads + local string tables
-    for (let i = 0; i < shardCount; i++) {
-      bytes.set(this._shards[i].bytes, shardPayloadOffsets[i]);
-      bytes.set(this._shards[i].stringTableBytes, shardStringTableOffsets[i]);
-    }
-
-    // Footer
-    const footerOff = totalBytes - FOOTER_BYTES;
-    dv.setUint32(footerOff + 0, 0xFFFFFFFF, true);
+  _emitFooterInto(dv, bytes, footerOff, crcVal) {
+    dv.setUint32(footerOff + 0, crcVal >>> 0, true);
     dv.setUint32(footerOff + 4, 0, true);
     bytes[footerOff + 8]  = 0x31;
     bytes[footerOff + 9]  = 0x4B;
     bytes[footerOff + 10] = 0x42;
     bytes[footerOff + 11] = 0x4C;
     dv.setUint32(footerOff + 12, FOOTER_BYTES, true);
+  }
+
+  // Classic prefix layout: header | schema | directory | zone maps | payloads |
+  // footer, in ONE contiguous buffer. Byte-for-byte identical to the pre-M6
+  // assembler when crc is off; the optional CRC-32C covers [0, footer_off).
+  _buildPrefixBytes(crcOn) {
+    const sb = this._computeSchemaBlock();
+    const shardCount = this._shards.length;
+    const shardDirBytes = shardCount * SHARD_ENTRY_BYTES;
+    const schemaBlockOff = CONTAINER_HEADER_BYTES;
+    const shardDirOff = schemaBlockOff + sb.schemaBlockBytes;
+    const T = this._trackedFieldIndices.length;
+    const zoneMapsEnabled = T > 0 && shardCount > 0;
+    let zoneMapsOff = 0, zoneMapsBytes = 0;
+    if (zoneMapsEnabled) {
+      zoneMapsOff = shardDirOff + shardDirBytes;
+      zoneMapsBytes = this._zoneMapsByteLen(T, shardCount);
+    }
+    const firstShardOff = shardDirOff + shardDirBytes + zoneMapsBytes;
+    let cursor = firstShardOff;
+    const payloadOffs = new Array(shardCount);
+    const stOffs = new Array(shardCount);
+    for (let i = 0; i < shardCount; i++) {
+      const s = this._shards[i];
+      payloadOffs[i] = cursor;
+      const payloadLen = s.bytes.length;
+      const payloadPad = (8 - (payloadLen & 7)) & 7;
+      cursor += payloadLen + payloadPad;
+      stOffs[i] = cursor;
+      cursor += s.stringTableBytes.length;   // stringTable.serialize() is already 8-padded
+    }
+    const footerOff = cursor;
+    const totalBytes = cursor + FOOTER_BYTES;
+    const buffer = new ArrayBuffer(totalBytes);
+    const dv = new DataView(buffer);
+    const bytes = new Uint8Array(buffer);
+
+    this._emitHeaderInto(dv, bytes, 0, {
+      schemaBlockOff, metadataOff: zoneMapsEnabled ? zoneMapsOff : 0,
+      shardDirOff, shardCount, totalRows: this._totalRows,
+    });
+    this._emitSchemaInto(dv, bytes, schemaBlockOff, sb);
+    for (let i = 0; i < shardCount; i++) {
+      const s = this._shards[i];
+      this._emitDirEntryInto(dv, shardDirOff + i * SHARD_ENTRY_BYTES,
+        payloadOffs[i], s.bytes.length, s.rowCount, stOffs[i], s.stringTableBytes.length);
+    }
+    if (zoneMapsEnabled) this._emitZoneMapsInto(dv, bytes, zoneMapsOff, T, shardCount, this._shards);
+    for (let i = 0; i < shardCount; i++) {
+      bytes.set(this._shards[i].bytes, payloadOffs[i]);
+      bytes.set(this._shards[i].stringTableBytes, stOffs[i]);
+    }
+    let crcVal = CRC_ABSENT;
+    if (crcOn) crcVal = crc32cFinal(crc32cUpdate(crc32cInit(), bytes, 0, footerOff));
+    this._emitFooterInto(dv, bytes, footerOff, crcVal);
+    return bytes;
+  }
 
-    return { buffer: container, totalRows: this._totalRows, shardCount, schema: this._schema };
+  // PUBLIC. Emit the container to a caller sink instead of returning a buffer.
+  //   layout:'prefix' -- the classic layout, written in one sink.write().
+  //   layout:'stream' -- payloads stream first, then the trailer + footer, with
+  //                      one sink.writeAt(header, 0) backpatch (needs writeAt).
+  // Returns { totalRows, shardCount, schema|mode, bytesWritten, layout }. When
+  // the writer was pre-bound to this sink via beginStream (bounded-RAM ingest),
+  // the shards are already emitted and this only writes the trailer.
+  finalizeToSink(sink, opts) {
+    if (this._finalized) throw new WriterError('W_FINALIZED', 'writer already finalized');
+    checkOpts('Writer.finalizeToSink', opts, FINALIZE_TO_SINK_OPTS, raiseWriter);
+    opts = opts || {};
+
+    if (this._sink !== null) {
+      // Pre-bound streaming: shards were emitted during ingest. Finish the input
+      // (streams the trailing shard), then write the trailer.
+      if (sink !== this._sink) raiseWriter('W_BAD_SINK', 'finalizeToSink sink differs from the streaming sink');
+      if (opts.layout !== undefined && opts.layout !== 'stream')
+        raiseWriter('W_BAD_SINK', "a streaming writer must be finalized with layout:'stream'");
+      const crcOn = opts.crc !== undefined ? opts.crc === true : this._sinkCrcOn;
+      this._sinkCrcOn = crcOn;
+      this._completeInput();
+      return this._finishStream(sink, crcOn);
+    }
+
+    const layout = opts.layout !== undefined ? opts.layout : 'stream';
+    const crcOn = opts.crc !== undefined ? opts.crc === true : this._crc;
+    validateSink(sink, layout === 'stream', raiseWriter);
+    this._completeInput();
+
+    if (layout === 'prefix') {
+      const bytes = this._buildPrefixBytes(crcOn);
+      let ret;
+      try { ret = sink.write(bytes); }
+      catch (e) { this._finalized = true; throw e; }
+      if (isThenable(ret)) { this._finalized = true; raiseWriter('W_BAD_SINK', 'sink.write returned a thenable; sinks must be synchronous'); }
+      this._finalized = true;
+      this._recordDepth = FINALIZED_DEPTH;
+      return { totalRows: this._totalRows, shardCount: this._shards.length, schema: this._schema, bytesWritten: bytes.length, layout: 'prefix' };
+    }
+
+    // layout 'stream', buffered shards: stream them out now, dropping each.
+    const buffered = this._shards;
+    this._shards = [];
+    this._sink = sink;
+    this._sinkCrcOn = crcOn;
+    this._sinkPos = 0;
+    this._sinkCrc = crc32cInit();
+    this._sinkWrite(new Uint8Array(CONTAINER_HEADER_BYTES), false);
+    this._sinkPos = CONTAINER_HEADER_BYTES;
+    for (let i = 0; i < buffered.length; i++) {
+      const s = buffered[i];
+      this._streamEmitShard(s.bytes, s.stringTableBytes, s.rowCount, s.mins, s.maxes);
+      buffered[i] = null;
+    }
+    return this._finishStream(sink, crcOn);
+  }
+
+  // Write the schema/directory/zone-map trailer + footer and backpatch the header.
+  // _shards holds one scalar descriptor per emitted shard; _sinkPos/_sinkCrc are
+  // the current sink position and running body CRC over [48, pos).
+  _finishStream(sink, crcOn) {
+    const shardCount = this._shards.length;
+    const sb = this._computeSchemaBlock();
+    const schemaBlockOff = this._sinkPos;   // 8-aligned: every payload region is 8-padded
+    const shardDirBytes = shardCount * SHARD_ENTRY_BYTES;
+    const shardDirOff = schemaBlockOff + sb.schemaBlockBytes;
+    const T = this._trackedFieldIndices.length;
+    const zoneMapsEnabled = T > 0 && shardCount > 0;
+    let zoneMapsOff = 0, zoneMapsBytes = 0;
+    if (zoneMapsEnabled) {
+      zoneMapsOff = shardDirOff + shardDirBytes;
+      zoneMapsBytes = this._zoneMapsByteLen(T, shardCount);
+    }
+    const trailerLen = sb.schemaBlockBytes + shardDirBytes + zoneMapsBytes;
+    const footerOff = schemaBlockOff + trailerLen;
+
+    const trailer = new Uint8Array(trailerLen);
+    const tdv = new DataView(trailer.buffer);
+    this._emitSchemaInto(tdv, trailer, 0, sb);
+    for (let i = 0; i < shardCount; i++) {
+      const d = this._shards[i];
+      this._emitDirEntryInto(tdv, sb.schemaBlockBytes + i * SHARD_ENTRY_BYTES,
+        d.payloadOff, d.payloadLen, d.rowCount, d.stringOff, d.stringLen);
+    }
+    if (zoneMapsEnabled) this._emitZoneMapsInto(tdv, trailer, sb.schemaBlockBytes + shardDirBytes, T, shardCount, this._shards);
+    this._sinkWrite(trailer, true);
+    this._sinkPos += trailerLen;
+
+    const header = new Uint8Array(CONTAINER_HEADER_BYTES);
+    const hdv = new DataView(header.buffer);
+    this._emitHeaderInto(hdv, header, 0, {
+      schemaBlockOff, metadataOff: zoneMapsEnabled ? zoneMapsOff : 0,
+      shardDirOff, shardCount, totalRows: this._totalRows,
+    });
+
+    let crcVal = CRC_ABSENT;
+    if (crcOn) {
+      const headerCrc = crc32cFinal(crc32cUpdate(crc32cInit(), header, 0, CONTAINER_HEADER_BYTES));
+      const suffixCrc = crc32cFinal(this._sinkCrc);
+      crcVal = crc32cCombine(headerCrc, suffixCrc, footerOff - CONTAINER_HEADER_BYTES);
+    }
+    const footer = new Uint8Array(FOOTER_BYTES);
+    const fdv = new DataView(footer.buffer);
+    this._emitFooterInto(fdv, footer, 0, crcVal);
+    this._sinkWrite(footer, false);
+    this._sinkPos += FOOTER_BYTES;
+
+    let ret;
+    try { ret = sink.writeAt(header, 0); }
+    catch (e) { this._finalized = true; throw e; }
+    if (isThenable(ret)) { this._finalized = true; raiseWriter('W_BAD_SINK', 'sink.writeAt returned a thenable; sinks must be synchronous'); }
+
+    this._finalized = true;
+    this._recordDepth = FINALIZED_DEPTH;
+    return { totalRows: this._totalRows, shardCount, schema: this._schema, bytesWritten: this._sinkPos, layout: 'stream' };
   }
 
   get schema()      { return this._schema; }

package/src/index.js

@@ -28,7 +28,7 @@ export {
   PreserveWriter, PreserveWriterError,
   PreserveReader, PreserveReaderError,
 };
-export const VERSION = '1.5.0';
+export const VERSION = '1.6.0';
 
 const encoder = new TextEncoder();
 
@@ -188,12 +188,12 @@ async function _serializeAsyncIterablePreserve(iterable, opts) {
 
 // Deserialize LBK1 bytes into the right Reader for the container's mode.
 // Auto-detects preserve vs schema via header flag byte at offset 7 bit 0.
-export function deserialize(bytes) {
+export function deserialize(bytes, opts) {
   const buffer = toContainerBuffer(bytes, 'deserialize');
   if (buffer.byteLength < 8) {
     throw new ReaderError('R_TRUNCATED', 'container too small to inspect header flags');
   }
   const flags = new Uint8Array(buffer, 7, 1)[0];
-  if (flags & 0x01) return new PreserveReader(buffer);
-  return new Reader(buffer);
+  if (flags & 0x01) return new PreserveReader(buffer, opts);
+  return new Reader(buffer, opts);
 }

package/types/PreserveReader.d.ts

@@ -13,9 +13,17 @@ export interface PreserveShardHandle {
   endRow: number;
 }
 
+export interface PreserveReaderOptions {
+  /** Verify the footer CRC-32C at open; throws R_BAD_CRC on mismatch, R_CRC_ABSENT if absent. */
+  verifyCrc?: boolean;
+}
+
 export class PreserveReader {
-  static fromBuffer(input: ArrayBuffer | Uint8Array): PreserveReader;
-  constructor(buffer: ArrayBuffer);
+  static fromBuffer(input: ArrayBuffer | Uint8Array, opts?: PreserveReaderOptions): PreserveReader;
+  constructor(buffer: ArrayBuffer, opts?: PreserveReaderOptions);
+
+  /** Recompute the footer CRC-32C over [0, footer_off): 'ok' | 'absent'; throws R_BAD_CRC on mismatch. */
+  verifyCrc(): 'ok' | 'absent';
 
   readonly totalRows: number;
   readonly shardCount: number;