@zakkster/lite-bake-stream 1.4.1 → 1.6.0

package/src/Reader.js

@@ -21,16 +21,31 @@
 //   R_BAD_METADATA           - metadata_off is non-zero but the zone-map segment is unparseable
 //   R_INVALID                - a structure is internally inconsistent but in-bounds
 //   R_UNKNOWN_FIELD          - get()/findShards()/shardBounds() called with an unknown field name
+//   R_OFFSET_TOO_LARGE       - a u64 header/schema/directory offset exceeds 2^53-1
+//                              (Number() would lose precision; fail closed, BS-05)
+//   R_ROW_OUT_OF_RANGE       - get(rowIdx) with rowIdx negative, fractional, NaN,
+//                              or >= totalRows (BS-32)
+//
+// Row-index policy (BS-32): get() range-checks rowIdx; the SHARD-index escape
+// hatches (shardPayload, shardF64, shardStringTable) do NOT -- their contract is
+// raw indexed access with no per-call checking. shardBounds returns null for an
+// out-of-range shard by decided policy.
 
 import { StringTable } from './StringTable.js';
 import { toContainerBuffer } from './Views.js';
+import { checkOpts } from './Opts.js';
+import { crc32cInit, crc32cUpdate, crc32cFinal } from './Crc32c.js';
 
-export const VERSION = '1.4.1';
+export const VERSION = '1.6.0';
 
 const CONTAINER_HEADER_BYTES = 48;
 const SHARD_ENTRY_BYTES = 40;
 const FIELD_DESCRIPTOR_BYTES = 24;
 const FOOTER_BYTES = 16;
+const CRC_ABSENT = 0xFFFFFFFF;
+
+const READER_OPTS = { verifyCrc: { t: 'bool' } };
+function raiseReaderOpt(code, msg) { throw new ReaderError(code, msg); }
 
 const LANE_F64 = 1;
 const LANE_U32 = 3;
@@ -42,6 +57,19 @@ export class ReaderError extends Error {
   constructor(code, msg) { super(msg); this.code = code; this.name = 'ReaderError'; }
 }
 
+// Read a u64 header/directory/schema field and narrow it to a JS number, failing
+// closed if it exceeds Number.MAX_SAFE_INTEGER (2^53-1). Past that a Number()
+// cast loses precision and every downstream bounds check reads a corrupted
+// offset -- an in-bounds lie is worse than a loud refusal (BS-05). Cold: called
+// only during header/schema/directory parse, never per row.
+function u64(dv, off, what) {
+  const v = dv.getBigUint64(off, true);
+  if (v > 9007199254740991n)
+    throw new ReaderError('R_OFFSET_TOO_LARGE',
+      what + ' value ' + v + ' exceeds the safe-integer ceiling 9007199254740991');
+  return Number(v);
+}
+
 // Validate a local string table's shape BEFORE StringTable.parse casts a
 // Uint32Array over it (T-1..T-4): the offsets array must be in bounds, the
 // count/blob must fit, offsets must be non-decreasing, and the trailing
@@ -71,11 +99,12 @@ function validateStringTable(bytes, off, len, label) {
 }
 
 export class Reader {
-  static fromBuffer(input) {
-    return new Reader(toContainerBuffer(input, 'Reader.fromBuffer'));
+  static fromBuffer(input, opts) {
+    return new Reader(toContainerBuffer(input, 'Reader.fromBuffer'), opts);
   }
 
-  constructor(buffer) {
+  constructor(buffer, opts) {
+    checkOpts('Reader', opts, READER_OPTS, raiseReaderOpt);
     this._buffer = buffer;
     this._dv = new DataView(buffer);
     this._bytes = new Uint8Array(buffer);
@@ -85,6 +114,27 @@ export class Reader {
     this._parseShardDirectory();
     this._parseZoneMaps();      // M7 -- no-op if metadata_off is 0
     this._buildFieldIndex();
+    // Optional open-time verification (fail closed on BOTH mismatch AND absence:
+    // the caller demanded verification, so an unverifiable container is an
+    // unverified state -- null is not zero).
+    if (opts && opts.verifyCrc === true) {
+      const status = this.verifyCrc();
+      if (status === 'absent')
+        throw new ReaderError('R_CRC_ABSENT', 'verifyCrc:true but the container carries no CRC (footer CRC is absent, 0xFFFFFFFF)');
+    }
+  }
+
+  // SPEC 3.7 integrity: recompute CRC-32C over [0, footer_off) and compare to the
+  // stored footer CRC. Returns 'ok' when they match, 'absent' when the container
+  // carries no CRC (footer CRC == 0xFFFFFFFF). A MISMATCH throws R_BAD_CRC.
+  verifyCrc() {
+    const footerOff = this._buffer.byteLength - FOOTER_BYTES;
+    const stored = this._dv.getUint32(footerOff, true) >>> 0;
+    if (stored === CRC_ABSENT) return 'absent';
+    const actual = crc32cFinal(crc32cUpdate(crc32cInit(), this._bytes, 0, footerOff));
+    if (actual !== stored)
+      throw new ReaderError('R_BAD_CRC', 'container CRC mismatch: stored 0x' + stored.toString(16) + ' != computed 0x' + actual.toString(16));
+    return 'ok';
   }
 
   _parseHeader() {
@@ -118,11 +168,11 @@ export class Reader {
     if (reserved1 !== 0)
       throw new ReaderError('R_RESERVED_NONZERO', 'header reserved1 at offset 36 must be 0, got ' + reserved1);
 
-    this._schemaBlockOff  = Number(this._dv.getBigUint64(8, true));
-    this._metadataOff     = Number(this._dv.getBigUint64(16, true));   // 0 = no metadata block; M7+ zone maps
-    this._shardDirOff     = Number(this._dv.getBigUint64(24, true));
+    this._schemaBlockOff  = u64(this._dv, 8, 'schema_block_off');
+    this._metadataOff     = u64(this._dv, 16, 'metadata_off');   // 0 = no metadata block; M7+ zone maps
+    this._shardDirOff     = u64(this._dv, 24, 'shard_directory_off');
     this._shardCount      = this._dv.getUint32(32, true);
-    this._totalRows       = Number(this._dv.getBigUint64(40, true));
+    this._totalRows       = u64(this._dv, 40, 'total_rows');
     this._formatVersion   = version;
 
     const len = this._buffer.byteLength;
@@ -180,7 +230,10 @@ export class Reader {
       const laneKind    = this._bytes[descOff + 4];
       const flags       = this._bytes[descOff + 5];
       const reserved2   = this._dv.getUint16(descOff + 6, true);
-      const nameStrOff  = Number(this._dv.getBigUint64(descOff + 8, true));
+      const nameStrOff  = u64(this._dv, descOff + 8, 'field ' + i + ' name_str_off');
+      // reserved3 is compared as BigInt vs 0n below and never Number()-cast, so
+      // it needs no u64() narrowing guard -- a value past 2^53 is caught by the
+      // reserved-must-be-zero check, not by precision loss (BS-05).
       const reserved3   = this._dv.getBigUint64(descOff + 16, true);
 
       if (flags !== 0) throw new ReaderError('R_BAD_FIELD_FLAGS', 'field ' + i + ' has non-zero flags (v2+ reserved)');
@@ -212,14 +265,14 @@ export class Reader {
     let rowSum = 0;
     for (let i = 0; i < this._shardCount; i++) {
       const entryOff = off + i * SHARD_ENTRY_BYTES;
-      const payloadOff    = Number(this._dv.getBigUint64(entryOff + 0, true));
+      const payloadOff    = u64(this._dv, entryOff + 0, 'shard ' + i + ' payload_off');
       const payloadLen    = this._dv.getUint32(entryOff + 8, true);
       const rowCount      = this._dv.getUint32(entryOff + 12, true);
       const minReaderVer  = this._dv.getUint16(entryOff + 16, true);
       const shardFlags    = this._dv.getUint16(entryOff + 18, true);
       const shardReserved = this._dv.getUint32(entryOff + 20, true);
-      const localStrOff   = Number(this._dv.getBigUint64(entryOff + 24, true));
-      const localStrLen   = Number(this._dv.getBigUint64(entryOff + 32, true));
+      const localStrOff   = u64(this._dv, entryOff + 24, 'shard ' + i + ' local_string_off');
+      const localStrLen   = u64(this._dv, entryOff + 32, 'shard ' + i + ' local_string_len');
       if (minReaderVer > READER_VERSION) {
         throw new ReaderError('R_SHARD_VERSION_TOO_NEW',
           'shard ' + i + ' requires reader version ' + minReaderVer);
@@ -396,13 +449,24 @@ export class Reader {
     return i;
   }
 
+  // Cold: the BS-32 row-range refusal body, kept out of get()'s hot frame.
+  _badRow(rowIdx) {
+    throw new ReaderError('R_ROW_OUT_OF_RANGE',
+      'rowIdx ' + rowIdx + ' out of range [0, ' + this._totalRows + ')');
+  }
+
   // Random-access get across all shards. Returns the field's decoded value:
   //   F64 lanes -> number
   //   U32 lanes -> JS string (resolved via the shard's local string table)
   // Not zero-alloc; intended for testing/introspection.
   get(rowIdx, fieldName) {
-    let remaining = rowIdx;
     const fieldIdx = this.fieldIndex(fieldName);
+    // BS-32 row-range door. `(rowIdx >>> 0) !== rowIdx` rejects negative,
+    // fractional, NaN and >= 2^32 in one compare; a whole ArrayBuffer cannot hold
+    // 2^32 rows, so the totalRows compare fires first for every reachable
+    // container. Throw body hoisted to the cold _badRow helper.
+    if ((rowIdx >>> 0) !== rowIdx || rowIdx >= this._totalRows) this._badRow(rowIdx);
+    let remaining = rowIdx;
     const field = this._schema.fields[fieldIdx];
     for (let s = 0; s < this._shards.length; s++) {
       const shard = this._shards[s];

package/src/Split.js

@@ -36,8 +36,9 @@ import { Writer, WriterError } from './Writer.js';
 import { Reader, ReaderError } from './Reader.js';
 import { StringTable } from './StringTable.js';
 import { checkOpts } from './Opts.js';
+import { crc32cInit, crc32cUpdate, crc32cFinal } from './Crc32c.js';
 
-export const VERSION = '1.4.1';
+export const VERSION = '1.6.0';
 
 const LF = 0x0A;
 const CONTAINER_HEADER_BYTES = 48;
@@ -219,6 +220,11 @@ export function mergeContainers(containers) {
   // Compute the target layout. Schema block is copied from container 0 verbatim.
   // Reader gives us schemaBlockOff and shardDirOff; the schema-block byte range
   // is [schemaBlockOff, shardDirOff). We reuse this slice.
+  // These Number(getBigUint64) reads carry NO u64 narrowing guard by design
+  // (BS-05 dominance): `new Reader(...)` at :193 above parsed every part first,
+  // and Reader's own guarded u64() over the same header bytes (schema_block_off,
+  // shard_directory_off) already threw R_OFFSET_TOO_LARGE for any over-2^53
+  // value. Split mints nothing here -- an untriggerable code would be dead.
   const src0 = parts[0];
   const dv0 = new DataView(src0.buffer, src0.byteOffset, src0.byteLength);
   const schemaBlockOff = Number(dv0.getBigUint64(8, true));
@@ -350,9 +356,20 @@ export function mergeContainers(containers) {
     }
   }
 
-  // Footer
+  // Footer. CRC policy (D4): recompute a fresh CRC-32C over the merged body iff
+  // EVERY input carried one; if any input's CRC is absent the merged CRC is
+  // absent too (an integrity guarantee only the inputs all shared can be
+  // honestly re-asserted -- never fabricated over an unverified part).
   const footerOff = totalBytes - FOOTER_BYTES;
-  outDv.setUint32(footerOff + 0, 0xFFFFFFFF, true);   // CRC absent
+  let mergedCrc = 0xFFFFFFFF;
+  let allHaveCrc = true;
+  for (const p of parts) {
+    const pDv = new DataView(p.buffer, p.byteOffset, p.byteLength);
+    const pCrc = pDv.getUint32(p.byteLength - FOOTER_BYTES, true) >>> 0;
+    if (pCrc === 0xFFFFFFFF) { allHaveCrc = false; break; }
+  }
+  if (allHaveCrc) mergedCrc = crc32cFinal(crc32cUpdate(crc32cInit(), out, 0, footerOff));
+  outDv.setUint32(footerOff + 0, mergedCrc >>> 0, true);
   outDv.setUint32(footerOff + 4, 0, true);
   out[footerOff + 8]  = 0x31;
   out[footerOff + 9]  = 0x4B;
@@ -383,7 +400,10 @@ function _schemasEqual(a, b) {
   return true;
 }
 
-// Look up the source string-table byte length for a shard.
+// Look up the source string-table byte length for a shard. No u64 narrowing
+// guard here (BS-05 dominance): `reader` is a fully-parsed Reader, so Reader's
+// guarded u64() over this same local_string_len field already threw
+// R_OFFSET_TOO_LARGE for any over-2^53 value during construction.
 function _sourceStringTableLen(reader, shardIdx) {
   const dv = new DataView(reader.buffer);
   const entryOff = reader.shardDirectoryOffset + shardIdx * SHARD_ENTRY_BYTES;

package/src/StringTable.js

@@ -19,8 +19,26 @@
 // Entry 0 is always the empty string: the table reserves it in the constructor
 // and at every reset(), so an absent U32 row cell (which is 0) decodes as ""
 // rather than aliasing the shard's first-interned string (SPEC 3.3, SPEC 7).
+//
+// Serialization ceilings (BS-31). serialize() writes entry_count and
+// blob_length as u32; a table that grew past either would silently wrap on
+// setUint32 and mint a corrupt-but-in-bounds container. Two cold guards in
+// _insertNew fail closed instead:
+//   ST_BLOB_OVERFLOW  - the blob would exceed the u32 blob_length ceiling, OR
+//                       the entry count would exceed the entry_count ceiling
+//                       (one code, two messages -- both name their ceiling).
+// The doubling clamps in _growBlob / _growOffsets cap allocation at the
+// ceiling so a crossing write ALWAYS re-enters the grow branch where the guard
+// lives; guard placement is then provably complete (no path reaches serialize
+// with an out-of-range count/blob). Zero hot cost: the guards read only inside
+// the cold grow arms.
+//
+// __setStringTableLimits({blobBytes, entryCount}) lowers the ceilings for a
+// cheap gated crossing on the real grow path and returns the previous pair.
+// TEST-ONLY: not re-exported from index.js, absent from the .d.ts and docs,
+// and carrying no semver guarantee.
 
-export const VERSION = '1.4.1';
+export const VERSION = '1.6.0';
 
 const EMPTY_SLOT = 0xFFFFFFFF;   // MUST be unsigned; typed-array reads are unsigned
 const INITIAL_BLOB_BYTES  = 64 * 1024;
@@ -29,6 +47,26 @@ const INITIAL_HASH_CAP    = 2048;      // load factor target 50%
 const HASH_MAX_LOAD_NUM   = 1;         // 50% load factor: num/den = 1/2
 const HASH_MAX_LOAD_DEN   = 2;
 
+// u32 serialization ceilings. blob_length is a u32 (max 4294967295). entry_count
+// is a u32, but the offsets array carries entry_count+1 slots (the sentinel), so
+// the last addressable entry_count is 4294967294 to keep offsets_len a valid u32.
+// `let`, not `const`: __setStringTableLimits lowers them for gated crossings.
+let LIMIT_BLOB = 4294967295;
+let LIMIT_ENTRIES = 4294967294;
+
+// TEST-ONLY seam (BS-31). Assigns whichever of {blobBytes, entryCount} are
+// present, returns the previous pair. Not part of the public API.
+export function __setStringTableLimits(next) {
+  const prev = { blobBytes: LIMIT_BLOB, entryCount: LIMIT_ENTRIES };
+  if (next && next.blobBytes !== undefined) LIMIT_BLOB = next.blobBytes;
+  if (next && next.entryCount !== undefined) LIMIT_ENTRIES = next.entryCount;
+  return prev;
+}
+
+export class StringTableError extends Error {
+  constructor(code, msg) { super(msg); this.code = code; this.name = 'StringTableError'; }
+}
+
 // Reserved entry 0. A zero-length range: no blob bytes, no growth, one hash
 // slot. Module-level so reset() reserves without allocating.
 const EMPTY = new Uint8Array(0);
@@ -94,10 +132,23 @@ export class StringTable {
 
   _insertNew(bytes, from, to, slot) {
     const n = to - from;
-    // Grow blob if needed
-    if (this._blobLen + n > this._blob.length) this._growBlob(n);
-    // Grow offsets if needed (leave room for +1 sentinel)
-    if (this._count + 1 >= this._offsets.length) this._growOffsets();
+    // Grow blob if needed. The clamp keeps _blob.length <= LIMIT_BLOB, so any
+    // write pushing past the ceiling necessarily re-enters this branch and is
+    // caught by the cold guard (fail closed -- serialize would wrap otherwise).
+    if (this._blobLen + n > this._blob.length) {
+      if (this._blobLen + n > LIMIT_BLOB)
+        throw new StringTableError('ST_BLOB_OVERFLOW',
+          'string blob ' + this._blobLen + ' + ' + n + ' would exceed the u32 blob_length ceiling ' + LIMIT_BLOB);
+      this._growBlob(n);
+    }
+    // Grow offsets if needed (leave room for +1 sentinel). Same completeness
+    // argument via the _growOffsets clamp on the entry_count ceiling.
+    if (this._count + 1 >= this._offsets.length) {
+      if (this._count >= LIMIT_ENTRIES)
+        throw new StringTableError('ST_BLOB_OVERFLOW',
+          'string table entry count ' + this._count + ' would exceed the entry_count ceiling ' + LIMIT_ENTRIES);
+      this._growOffsets();
+    }
 
     const idx = this._count;
     this._offsets[idx] = this._blobLen;
@@ -123,13 +174,18 @@ export class StringTable {
   _growBlob(needed) {
     let cap = this._blob.length;
     while (cap < this._blobLen + needed) cap *= 2;
+    // Clamp so allocated length never exceeds the u32 ceiling (see _insertNew).
+    if (cap > LIMIT_BLOB) cap = LIMIT_BLOB;
     const nb = new Uint8Array(cap);
     nb.set(this._blob);
     this._blob = nb;
   }
 
   _growOffsets() {
-    const nb = new Uint32Array(this._offsets.length * 2);
+    let len = this._offsets.length * 2;
+    // Clamp so the offsets array (entry_count + 1 slots) never exceeds a valid u32.
+    if (len > LIMIT_ENTRIES + 1) len = LIMIT_ENTRIES + 1;
+    const nb = new Uint32Array(len);
     nb.set(this._offsets);
     this._offsets = nb;
   }

package/src/Tokenizer.js

@@ -30,7 +30,7 @@
 
 import { checkOpts } from './Opts.js';
 
-export const VERSION = '1.4.1';
+export const VERSION = '1.6.0';
 
 const U32_MAX = 4294967295;
 const TOKENIZER_OPTS = {

package/src/Views.js

@@ -23,3 +23,65 @@ export function toContainerBuffer(input, label) {
   throw new TypeError(label + ': expected ArrayBuffer or Uint8Array, got ' +
     (input === null ? 'null' : typeof input));
 }
+
+// ---- streaming sink support (M6) --------------------------------------------
+//
+// A sink is any object with a synchronous write(bytes) method; the streaming
+// (layout:'stream') path additionally needs writeAt(bytes, position) for the
+// single header backpatch. MemorySink is the in-RAM reference implementation
+// that finalize() drives internally so it can return the identical container
+// buffer the classic assembler produced.
+
+// Growable in-RAM sink. write() appends; writeAt() places bytes at an absolute
+// position (growing the logical length if needed); toArrayBuffer() returns an
+// EXACT-length ArrayBuffer copy of the bytes written.
+export class MemorySink {
+  constructor(hint) {
+    const cap = (typeof hint === 'number' && hint > 0) ? hint : 64 * 1024;
+    this._buf = new Uint8Array(cap);
+    this._len = 0;
+  }
+  _ensure(need) {
+    if (need <= this._buf.length) return;
+    let cap = this._buf.length;
+    while (cap < need) cap *= 2;
+    const next = new Uint8Array(cap);
+    next.set(this._buf.subarray(0, this._len));
+    this._buf = next;
+  }
+  write(bytes) {
+    this._ensure(this._len + bytes.length);
+    this._buf.set(bytes, this._len);
+    this._len += bytes.length;
+  }
+  writeAt(bytes, position) {
+    this._ensure(position + bytes.length);
+    this._buf.set(bytes, position);
+    if (position + bytes.length > this._len) this._len = position + bytes.length;
+  }
+  toArrayBuffer() {
+    return this._buf.buffer.slice(0, this._len);
+  }
+}
+
+// Cold prologue guard for finalizeToSink (raised as W_BAD_SINK by the writer):
+// the sink must be an object with write(); layout:'stream' also needs writeAt().
+export function validateSink(sink, needsWriteAt, raise) {
+  if (sink === null || typeof sink !== 'object') {
+    raise('W_BAD_SINK', 'sink must be an object with a write(bytes) method; got ' +
+      (sink === null ? 'null' : typeof sink));
+  }
+  if (typeof sink.write !== 'function') {
+    raise('W_BAD_SINK', 'sink is missing a write(bytes) method');
+  }
+  if (needsWriteAt && typeof sink.writeAt !== 'function') {
+    raise('W_BAD_SINK', "sink is missing a writeAt(bytes, position) method required for layout:'stream'");
+  }
+}
+
+// A sink write that returns a thenable is an async sink; the contract is
+// synchronous, so the writer treats it as a malformed sink (W_BAD_SINK).
+export function isThenable(v) {
+  return v !== null && (typeof v === 'object' || typeof v === 'function') &&
+    typeof v.then === 'function';
+}