@zakkster/lite-bake-stream 1.4.1 → 1.6.0

package/src/index.js

@@ -12,7 +12,7 @@
 import { Tokenizer, TokenizerError } from './Tokenizer.js';
 import { Writer, WriterError } from './Writer.js';
 import { Reader, ReaderError } from './Reader.js';
-import { StringTable } from './StringTable.js';
+import { StringTable, StringTableError } from './StringTable.js';
 import { PreserveTokenizer, PreserveTokenizerError } from './PreserveTokenizer.js';
 import { PreserveWriter, PreserveWriterError } from './PreserveWriter.js';
 import { PreserveReader, PreserveReaderError } from './PreserveReader.js';
@@ -23,12 +23,12 @@ export {
   Tokenizer, TokenizerError,
   Writer, WriterError,
   Reader, ReaderError,
-  StringTable,
+  StringTable, StringTableError,
   PreserveTokenizer, PreserveTokenizerError,
   PreserveWriter, PreserveWriterError,
   PreserveReader, PreserveReaderError,
 };
-export const VERSION = '1.4.1';
+export const VERSION = '1.6.0';
 
 const encoder = new TextEncoder();
 
@@ -188,12 +188,12 @@ async function _serializeAsyncIterablePreserve(iterable, opts) {
 
 // Deserialize LBK1 bytes into the right Reader for the container's mode.
 // Auto-detects preserve vs schema via header flag byte at offset 7 bit 0.
-export function deserialize(bytes) {
+export function deserialize(bytes, opts) {
   const buffer = toContainerBuffer(bytes, 'deserialize');
   if (buffer.byteLength < 8) {
     throw new ReaderError('R_TRUNCATED', 'container too small to inspect header flags');
   }
   const flags = new Uint8Array(buffer, 7, 1)[0];
-  if (flags & 0x01) return new PreserveReader(buffer);
-  return new Reader(buffer);
+  if (flags & 0x01) return new PreserveReader(buffer, opts);
+  return new Reader(buffer, opts);
 }

package/types/PreserveReader.d.ts

@@ -13,9 +13,17 @@ export interface PreserveShardHandle {
   endRow: number;
 }
 
+export interface PreserveReaderOptions {
+  /** Verify the footer CRC-32C at open; throws R_BAD_CRC on mismatch, R_CRC_ABSENT if absent. */
+  verifyCrc?: boolean;
+}
+
 export class PreserveReader {
-  static fromBuffer(input: ArrayBuffer | Uint8Array): PreserveReader;
-  constructor(buffer: ArrayBuffer);
+  static fromBuffer(input: ArrayBuffer | Uint8Array, opts?: PreserveReaderOptions): PreserveReader;
+  constructor(buffer: ArrayBuffer, opts?: PreserveReaderOptions);
+
+  /** Recompute the footer CRC-32C over [0, footer_off): 'ok' | 'absent'; throws R_BAD_CRC on mismatch. */
+  verifyCrc(): 'ok' | 'absent';
 
   readonly totalRows: number;
   readonly shardCount: number;

package/types/PreserveWriter.d.ts

@@ -8,6 +8,8 @@ export interface PreserveWriterOptions {
   targetShardBytes?: number;
   /** Passed to PreserveTokenizer via the top-level API. */
   maxRecordBytes?: number;
+  /** Emit a CRC-32C over [0, footer_off) into the footer. Default false. */
+  crc?: boolean;
 }
 
 export interface PreserveContainer {
@@ -17,6 +19,24 @@ export interface PreserveContainer {
   mode: 'preserve';
 }
 
+export interface PreserveContainerSink {
+  write(bytes: Uint8Array): void;
+  writeAt?(bytes: Uint8Array, position: number): void;
+}
+
+export interface PreserveFinalizeToSinkOptions {
+  layout?: 'prefix' | 'stream';
+  crc?: boolean;
+}
+
+export interface PreserveSinkResult {
+  totalRows: number;
+  shardCount: number;
+  mode: 'preserve';
+  bytesWritten: number;
+  layout: 'prefix' | 'stream';
+}
+
 export class PreserveWriter {
   constructor(opts?: PreserveWriterOptions);
   /** Sink protocol. Called by PreserveTokenizer for each detected record. */
@@ -24,6 +44,16 @@ export class PreserveWriter {
   /** Public API for direct record writes. */
   writeRecord(bytes: Uint8Array): void;
   finalize(): PreserveContainer;
+  /**
+   * Bind a sink and stream shards as they finalize (bounded RAM). Call BEFORE
+   * feeding, then finish with finalizeToSink. opts.layout must be 'stream'.
+   */
+  beginStream(sink: PreserveContainerSink, opts?: PreserveFinalizeToSinkOptions): void;
+  /**
+   * Emit to a caller sink. Preceded by beginStream it writes only the trailer
+   * (O(shard)); called alone it buffers first (O(container)).
+   */
+  finalizeToSink(sink: PreserveContainerSink, opts?: PreserveFinalizeToSinkOptions): PreserveSinkResult;
   readonly totalRows: number;
   readonly shardCount: number;
 }

package/types/RangeReader.d.ts

@@ -36,6 +36,8 @@ export class MockRangeAdapter implements IOAdapter {
 export interface RangeReaderOptions {
   /** Bounded LRU cache size. Default 8. */
   maxCachedShards?: number;
+  /** Verify the footer CRC-32C at open; rejects R_BAD_CRC on mismatch, R_CRC_ABSENT if absent. */
+  verifyCrc?: boolean;
 }
 
 export interface RangeShardHandle {
@@ -60,6 +62,9 @@ export class RangeReader {
   readonly cachedShardCount: number;
   readonly hasZoneMaps: boolean;
 
+  /** Ranged-fetch the body [0, footer_off) and verify its CRC-32C: 'ok' | 'absent'; rejects R_BAD_CRC on mismatch. */
+  verifyCrc(): Promise<'ok' | 'absent'>;
+
   fieldIndex(name: string): number;
 
   /** Fetch a shard's payload+string-table (one range request), cache, return record. */

package/types/Reader.d.ts

@@ -42,9 +42,17 @@ export interface FindShardsOptions {
   max?: number;
 }
 
+export interface ReaderOptions {
+  /** Verify the footer CRC-32C at open; throws R_BAD_CRC on mismatch, R_CRC_ABSENT if absent. */
+  verifyCrc?: boolean;
+}
+
 export class Reader {
-  static fromBuffer(bufferOrArrayBuffer: ArrayBuffer | Uint8Array): Reader;
-  constructor(buffer: ArrayBuffer);
+  static fromBuffer(bufferOrArrayBuffer: ArrayBuffer | Uint8Array, opts?: ReaderOptions): Reader;
+  constructor(buffer: ArrayBuffer, opts?: ReaderOptions);
+
+  /** Recompute the footer CRC-32C over [0, footer_off): 'ok' | 'absent'; throws R_BAD_CRC on mismatch. */
+  verifyCrc(): 'ok' | 'absent';
 
   readonly schema: FrozenSchema;
   readonly totalRows: number;

package/types/StringTable.d.ts

@@ -3,6 +3,11 @@
 
 export const VERSION: string;
 
+export class StringTableError extends Error {
+  readonly code: string;
+  readonly name: 'StringTableError';
+}
+
 export class StringTable {
   constructor();
   /** Intern a byte range. Zero-alloc on the hit path. Returns u32 index. */

package/types/Writer.d.ts

@@ -30,6 +30,30 @@ export interface WriterOptions {
    * Default = targetShardBytes.
    */
   sampleBytes?: number;
+  /** Emit a CRC-32C over [0, footer_off) into the footer. Default false. */
+  crc?: boolean;
+}
+
+/** A synchronous byte sink for `finalizeToSink`. */
+export interface ContainerSink {
+  write(bytes: Uint8Array): void;
+  /** Required for layout:'stream' (one header backpatch at position 0). */
+  writeAt?(bytes: Uint8Array, position: number): void;
+}
+
+export interface FinalizeToSinkOptions {
+  /** 'prefix' (classic, byte-identical to finalize()) or 'stream'. Default 'stream'. */
+  layout?: 'prefix' | 'stream';
+  /** Override the constructor `crc`. */
+  crc?: boolean;
+}
+
+export interface SinkResult {
+  totalRows: number;
+  shardCount: number;
+  schema: FrozenSchema;
+  bytesWritten: number;
+  layout: 'prefix' | 'stream';
 }
 
 export interface FrozenField {
@@ -85,6 +109,19 @@ export class Writer implements TokenizerSinkForWriter {
    * Tokenizer has been given `end()`.
    */
   finalize(): Container;
+  /**
+   * Bind a sink and stream shards as they finalize, for bounded-RAM output.
+   * Call BEFORE feeding the tokenizer, then finish with finalizeToSink. Peak
+   * memory is O(targetShardBytes + directory). opts.layout must be 'stream'.
+   */
+  beginStream(sink: ContainerSink, opts?: FinalizeToSinkOptions): void;
+  /**
+   * Emit the container to a caller sink instead of returning a buffer. Preceded
+   * by beginStream it writes only the trailer (bounded RAM, O(shard)); called
+   * alone it buffers the shards first (O(container)). With layout:'prefix' the
+   * emitted bytes are identical to finalize().
+   */
+  finalizeToSink(sink: ContainerSink, opts?: FinalizeToSinkOptions): SinkResult;
   readonly schema: FrozenSchema | null;
   readonly totalRows: number;
   readonly shardCount: number;

package/types/index.d.ts

@@ -13,7 +13,7 @@ export { Reader, ReaderError } from './Reader.d.ts';
 export type {
   ShardHandle, StringTableView, Bounds, FindShardsOptions,
 } from './Reader.d.ts';
-export { StringTable } from './StringTable.d.ts';
+export { StringTable, StringTableError } from './StringTable.d.ts';
 export { PreserveTokenizer, PreserveTokenizerError } from './PreserveTokenizer.d.ts';
 export type { PreserveSink, PreserveTokenizerOptions } from './PreserveTokenizer.d.ts';
 export { PreserveWriter, PreserveWriterError } from './PreserveWriter.d.ts';
@@ -54,5 +54,6 @@ export function serialize(
  * Auto-detects preserve vs schema via the header flag byte.
  */
 export function deserialize(
-  bytes: Uint8Array | ArrayBuffer
+  bytes: Uint8Array | ArrayBuffer,
+  opts?: { verifyCrc?: boolean }
 ): import('./Reader.d.ts').Reader | import('./PreserveReader.d.ts').PreserveReader;