@zakkster/lite-bake-stream 1.5.0 → 1.6.1

package/src/index.js

@@ -8,6 +8,10 @@
 //                           Set opts.preserve = true. Bytes in, same bytes out.
 // deserialize() auto-detects the mode via the container's flag bit and
 // returns the appropriate Reader.
+//
+// Error codes:
+//   E_OPTION_CONFLICT  - serialize() opts combine preserve with an incompatible framing
+//   R_TRUNCATED        - deserialize() container too small to inspect header flags
 
 import { Tokenizer, TokenizerError } from './Tokenizer.js';
 import { Writer, WriterError } from './Writer.js';
@@ -28,7 +32,7 @@ export {
   PreserveWriter, PreserveWriterError,
   PreserveReader, PreserveReaderError,
 };
-export const VERSION = '1.5.0';
+export const VERSION = '1.6.1';
 
 const encoder = new TextEncoder();
 
@@ -188,12 +192,12 @@ async function _serializeAsyncIterablePreserve(iterable, opts) {
 
 // Deserialize LBK1 bytes into the right Reader for the container's mode.
 // Auto-detects preserve vs schema via header flag byte at offset 7 bit 0.
-export function deserialize(bytes) {
+export function deserialize(bytes, opts) {
   const buffer = toContainerBuffer(bytes, 'deserialize');
   if (buffer.byteLength < 8) {
     throw new ReaderError('R_TRUNCATED', 'container too small to inspect header flags');
   }
   const flags = new Uint8Array(buffer, 7, 1)[0];
-  if (flags & 0x01) return new PreserveReader(buffer);
-  return new Reader(buffer);
+  if (flags & 0x01) return new PreserveReader(buffer, opts);
+  return new Reader(buffer, opts);
 }

package/types/MultiReader.d.ts

@@ -24,7 +24,7 @@ export class MultiReader {
   /** Which sub-Reader owns a given global shard? */
   readerForShard(globalShardIdx: number): { readerIdx: number; localShard: number } | null;
 
-  // Zone maps — global shard indices merged across sub-readers
+  // Zone maps -- global shard indices merged across sub-readers
   shardBounds(globalShardIdx: number, fieldName: string): Bounds | null;
   findShards(fieldName: string, opts?: FindShardsOptions): number[];
 }

package/types/PreserveReader.d.ts

@@ -13,9 +13,17 @@ export interface PreserveShardHandle {
   endRow: number;
 }
 
+export interface PreserveReaderOptions {
+  /** Verify the footer CRC-32C at open; throws R_BAD_CRC on mismatch, R_CRC_ABSENT if absent. */
+  verifyCrc?: boolean;
+}
+
 export class PreserveReader {
-  static fromBuffer(input: ArrayBuffer | Uint8Array): PreserveReader;
-  constructor(buffer: ArrayBuffer);
+  static fromBuffer(input: ArrayBuffer | Uint8Array, opts?: PreserveReaderOptions): PreserveReader;
+  constructor(buffer: ArrayBuffer, opts?: PreserveReaderOptions);
+
+  /** Recompute the footer CRC-32C over [0, footer_off): 'ok' | 'absent'; throws R_BAD_CRC on mismatch. */
+  verifyCrc(): 'ok' | 'absent';
 
   readonly totalRows: number;
   readonly shardCount: number;

package/types/PreserveWriter.d.ts

@@ -8,6 +8,8 @@ export interface PreserveWriterOptions {
   targetShardBytes?: number;
   /** Passed to PreserveTokenizer via the top-level API. */
   maxRecordBytes?: number;
+  /** Emit a CRC-32C over [0, footer_off) into the footer. Default false. */
+  crc?: boolean;
 }
 
 export interface PreserveContainer {
@@ -17,6 +19,24 @@ export interface PreserveContainer {
   mode: 'preserve';
 }
 
+export interface PreserveContainerSink {
+  write(bytes: Uint8Array): void;
+  writeAt?(bytes: Uint8Array, position: number): void;
+}
+
+export interface PreserveFinalizeToSinkOptions {
+  layout?: 'prefix' | 'stream';
+  crc?: boolean;
+}
+
+export interface PreserveSinkResult {
+  totalRows: number;
+  shardCount: number;
+  mode: 'preserve';
+  bytesWritten: number;
+  layout: 'prefix' | 'stream';
+}
+
 export class PreserveWriter {
   constructor(opts?: PreserveWriterOptions);
   /** Sink protocol. Called by PreserveTokenizer for each detected record. */
@@ -24,6 +44,16 @@ export class PreserveWriter {
   /** Public API for direct record writes. */
   writeRecord(bytes: Uint8Array): void;
   finalize(): PreserveContainer;
+  /**
+   * Bind a sink and stream shards as they finalize (bounded RAM). Call BEFORE
+   * feeding, then finish with finalizeToSink. opts.layout must be 'stream'.
+   */
+  beginStream(sink: PreserveContainerSink, opts?: PreserveFinalizeToSinkOptions): void;
+  /**
+   * Emit to a caller sink. Preceded by beginStream it writes only the trailer
+   * (O(shard)); called alone it buffers first (O(container)).
+   */
+  finalizeToSink(sink: PreserveContainerSink, opts?: PreserveFinalizeToSinkOptions): PreserveSinkResult;
   readonly totalRows: number;
   readonly shardCount: number;
 }

package/types/RangeReader.d.ts

@@ -36,6 +36,8 @@ export class MockRangeAdapter implements IOAdapter {
 export interface RangeReaderOptions {
   /** Bounded LRU cache size. Default 8. */
   maxCachedShards?: number;
+  /** Verify the footer CRC-32C at open; rejects R_BAD_CRC on mismatch, R_CRC_ABSENT if absent. */
+  verifyCrc?: boolean;
 }
 
 export interface RangeShardHandle {
@@ -60,6 +62,9 @@ export class RangeReader {
   readonly cachedShardCount: number;
   readonly hasZoneMaps: boolean;
 
+  /** Ranged-fetch the body [0, footer_off) and verify its CRC-32C: 'ok' | 'absent'; rejects R_BAD_CRC on mismatch. */
+  verifyCrc(): Promise<'ok' | 'absent'>;
+
   fieldIndex(name: string): number;
 
   /** Fetch a shard's payload+string-table (one range request), cache, return record. */
@@ -81,7 +86,7 @@ export class RangeReader {
     get(rowIdx: number, fieldName: string): number | string | undefined;
   };
 
-  // Zone maps (M7) — synchronous, populated at open()
+  // Zone maps (M7) -- synchronous, populated at open()
   shardBounds(shardIdx: number, fieldName: string): Bounds | null;
   findShards(fieldName: string, opts?: FindShardsOptions): number[];
 }

package/types/Reader.d.ts

@@ -42,9 +42,17 @@ export interface FindShardsOptions {
   max?: number;
 }
 
+export interface ReaderOptions {
+  /** Verify the footer CRC-32C at open; throws R_BAD_CRC on mismatch, R_CRC_ABSENT if absent. */
+  verifyCrc?: boolean;
+}
+
 export class Reader {
-  static fromBuffer(bufferOrArrayBuffer: ArrayBuffer | Uint8Array): Reader;
-  constructor(buffer: ArrayBuffer);
+  static fromBuffer(bufferOrArrayBuffer: ArrayBuffer | Uint8Array, opts?: ReaderOptions): Reader;
+  constructor(buffer: ArrayBuffer, opts?: ReaderOptions);
+
+  /** Recompute the footer CRC-32C over [0, footer_off): 'ok' | 'absent'; throws R_BAD_CRC on mismatch. */
+  verifyCrc(): 'ok' | 'absent';
 
   readonly schema: FrozenSchema;
   readonly totalRows: number;
@@ -55,7 +63,7 @@ export class Reader {
   fieldIndex(name: string): number;
 
   /**
-   * Get value at (rowIdx, fieldName). F64 lane → number; U32 lane → string
+   * Get value at (rowIdx, fieldName). F64 lane -> number; U32 lane -> string
    * (resolved via the shard's local string table).
    */
   get(rowIdx: number, fieldName: string): number | string | undefined;

package/types/Split.d.ts

@@ -28,7 +28,7 @@ export interface CompileInPartsOptions extends SplitOptions, CompilePartOptions
 
 /**
  * Divide NDJSON bytes into N ranges at safe line boundaries. Every returned
- * range contains complete records — no split mid-line. The union of ranges
+ * range contains complete records -- no split mid-line. The union of ranges
  * equals the original bytes (no gaps, no overlaps).
  */
 export function splitNDJSON(bytes: Uint8Array, opts?: SplitOptions): SplitRange[];

package/types/Tokenizer.d.ts

@@ -5,7 +5,7 @@ export const VERSION: string;
 
 /**
  * Sink protocol the Tokenizer emits into. Byte ranges (bytes[from, to)) are
- * ephemeral — valid only for the duration of the call. Consumers that need
+ * ephemeral -- valid only for the duration of the call. Consumers that need
  * to retain content must copy or decode inside the sink method.
  */
 export interface TokenizerSink {

package/types/Writer.d.ts

@@ -30,6 +30,30 @@ export interface WriterOptions {
    * Default = targetShardBytes.
    */
   sampleBytes?: number;
+  /** Emit a CRC-32C over [0, footer_off) into the footer. Default false. */
+  crc?: boolean;
+}
+
+/** A synchronous byte sink for `finalizeToSink`. */
+export interface ContainerSink {
+  write(bytes: Uint8Array): void;
+  /** Required for layout:'stream' (one header backpatch at position 0). */
+  writeAt?(bytes: Uint8Array, position: number): void;
+}
+
+export interface FinalizeToSinkOptions {
+  /** 'prefix' (classic, byte-identical to finalize()) or 'stream'. Default 'stream'. */
+  layout?: 'prefix' | 'stream';
+  /** Override the constructor `crc`. */
+  crc?: boolean;
+}
+
+export interface SinkResult {
+  totalRows: number;
+  shardCount: number;
+  schema: FrozenSchema;
+  bytesWritten: number;
+  layout: 'prefix' | 'stream';
 }
 
 export interface FrozenField {
@@ -54,7 +78,7 @@ export interface Container {
 export class Writer implements TokenizerSinkForWriter {
   constructor(opts?: WriterOptions);
   /**
-   * Sink protocol — the Tokenizer calls these. Byte ranges are ephemeral.
+   * Sink protocol -- the Tokenizer calls these. Byte ranges are ephemeral.
    */
   onStartObject(): void;
   onEndObject(): void;
@@ -85,12 +109,25 @@ export class Writer implements TokenizerSinkForWriter {
    * Tokenizer has been given `end()`.
    */
   finalize(): Container;
+  /**
+   * Bind a sink and stream shards as they finalize, for bounded-RAM output.
+   * Call BEFORE feeding the tokenizer, then finish with finalizeToSink. Peak
+   * memory is O(targetShardBytes + directory). opts.layout must be 'stream'.
+   */
+  beginStream(sink: ContainerSink, opts?: FinalizeToSinkOptions): void;
+  /**
+   * Emit the container to a caller sink instead of returning a buffer. Preceded
+   * by beginStream it writes only the trailer (bounded RAM, O(shard)); called
+   * alone it buffers the shards first (O(container)). With layout:'prefix' the
+   * emitted bytes are identical to finalize().
+   */
+  finalizeToSink(sink: ContainerSink, opts?: FinalizeToSinkOptions): SinkResult;
   readonly schema: FrozenSchema | null;
   readonly totalRows: number;
   readonly shardCount: number;
 }
 
-// Structural type — the Writer conforms to the Tokenizer's sink protocol.
+// Structural type -- the Writer conforms to the Tokenizer's sink protocol.
 // (Redeclared here so the /writer subpath is self-contained.)
 interface TokenizerSinkForWriter {
   onStartObject(): void;

package/types/index.d.ts

@@ -54,5 +54,6 @@ export function serialize(
  * Auto-detects preserve vs schema via the header flag byte.
  */
 export function deserialize(
-  bytes: Uint8Array | ArrayBuffer
+  bytes: Uint8Array | ArrayBuffer,
+  opts?: { verifyCrc?: boolean }
 ): import('./Reader.d.ts').Reader | import('./PreserveReader.d.ts').PreserveReader;