@zakkster/lite-bake-stream 1.0.0

package/src/index.js

@@ -0,0 +1,193 @@
+// @zakkster/lite-bake-stream / top-level convenience API
+// Copyright (c) 2026 Zahary Shinikchiev. MIT.
+//
+// The one-liners most consumers want. Two modes are dispatched from a single
+// API:
+//   schema mode (default) - Tokenizer + Writer, F64/U32 lanes, zone maps, query pruning.
+//   preserve mode         - PreserveTokenizer + PreserveWriter, opaque byte blobs.
+//                           Set opts.preserve = true. Bytes in, same bytes out.
+// deserialize() auto-detects the mode via the container's flag bit and
+// returns the appropriate Reader.
+
+import { Tokenizer, TokenizerError } from './Tokenizer.js';
+import { Writer, WriterError } from './Writer.js';
+import { Reader, ReaderError } from './Reader.js';
+import { StringTable } from './StringTable.js';
+import { PreserveTokenizer, PreserveTokenizerError } from './PreserveTokenizer.js';
+import { PreserveWriter, PreserveWriterError } from './PreserveWriter.js';
+import { PreserveReader, PreserveReaderError } from './PreserveReader.js';
+
+export {
+  Tokenizer, TokenizerError,
+  Writer, WriterError,
+  Reader, ReaderError,
+  StringTable,
+  PreserveTokenizer, PreserveTokenizerError,
+  PreserveWriter, PreserveWriterError,
+  PreserveReader, PreserveReaderError,
+};
+export const VERSION = '1.0.0';
+
+const encoder = new TextEncoder();
+
+// Serialize input data into an LBK1 container. Input may be:
+//   - Uint8Array         (raw NDJSON bytes)
+//   - string             (NDJSON text)
+//   - Iterable<object>   (records; JSON.stringify'd internally). Schema mode only.
+//   - AsyncIterable<Uint8Array> or ReadableStream<Uint8Array>
+//
+// Options:
+//   preserve:  when true, use preserve mode (opaque byte blobs, any JSON shape
+//              allowed). When false (default), schema mode with lane packing.
+//   framing:   'ndjson' | 'array' | 'auto'. Preserve mode is NDJSON-only.
+//   writer:    writer options. Schema mode: { schema, targetShardBytes, sampleBytes }.
+//              Preserve mode: { targetShardBytes, maxRecordBytes }.
+export function serialize(input, opts) {
+  opts = opts || {};
+  const preserve = opts.preserve === true;
+
+  if (input && typeof input.getReader === 'function') {
+    return preserve
+      ? _serializeReadableStreamPreserve(input, opts)
+      : _serializeReadableStream(input, opts.writer || {}, opts.framing || 'ndjson');
+  }
+  if (input && typeof input[Symbol.asyncIterator] === 'function') {
+    return preserve
+      ? _serializeAsyncIterablePreserve(input, opts)
+      : _serializeAsyncIterable(input, opts.writer || {}, opts.framing || 'ndjson');
+  }
+
+  if (preserve) return _serializeSyncPreserve(input, opts);
+  return _serializeSyncSchema(input, opts.writer || {}, opts.framing || 'ndjson');
+}
+
+function _serializeSyncSchema(input, writerOpts, framing) {
+  const w = new Writer(writerOpts);
+  const t = new Tokenizer(w, { framing });
+  if (input instanceof Uint8Array) {
+    t.feed(input);
+  } else if (typeof input === 'string') {
+    t.feed(encoder.encode(input));
+  } else if (input && typeof input[Symbol.iterator] === 'function') {
+    for (const record of input) {
+      if (record === null || typeof record !== 'object') {
+        throw new TypeError('serialize: iterable must yield objects, got ' + typeof record);
+      }
+      t.feed(encoder.encode(JSON.stringify(record) + '\n'));
+    }
+  } else {
+    throw new TypeError('serialize: unsupported input type');
+  }
+  t.end();
+  return new Uint8Array(w.finalize().buffer);
+}
+
+function _serializeSyncPreserve(input, opts) {
+  const w = new PreserveWriter(opts.writer || {});
+  const t = new PreserveTokenizer(w, {
+    framing: 'ndjson',
+    maxRecordBytes: (opts.writer && opts.writer.maxRecordBytes) || 0,
+  });
+  if (input instanceof Uint8Array) {
+    t.feed(input);
+  } else if (typeof input === 'string') {
+    t.feed(encoder.encode(input));
+  } else if (input && typeof input[Symbol.iterator] === 'function') {
+    // Iterable of records: JSON.stringify each and feed as NDJSON.
+    for (const record of input) {
+      if (record === null || typeof record !== 'object') {
+        throw new TypeError('serialize (preserve): iterable must yield objects, got ' + typeof record);
+      }
+      t.feed(encoder.encode(JSON.stringify(record) + '\n'));
+    }
+  } else {
+    throw new TypeError('serialize: unsupported input type');
+  }
+  t.end();
+  return new Uint8Array(w.finalize().buffer);
+}
+
+async function _serializeReadableStream(stream, writerOpts, framing) {
+  const w = new Writer(writerOpts);
+  const t = new Tokenizer(w, { framing });
+  const reader = stream.getReader();
+  try {
+    while (true) {
+      const { value, done } = await reader.read();
+      if (done) break;
+      if (!(value instanceof Uint8Array)) throw new TypeError('serialize: stream must yield Uint8Array');
+      t.feed(value);
+    }
+  } finally { reader.releaseLock(); }
+  t.end();
+  return new Uint8Array(w.finalize().buffer);
+}
+
+async function _serializeReadableStreamPreserve(stream, opts) {
+  const w = new PreserveWriter(opts.writer || {});
+  const t = new PreserveTokenizer(w, {
+    framing: 'ndjson',
+    maxRecordBytes: (opts.writer && opts.writer.maxRecordBytes) || 0,
+  });
+  const reader = stream.getReader();
+  try {
+    while (true) {
+      const { value, done } = await reader.read();
+      if (done) break;
+      if (!(value instanceof Uint8Array)) throw new TypeError('serialize: stream must yield Uint8Array');
+      t.feed(value);
+    }
+  } finally { reader.releaseLock(); }
+  t.end();
+  return new Uint8Array(w.finalize().buffer);
+}
+
+async function _serializeAsyncIterable(iterable, writerOpts, framing) {
+  const w = new Writer(writerOpts);
+  const t = new Tokenizer(w, { framing });
+  for await (const chunk of iterable) {
+    if (!(chunk instanceof Uint8Array)) throw new TypeError('serialize: async iterable must yield Uint8Array');
+    t.feed(chunk);
+  }
+  t.end();
+  return new Uint8Array(w.finalize().buffer);
+}
+
+async function _serializeAsyncIterablePreserve(iterable, opts) {
+  const w = new PreserveWriter(opts.writer || {});
+  const t = new PreserveTokenizer(w, {
+    framing: 'ndjson',
+    maxRecordBytes: (opts.writer && opts.writer.maxRecordBytes) || 0,
+  });
+  for await (const chunk of iterable) {
+    if (!(chunk instanceof Uint8Array)) throw new TypeError('serialize: async iterable must yield Uint8Array');
+    t.feed(chunk);
+  }
+  t.end();
+  return new Uint8Array(w.finalize().buffer);
+}
+
+// Deserialize LBK1 bytes into the right Reader for the container's mode.
+// Auto-detects preserve vs schema via header flag byte at offset 7 bit 0.
+export function deserialize(bytes) {
+  let buffer;
+  if (bytes instanceof ArrayBuffer) {
+    buffer = bytes;
+  } else if (bytes instanceof Uint8Array) {
+    if (bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength) {
+      buffer = bytes.buffer;
+    } else {
+      const copy = new Uint8Array(bytes.byteLength);
+      copy.set(bytes);
+      buffer = copy.buffer;
+    }
+  } else {
+    throw new TypeError('deserialize: expected Uint8Array or ArrayBuffer, got ' + typeof bytes);
+  }
+  if (buffer.byteLength < 8) {
+    throw new ReaderError('R_TRUNCATED', 'container too small to inspect header flags');
+  }
+  const flags = new Uint8Array(buffer, 7, 1)[0];
+  if (flags & 0x01) return new PreserveReader(buffer);
+  return new Reader(buffer);
+}

package/types/FileIngest.d.ts

@@ -0,0 +1,44 @@
+// Type declarations for @zakkster/lite-bake-stream/file-ingest
+// Copyright (c) 2026 Zahary Shinikchiev. MIT.
+
+import type { Reader } from './Reader.d.ts';
+import type { WriterOptions } from './Writer.d.ts';
+
+export const VERSION: string;
+
+export interface IngestProgress {
+  bytesIngested: number;
+  totalBytes: number;
+  rowsWritten: number;
+  shardsCommitted: number;
+  chunkBytes: number;
+  chunkCount: number;
+  elapsedMs: number;
+}
+
+export interface IngestStreamOptions {
+  /** 'ndjson' | 'array' | 'auto'. Default 'ndjson'. */
+  framing?: 'ndjson' | 'array' | 'auto';
+  writer?: WriterOptions;
+  /** Called after each chunk with progress state. */
+  onProgress?: (state: IngestProgress) => void;
+  /** Pass file.size for progress percentage. -1 for indeterminate streams. */
+  totalBytes?: number;
+}
+
+/**
+ * Ingest a ReadableStream<Uint8Array> into an LBK1 Reader.
+ * Works with File.stream(), fetch().body, and any Web Streams source.
+ */
+export function ingestStream(
+  readableStream: ReadableStream<Uint8Array>,
+  opts?: IngestStreamOptions
+): Promise<Reader>;
+
+/**
+ * Convenience for Blob/File inputs. Pulls .stream() and .size automatically.
+ */
+export function ingestFile(
+  file: Blob | File,
+  opts?: IngestStreamOptions
+): Promise<Reader>;

package/types/MultiReader.d.ts

@@ -0,0 +1,36 @@
+// Type declarations for @zakkster/lite-bake-stream/multi-reader
+// Copyright (c) 2026 Zahary Shinikchiev. MIT.
+
+import type { Reader, FrozenSchema, Bounds, FindShardsOptions } from './Reader.d.ts';
+
+export const VERSION: string;
+
+export class MultiReader {
+  constructor(readers: Reader[]);
+
+  readonly schema: FrozenSchema;
+  readonly totalRows: number;
+  readonly shardCount: number;
+  readonly readerCount: number;
+  readonly hasZoneMaps: boolean;
+
+  fieldIndex(name: string): number;
+  get(rowIdx: number, fieldName: string): number | string | undefined;
+
+  /** Access a sub-Reader by index. */
+  reader(readerIdx: number): Reader;
+  /** Which sub-Reader owns a given global row? */
+  readerForRow(rowIdx: number): { readerIdx: number; localRow: number };
+  /** Which sub-Reader owns a given global shard? */
+  readerForShard(globalShardIdx: number): { readerIdx: number; localShard: number } | null;
+
+  // Zone maps — global shard indices merged across sub-readers
+  shardBounds(globalShardIdx: number, fieldName: string): Bounds | null;
+  findShards(fieldName: string, opts?: FindShardsOptions): number[];
+}
+
+export class MultiReaderError extends Error {
+  code: string;
+  name: 'MultiReaderError';
+  constructor(code: string, message: string);
+}

package/types/PreserveReader.d.ts

@@ -0,0 +1,44 @@
+// Type declarations for @zakkster/lite-bake-stream/preserve-reader
+// Copyright (c) 2026 Zahary Shinikchiev. MIT.
+
+export const VERSION: string;
+
+export interface PreserveShardHandle {
+  payloadOff: number;
+  payloadLen: number;
+  rowCount: number;
+  blobLen: number;
+  offsetTableOff: number;
+  firstRow: number;
+  endRow: number;
+}
+
+export class PreserveReader {
+  static fromBuffer(input: ArrayBuffer | Uint8Array): PreserveReader;
+  constructor(buffer: ArrayBuffer);
+
+  readonly totalRows: number;
+  readonly shardCount: number;
+  readonly shards: PreserveShardHandle[];
+  readonly buffer: ArrayBuffer;
+  readonly mode: 'preserve';
+  readonly hasZoneMaps: false;
+
+  /**
+   * Zero-alloc: returns a Uint8Array subarray view into the container buffer.
+   * Do NOT mutate. Do NOT retain past the reader's lifetime.
+   */
+  getBytes(rowIdx: number): Uint8Array;
+
+  /** Allocates one string via TextDecoder. */
+  getString(rowIdx: number): string;
+
+  /** Allocates the parsed value graph via JSON.parse. */
+  getJSON(rowIdx: number): unknown;
+}
+
+export class PreserveReaderError extends Error {
+  code: string;
+  name: 'PreserveReaderError';
+  constructor(code: string, message: string);
+}

package/types/PreserveTokenizer.d.ts

@@ -0,0 +1,27 @@
+// Type declarations for @zakkster/lite-bake-stream/preserve-tokenizer
+// Copyright (c) 2026 Zahary Shinikchiev. MIT.
+
+export const VERSION: string;
+
+export interface PreserveSink {
+  onRecord(bytes: Uint8Array, from: number, to: number): void;
+}
+
+export interface PreserveTokenizerOptions {
+  /** Only 'ndjson' is supported in v1. */
+  framing?: 'ndjson';
+  /** If > 0, throw E_RECORD_TOO_LARGE when a single record exceeds this. */
+  maxRecordBytes?: number;
+}
+
+export class PreserveTokenizer {
+  constructor(sink: PreserveSink, opts?: PreserveTokenizerOptions);
+  feed(chunk: Uint8Array): void;
+  end(): void;
+}
+
+export class PreserveTokenizerError extends Error {
+  code: string;
+  name: 'PreserveTokenizerError';
+  constructor(code: string, message: string);
+}

package/types/PreserveWriter.d.ts

@@ -0,0 +1,35 @@
+// Type declarations for @zakkster/lite-bake-stream/preserve-writer
+// Copyright (c) 2026 Zahary Shinikchiev. MIT.
+
+export const VERSION: string;
+
+export interface PreserveWriterOptions {
+  /** Target output bytes per shard. Default 8 MiB. */
+  targetShardBytes?: number;
+  /** Passed to PreserveTokenizer via the top-level API. */
+  maxRecordBytes?: number;
+}
+
+export interface PreserveContainer {
+  buffer: ArrayBuffer;
+  totalRows: number;
+  shardCount: number;
+  mode: 'preserve';
+}
+
+export class PreserveWriter {
+  constructor(opts?: PreserveWriterOptions);
+  /** Sink protocol. Called by PreserveTokenizer for each detected record. */
+  onRecord(bytes: Uint8Array, from: number, to: number): void;
+  /** Public API for direct record writes. */
+  writeRecord(bytes: Uint8Array): void;
+  finalize(): PreserveContainer;
+  readonly totalRows: number;
+  readonly shardCount: number;
+}
+
+export class PreserveWriterError extends Error {
+  code: string;
+  name: 'PreserveWriterError';
+  constructor(code: string, message: string);
+}

package/types/RangeReader.d.ts

@@ -0,0 +1,93 @@
+// Type declarations for @zakkster/lite-bake-stream/range-reader
+// Copyright (c) 2026 Zahary Shinikchiev. MIT.
+
+import type { FrozenSchema, Bounds, FindShardsOptions } from './Reader.d.ts';
+import type { StringTableView } from './StringTable.d.ts';
+
+export const VERSION: string;
+
+export interface IOAdapter {
+  size: number;
+  fetch(byteOffset: number, byteLength: number): Promise<Uint8Array>;
+}
+
+export interface AdapterStats {
+  requests: number;
+  bytesFetched: number;
+}
+
+export class HTTPRangeAdapter implements IOAdapter {
+  static open(url: string, opts?: { fetch?: typeof fetch }): Promise<HTTPRangeAdapter>;
+  readonly url: string;
+  size: number;
+  stats: AdapterStats;
+  fetch(byteOffset: number, byteLength: number): Promise<Uint8Array>;
+}
+
+export class MockRangeAdapter implements IOAdapter {
+  constructor(bytes: Uint8Array | ArrayBuffer);
+  size: number;
+  /** Fetch log: [[offset, length], ...] for test assertions. */
+  log: [number, number][];
+  stats: AdapterStats;
+  fetch(byteOffset: number, byteLength: number): Promise<Uint8Array>;
+}
+
+export interface RangeReaderOptions {
+  /** Bounded LRU cache size. Default 8. */
+  maxCachedShards?: number;
+}
+
+export interface RangeShardHandle {
+  payloadOff: number;
+  payloadLen: number;
+  rowCount: number;
+  localStrOff: number;
+  localStrLen: number;
+  firstRow: number;
+  endRow: number;
+}
+
+export class RangeReader {
+  static open(adapter: IOAdapter, opts?: RangeReaderOptions): Promise<RangeReader>;
+
+  readonly adapter: IOAdapter;
+  readonly maxCachedShards: number;
+  readonly schema: FrozenSchema;
+  readonly totalRows: number;
+  readonly shardCount: number;
+  readonly shards: RangeShardHandle[];
+  readonly cachedShardCount: number;
+  readonly hasZoneMaps: boolean;
+
+  fieldIndex(name: string): number;
+
+  /** Fetch a shard's payload+string-table (one range request), cache, return record. */
+  loadShard(shardIdx: number): Promise<{
+    payloadBytes: Uint8Array;
+    payloadDv: DataView;
+    stringTable: StringTableView | null;
+    firstRow: number;
+    rowCount: number;
+  }>;
+
+  get(rowIdx: number, fieldName: string): Promise<number | string | undefined>;
+
+  /** Ensure the shards covering [firstRow, lastRow) are cached. */
+  prefetchRange(firstRow: number, lastRow: number): Promise<void>;
+
+  /** Await-free reads after prefetchRange. Throws if any covering shard is uncached. */
+  syncRange(firstRow: number, lastRow: number): {
+    get(rowIdx: number, fieldName: string): number | string | undefined;
+  };
+
+  // Zone maps (M7) — synchronous, populated at open()
+  shardBounds(shardIdx: number, fieldName: string): Bounds | null;
+  findShards(fieldName: string, opts?: FindShardsOptions): number[];
+}
+
+export class RangeReaderError extends Error {
+  code: string;
+  name: 'RangeReaderError';
+  constructor(code: string, message: string);
+}

package/types/Reader.d.ts

@@ -0,0 +1,85 @@
+// Type declarations for @zakkster/lite-bake-stream/reader
+// Copyright (c) 2026 Zahary Shinikchiev. MIT.
+
+export const VERSION: string;
+
+export type LaneKindNumeric = 1 | 3;
+
+export interface FrozenField {
+  name: string;
+  /** 1 = F64, 3 = U32. */
+  laneKind: LaneKindNumeric;
+  offsetInRow: number;
+}
+
+export interface FrozenSchema {
+  fields: FrozenField[];
+  rowStride: number;
+}
+
+export interface ShardHandle {
+  payloadOff: number;
+  payloadLen: number;
+  rowCount: number;
+  payloadBytes: Uint8Array;
+  payloadDv: DataView;
+  stringTable: StringTableView | null;
+}
+
+export interface StringTableView {
+  readonly count: number;
+  get(idx: number): string | undefined;
+  bytesAt(idx: number): Uint8Array | undefined;
+}
+
+export interface Bounds {
+  min: number;
+  max: number;
+}
+
+export interface FindShardsOptions {
+  min?: number;
+  max?: number;
+}
+
+export class Reader {
+  static fromBuffer(bufferOrArrayBuffer: ArrayBuffer | Uint8Array): Reader;
+  constructor(buffer: ArrayBuffer);
+
+  readonly schema: FrozenSchema;
+  readonly totalRows: number;
+  readonly shardCount: number;
+  readonly shards: ShardHandle[];
+  readonly buffer: ArrayBuffer;
+
+  fieldIndex(name: string): number;
+
+  /**
+   * Get value at (rowIdx, fieldName). F64 lane → number; U32 lane → string
+   * (resolved via the shard's local string table).
+   */
+  get(rowIdx: number, fieldName: string): number | string | undefined;
+
+  // Hot-loop friendly accessors
+  shardPayload(shardIdx: number): DataView;
+  strideBytes(): number;
+  offsetBytes(fieldName: string): number;
+  laneKind(fieldName: string): LaneKindNumeric;
+  shardStringTable(shardIdx: number): StringTableView | null;
+
+  // Back-compat with all-F64 schemas (M2 API surface)
+  shardF64(shardIdx: number): Float64Array;
+  strideF64(): number;
+  offsetF64(fieldName: string): number;
+
+  // Zone maps (M7)
+  readonly hasZoneMaps: boolean;
+  shardBounds(shardIdx: number, fieldName: string): Bounds | null;
+  findShards(fieldName: string, opts?: FindShardsOptions): number[];
+}
+
+export class ReaderError extends Error {
+  code: string;
+  name: 'ReaderError';
+  constructor(code: string, message: string);
+}

package/types/Split.d.ts

@@ -0,0 +1,66 @@
+// Type declarations for @zakkster/lite-bake-stream/split
+// Copyright (c) 2026 Zahary Shinikchiev. MIT.
+
+import type { WriterOptions } from './Writer.d.ts';
+
+export const VERSION: string;
+
+export interface SplitRange {
+  /** Byte offset (inclusive). */
+  start: number;
+  /** Byte offset (exclusive). */
+  end: number;
+}
+
+export interface SplitOptions {
+  /** Desired part count. Actual count may be less on tiny inputs. Default 4. */
+  targetParts?: number;
+  /** Optional ceiling on part size. Splitter emits more parts to honor it. */
+  maxPartBytes?: number;
+}
+
+export interface CompilePartOptions {
+  framing?: 'ndjson' | 'array' | 'auto';
+  writer?: WriterOptions;
+}
+
+export interface CompileInPartsOptions extends SplitOptions, CompilePartOptions {}
+
+/**
+ * Divide NDJSON bytes into N ranges at safe line boundaries. Every returned
+ * range contains complete records — no split mid-line. The union of ranges
+ * equals the original bytes (no gaps, no overlaps).
+ */
+export function splitNDJSON(bytes: Uint8Array, opts?: SplitOptions): SplitRange[];
+
+/**
+ * Compile one byte range into a standalone LBK1 container. Trivially
+ * Transferable across worker boundaries via `postMessage(buf, [buf])`.
+ * For multi-part compilation, ALL parts MUST share the same explicit
+ * schema (see module docs); otherwise merge will fail with S_SCHEMA_MISMATCH.
+ */
+export function compilePart(bytes: Uint8Array, opts?: CompilePartOptions): Uint8Array;
+
+/**
+ * Sequential convenience: split + compile each part serially.
+ * Returns one Uint8Array per part. Use this when you don't have workers,
+ * or as a deterministic baseline against a worker-parallelized run.
+ */
+export function compileInParts(bytes: Uint8Array, opts?: CompileInPartsOptions): Uint8Array[];
+
+/**
+ * Concatenate N LBK1 containers into ONE. All containers MUST share an
+ * identical schema. Preserves per-shard string tables and zone maps
+ * verbatim; rewrites the shard directory and header offsets.
+ *
+ * For query workloads over multiple parts, prefer MultiReader (M4) which
+ * avoids the copy. Use mergeContainers when you need a single-file
+ * distributable.
+ */
+export function mergeContainers(containers: (Uint8Array | ArrayBuffer)[]): Uint8Array;
+
+export class SplitError extends Error {
+  code: string;
+  name: 'SplitError';
+  constructor(code: string, message: string);
+}

package/types/StringTable.d.ts

@@ -0,0 +1,23 @@
+// Type declarations for @zakkster/lite-bake-stream/string-table
+// Copyright (c) 2026 Zahary Shinikchiev. MIT.
+
+export const VERSION: string;
+
+export class StringTable {
+  constructor();
+  /** Intern a byte range. Zero-alloc on the hit path. Returns u32 index. */
+  intern(bytes: Uint8Array, from: number, to: number): number;
+  /** Zero-copy byte range for entry idx. */
+  bytesAt(idx: number): Uint8Array | undefined;
+  reset(): void;
+  serialize(): { bytes: Uint8Array; byteLength: number };
+  static parse(bytes: Uint8Array, byteOffset: number): StringTableView;
+  readonly count: number;
+  readonly blobLen: number;
+}
+
+export class StringTableView {
+  readonly count: number;
+  get(idx: number): string | undefined;
+  bytesAt(idx: number): Uint8Array | undefined;
+}

package/types/Tokenizer.d.ts

@@ -0,0 +1,42 @@
+// Type declarations for @zakkster/lite-bake-stream/tokenizer
+// Copyright (c) 2026 Zahary Shinikchiev. MIT.
+
+export const VERSION: string;
+
+/**
+ * Sink protocol the Tokenizer emits into. Byte ranges (bytes[from, to)) are
+ * ephemeral — valid only for the duration of the call. Consumers that need
+ * to retain content must copy or decode inside the sink method.
+ */
+export interface TokenizerSink {
+  onStartObject?(): void;
+  onEndObject?(): void;
+  onStartArray?(): void;
+  onEndArray?(): void;
+  onKey?(bytes: Uint8Array, from: number, to: number): void;
+  onNumber?(value: number): void;
+  onString?(bytes: Uint8Array, from: number, to: number): void;
+  onTrue?(): void;
+  onFalse?(): void;
+  onNull?(): void;
+  onEnd?(): void;
+}
+
+export interface TokenizerOptions {
+  /** 'ndjson' | 'array' | 'auto'. Default 'ndjson'. */
+  framing?: 'ndjson' | 'array' | 'auto';
+}
+
+export class Tokenizer {
+  constructor(sink: TokenizerSink, opts?: TokenizerOptions);
+  /** Feed a chunk of UTF-8 bytes. Safe to split at any byte boundary. */
+  feed(chunk: Uint8Array): void;
+  /** Signal end of input. Any pending record is flushed to the sink. */
+  end(): void;
+}
+
+export class TokenizerError extends Error {
+  code: string;
+  name: 'TokenizerError';
+  constructor(code: string, message: string);
+}