@zakkster/lite-bake-stream 1.6.0 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +148 -128
- package/README.md +291 -109
- package/SPEC.md +29 -29
- package/llms.txt +10 -13
- package/package.json +2 -2
- package/src/FileIngest.js +6 -2
- package/src/MultiReader.js +1 -1
- package/src/Opts.js +4 -0
- package/src/PreserveReader.js +3 -1
- package/src/PreserveTokenizer.js +12 -3
- package/src/PreserveWriter.js +8 -2
- package/src/RangeReader.js +3 -1
- package/src/Reader.js +3 -1
- package/src/Split.js +8 -4
- package/src/StringTable.js +7 -4
- package/src/Tokenizer.js +3 -2
- package/src/Views.js +3 -0
- package/src/Writer.js +6 -4
- package/src/index.js +5 -1
- package/types/MultiReader.d.ts +1 -1
- package/types/RangeReader.d.ts +1 -1
- package/types/Reader.d.ts +1 -1
- package/types/Split.d.ts +1 -1
- package/types/Tokenizer.d.ts +1 -1
- package/types/Writer.d.ts +2 -2
package/llms.txt
CHANGED
|
@@ -1,20 +1,20 @@
|
|
|
1
1
|
# @zakkster/lite-bake-stream
|
|
2
2
|
|
|
3
|
-
Streaming byte-level JSON to
|
|
3
|
+
Streaming byte-level JSON to LBK1 binary containers for JS runtimes.
|
|
4
4
|
|
|
5
5
|
## Purpose
|
|
6
6
|
|
|
7
|
-
Ingest gigabyte-scale JSON (top-level array or NDJSON) into the
|
|
7
|
+
Ingest gigabyte-scale JSON (top-level array or NDJSON) into the LBK1 binary container format without materializing the intermediate object graph. The container is flat, interleaved, and zero-GC to read; a Reader gives random access to any row without re-parsing JSON.
|
|
8
8
|
|
|
9
9
|
## Status
|
|
10
10
|
|
|
11
|
-
v1.6.
|
|
11
|
+
v1.6.1 -- stable. LBK1 format frozen at `format_version: 1`. Qualified on an 8 GB soak (M1 MacBook Pro): 98.37M rows, 4.89 GB container, zero major GC, zero minor GC, 499 KB total heap allocation, 590.21M cells verified byte-exact, zero mismatches. Tokenizer benches at 222-237 MB/s (~55% of JSON.parse, with no object graph allocated).
|
|
12
12
|
|
|
13
13
|
Public API follows semver from 1.0.0. Future additions (I64 lane, columnar payload mode, container-level string table) land via the format's forward-compat seams -- `min_reader_version` on ShardEntry, reserved FieldDescriptor flags, the `metadata_off` block wrapper -- without a format_version bump.
|
|
14
14
|
|
|
15
15
|
See SPEC.md for the LBK1 container format, section 3.6 for zone maps, section 4.3 for the reserved field flags.
|
|
16
16
|
|
|
17
|
-
## Public API (v1.6.
|
|
17
|
+
## Public API (v1.6.1)
|
|
18
18
|
|
|
19
19
|
Two ingest modes share one top-level API:
|
|
20
20
|
|
|
@@ -34,21 +34,18 @@ Schema-mode classes:
|
|
|
34
34
|
Preserve-mode classes:
|
|
35
35
|
- `PreserveTokenizer` (`/preserve-tokenizer`): NDJSON record-boundary scanner, JSON-aware depth tracking, chunk-safe.
|
|
36
36
|
- `PreserveWriter` (`/preserve-writer`): opaque byte-blob sink, pre-allocated shard buffer, zero-GC record path.
|
|
37
|
-
- `PreserveReader` (`/preserve-reader`): tri-API
|
|
37
|
+
- `PreserveReader` (`/preserve-reader`): tri-API -- `getBytes(i)` (zero-alloc view), `getString(i)`, `getJSON(i)`. A returned view pins the container's ArrayBuffer (plain subarray semantics); copy the bytes out if you need the container to be collectable.
|
|
38
38
|
|
|
39
39
|
Shared:
|
|
40
40
|
- `StringTable` (`/string-table`): byte-level UTF-8 interning primitive.
|
|
41
|
-
- `ingestStream`, `ingestFile` (`/file-ingest`): browser helpers piping a `ReadableStream<Uint8Array>` through the pipeline. `preserve` option dispatches to the right writer.
|
|
41
|
+
- `ingestStream`, `ingestFile` (`/file-ingest`): browser helpers piping a `ReadableStream<Uint8Array>` (e.g. `File.stream()`) through the pipeline. `preserve` option dispatches to the right writer. Per-chunk `onProgress` callback; its state object is reused across calls (mutated in place, zero per-chunk allocation) -- copy it if retained past the callback.
|
|
42
42
|
|
|
43
43
|
Error classes with stable `code`: `TokenizerError`, `WriterError`, `ReaderError`, `RangeReaderError`, `MultiReaderError`, `SplitError`, `PreserveTokenizerError`, `PreserveWriterError`, `PreserveReaderError`.
|
|
44
|
-
- `ingestStream`, `ingestFile` (`/file-ingest`): browser helpers piping a `ReadableStream<Uint8Array>` (e.g. `File.stream()`) through the Tokenizer + Writer, returning a Reader. Per-chunk `onProgress` callback; its state object is reused across calls (mutated in place, zero per-chunk allocation) -- copy it if retained past the callback.
|
|
45
|
-
- `TokenizerError`, `WriterError`, `ReaderError`, `RangeReaderError`: thrown on parse/write/read errors with stable `code`.
|
|
46
|
-
- `VERSION` const per subpath.
|
|
47
44
|
|
|
48
45
|
## Schema forms
|
|
49
46
|
|
|
50
|
-
- `{ fields: ['id', 'x', 'y'] }`
|
|
51
|
-
- `{ fields: [{name:'id', laneKind:'f64'}, {name:'tag', laneKind:'u32'}] }`
|
|
47
|
+
- `{ fields: ['id', 'x', 'y'] }` -- all-F64 shorthand, back-compat with M2.
|
|
48
|
+
- `{ fields: [{name:'id', laneKind:'f64'}, {name:'tag', laneKind:'u32'}] }` -- mixed lanes (M3).
|
|
52
49
|
- Sample-and-infer (default when no schema is passed): the first `sampleBytes` of INPUT are observed, byte-true (BS-07) -- the window ends at the first record boundary at or after `sampleBytes` input bytes, read from the Tokenizer's `absOffset`, independent of chunking and record count (a hand-driven Writer with no wired source samples by record count). Fields with only numbers become F64, only strings become U32. null is lane-neutral (BS-20): it sets no kind, so null+string infers U32 (no longer "mixed") and null+number infers F64; post-freeze null reads 0 on an F64 lane and "" on a U32 lane. A field that saw both a real number and a real string raises `W_MIXED_LANE_TYPES` at freeze. Sample memory is O(`sampleBytes`): columnar staging plus a shared string table, independent of total input size.
|
|
53
50
|
|
|
54
51
|
## Contract
|
|
@@ -60,7 +57,7 @@ Error classes with stable `code`: `TokenizerError`, `WriterError`, `ReaderError`
|
|
|
60
57
|
|
|
61
58
|
## Preservation contract
|
|
62
59
|
|
|
63
|
-
Every declared field of every row round-trips. F64 lanes: bit-exact for numeric literals with
|
|
60
|
+
Every declared field of every row round-trips. F64 lanes: bit-exact for numeric literals with <=15 significant digits and |exponent| <= 22 (Clinger's fast path); <=1 ULP drift outside that domain (documented, pinned by NumericBoundary.test.js). U32 lanes: byte-exact UTF-8, unconditional. Missing fields -> documented defaults. Unknown keys -> silently dropped. Wrong-type value on post-freeze schema -> W_LANE_MISMATCH error, no corrupt container produced. See SPEC section 7 for the full table. Asserted by tests, property-based fuzz, AND torture-scale verification (~103,000 rows across 20 scenarios per fast-tier run).
|
|
64
61
|
|
|
65
62
|
## Row-index and refusal codes
|
|
66
63
|
|
|
@@ -85,7 +82,7 @@ F64 only. Values exceeding IEEE 754 double range are rejected as `E_NUMBER_OVERF
|
|
|
85
82
|
|
|
86
83
|
## Tree-shaking
|
|
87
84
|
|
|
88
|
-
Subpath entries per SPEC section 6. `sideEffects: false`. Consumers import only the path they need; the browser reader never pulls the writer.
|
|
85
|
+
Subpath entries per SPEC section 6. `sideEffects: false`. Consumers import only the path they need; the browser reader never pulls the writer. Every subpath exports a `VERSION` const.
|
|
89
86
|
|
|
90
87
|
## Streaming emission and integrity (M6)
|
|
91
88
|
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zakkster/lite-bake-stream",
|
|
3
|
-
"version": "1.6.
|
|
4
|
-
"description": "Streaming byte-level JSON to
|
|
3
|
+
"version": "1.6.1",
|
|
4
|
+
"description": "Streaming byte-level JSON to LBK1 binary containers. Zero-GC, tree-shakeable, gigabyte-scale.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"sideEffects": false,
|
|
7
7
|
"exports": {
|
package/src/FileIngest.js
CHANGED
|
@@ -2,11 +2,15 @@
|
|
|
2
2
|
// Browser-side helper: File.stream() -> Tokenizer -> Writer -> Reader.
|
|
3
3
|
// Copyright (c) 2026 Zahary Shinikchiev. MIT.
|
|
4
4
|
//
|
|
5
|
+
// Error codes:
|
|
6
|
+
// E_OPTION_CONFLICT - preserve mode requested with a non-ndjson framing
|
|
7
|
+
//
|
|
5
8
|
// The natural browser ingest path is:
|
|
6
9
|
//
|
|
7
10
|
// const file = fileInputEl.files[0];
|
|
8
|
-
// const reader = await ingestStream(file.stream(),
|
|
11
|
+
// const reader = await ingestStream(file.stream(), {
|
|
9
12
|
// framing: 'ndjson',
|
|
13
|
+
// totalBytes: file.size,
|
|
10
14
|
// onProgress: ({bytesIngested, totalBytes, rowsWritten, shardsCommitted}) => {...}
|
|
11
15
|
// });
|
|
12
16
|
// // reader is a fully-materialized in-memory LBK1 Reader.
|
|
@@ -27,7 +31,7 @@ import { PreserveWriter } from './PreserveWriter.js';
|
|
|
27
31
|
import { PreserveReader } from './PreserveReader.js';
|
|
28
32
|
import { checkOpts } from './Opts.js';
|
|
29
33
|
|
|
30
|
-
export const VERSION = '1.6.
|
|
34
|
+
export const VERSION = '1.6.1';
|
|
31
35
|
|
|
32
36
|
const U32_MAX = 4294967295;
|
|
33
37
|
const INGEST_OPTS = {
|
package/src/MultiReader.js
CHANGED
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
// M_ROW_OUT_OF_RANGE - rowIdx >= totalRows
|
|
26
26
|
// M_TOO_MANY_ROWS - cumulative row count exceeds Number.MAX_SAFE_INTEGER
|
|
27
27
|
|
|
28
|
-
export const VERSION = '1.6.
|
|
28
|
+
export const VERSION = '1.6.1';
|
|
29
29
|
|
|
30
30
|
export class MultiReaderError extends Error {
|
|
31
31
|
constructor(code, msg) { super(msg); this.code = code; this.name = 'MultiReaderError'; }
|
package/src/Opts.js
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
// @zakkster/lite-bake-stream / Opts (internal, not exported from index.js)
|
|
2
2
|
// Copyright (c) 2026 Zahary Shinikchiev. MIT.
|
|
3
3
|
//
|
|
4
|
+
// Error codes:
|
|
5
|
+
// E_UNKNOWN_OPTION - an unknown option key (fails closed with a did-you-mean hint)
|
|
6
|
+
// E_OPTION_VALUE - an option value falls outside its declared domain
|
|
7
|
+
//
|
|
4
8
|
// Shared construction-time options validator. Every public constructor and
|
|
5
9
|
// entry function that takes an opts object runs checkOpts once, at construction
|
|
6
10
|
// or in a synchronous prologue -- never per byte, never per record. An unknown
|
package/src/PreserveReader.js
CHANGED
|
@@ -29,13 +29,15 @@
|
|
|
29
29
|
// R_INVALID - a structure is internally inconsistent but in-bounds
|
|
30
30
|
// R_SHARD_VERSION_TOO_NEW - a shard's min_reader_version > this reader
|
|
31
31
|
// R_ROW_OUT_OF_RANGE - rowIdx negative, fractional, NaN, or >= totalRows (BS-32)
|
|
32
|
+
// R_BAD_CRC - CRC-32C verify requested and the stored checksum mismatched
|
|
33
|
+
// R_CRC_ABSENT - CRC verify requested but the container carries no checksum
|
|
32
34
|
// R_OFFSET_TOO_LARGE - a u64 header/directory offset exceeds 2^53-1
|
|
33
35
|
|
|
34
36
|
import { toContainerBuffer } from './Views.js';
|
|
35
37
|
import { checkOpts } from './Opts.js';
|
|
36
38
|
import { crc32cInit, crc32cUpdate, crc32cFinal } from './Crc32c.js';
|
|
37
39
|
|
|
38
|
-
export const VERSION = '1.6.
|
|
40
|
+
export const VERSION = '1.6.1';
|
|
39
41
|
|
|
40
42
|
const CONTAINER_HEADER_BYTES = 48;
|
|
41
43
|
const SHARD_ENTRY_BYTES = 40;
|
package/src/PreserveTokenizer.js
CHANGED
|
@@ -1,11 +1,20 @@
|
|
|
1
1
|
// @zakkster/lite-bake-stream / PreserveTokenizer
|
|
2
2
|
// NDJSON record-boundary scanner for preserve-mode. Copyright (c) 2026 Zahary Shinikchiev. MIT.
|
|
3
3
|
//
|
|
4
|
+
// Error codes:
|
|
5
|
+
// E_UNSUPPORTED_FRAMING - preserve mode fed a framing other than ndjson
|
|
6
|
+
// E_RECORD_TOO_LARGE - a single record exceeds maxRecordBytes
|
|
7
|
+
// E_UNBALANCED - a record's JSON structure never closes
|
|
8
|
+
// E_TRUNCATED - input ends mid-record
|
|
9
|
+
// E_ENDED - feed() called after end()
|
|
10
|
+
// E_POISONED - reuse after a prior throw
|
|
11
|
+
//
|
|
4
12
|
// The schema-mode Tokenizer walks the full JSON AST (numbers, strings, keys,
|
|
5
13
|
// object/array structure) so the Writer can pack values into typed lanes.
|
|
6
|
-
// Preserve-mode doesn't crack open records
|
|
14
|
+
// Preserve-mode doesn't crack open records -- it just needs to find where one
|
|
7
15
|
// record ends and the next begins, so the record's original bytes can be
|
|
8
16
|
// shoved into a shard intact.
|
|
17
|
+
// See decisions/0010-preserve-mode.md.
|
|
9
18
|
//
|
|
10
19
|
// Contract:
|
|
11
20
|
// - Input: NDJSON bytes. One JSON value per line, delimited by 0x0A.
|
|
@@ -32,7 +41,7 @@
|
|
|
32
41
|
|
|
33
42
|
import { checkOpts } from './Opts.js';
|
|
34
43
|
|
|
35
|
-
export const VERSION = '1.6.
|
|
44
|
+
export const VERSION = '1.6.1';
|
|
36
45
|
|
|
37
46
|
const U32_MAX = 4294967295;
|
|
38
47
|
const PRESERVE_TOKENIZER_OPTS = {
|
|
@@ -57,7 +66,7 @@ const BYTE_RBRACK = 0x5D;
|
|
|
57
66
|
const BYTE_SPACE = 0x20;
|
|
58
67
|
const BYTE_TAB = 0x09;
|
|
59
68
|
|
|
60
|
-
const INITIAL_BUF = 1 << 16; // 64 KiB
|
|
69
|
+
const INITIAL_BUF = 1 << 16; // 64 KiB -- grows on demand for larger records
|
|
61
70
|
|
|
62
71
|
export class PreserveTokenizer {
|
|
63
72
|
constructor(sink, opts) {
|
package/src/PreserveWriter.js
CHANGED
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
// @zakkster/lite-bake-stream / PreserveWriter
|
|
2
2
|
// Copyright (c) 2026 Zahary Shinikchiev. MIT.
|
|
3
3
|
//
|
|
4
|
+
// Error codes:
|
|
5
|
+
// W_EMPTY_INPUT - finalize with no records written
|
|
6
|
+
// W_BAD_SINK - sink is not an object / missing write / async return
|
|
7
|
+
// W_FINALIZED - write after the writer was finalized
|
|
8
|
+
// ST_BLOB_OVERFLOW - a shard payload_len would exceed the u32 ceiling
|
|
9
|
+
//
|
|
4
10
|
// Preserve-mode sink: opaque record blobs packed into shards, with a trailing
|
|
5
11
|
// u32 offset table per shard. No schema, no lanes, no string table, no zone
|
|
6
12
|
// maps. Bytes in, same bytes out.
|
|
@@ -20,7 +26,7 @@ import { checkOpts } from './Opts.js';
|
|
|
20
26
|
import { crc32cInit, crc32cUpdate, crc32cFinal, crc32cCombine } from './Crc32c.js';
|
|
21
27
|
import { validateSink, isThenable } from './Views.js';
|
|
22
28
|
|
|
23
|
-
export const VERSION = '1.6.
|
|
29
|
+
export const VERSION = '1.6.1';
|
|
24
30
|
|
|
25
31
|
export class PreserveWriterError extends Error {
|
|
26
32
|
constructor(code, msg) { super(msg); this.code = code; this.name = 'PreserveWriterError'; }
|
|
@@ -35,7 +41,7 @@ const FINALIZE_TO_SINK_OPTS = {
|
|
|
35
41
|
layout: { t: 'enum', values: ['prefix', 'stream'] },
|
|
36
42
|
crc: { t: 'bool' },
|
|
37
43
|
};
|
|
38
|
-
const DEFAULT_TARGET_SHARD_BYTES = 8 * 1024 * 1024; // 8 MiB
|
|
44
|
+
const DEFAULT_TARGET_SHARD_BYTES = 8 * 1024 * 1024; // 8 MiB -- smaller than schema mode
|
|
39
45
|
const INITIAL_OFFSETS_CAP = 4096;
|
|
40
46
|
const U32_MAX = 4294967295;
|
|
41
47
|
// maxRecordBytes is a declared, validated, deliberately-unused key here: the
|
package/src/RangeReader.js
CHANGED
|
@@ -33,6 +33,8 @@
|
|
|
33
33
|
// R_ADAPTER_SHORT_READ -- adapter returned fewer bytes than requested, OR a
|
|
34
34
|
// non-Uint8Array / wrong-length adapter return (T7c)
|
|
35
35
|
// R_ROW_OUT_OF_RANGE -- rowIdx negative, fractional, NaN, or >= totalRows (BS-32)
|
|
36
|
+
// R_BAD_CRC -- CRC-32C verify requested and the stored checksum mismatched
|
|
37
|
+
// R_CRC_ABSENT -- CRC verify requested but the container carries no checksum
|
|
36
38
|
// R_OFFSET_TOO_LARGE -- a u64 header/schema/directory offset exceeds 2^53-1
|
|
37
39
|
// R_NOT_PREFETCHED -- syncRange over a shard that prefetchRange has not cached
|
|
38
40
|
// R_WRONG_MODE -- preserve-mode container fed to the schema RangeReader
|
|
@@ -46,7 +48,7 @@ import { StringTable } from './StringTable.js';
|
|
|
46
48
|
import { checkOpts } from './Opts.js';
|
|
47
49
|
import { crc32cInit, crc32cUpdate, crc32cFinal } from './Crc32c.js';
|
|
48
50
|
|
|
49
|
-
export const VERSION = '1.6.
|
|
51
|
+
export const VERSION = '1.6.1';
|
|
50
52
|
|
|
51
53
|
const CONTAINER_HEADER_BYTES = 48;
|
|
52
54
|
const SHARD_ENTRY_BYTES = 40;
|
package/src/Reader.js
CHANGED
|
@@ -25,6 +25,8 @@
|
|
|
25
25
|
// (Number() would lose precision; fail closed, BS-05)
|
|
26
26
|
// R_ROW_OUT_OF_RANGE - get(rowIdx) with rowIdx negative, fractional, NaN,
|
|
27
27
|
// or >= totalRows (BS-32)
|
|
28
|
+
// R_BAD_CRC - CRC-32C verify requested and the stored checksum mismatched
|
|
29
|
+
// R_CRC_ABSENT - CRC verify requested but the container carries no checksum
|
|
28
30
|
//
|
|
29
31
|
// Row-index policy (BS-32): get() range-checks rowIdx; the SHARD-index escape
|
|
30
32
|
// hatches (shardPayload, shardF64, shardStringTable) do NOT -- their contract is
|
|
@@ -36,7 +38,7 @@ import { toContainerBuffer } from './Views.js';
|
|
|
36
38
|
import { checkOpts } from './Opts.js';
|
|
37
39
|
import { crc32cInit, crc32cUpdate, crc32cFinal } from './Crc32c.js';
|
|
38
40
|
|
|
39
|
-
export const VERSION = '1.6.
|
|
41
|
+
export const VERSION = '1.6.1';
|
|
40
42
|
|
|
41
43
|
const CONTAINER_HEADER_BYTES = 48;
|
|
42
44
|
const SHARD_ENTRY_BYTES = 40;
|
package/src/Split.js
CHANGED
|
@@ -2,6 +2,10 @@
|
|
|
2
2
|
// Splitters + per-part compilation + container merge for parallel/checkpointed ingest.
|
|
3
3
|
// Copyright (c) 2026 Zahary Shinikchiev. MIT.
|
|
4
4
|
//
|
|
5
|
+
// Error codes:
|
|
6
|
+
// S_MERGE_EMPTY - mergeContainers called with no inputs
|
|
7
|
+
// S_SCHEMA_MISMATCH - parts carry incompatible schemas
|
|
8
|
+
//
|
|
5
9
|
// M6 covers three primitives that together enable worker-parallel or checkpoint-
|
|
6
10
|
// resumable ingest against LBK1:
|
|
7
11
|
//
|
|
@@ -38,7 +42,7 @@ import { StringTable } from './StringTable.js';
|
|
|
38
42
|
import { checkOpts } from './Opts.js';
|
|
39
43
|
import { crc32cInit, crc32cUpdate, crc32cFinal } from './Crc32c.js';
|
|
40
44
|
|
|
41
|
-
export const VERSION = '1.6.
|
|
45
|
+
export const VERSION = '1.6.1';
|
|
42
46
|
|
|
43
47
|
const LF = 0x0A;
|
|
44
48
|
const CONTAINER_HEADER_BYTES = 48;
|
|
@@ -143,7 +147,7 @@ export function compilePart(bytes, opts) {
|
|
|
143
147
|
// ---------- compileInParts ----------
|
|
144
148
|
|
|
145
149
|
// Sequential convenience: split then compile each part serially.
|
|
146
|
-
// Returns Uint8Array[]
|
|
150
|
+
// Returns Uint8Array[] -- one container per part. Equivalent output to running
|
|
147
151
|
// each part through a worker (or any parallel executor); use this when workers
|
|
148
152
|
// aren't available or for testing.
|
|
149
153
|
export function compileInParts(bytes, opts) {
|
|
@@ -237,7 +241,7 @@ export function mergeContainers(containers) {
|
|
|
237
241
|
const outShardDirBytes = totalShards * SHARD_ENTRY_BYTES;
|
|
238
242
|
|
|
239
243
|
// Zone maps: emit iff container 0 has them AND all others do too.
|
|
240
|
-
// (MultiReader uses the same "all-or-none" policy
|
|
244
|
+
// (MultiReader uses the same "all-or-none" policy -- see MultiReader.d.ts.)
|
|
241
245
|
const outHasZoneMaps = readers.every((r) => r.hasZoneMaps);
|
|
242
246
|
const zoneMapsRaw = readers.map((r) => r.zoneMapsRaw());
|
|
243
247
|
const T = outHasZoneMaps ? zoneMapsRaw[0].trackedFields.length : 0;
|
|
@@ -321,7 +325,7 @@ export function mergeContainers(containers) {
|
|
|
321
325
|
outDv.setUint32(outMetadataOff + 4, totalShards, true);
|
|
322
326
|
outDv.setUint32(outMetadataOff + 8, T, true);
|
|
323
327
|
outDv.setUint32(outMetadataOff + 12, 0, true);
|
|
324
|
-
// Field index table (copied from container 0
|
|
328
|
+
// Field index table (copied from container 0 -- all containers share the schema)
|
|
325
329
|
const fieldTableOut = outMetadataOff + zoneMapsHeaderLen;
|
|
326
330
|
const tracked0 = zoneMapsRaw[0].trackedFields;
|
|
327
331
|
for (let t = 0; t < T; t++) outDv.setUint16(fieldTableOut + t * 2, tracked0[t], true);
|
package/src/StringTable.js
CHANGED
|
@@ -2,6 +2,9 @@
|
|
|
2
2
|
// Byte-level UTF-8 string interning with zero-alloc hot path.
|
|
3
3
|
// Copyright (c) 2026 Zahary Shinikchiev. MIT.
|
|
4
4
|
//
|
|
5
|
+
// Error codes:
|
|
6
|
+
// ST_BLOB_OVERFLOW - the blob or entry count would cross its u32 ceiling
|
|
7
|
+
//
|
|
5
8
|
// intern(bytes, from, to) -> u32 index, deduplicating identical byte ranges.
|
|
6
9
|
// The hash path never allocates: bytes stay in the caller's buffer during
|
|
7
10
|
// lookup, and internal storage grows in doubling steps.
|
|
@@ -14,7 +17,7 @@
|
|
|
14
17
|
// u8 blob[blob_length_bytes]
|
|
15
18
|
//
|
|
16
19
|
// The trailing sentinel makes `len(i) = offsets[i+1] - offsets[i]` uniform for
|
|
17
|
-
// all i including the last
|
|
20
|
+
// all i including the last -- no branch in the reader hot path.
|
|
18
21
|
//
|
|
19
22
|
// Entry 0 is always the empty string: the table reserves it in the constructor
|
|
20
23
|
// and at every reset(), so an absent U32 row cell (which is 0) decodes as ""
|
|
@@ -38,7 +41,7 @@
|
|
|
38
41
|
// TEST-ONLY: not re-exported from index.js, absent from the .d.ts and docs,
|
|
39
42
|
// and carrying no semver guarantee.
|
|
40
43
|
|
|
41
|
-
export const VERSION = '1.6.
|
|
44
|
+
export const VERSION = '1.6.1';
|
|
42
45
|
|
|
43
46
|
const EMPTY_SLOT = 0xFFFFFFFF; // MUST be unsigned; typed-array reads are unsigned
|
|
44
47
|
const INITIAL_BLOB_BYTES = 64 * 1024;
|
|
@@ -256,7 +259,7 @@ export class StringTable {
|
|
|
256
259
|
}
|
|
257
260
|
|
|
258
261
|
// Static: parse a serialized table from a byte view. Returns a read-only
|
|
259
|
-
// accessor object (not a live StringTable
|
|
262
|
+
// accessor object (not a live StringTable -- cheaper for the Reader path).
|
|
260
263
|
static parse(bytes, byteOffset) {
|
|
261
264
|
const dv = new DataView(bytes.buffer, bytes.byteOffset + byteOffset, bytes.byteLength - byteOffset);
|
|
262
265
|
const entryCount = dv.getUint32(0, true);
|
|
@@ -277,7 +280,7 @@ export class StringTableView {
|
|
|
277
280
|
this._count = count;
|
|
278
281
|
this._blob = blob;
|
|
279
282
|
this._offsets = offsets;
|
|
280
|
-
this._decoder = null; // lazy
|
|
283
|
+
this._decoder = null; // lazy -- created on first get()
|
|
281
284
|
}
|
|
282
285
|
|
|
283
286
|
get count() { return this._count; }
|
package/src/Tokenizer.js
CHANGED
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
|
|
31
31
|
import { checkOpts } from './Opts.js';
|
|
32
32
|
|
|
33
|
-
export const VERSION = '1.6.
|
|
33
|
+
export const VERSION = '1.6.1';
|
|
34
34
|
|
|
35
35
|
const U32_MAX = 4294967295;
|
|
36
36
|
const TOKENIZER_OPTS = {
|
|
@@ -88,6 +88,7 @@ const NS_EXP = 7; // in exponent digits
|
|
|
88
88
|
// correctly-rounded F64 via a single multiplication (positive exp) or single
|
|
89
89
|
// division (negative exp) using this table. Outside that domain, correctly
|
|
90
90
|
// rounded parsing requires David Gay's strtod, deferred to M4+.
|
|
91
|
+
// See decisions/0012-clinger-fast-path.md.
|
|
91
92
|
const POW10 = new Float64Array(23);
|
|
92
93
|
POW10[0] = 1;
|
|
93
94
|
for (let _pi = 1; _pi < 23; _pi++) POW10[_pi] = POW10[_pi - 1] * 10;
|
|
@@ -691,7 +692,7 @@ export class Tokenizer {
|
|
|
691
692
|
const need = to - from;
|
|
692
693
|
if (this._strLen + need > this._maxStringBytes) this._err('E_STRING_TOO_LONG', 'string exceeds maxStringBytes cap (' + (this._strLen + need) + ' > ' + this._maxStringBytes + ')');
|
|
693
694
|
if (this._strLen + need > this._strBuf.length) this._growStrBuf(need);
|
|
694
|
-
// Manual copy loop
|
|
695
|
+
// Manual copy loop -- Uint8Array.set(source) via subarray allocates a small
|
|
695
696
|
// view header per call, which turns into MB-scale GC pressure across a
|
|
696
697
|
// string-heavy fixture. The loop is boring but keeps the hot path allocation-free.
|
|
697
698
|
const dst = this._strBuf;
|
package/src/Views.js
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
// @zakkster/lite-bake-stream / Views (internal)
|
|
2
2
|
// Copyright (c) 2026 Zahary Shinikchiev. MIT.
|
|
3
3
|
//
|
|
4
|
+
// Error codes:
|
|
5
|
+
// W_BAD_SINK - shared finalizeToSink guard: sink missing write/writeAt or async
|
|
6
|
+
//
|
|
4
7
|
// One shared helper so every container entry point (deserialize,
|
|
5
8
|
// Reader.fromBuffer, PreserveReader.fromBuffer) resolves caller-supplied bytes
|
|
6
9
|
// to an ArrayBuffer by exactly the SAME rule (BS-09). A Uint8Array that is a
|
package/src/Writer.js
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
// setInputSource; hand-driven Writers with no source sample by record
|
|
12
12
|
// count) buffered in a columnar staging area, per-field lane kind inferred
|
|
13
13
|
// from observed value types (number->F64, string->U32-into-string-table).
|
|
14
|
+
// See decisions/0011-sample-drain-reintern.md.
|
|
14
15
|
// null is lane-neutral (BS-20): it marks the field but sets no kind, so
|
|
15
16
|
// null+string infers U32 and null+number infers F64. A field that saw both
|
|
16
17
|
// a real number and a real string raises W_MIXED_LANE_TYPES at freeze.
|
|
@@ -52,6 +53,7 @@
|
|
|
52
53
|
// W_SCHEMA_TOO_WIDE - a field offset exceeds the u16 offset_in_row ceiling
|
|
53
54
|
// W_FIELD_NAME_INVALID - field name empty, non-string, or > 255 UTF-8 bytes
|
|
54
55
|
// W_FINALIZED - a sink event or finalize() after finalize()
|
|
56
|
+
// W_BAD_SINK - finalizeToSink given a sink missing write/writeAt or async
|
|
55
57
|
// E_UNKNOWN_OPTION - unknown constructor option key
|
|
56
58
|
// E_OPTION_VALUE - constructor option value out of domain
|
|
57
59
|
|
|
@@ -60,7 +62,7 @@ import { checkOpts } from './Opts.js';
|
|
|
60
62
|
import { crc32cInit, crc32cUpdate, crc32cFinal, crc32cCombine } from './Crc32c.js';
|
|
61
63
|
import { validateSink, isThenable } from './Views.js';
|
|
62
64
|
|
|
63
|
-
export const VERSION = '1.6.
|
|
65
|
+
export const VERSION = '1.6.1';
|
|
64
66
|
|
|
65
67
|
const U32_MAX = 4294967295;
|
|
66
68
|
// Post-finalize sentinel for _recordDepth. Chosen = 2 so every post-finalize
|
|
@@ -253,8 +255,8 @@ export class Writer {
|
|
|
253
255
|
this._schema = null; // { fields: [{name, laneKind, offsetInRow}], rowStride }
|
|
254
256
|
this._fieldNamesUtf8 = null;
|
|
255
257
|
this._fieldNameHashes = null;
|
|
256
|
-
this._fieldLaneKinds = null; // Uint8Array
|
|
257
|
-
this._fieldOffsets = null; // Uint16Array
|
|
258
|
+
this._fieldLaneKinds = null; // Uint8Array -- per-field LANE_* (fast dispatch)
|
|
259
|
+
this._fieldOffsets = null; // Uint16Array -- per-field byte offset in row
|
|
258
260
|
this._rowValueSlotsF64 = null; // Float64Array<fieldCount> scratch
|
|
259
261
|
this._rowValueSlotsU32 = null; // Uint32Array<fieldCount> scratch
|
|
260
262
|
|
|
@@ -336,7 +338,7 @@ export class Writer {
|
|
|
336
338
|
} else {
|
|
337
339
|
// Sample window: decode to a JS string NOW. The tokenizer will reuse this
|
|
338
340
|
// buffer for the value bytes before onNumber/onString fires, so we cannot
|
|
339
|
-
// defer decoding. String allocation here is expected
|
|
341
|
+
// defer decoding. String allocation here is expected -- bounded to the
|
|
340
342
|
// sample window; steady-state post-freeze remains zero-alloc.
|
|
341
343
|
this._currentKeyName = KEY_DECODER.decode(bytes.subarray(from, to));
|
|
342
344
|
}
|
package/src/index.js
CHANGED
|
@@ -8,6 +8,10 @@
|
|
|
8
8
|
// Set opts.preserve = true. Bytes in, same bytes out.
|
|
9
9
|
// deserialize() auto-detects the mode via the container's flag bit and
|
|
10
10
|
// returns the appropriate Reader.
|
|
11
|
+
//
|
|
12
|
+
// Error codes:
|
|
13
|
+
// E_OPTION_CONFLICT - serialize() opts combine preserve with an incompatible framing
|
|
14
|
+
// R_TRUNCATED - deserialize() container too small to inspect header flags
|
|
11
15
|
|
|
12
16
|
import { Tokenizer, TokenizerError } from './Tokenizer.js';
|
|
13
17
|
import { Writer, WriterError } from './Writer.js';
|
|
@@ -28,7 +32,7 @@ export {
|
|
|
28
32
|
PreserveWriter, PreserveWriterError,
|
|
29
33
|
PreserveReader, PreserveReaderError,
|
|
30
34
|
};
|
|
31
|
-
export const VERSION = '1.6.
|
|
35
|
+
export const VERSION = '1.6.1';
|
|
32
36
|
|
|
33
37
|
const encoder = new TextEncoder();
|
|
34
38
|
|
package/types/MultiReader.d.ts
CHANGED
|
@@ -24,7 +24,7 @@ export class MultiReader {
|
|
|
24
24
|
/** Which sub-Reader owns a given global shard? */
|
|
25
25
|
readerForShard(globalShardIdx: number): { readerIdx: number; localShard: number } | null;
|
|
26
26
|
|
|
27
|
-
// Zone maps
|
|
27
|
+
// Zone maps -- global shard indices merged across sub-readers
|
|
28
28
|
shardBounds(globalShardIdx: number, fieldName: string): Bounds | null;
|
|
29
29
|
findShards(fieldName: string, opts?: FindShardsOptions): number[];
|
|
30
30
|
}
|
package/types/RangeReader.d.ts
CHANGED
|
@@ -86,7 +86,7 @@ export class RangeReader {
|
|
|
86
86
|
get(rowIdx: number, fieldName: string): number | string | undefined;
|
|
87
87
|
};
|
|
88
88
|
|
|
89
|
-
// Zone maps (M7)
|
|
89
|
+
// Zone maps (M7) -- synchronous, populated at open()
|
|
90
90
|
shardBounds(shardIdx: number, fieldName: string): Bounds | null;
|
|
91
91
|
findShards(fieldName: string, opts?: FindShardsOptions): number[];
|
|
92
92
|
}
|
package/types/Reader.d.ts
CHANGED
|
@@ -63,7 +63,7 @@ export class Reader {
|
|
|
63
63
|
fieldIndex(name: string): number;
|
|
64
64
|
|
|
65
65
|
/**
|
|
66
|
-
* Get value at (rowIdx, fieldName). F64 lane
|
|
66
|
+
* Get value at (rowIdx, fieldName). F64 lane -> number; U32 lane -> string
|
|
67
67
|
* (resolved via the shard's local string table).
|
|
68
68
|
*/
|
|
69
69
|
get(rowIdx: number, fieldName: string): number | string | undefined;
|
package/types/Split.d.ts
CHANGED
|
@@ -28,7 +28,7 @@ export interface CompileInPartsOptions extends SplitOptions, CompilePartOptions
|
|
|
28
28
|
|
|
29
29
|
/**
|
|
30
30
|
* Divide NDJSON bytes into N ranges at safe line boundaries. Every returned
|
|
31
|
-
* range contains complete records
|
|
31
|
+
* range contains complete records -- no split mid-line. The union of ranges
|
|
32
32
|
* equals the original bytes (no gaps, no overlaps).
|
|
33
33
|
*/
|
|
34
34
|
export function splitNDJSON(bytes: Uint8Array, opts?: SplitOptions): SplitRange[];
|
package/types/Tokenizer.d.ts
CHANGED
|
@@ -5,7 +5,7 @@ export const VERSION: string;
|
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
7
|
* Sink protocol the Tokenizer emits into. Byte ranges (bytes[from, to)) are
|
|
8
|
-
* ephemeral
|
|
8
|
+
* ephemeral -- valid only for the duration of the call. Consumers that need
|
|
9
9
|
* to retain content must copy or decode inside the sink method.
|
|
10
10
|
*/
|
|
11
11
|
export interface TokenizerSink {
|
package/types/Writer.d.ts
CHANGED
|
@@ -78,7 +78,7 @@ export interface Container {
|
|
|
78
78
|
export class Writer implements TokenizerSinkForWriter {
|
|
79
79
|
constructor(opts?: WriterOptions);
|
|
80
80
|
/**
|
|
81
|
-
* Sink protocol
|
|
81
|
+
* Sink protocol -- the Tokenizer calls these. Byte ranges are ephemeral.
|
|
82
82
|
*/
|
|
83
83
|
onStartObject(): void;
|
|
84
84
|
onEndObject(): void;
|
|
@@ -127,7 +127,7 @@ export class Writer implements TokenizerSinkForWriter {
|
|
|
127
127
|
readonly shardCount: number;
|
|
128
128
|
}
|
|
129
129
|
|
|
130
|
-
// Structural type
|
|
130
|
+
// Structural type -- the Writer conforms to the Tokenizer's sink protocol.
|
|
131
131
|
// (Redeclared here so the /writer subpath is self-contained.)
|
|
132
132
|
interface TokenizerSinkForWriter {
|
|
133
133
|
onStartObject(): void;
|