@zakkster/lite-bake-stream 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/Split.js ADDED
@@ -0,0 +1,359 @@
1
+ // @zakkster/lite-bake-stream / Split (M6)
2
+ // Splitters + per-part compilation + container merge for parallel/checkpointed ingest.
3
+ // Copyright (c) 2026 Zahary Shinikchiev. MIT.
4
+ //
5
+ // M6 covers three primitives that together enable worker-parallel or checkpoint-
6
+ // resumable ingest against LBK1:
7
+ //
8
+ // splitNDJSON(bytes, opts) -> [{start, end}, ...]
9
+ // Divide input into N byte ranges that each contain complete NDJSON records.
10
+ //
11
+ // compilePart(bytes, opts) -> Uint8Array
12
+ // Compile ONE byte range into a standalone LBK1 container. Trivially
13
+ // Transferable across worker boundaries via postMessage(buf, [buf]).
14
+ //
15
+ // compileInParts(bytes, opts) -> Uint8Array[]
16
+ // Sequential convenience: split + compile each part. Same output shape as
17
+ // running each part through a worker; deterministic and easy to test.
18
+ // Skip this when you have workers; use it as a fallback when you don't.
19
+ //
20
+ // mergeContainers([bytes, ...]) -> Uint8Array
21
+ // Concatenate N LBK1 containers into ONE. All must share the same schema.
22
+ // Preserves per-shard string tables and zone maps verbatim; rewrites the
23
+ // shard directory and header offsets. For query workloads over multiple
24
+ // parts, prefer MultiReader (M4) which avoids the copy; use mergeContainers
25
+ // when you need a single-file distributable.
26
+ //
27
+ // The "parallelism-safe" contract: every part MUST use the SAME explicit
28
+ // schema. Two workers running sample-and-infer independently could infer
29
+ // different field sets or lane kinds, and their outputs would then fail to
30
+ // merge. To use sample-and-infer with parts, compile part 0 first, extract
31
+ // its schema via `new Reader(container.buffer).schema`, and pass that as the
32
+ // explicit schema for parts 1..N.
33
+
34
+ import { Tokenizer, TokenizerError } from './Tokenizer.js';
35
+ import { Writer, WriterError } from './Writer.js';
36
+ import { Reader, ReaderError } from './Reader.js';
37
+ import { StringTable } from './StringTable.js';
38
+
39
+ export const VERSION = '1.0.0';
40
+
41
+ const LF = 0x0A;
42
+ const CONTAINER_HEADER_BYTES = 48;
43
+ const SHARD_ENTRY_BYTES = 40;
44
+ const FIELD_DESCRIPTOR_BYTES = 24;
45
+ const FOOTER_BYTES = 16;
46
+ const LANE_F64 = 1;
47
+
48
+ export class SplitError extends Error {
49
+ constructor(code, msg) { super(msg); this.code = code; this.name = 'SplitError'; }
50
+ }
51
+
52
+ // ---------- splitNDJSON ----------
53
+
54
+ // Divide NDJSON bytes into N ranges, each ending at (and including) a newline.
55
+ // Guarantees:
56
+ // - Every returned {start, end} covers complete records (no split mid-line).
57
+ // - Union of ranges = original bytes (no gaps, no overlaps).
58
+ // - Concatenating bytes.subarray(start, end) across parts reproduces the
59
+ // original byte-for-byte.
60
+ // Options:
61
+ // targetParts: desired part count (default 4). Actual count may be less
62
+ // if the input is too small or has too few newlines.
63
+ // maxPartBytes: optional cap on part size. If set, splitter emits enough
64
+ // parts to keep each under this ceiling.
65
+ export function splitNDJSON(bytes, opts) {
66
+ if (!(bytes instanceof Uint8Array)) {
67
+ throw new TypeError('splitNDJSON: expected Uint8Array');
68
+ }
69
+ opts = opts || {};
70
+ const targetParts = Math.max(1, opts.targetParts || 4);
71
+ const maxPartBytes = opts.maxPartBytes || Infinity;
72
+ const total = bytes.length;
73
+ if (total === 0) return [];
74
+
75
+ // Compute effective number of parts: honor targetParts, then bump up if
76
+ // any part would exceed maxPartBytes.
77
+ let parts = targetParts;
78
+ if (Number.isFinite(maxPartBytes)) {
79
+ const minPartsByCap = Math.ceil(total / maxPartBytes);
80
+ if (minPartsByCap > parts) parts = minPartsByCap;
81
+ }
82
+
83
+ const ranges = [];
84
+ let cursor = 0;
85
+ for (let i = 1; i < parts; i++) {
86
+ // Target boundary at even split; slide forward to the NEXT newline so we
87
+ // don't cut a record. If no newline exists between cursor and end, this
88
+ // part absorbs the remainder and we stop.
89
+ const target = Math.floor((total * i) / parts);
90
+ if (target <= cursor) continue; // parts overlap in tiny inputs; skip
91
+ let boundary = target;
92
+ while (boundary < total && bytes[boundary] !== LF) boundary++;
93
+ if (boundary >= total) break;
94
+ boundary++; // include the newline in this part
95
+ ranges.push({ start: cursor, end: boundary });
96
+ cursor = boundary;
97
+ }
98
+ // Last part absorbs everything left.
99
+ if (cursor < total) ranges.push({ start: cursor, end: total });
100
+ return ranges;
101
+ }
102
+
103
+ // ---------- compilePart ----------
104
+
105
+ // Compile one byte range into a standalone LBK1 container.
106
+ // Same options as Writer; a schema is STRONGLY recommended when this is a
107
+ // part of a larger multi-part compile (see module doc).
108
+ export function compilePart(bytes, opts) {
109
+ if (!(bytes instanceof Uint8Array)) {
110
+ throw new TypeError('compilePart: expected Uint8Array');
111
+ }
112
+ opts = opts || {};
113
+ const framing = opts.framing || 'ndjson';
114
+ const w = new Writer(opts.writer || {});
115
+ const t = new Tokenizer(w, { framing });
116
+ t.feed(bytes);
117
+ t.end();
118
+ return new Uint8Array(w.finalize().buffer);
119
+ }
120
+
121
+ // ---------- compileInParts ----------
122
+
123
+ // Sequential convenience: split then compile each part serially.
124
+ // Returns Uint8Array[] — one container per part. Equivalent output to running
125
+ // each part through a worker (or any parallel executor); use this when workers
126
+ // aren't available or for testing.
127
+ export function compileInParts(bytes, opts) {
128
+ opts = opts || {};
129
+ const splitOpts = {
130
+ targetParts: opts.targetParts,
131
+ maxPartBytes: opts.maxPartBytes,
132
+ };
133
+ const partOpts = {
134
+ framing: opts.framing || 'ndjson',
135
+ writer: opts.writer || {},
136
+ };
137
+ const ranges = splitNDJSON(bytes, splitOpts);
138
+ const containers = new Array(ranges.length);
139
+ for (let i = 0; i < ranges.length; i++) {
140
+ const { start, end } = ranges[i];
141
+ containers[i] = compilePart(bytes.subarray(start, end), partOpts);
142
+ }
143
+ return containers;
144
+ }
145
+
146
+ // ---------- mergeContainers ----------
147
+
148
+ // Concatenate N LBK1 containers into ONE. All containers must share an
149
+ // identical schema (byte-compare over the schema block). Preserves per-shard
150
+ // string tables and zone maps verbatim; rewrites the shard directory + header
151
+ // offsets to make the combined layout self-consistent.
152
+ //
153
+ // The result is byte-identical (except for the shard directory / header
154
+ // offsets) to what a single Writer would have produced if it had emitted the
155
+ // same shards in the same order. Query behavior via `new Reader(merged)` and
156
+ // via `new MultiReader([readers...])` is equivalent (both return the same
157
+ // values for the same global row indices).
158
+ export function mergeContainers(containers) {
159
+ if (!Array.isArray(containers) || containers.length === 0) {
160
+ throw new SplitError('S_MERGE_EMPTY', 'mergeContainers requires a non-empty array');
161
+ }
162
+ // Normalize inputs to Uint8Array
163
+ const parts = containers.map((c, i) => {
164
+ if (c instanceof Uint8Array) return c;
165
+ if (c instanceof ArrayBuffer) return new Uint8Array(c);
166
+ throw new TypeError('mergeContainers: container ' + i + ' is not Uint8Array or ArrayBuffer');
167
+ });
168
+
169
+ // Parse each via Reader for authoritative metadata. This also validates
170
+ // the LBK1 magic and version.
171
+ const readers = parts.map((p) => new Reader(_toArrayBuffer(p)));
172
+
173
+ // Schemas must match. We compare the parsed schema (byte-exact would work
174
+ // too but this catches equivalent-but-differently-padded cases).
175
+ const schema0 = readers[0].schema;
176
+ for (let i = 1; i < readers.length; i++) {
177
+ if (!_schemasEqual(schema0, readers[i].schema)) {
178
+ throw new SplitError('S_SCHEMA_MISMATCH',
179
+ 'mergeContainers: container ' + i + ' schema differs from container 0');
180
+ }
181
+ }
182
+
183
+ // Total shards and rows
184
+ let totalShards = 0, totalRows = 0;
185
+ for (const r of readers) { totalShards += r.shardCount; totalRows += r.totalRows; }
186
+
187
+ // Compute the target layout. Schema block is copied from container 0 verbatim.
188
+ // Reader gives us schemaBlockOff and shardDirOff; the schema-block byte range
189
+ // is [schemaBlockOff, shardDirOff). We reuse this slice.
190
+ const src0 = parts[0];
191
+ const dv0 = new DataView(src0.buffer, src0.byteOffset, src0.byteLength);
192
+ const schemaBlockOff = Number(dv0.getBigUint64(8, true));
193
+ const shardDirOff0 = Number(dv0.getBigUint64(24, true));
194
+ const schemaBlockLen = shardDirOff0 - schemaBlockOff;
195
+
196
+ // New offsets
197
+ const outSchemaBlockOff = CONTAINER_HEADER_BYTES;
198
+ const outShardDirOff = outSchemaBlockOff + schemaBlockLen;
199
+ const outShardDirBytes = totalShards * SHARD_ENTRY_BYTES;
200
+
201
+ // Zone maps: emit iff container 0 has them AND all others do too.
202
+ // (MultiReader uses the same "all-or-none" policy — see MultiReader.d.ts.)
203
+ const outHasZoneMaps = readers.every((r) => r.hasZoneMaps);
204
+ const zoneMapsRaw = readers.map((r) => r.zoneMapsRaw());
205
+ const T = outHasZoneMaps ? zoneMapsRaw[0].trackedFields.length : 0;
206
+ const zoneMapsHeaderLen = 16;
207
+ const zoneMapsFieldTableLen = T * 2;
208
+ const zoneMapsFieldTablePad = (8 - (zoneMapsFieldTableLen & 7)) & 7;
209
+ const zoneMapsMinsMaxesLen = outHasZoneMaps ? totalShards * T * 8 * 2 : 0;
210
+ const outMetadataOff = outHasZoneMaps ? (outShardDirOff + outShardDirBytes) : 0;
211
+ const outZoneMapsBytes = outHasZoneMaps
212
+ ? zoneMapsHeaderLen + zoneMapsFieldTableLen + zoneMapsFieldTablePad + zoneMapsMinsMaxesLen
213
+ : 0;
214
+ const outFirstShardOff = outShardDirOff + outShardDirBytes + outZoneMapsBytes;
215
+
216
+ // Compute per-source-shard payload/string-table extents in the OUTPUT buffer.
217
+ // Layout mirrors the writer's: each shard's payload followed by its local
218
+ // string table, padded to 8 between shards.
219
+ let cursor = outFirstShardOff;
220
+ const shardOutMap = []; // array of {payloadOff, payloadLen, stringOff, stringLen, srcContainerIdx, srcShardIdx, rowCount, mins?, maxes?}
221
+ for (let ci = 0; ci < readers.length; ci++) {
222
+ const r = readers[ci];
223
+ for (let si = 0; si < r.shardCount; si++) {
224
+ const shard = r.shards[si];
225
+ // Source shard payload lives at shard.payloadOff in the source container.
226
+ const payloadLen = shard.payloadLen;
227
+ const payloadPad = (8 - (payloadLen & 7)) & 7;
228
+ const stringLen = shard.stringTable ? _sourceStringTableLen(readers[ci], si) : 0;
229
+ const outPayloadOff = cursor;
230
+ const outStringOff = outPayloadOff + payloadLen + payloadPad;
231
+ shardOutMap.push({
232
+ payloadOff: outPayloadOff,
233
+ payloadLen,
234
+ stringOff: outStringOff,
235
+ stringLen,
236
+ srcContainerIdx: ci,
237
+ srcShardIdx: si,
238
+ rowCount: shard.rowCount,
239
+ });
240
+ cursor = outStringOff + stringLen;
241
+ }
242
+ }
243
+ const totalBytes = cursor + FOOTER_BYTES;
244
+
245
+ // Allocate output
246
+ const out = new Uint8Array(totalBytes);
247
+ const outDv = new DataView(out.buffer);
248
+
249
+ // Header
250
+ out[0] = 0x4C; out[1] = 0x42; out[2] = 0x4B; out[3] = 0x31; // 'LBK1'
251
+ outDv.setUint16(4, 1, true); // format_version
252
+ out[6] = 1; // endian LE
253
+ out[7] = 0;
254
+ outDv.setBigUint64(8, BigInt(outSchemaBlockOff), true);
255
+ outDv.setBigUint64(16, BigInt(outMetadataOff), true);
256
+ outDv.setBigUint64(24, BigInt(outShardDirOff), true);
257
+ outDv.setUint32(32, totalShards, true);
258
+ outDv.setUint32(36, 0, true);
259
+ outDv.setBigUint64(40, BigInt(totalRows), true);
260
+
261
+ // Schema block: copy verbatim from container 0
262
+ out.set(parts[0].subarray(schemaBlockOff, schemaBlockOff + schemaBlockLen), outSchemaBlockOff);
263
+
264
+ // Shard directory: rewrite ShardEntry for every output shard
265
+ for (let s = 0; s < shardOutMap.length; s++) {
266
+ const m = shardOutMap[s];
267
+ const entryOff = outShardDirOff + s * SHARD_ENTRY_BYTES;
268
+ outDv.setBigUint64(entryOff + 0, BigInt(m.payloadOff), true);
269
+ outDv.setUint32(entryOff + 8, m.payloadLen, true);
270
+ outDv.setUint32(entryOff + 12, m.rowCount, true);
271
+ outDv.setUint16(entryOff + 16, 1, true); // min_reader_version
272
+ outDv.setUint16(entryOff + 18, 0, true); // flags
273
+ outDv.setUint32(entryOff + 20, 0, true); // reserved
274
+ outDv.setBigUint64(entryOff + 24, BigInt(m.stringLen > 0 ? m.stringOff : 0), true);
275
+ outDv.setBigUint64(entryOff + 32, BigInt(m.stringLen), true);
276
+ }
277
+
278
+ // Zone maps: header + field table + interleaved mins/maxes.
279
+ if (outHasZoneMaps) {
280
+ // Segment header
281
+ out[outMetadataOff + 0] = 0x30; out[outMetadataOff + 1] = 0x5A;
282
+ out[outMetadataOff + 2] = 0x4D; out[outMetadataOff + 3] = 0x31; // 'ZM01'
283
+ outDv.setUint32(outMetadataOff + 4, totalShards, true);
284
+ outDv.setUint32(outMetadataOff + 8, T, true);
285
+ outDv.setUint32(outMetadataOff + 12, 0, true);
286
+ // Field index table (copied from container 0 — all containers share the schema)
287
+ const fieldTableOut = outMetadataOff + zoneMapsHeaderLen;
288
+ const tracked0 = zoneMapsRaw[0].trackedFields;
289
+ for (let t = 0; t < T; t++) outDv.setUint16(fieldTableOut + t * 2, tracked0[t], true);
290
+ // Mins / maxes concatenated in output-shard order
291
+ const minsOut = fieldTableOut + zoneMapsFieldTableLen + zoneMapsFieldTablePad;
292
+ const maxesOut = minsOut + totalShards * T * 8;
293
+ for (let s = 0; s < shardOutMap.length; s++) {
294
+ const m = shardOutMap[s];
295
+ const zm = zoneMapsRaw[m.srcContainerIdx];
296
+ for (let t = 0; t < T; t++) {
297
+ const srcPos = m.srcShardIdx * zm.laneCount + t;
298
+ outDv.setFloat64(minsOut + (s * T + t) * 8, zm.mins[srcPos], true);
299
+ outDv.setFloat64(maxesOut + (s * T + t) * 8, zm.maxes[srcPos], true);
300
+ }
301
+ }
302
+ }
303
+
304
+ // Shard payloads + string tables: bulk-copy each source range to its new offset
305
+ for (const m of shardOutMap) {
306
+ const src = parts[m.srcContainerIdx];
307
+ const r = readers[m.srcContainerIdx];
308
+ const srcShard = r.shards[m.srcShardIdx];
309
+ // Payload
310
+ out.set(src.subarray(srcShard.payloadOff, srcShard.payloadOff + m.payloadLen), m.payloadOff);
311
+ // String table (if any)
312
+ if (m.stringLen > 0) {
313
+ // In source, the string table starts at localStrOff (from the ShardEntry).
314
+ const srcDv = new DataView(src.buffer, src.byteOffset, src.byteLength);
315
+ const srcShardEntry = r.shardDirectoryOffset + m.srcShardIdx * SHARD_ENTRY_BYTES;
316
+ const srcLocalStrOff = Number(srcDv.getBigUint64(srcShardEntry + 24, true));
317
+ out.set(src.subarray(srcLocalStrOff, srcLocalStrOff + m.stringLen), m.stringOff);
318
+ }
319
+ }
320
+
321
+ // Footer
322
+ const footerOff = totalBytes - FOOTER_BYTES;
323
+ outDv.setUint32(footerOff + 0, 0xFFFFFFFF, true); // CRC absent
324
+ outDv.setUint32(footerOff + 4, 0, true);
325
+ out[footerOff + 8] = 0x31;
326
+ out[footerOff + 9] = 0x4B;
327
+ out[footerOff + 10] = 0x42;
328
+ out[footerOff + 11] = 0x4C;
329
+ outDv.setUint32(footerOff + 12, FOOTER_BYTES, true);
330
+
331
+ return out;
332
+ }
333
+
334
+ // ---------- internals ----------
335
+
336
+ function _toArrayBuffer(u8) {
337
+ if (u8.byteOffset === 0 && u8.byteLength === u8.buffer.byteLength) return u8.buffer;
338
+ const copy = new Uint8Array(u8.byteLength);
339
+ copy.set(u8);
340
+ return copy.buffer;
341
+ }
342
+
343
+ function _schemasEqual(a, b) {
344
+ if (a.fields.length !== b.fields.length) return false;
345
+ if (a.rowStride !== b.rowStride) return false;
346
+ for (let i = 0; i < a.fields.length; i++) {
347
+ if (a.fields[i].name !== b.fields[i].name) return false;
348
+ if (a.fields[i].laneKind !== b.fields[i].laneKind) return false;
349
+ if (a.fields[i].offsetInRow !== b.fields[i].offsetInRow) return false;
350
+ }
351
+ return true;
352
+ }
353
+
354
+ // Look up the source string-table byte length for a shard.
355
+ function _sourceStringTableLen(reader, shardIdx) {
356
+ const dv = new DataView(reader.buffer);
357
+ const entryOff = reader.shardDirectoryOffset + shardIdx * SHARD_ENTRY_BYTES;
358
+ return Number(dv.getBigUint64(entryOff + 32, true));
359
+ }
@@ -0,0 +1,225 @@
1
+ // @zakkster/lite-bake-stream / StringTable
2
+ // Byte-level UTF-8 string interning with zero-alloc hot path.
3
+ // Copyright (c) 2026 Zahary Shinikchiev. MIT.
4
+ //
5
+ // intern(bytes, from, to) -> u32 index, deduplicating identical byte ranges.
6
+ // The hash path never allocates: bytes stay in the caller's buffer during
7
+ // lookup, and internal storage grows in doubling steps.
8
+ //
9
+ // Serializes to the LBK1 per-shard string-table layout (SPEC.md section 3.3):
10
+ // u32 entry_count
11
+ // u32 blob_length_bytes
12
+ // u32 offsets[entry_count] (start of each string in the blob)
13
+ // u32 blob_end_sentinel (= blob_length_bytes, for len derivation)
14
+ // u8 blob[blob_length_bytes]
15
+ //
16
+ // The trailing sentinel makes `len(i) = offsets[i+1] - offsets[i]` uniform for
17
+ // all i including the last — no branch in the reader hot path.
18
+
19
+ export const VERSION = '1.0.0';
20
+
21
+ const EMPTY_SLOT = 0xFFFFFFFF; // MUST be unsigned; typed-array reads are unsigned
22
+ const INITIAL_BLOB_BYTES = 64 * 1024;
23
+ const INITIAL_ENTRIES = 1024;
24
+ const INITIAL_HASH_CAP = 2048; // load factor target 50%
25
+ const HASH_MAX_LOAD_NUM = 1; // 50% load factor: num/den = 1/2
26
+ const HASH_MAX_LOAD_DEN = 2;
27
+
28
+ // FNV-1a 32-bit over a byte range.
29
+ function fnv1a(bytes, from, to) {
30
+ let h = 0x811c9dc5 | 0;
31
+ for (let i = from; i < to; i++) {
32
+ h = (h ^ bytes[i]) | 0;
33
+ h = Math.imul(h, 0x01000193) | 0;
34
+ }
35
+ return h >>> 0;
36
+ }
37
+
38
+ export class StringTable {
39
+ constructor() {
40
+ this._blob = new Uint8Array(INITIAL_BLOB_BYTES);
41
+ this._blobLen = 0;
42
+ this._offsets = new Uint32Array(INITIAL_ENTRIES + 1); // +1 for sentinel slot
43
+ this._count = 0;
44
+ this._hashCap = INITIAL_HASH_CAP;
45
+ this._hashMask = INITIAL_HASH_CAP - 1;
46
+ this._hashKeys = new Uint32Array(INITIAL_HASH_CAP).fill(EMPTY_SLOT);
47
+ }
48
+
49
+ // Returns u32 index of the string with these bytes, adding a new entry if
50
+ // this exact byte sequence hasn't been seen before. Zero-alloc on the hit path.
51
+ intern(bytes, from, to) {
52
+ const h = fnv1a(bytes, from, to);
53
+ let slot = h & this._hashMask;
54
+ // Linear probing with wraparound
55
+ for (;;) {
56
+ const stored = this._hashKeys[slot];
57
+ if (stored === EMPTY_SLOT) {
58
+ // Miss: this is a new string; append + record + insert.
59
+ return this._insertNew(bytes, from, to, slot);
60
+ }
61
+ // Occupied: check for byte-equal match at stored index
62
+ if (this._equalsAt(stored, bytes, from, to)) return stored;
63
+ slot = (slot + 1) & this._hashMask;
64
+ }
65
+ }
66
+
67
+ _equalsAt(idx, bytes, from, to) {
68
+ const start = this._offsets[idx];
69
+ const end = this._offsets[idx + 1]; // sentinel makes this valid for the last entry too
70
+ const n = to - from;
71
+ if ((end - start) !== n) return false;
72
+ const blob = this._blob;
73
+ for (let i = 0; i < n; i++) {
74
+ if (blob[start + i] !== bytes[from + i]) return false;
75
+ }
76
+ return true;
77
+ }
78
+
79
+ _insertNew(bytes, from, to, slot) {
80
+ const n = to - from;
81
+ // Grow blob if needed
82
+ if (this._blobLen + n > this._blob.length) this._growBlob(n);
83
+ // Grow offsets if needed (leave room for +1 sentinel)
84
+ if (this._count + 1 >= this._offsets.length) this._growOffsets();
85
+
86
+ const idx = this._count;
87
+ this._offsets[idx] = this._blobLen;
88
+ // append bytes
89
+ const dst = this._blob;
90
+ let d = this._blobLen;
91
+ for (let s = from; s < to; s++) dst[d++] = bytes[s];
92
+ this._blobLen = d;
93
+ // update sentinel at idx+1
94
+ this._offsets[idx + 1] = this._blobLen;
95
+ this._count++;
96
+
97
+ // Insert into hash table
98
+ this._hashKeys[slot] = idx;
99
+
100
+ // Resize hash table if load factor exceeded
101
+ if (this._count * HASH_MAX_LOAD_DEN > this._hashCap * HASH_MAX_LOAD_NUM) {
102
+ this._resizeHash();
103
+ }
104
+ return idx;
105
+ }
106
+
107
+ _growBlob(needed) {
108
+ let cap = this._blob.length;
109
+ while (cap < this._blobLen + needed) cap *= 2;
110
+ const nb = new Uint8Array(cap);
111
+ nb.set(this._blob);
112
+ this._blob = nb;
113
+ }
114
+
115
+ _growOffsets() {
116
+ const nb = new Uint32Array(this._offsets.length * 2);
117
+ nb.set(this._offsets);
118
+ this._offsets = nb;
119
+ }
120
+
121
+ _resizeHash() {
122
+ const newCap = this._hashCap * 2;
123
+ const newMask = newCap - 1;
124
+ const newKeys = new Uint32Array(newCap).fill(EMPTY_SLOT);
125
+ // Rehash every occupied slot by looking up its string in the blob.
126
+ const oldKeys = this._hashKeys;
127
+ for (let i = 0; i < oldKeys.length; i++) {
128
+ const idx = oldKeys[i];
129
+ if (idx === EMPTY_SLOT) continue;
130
+ const start = this._offsets[idx];
131
+ const end = this._offsets[idx + 1];
132
+ const h = fnv1a(this._blob, start, end);
133
+ let slot = h & newMask;
134
+ while (newKeys[slot] !== EMPTY_SLOT) slot = (slot + 1) & newMask;
135
+ newKeys[slot] = idx;
136
+ }
137
+ this._hashCap = newCap;
138
+ this._hashMask = newMask;
139
+ this._hashKeys = newKeys;
140
+ }
141
+
142
+ get count() { return this._count; }
143
+ get blobLen() { return this._blobLen; }
144
+
145
+ // Zero-copy byte range for entry `idx`. Used by consumers that want the raw
146
+ // UTF-8 bytes without materializing a JS string (e.g. re-interning across
147
+ // string tables during sample drain).
148
+ bytesAt(idx) {
149
+ if (idx >= this._count) return undefined;
150
+ return this._blob.subarray(this._offsets[idx], this._offsets[idx + 1]);
151
+ }
152
+
153
+ // Reset in-place, keeping allocated arrays (per-shard reuse pattern).
154
+ reset() {
155
+ this._blobLen = 0;
156
+ this._count = 0;
157
+ // clear the sentinel/offsets that were in use to avoid stale data
158
+ // (we only touched [0..count], so the fill is bounded)
159
+ this._offsets[0] = 0;
160
+ this._hashKeys.fill(EMPTY_SLOT);
161
+ }
162
+
163
+ // Serialize into the LBK1 per-shard string-table byte layout described at the
164
+ // top of this file. Returns { bytes: Uint8Array, byteLength: number }.
165
+ // byteLength includes 8-byte pad so the payload aligns cleanly.
166
+ serialize() {
167
+ const entryCount = this._count;
168
+ const blobLen = this._blobLen;
169
+ // header: 4 (entry_count) + 4 (blob_len) + 4*(entry_count+1) (offsets incl. sentinel) + blob
170
+ const bodyBytes = 8 + 4 * (entryCount + 1) + blobLen;
171
+ const padded = (bodyBytes + 7) & ~7;
172
+ const bytes = new Uint8Array(padded);
173
+ const dv = new DataView(bytes.buffer);
174
+ dv.setUint32(0, entryCount, true);
175
+ dv.setUint32(4, blobLen, true);
176
+ let off = 8;
177
+ for (let i = 0; i <= entryCount; i++) { // <=: include the sentinel
178
+ dv.setUint32(off, this._offsets[i], true);
179
+ off += 4;
180
+ }
181
+ bytes.set(this._blob.subarray(0, blobLen), off);
182
+ return { bytes, byteLength: padded };
183
+ }
184
+
185
+ // Static: parse a serialized table from a byte view. Returns a read-only
186
+ // accessor object (not a live StringTable — cheaper for the Reader path).
187
+ static parse(bytes, byteOffset) {
188
+ const dv = new DataView(bytes.buffer, bytes.byteOffset + byteOffset, bytes.byteLength - byteOffset);
189
+ const entryCount = dv.getUint32(0, true);
190
+ const blobLen = dv.getUint32(4, true);
191
+ // offsets live at bytes[byteOffset + 8 ..], length = (entryCount+1)*4
192
+ const offsetsByteOff = byteOffset + 8;
193
+ const offsets = new Uint32Array(bytes.buffer, bytes.byteOffset + offsetsByteOff, entryCount + 1);
194
+ const blobByteOff = offsetsByteOff + 4 * (entryCount + 1);
195
+ const blob = new Uint8Array(bytes.buffer, bytes.byteOffset + blobByteOff, blobLen);
196
+ return new StringTableView(entryCount, blob, offsets);
197
+ }
198
+ }
199
+
200
+ // Read-only view over a serialized string table. Zero-copy over the underlying
201
+ // buffer; JS string materialized only on .get(i) call.
202
+ export class StringTableView {
203
+ constructor(count, blob, offsets) {
204
+ this._count = count;
205
+ this._blob = blob;
206
+ this._offsets = offsets;
207
+ this._decoder = null; // lazy — created on first get()
208
+ }
209
+
210
+ get count() { return this._count; }
211
+
212
+ get(idx) {
213
+ if (idx >= this._count) return undefined;
214
+ if (!this._decoder) this._decoder = new TextDecoder('utf-8');
215
+ const start = this._offsets[idx];
216
+ const end = this._offsets[idx + 1];
217
+ return this._decoder.decode(this._blob.subarray(start, end));
218
+ }
219
+
220
+ // Raw byte range for callers that want to skip UTF-8 decode.
221
+ bytesAt(idx) {
222
+ if (idx >= this._count) return undefined;
223
+ return this._blob.subarray(this._offsets[idx], this._offsets[idx + 1]);
224
+ }
225
+ }