@zakkster/lite-bake-stream 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +457 -0
- package/LICENSE +21 -0
- package/README.md +150 -0
- package/SPEC.md +364 -0
- package/llms.txt +81 -0
- package/package.json +117 -0
- package/src/FileIngest.js +104 -0
- package/src/MultiReader.js +160 -0
- package/src/PreserveReader.js +180 -0
- package/src/PreserveTokenizer.js +172 -0
- package/src/PreserveWriter.js +218 -0
- package/src/RangeReader.js +470 -0
- package/src/Reader.js +349 -0
- package/src/Split.js +359 -0
- package/src/StringTable.js +225 -0
- package/src/Tokenizer.js +691 -0
- package/src/Writer.js +713 -0
- package/src/index.js +193 -0
- package/types/FileIngest.d.ts +44 -0
- package/types/MultiReader.d.ts +36 -0
- package/types/PreserveReader.d.ts +44 -0
- package/types/PreserveTokenizer.d.ts +27 -0
- package/types/PreserveWriter.d.ts +35 -0
- package/types/RangeReader.d.ts +93 -0
- package/types/Reader.d.ts +85 -0
- package/types/Split.d.ts +66 -0
- package/types/StringTable.d.ts +23 -0
- package/types/Tokenizer.d.ts +42 -0
- package/types/Writer.d.ts +92 -0
- package/types/index.d.ts +58 -0
package/src/Split.js
ADDED
|
@@ -0,0 +1,359 @@
|
|
|
1
|
+
// @zakkster/lite-bake-stream / Split (M6)
|
|
2
|
+
// Splitters + per-part compilation + container merge for parallel/checkpointed ingest.
|
|
3
|
+
// Copyright (c) 2026 Zahary Shinikchiev. MIT.
|
|
4
|
+
//
|
|
5
|
+
// M6 covers three primitives that together enable worker-parallel or checkpoint-
|
|
6
|
+
// resumable ingest against LBK1:
|
|
7
|
+
//
|
|
8
|
+
// splitNDJSON(bytes, opts) -> [{start, end}, ...]
|
|
9
|
+
// Divide input into N byte ranges that each contain complete NDJSON records.
|
|
10
|
+
//
|
|
11
|
+
// compilePart(bytes, opts) -> Uint8Array
|
|
12
|
+
// Compile ONE byte range into a standalone LBK1 container. Trivially
|
|
13
|
+
// Transferable across worker boundaries via postMessage(buf, [buf]).
|
|
14
|
+
//
|
|
15
|
+
// compileInParts(bytes, opts) -> Uint8Array[]
|
|
16
|
+
// Sequential convenience: split + compile each part. Same output shape as
|
|
17
|
+
// running each part through a worker; deterministic and easy to test.
|
|
18
|
+
// Skip this when you have workers; use it as a fallback when you don't.
|
|
19
|
+
//
|
|
20
|
+
// mergeContainers([bytes, ...]) -> Uint8Array
|
|
21
|
+
// Concatenate N LBK1 containers into ONE. All must share the same schema.
|
|
22
|
+
// Preserves per-shard string tables and zone maps verbatim; rewrites the
|
|
23
|
+
// shard directory and header offsets. For query workloads over multiple
|
|
24
|
+
// parts, prefer MultiReader (M4) which avoids the copy; use mergeContainers
|
|
25
|
+
// when you need a single-file distributable.
|
|
26
|
+
//
|
|
27
|
+
// The "parallelism-safe" contract: every part MUST use the SAME explicit
|
|
28
|
+
// schema. Two workers running sample-and-infer independently could infer
|
|
29
|
+
// different field sets or lane kinds, and their outputs would then fail to
|
|
30
|
+
// merge. To use sample-and-infer with parts, compile part 0 first, extract
|
|
31
|
+
// its schema via `new Reader(container.buffer).schema`, and pass that as the
|
|
32
|
+
// explicit schema for parts 1..N.
|
|
33
|
+
|
|
34
|
+
import { Tokenizer, TokenizerError } from './Tokenizer.js';
|
|
35
|
+
import { Writer, WriterError } from './Writer.js';
|
|
36
|
+
import { Reader, ReaderError } from './Reader.js';
|
|
37
|
+
import { StringTable } from './StringTable.js';
|
|
38
|
+
|
|
39
|
+
export const VERSION = '1.0.0';
|
|
40
|
+
|
|
41
|
+
const LF = 0x0A;
|
|
42
|
+
const CONTAINER_HEADER_BYTES = 48;
|
|
43
|
+
const SHARD_ENTRY_BYTES = 40;
|
|
44
|
+
const FIELD_DESCRIPTOR_BYTES = 24;
|
|
45
|
+
const FOOTER_BYTES = 16;
|
|
46
|
+
const LANE_F64 = 1;
|
|
47
|
+
|
|
48
|
+
export class SplitError extends Error {
|
|
49
|
+
constructor(code, msg) { super(msg); this.code = code; this.name = 'SplitError'; }
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// ---------- splitNDJSON ----------
|
|
53
|
+
|
|
54
|
+
// Divide NDJSON bytes into N ranges, each ending at (and including) a newline.
|
|
55
|
+
// Guarantees:
|
|
56
|
+
// - Every returned {start, end} covers complete records (no split mid-line).
|
|
57
|
+
// - Union of ranges = original bytes (no gaps, no overlaps).
|
|
58
|
+
// - Concatenating bytes.subarray(start, end) across parts reproduces the
|
|
59
|
+
// original byte-for-byte.
|
|
60
|
+
// Options:
|
|
61
|
+
// targetParts: desired part count (default 4). Actual count may be less
|
|
62
|
+
// if the input is too small or has too few newlines.
|
|
63
|
+
// maxPartBytes: optional cap on part size. If set, splitter emits enough
|
|
64
|
+
// parts to keep each under this ceiling.
|
|
65
|
+
export function splitNDJSON(bytes, opts) {
|
|
66
|
+
if (!(bytes instanceof Uint8Array)) {
|
|
67
|
+
throw new TypeError('splitNDJSON: expected Uint8Array');
|
|
68
|
+
}
|
|
69
|
+
opts = opts || {};
|
|
70
|
+
const targetParts = Math.max(1, opts.targetParts || 4);
|
|
71
|
+
const maxPartBytes = opts.maxPartBytes || Infinity;
|
|
72
|
+
const total = bytes.length;
|
|
73
|
+
if (total === 0) return [];
|
|
74
|
+
|
|
75
|
+
// Compute effective number of parts: honor targetParts, then bump up if
|
|
76
|
+
// any part would exceed maxPartBytes.
|
|
77
|
+
let parts = targetParts;
|
|
78
|
+
if (Number.isFinite(maxPartBytes)) {
|
|
79
|
+
const minPartsByCap = Math.ceil(total / maxPartBytes);
|
|
80
|
+
if (minPartsByCap > parts) parts = minPartsByCap;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const ranges = [];
|
|
84
|
+
let cursor = 0;
|
|
85
|
+
for (let i = 1; i < parts; i++) {
|
|
86
|
+
// Target boundary at even split; slide forward to the NEXT newline so we
|
|
87
|
+
// don't cut a record. If no newline exists between cursor and end, this
|
|
88
|
+
// part absorbs the remainder and we stop.
|
|
89
|
+
const target = Math.floor((total * i) / parts);
|
|
90
|
+
if (target <= cursor) continue; // parts overlap in tiny inputs; skip
|
|
91
|
+
let boundary = target;
|
|
92
|
+
while (boundary < total && bytes[boundary] !== LF) boundary++;
|
|
93
|
+
if (boundary >= total) break;
|
|
94
|
+
boundary++; // include the newline in this part
|
|
95
|
+
ranges.push({ start: cursor, end: boundary });
|
|
96
|
+
cursor = boundary;
|
|
97
|
+
}
|
|
98
|
+
// Last part absorbs everything left.
|
|
99
|
+
if (cursor < total) ranges.push({ start: cursor, end: total });
|
|
100
|
+
return ranges;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// ---------- compilePart ----------
|
|
104
|
+
|
|
105
|
+
// Compile one byte range into a standalone LBK1 container.
|
|
106
|
+
// Same options as Writer; a schema is STRONGLY recommended when this is a
|
|
107
|
+
// part of a larger multi-part compile (see module doc).
|
|
108
|
+
export function compilePart(bytes, opts) {
|
|
109
|
+
if (!(bytes instanceof Uint8Array)) {
|
|
110
|
+
throw new TypeError('compilePart: expected Uint8Array');
|
|
111
|
+
}
|
|
112
|
+
opts = opts || {};
|
|
113
|
+
const framing = opts.framing || 'ndjson';
|
|
114
|
+
const w = new Writer(opts.writer || {});
|
|
115
|
+
const t = new Tokenizer(w, { framing });
|
|
116
|
+
t.feed(bytes);
|
|
117
|
+
t.end();
|
|
118
|
+
return new Uint8Array(w.finalize().buffer);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// ---------- compileInParts ----------
|
|
122
|
+
|
|
123
|
+
// Sequential convenience: split then compile each part serially.
|
|
124
|
+
// Returns Uint8Array[] — one container per part. Equivalent output to running
|
|
125
|
+
// each part through a worker (or any parallel executor); use this when workers
|
|
126
|
+
// aren't available or for testing.
|
|
127
|
+
export function compileInParts(bytes, opts) {
|
|
128
|
+
opts = opts || {};
|
|
129
|
+
const splitOpts = {
|
|
130
|
+
targetParts: opts.targetParts,
|
|
131
|
+
maxPartBytes: opts.maxPartBytes,
|
|
132
|
+
};
|
|
133
|
+
const partOpts = {
|
|
134
|
+
framing: opts.framing || 'ndjson',
|
|
135
|
+
writer: opts.writer || {},
|
|
136
|
+
};
|
|
137
|
+
const ranges = splitNDJSON(bytes, splitOpts);
|
|
138
|
+
const containers = new Array(ranges.length);
|
|
139
|
+
for (let i = 0; i < ranges.length; i++) {
|
|
140
|
+
const { start, end } = ranges[i];
|
|
141
|
+
containers[i] = compilePart(bytes.subarray(start, end), partOpts);
|
|
142
|
+
}
|
|
143
|
+
return containers;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// ---------- mergeContainers ----------
|
|
147
|
+
|
|
148
|
+
// Concatenate N LBK1 containers into ONE. All containers must share an
|
|
149
|
+
// identical schema (byte-compare over the schema block). Preserves per-shard
|
|
150
|
+
// string tables and zone maps verbatim; rewrites the shard directory + header
|
|
151
|
+
// offsets to make the combined layout self-consistent.
|
|
152
|
+
//
|
|
153
|
+
// The result is byte-identical (except for the shard directory / header
|
|
154
|
+
// offsets) to what a single Writer would have produced if it had emitted the
|
|
155
|
+
// same shards in the same order. Query behavior via `new Reader(merged)` and
|
|
156
|
+
// via `new MultiReader([readers...])` is equivalent (both return the same
|
|
157
|
+
// values for the same global row indices).
|
|
158
|
+
export function mergeContainers(containers) {
|
|
159
|
+
if (!Array.isArray(containers) || containers.length === 0) {
|
|
160
|
+
throw new SplitError('S_MERGE_EMPTY', 'mergeContainers requires a non-empty array');
|
|
161
|
+
}
|
|
162
|
+
// Normalize inputs to Uint8Array
|
|
163
|
+
const parts = containers.map((c, i) => {
|
|
164
|
+
if (c instanceof Uint8Array) return c;
|
|
165
|
+
if (c instanceof ArrayBuffer) return new Uint8Array(c);
|
|
166
|
+
throw new TypeError('mergeContainers: container ' + i + ' is not Uint8Array or ArrayBuffer');
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
// Parse each via Reader for authoritative metadata. This also validates
|
|
170
|
+
// the LBK1 magic and version.
|
|
171
|
+
const readers = parts.map((p) => new Reader(_toArrayBuffer(p)));
|
|
172
|
+
|
|
173
|
+
// Schemas must match. We compare the parsed schema (byte-exact would work
|
|
174
|
+
// too but this catches equivalent-but-differently-padded cases).
|
|
175
|
+
const schema0 = readers[0].schema;
|
|
176
|
+
for (let i = 1; i < readers.length; i++) {
|
|
177
|
+
if (!_schemasEqual(schema0, readers[i].schema)) {
|
|
178
|
+
throw new SplitError('S_SCHEMA_MISMATCH',
|
|
179
|
+
'mergeContainers: container ' + i + ' schema differs from container 0');
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// Total shards and rows
|
|
184
|
+
let totalShards = 0, totalRows = 0;
|
|
185
|
+
for (const r of readers) { totalShards += r.shardCount; totalRows += r.totalRows; }
|
|
186
|
+
|
|
187
|
+
// Compute the target layout. Schema block is copied from container 0 verbatim.
|
|
188
|
+
// Reader gives us schemaBlockOff and shardDirOff; the schema-block byte range
|
|
189
|
+
// is [schemaBlockOff, shardDirOff). We reuse this slice.
|
|
190
|
+
const src0 = parts[0];
|
|
191
|
+
const dv0 = new DataView(src0.buffer, src0.byteOffset, src0.byteLength);
|
|
192
|
+
const schemaBlockOff = Number(dv0.getBigUint64(8, true));
|
|
193
|
+
const shardDirOff0 = Number(dv0.getBigUint64(24, true));
|
|
194
|
+
const schemaBlockLen = shardDirOff0 - schemaBlockOff;
|
|
195
|
+
|
|
196
|
+
// New offsets
|
|
197
|
+
const outSchemaBlockOff = CONTAINER_HEADER_BYTES;
|
|
198
|
+
const outShardDirOff = outSchemaBlockOff + schemaBlockLen;
|
|
199
|
+
const outShardDirBytes = totalShards * SHARD_ENTRY_BYTES;
|
|
200
|
+
|
|
201
|
+
// Zone maps: emit iff container 0 has them AND all others do too.
|
|
202
|
+
// (MultiReader uses the same "all-or-none" policy — see MultiReader.d.ts.)
|
|
203
|
+
const outHasZoneMaps = readers.every((r) => r.hasZoneMaps);
|
|
204
|
+
const zoneMapsRaw = readers.map((r) => r.zoneMapsRaw());
|
|
205
|
+
const T = outHasZoneMaps ? zoneMapsRaw[0].trackedFields.length : 0;
|
|
206
|
+
const zoneMapsHeaderLen = 16;
|
|
207
|
+
const zoneMapsFieldTableLen = T * 2;
|
|
208
|
+
const zoneMapsFieldTablePad = (8 - (zoneMapsFieldTableLen & 7)) & 7;
|
|
209
|
+
const zoneMapsMinsMaxesLen = outHasZoneMaps ? totalShards * T * 8 * 2 : 0;
|
|
210
|
+
const outMetadataOff = outHasZoneMaps ? (outShardDirOff + outShardDirBytes) : 0;
|
|
211
|
+
const outZoneMapsBytes = outHasZoneMaps
|
|
212
|
+
? zoneMapsHeaderLen + zoneMapsFieldTableLen + zoneMapsFieldTablePad + zoneMapsMinsMaxesLen
|
|
213
|
+
: 0;
|
|
214
|
+
const outFirstShardOff = outShardDirOff + outShardDirBytes + outZoneMapsBytes;
|
|
215
|
+
|
|
216
|
+
// Compute per-source-shard payload/string-table extents in the OUTPUT buffer.
|
|
217
|
+
// Layout mirrors the writer's: each shard's payload followed by its local
|
|
218
|
+
// string table, padded to 8 between shards.
|
|
219
|
+
let cursor = outFirstShardOff;
|
|
220
|
+
const shardOutMap = []; // array of {payloadOff, payloadLen, stringOff, stringLen, srcContainerIdx, srcShardIdx, rowCount, mins?, maxes?}
|
|
221
|
+
for (let ci = 0; ci < readers.length; ci++) {
|
|
222
|
+
const r = readers[ci];
|
|
223
|
+
for (let si = 0; si < r.shardCount; si++) {
|
|
224
|
+
const shard = r.shards[si];
|
|
225
|
+
// Source shard payload lives at shard.payloadOff in the source container.
|
|
226
|
+
const payloadLen = shard.payloadLen;
|
|
227
|
+
const payloadPad = (8 - (payloadLen & 7)) & 7;
|
|
228
|
+
const stringLen = shard.stringTable ? _sourceStringTableLen(readers[ci], si) : 0;
|
|
229
|
+
const outPayloadOff = cursor;
|
|
230
|
+
const outStringOff = outPayloadOff + payloadLen + payloadPad;
|
|
231
|
+
shardOutMap.push({
|
|
232
|
+
payloadOff: outPayloadOff,
|
|
233
|
+
payloadLen,
|
|
234
|
+
stringOff: outStringOff,
|
|
235
|
+
stringLen,
|
|
236
|
+
srcContainerIdx: ci,
|
|
237
|
+
srcShardIdx: si,
|
|
238
|
+
rowCount: shard.rowCount,
|
|
239
|
+
});
|
|
240
|
+
cursor = outStringOff + stringLen;
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
const totalBytes = cursor + FOOTER_BYTES;
|
|
244
|
+
|
|
245
|
+
// Allocate output
|
|
246
|
+
const out = new Uint8Array(totalBytes);
|
|
247
|
+
const outDv = new DataView(out.buffer);
|
|
248
|
+
|
|
249
|
+
// Header
|
|
250
|
+
out[0] = 0x4C; out[1] = 0x42; out[2] = 0x4B; out[3] = 0x31; // 'LBK1'
|
|
251
|
+
outDv.setUint16(4, 1, true); // format_version
|
|
252
|
+
out[6] = 1; // endian LE
|
|
253
|
+
out[7] = 0;
|
|
254
|
+
outDv.setBigUint64(8, BigInt(outSchemaBlockOff), true);
|
|
255
|
+
outDv.setBigUint64(16, BigInt(outMetadataOff), true);
|
|
256
|
+
outDv.setBigUint64(24, BigInt(outShardDirOff), true);
|
|
257
|
+
outDv.setUint32(32, totalShards, true);
|
|
258
|
+
outDv.setUint32(36, 0, true);
|
|
259
|
+
outDv.setBigUint64(40, BigInt(totalRows), true);
|
|
260
|
+
|
|
261
|
+
// Schema block: copy verbatim from container 0
|
|
262
|
+
out.set(parts[0].subarray(schemaBlockOff, schemaBlockOff + schemaBlockLen), outSchemaBlockOff);
|
|
263
|
+
|
|
264
|
+
// Shard directory: rewrite ShardEntry for every output shard
|
|
265
|
+
for (let s = 0; s < shardOutMap.length; s++) {
|
|
266
|
+
const m = shardOutMap[s];
|
|
267
|
+
const entryOff = outShardDirOff + s * SHARD_ENTRY_BYTES;
|
|
268
|
+
outDv.setBigUint64(entryOff + 0, BigInt(m.payloadOff), true);
|
|
269
|
+
outDv.setUint32(entryOff + 8, m.payloadLen, true);
|
|
270
|
+
outDv.setUint32(entryOff + 12, m.rowCount, true);
|
|
271
|
+
outDv.setUint16(entryOff + 16, 1, true); // min_reader_version
|
|
272
|
+
outDv.setUint16(entryOff + 18, 0, true); // flags
|
|
273
|
+
outDv.setUint32(entryOff + 20, 0, true); // reserved
|
|
274
|
+
outDv.setBigUint64(entryOff + 24, BigInt(m.stringLen > 0 ? m.stringOff : 0), true);
|
|
275
|
+
outDv.setBigUint64(entryOff + 32, BigInt(m.stringLen), true);
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
// Zone maps: header + field table + interleaved mins/maxes.
|
|
279
|
+
if (outHasZoneMaps) {
|
|
280
|
+
// Segment header
|
|
281
|
+
out[outMetadataOff + 0] = 0x30; out[outMetadataOff + 1] = 0x5A;
|
|
282
|
+
out[outMetadataOff + 2] = 0x4D; out[outMetadataOff + 3] = 0x31; // 'ZM01'
|
|
283
|
+
outDv.setUint32(outMetadataOff + 4, totalShards, true);
|
|
284
|
+
outDv.setUint32(outMetadataOff + 8, T, true);
|
|
285
|
+
outDv.setUint32(outMetadataOff + 12, 0, true);
|
|
286
|
+
// Field index table (copied from container 0 — all containers share the schema)
|
|
287
|
+
const fieldTableOut = outMetadataOff + zoneMapsHeaderLen;
|
|
288
|
+
const tracked0 = zoneMapsRaw[0].trackedFields;
|
|
289
|
+
for (let t = 0; t < T; t++) outDv.setUint16(fieldTableOut + t * 2, tracked0[t], true);
|
|
290
|
+
// Mins / maxes concatenated in output-shard order
|
|
291
|
+
const minsOut = fieldTableOut + zoneMapsFieldTableLen + zoneMapsFieldTablePad;
|
|
292
|
+
const maxesOut = minsOut + totalShards * T * 8;
|
|
293
|
+
for (let s = 0; s < shardOutMap.length; s++) {
|
|
294
|
+
const m = shardOutMap[s];
|
|
295
|
+
const zm = zoneMapsRaw[m.srcContainerIdx];
|
|
296
|
+
for (let t = 0; t < T; t++) {
|
|
297
|
+
const srcPos = m.srcShardIdx * zm.laneCount + t;
|
|
298
|
+
outDv.setFloat64(minsOut + (s * T + t) * 8, zm.mins[srcPos], true);
|
|
299
|
+
outDv.setFloat64(maxesOut + (s * T + t) * 8, zm.maxes[srcPos], true);
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
// Shard payloads + string tables: bulk-copy each source range to its new offset
|
|
305
|
+
for (const m of shardOutMap) {
|
|
306
|
+
const src = parts[m.srcContainerIdx];
|
|
307
|
+
const r = readers[m.srcContainerIdx];
|
|
308
|
+
const srcShard = r.shards[m.srcShardIdx];
|
|
309
|
+
// Payload
|
|
310
|
+
out.set(src.subarray(srcShard.payloadOff, srcShard.payloadOff + m.payloadLen), m.payloadOff);
|
|
311
|
+
// String table (if any)
|
|
312
|
+
if (m.stringLen > 0) {
|
|
313
|
+
// In source, the string table starts at localStrOff (from the ShardEntry).
|
|
314
|
+
const srcDv = new DataView(src.buffer, src.byteOffset, src.byteLength);
|
|
315
|
+
const srcShardEntry = r.shardDirectoryOffset + m.srcShardIdx * SHARD_ENTRY_BYTES;
|
|
316
|
+
const srcLocalStrOff = Number(srcDv.getBigUint64(srcShardEntry + 24, true));
|
|
317
|
+
out.set(src.subarray(srcLocalStrOff, srcLocalStrOff + m.stringLen), m.stringOff);
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
// Footer
|
|
322
|
+
const footerOff = totalBytes - FOOTER_BYTES;
|
|
323
|
+
outDv.setUint32(footerOff + 0, 0xFFFFFFFF, true); // CRC absent
|
|
324
|
+
outDv.setUint32(footerOff + 4, 0, true);
|
|
325
|
+
out[footerOff + 8] = 0x31;
|
|
326
|
+
out[footerOff + 9] = 0x4B;
|
|
327
|
+
out[footerOff + 10] = 0x42;
|
|
328
|
+
out[footerOff + 11] = 0x4C;
|
|
329
|
+
outDv.setUint32(footerOff + 12, FOOTER_BYTES, true);
|
|
330
|
+
|
|
331
|
+
return out;
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
// ---------- internals ----------
|
|
335
|
+
|
|
336
|
+
function _toArrayBuffer(u8) {
|
|
337
|
+
if (u8.byteOffset === 0 && u8.byteLength === u8.buffer.byteLength) return u8.buffer;
|
|
338
|
+
const copy = new Uint8Array(u8.byteLength);
|
|
339
|
+
copy.set(u8);
|
|
340
|
+
return copy.buffer;
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
function _schemasEqual(a, b) {
|
|
344
|
+
if (a.fields.length !== b.fields.length) return false;
|
|
345
|
+
if (a.rowStride !== b.rowStride) return false;
|
|
346
|
+
for (let i = 0; i < a.fields.length; i++) {
|
|
347
|
+
if (a.fields[i].name !== b.fields[i].name) return false;
|
|
348
|
+
if (a.fields[i].laneKind !== b.fields[i].laneKind) return false;
|
|
349
|
+
if (a.fields[i].offsetInRow !== b.fields[i].offsetInRow) return false;
|
|
350
|
+
}
|
|
351
|
+
return true;
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
// Look up the source string-table byte length for a shard.
|
|
355
|
+
function _sourceStringTableLen(reader, shardIdx) {
|
|
356
|
+
const dv = new DataView(reader.buffer);
|
|
357
|
+
const entryOff = reader.shardDirectoryOffset + shardIdx * SHARD_ENTRY_BYTES;
|
|
358
|
+
return Number(dv.getBigUint64(entryOff + 32, true));
|
|
359
|
+
}
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
// @zakkster/lite-bake-stream / StringTable
|
|
2
|
+
// Byte-level UTF-8 string interning with zero-alloc hot path.
|
|
3
|
+
// Copyright (c) 2026 Zahary Shinikchiev. MIT.
|
|
4
|
+
//
|
|
5
|
+
// intern(bytes, from, to) -> u32 index, deduplicating identical byte ranges.
|
|
6
|
+
// The hash path never allocates: bytes stay in the caller's buffer during
|
|
7
|
+
// lookup, and internal storage grows in doubling steps.
|
|
8
|
+
//
|
|
9
|
+
// Serializes to the LBK1 per-shard string-table layout (SPEC.md section 3.3):
|
|
10
|
+
// u32 entry_count
|
|
11
|
+
// u32 blob_length_bytes
|
|
12
|
+
// u32 offsets[entry_count] (start of each string in the blob)
|
|
13
|
+
// u32 blob_end_sentinel (= blob_length_bytes, for len derivation)
|
|
14
|
+
// u8 blob[blob_length_bytes]
|
|
15
|
+
//
|
|
16
|
+
// The trailing sentinel makes `len(i) = offsets[i+1] - offsets[i]` uniform for
|
|
17
|
+
// all i including the last — no branch in the reader hot path.
|
|
18
|
+
|
|
19
|
+
export const VERSION = '1.0.0';
|
|
20
|
+
|
|
21
|
+
const EMPTY_SLOT = 0xFFFFFFFF; // MUST be unsigned; typed-array reads are unsigned
|
|
22
|
+
const INITIAL_BLOB_BYTES = 64 * 1024;
|
|
23
|
+
const INITIAL_ENTRIES = 1024;
|
|
24
|
+
const INITIAL_HASH_CAP = 2048; // load factor target 50%
|
|
25
|
+
const HASH_MAX_LOAD_NUM = 1; // 50% load factor: num/den = 1/2
|
|
26
|
+
const HASH_MAX_LOAD_DEN = 2;
|
|
27
|
+
|
|
28
|
+
// FNV-1a 32-bit over a byte range.
|
|
29
|
+
function fnv1a(bytes, from, to) {
|
|
30
|
+
let h = 0x811c9dc5 | 0;
|
|
31
|
+
for (let i = from; i < to; i++) {
|
|
32
|
+
h = (h ^ bytes[i]) | 0;
|
|
33
|
+
h = Math.imul(h, 0x01000193) | 0;
|
|
34
|
+
}
|
|
35
|
+
return h >>> 0;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export class StringTable {
|
|
39
|
+
constructor() {
|
|
40
|
+
this._blob = new Uint8Array(INITIAL_BLOB_BYTES);
|
|
41
|
+
this._blobLen = 0;
|
|
42
|
+
this._offsets = new Uint32Array(INITIAL_ENTRIES + 1); // +1 for sentinel slot
|
|
43
|
+
this._count = 0;
|
|
44
|
+
this._hashCap = INITIAL_HASH_CAP;
|
|
45
|
+
this._hashMask = INITIAL_HASH_CAP - 1;
|
|
46
|
+
this._hashKeys = new Uint32Array(INITIAL_HASH_CAP).fill(EMPTY_SLOT);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// Returns u32 index of the string with these bytes, adding a new entry if
|
|
50
|
+
// this exact byte sequence hasn't been seen before. Zero-alloc on the hit path.
|
|
51
|
+
intern(bytes, from, to) {
|
|
52
|
+
const h = fnv1a(bytes, from, to);
|
|
53
|
+
let slot = h & this._hashMask;
|
|
54
|
+
// Linear probing with wraparound
|
|
55
|
+
for (;;) {
|
|
56
|
+
const stored = this._hashKeys[slot];
|
|
57
|
+
if (stored === EMPTY_SLOT) {
|
|
58
|
+
// Miss: this is a new string; append + record + insert.
|
|
59
|
+
return this._insertNew(bytes, from, to, slot);
|
|
60
|
+
}
|
|
61
|
+
// Occupied: check for byte-equal match at stored index
|
|
62
|
+
if (this._equalsAt(stored, bytes, from, to)) return stored;
|
|
63
|
+
slot = (slot + 1) & this._hashMask;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
_equalsAt(idx, bytes, from, to) {
|
|
68
|
+
const start = this._offsets[idx];
|
|
69
|
+
const end = this._offsets[idx + 1]; // sentinel makes this valid for the last entry too
|
|
70
|
+
const n = to - from;
|
|
71
|
+
if ((end - start) !== n) return false;
|
|
72
|
+
const blob = this._blob;
|
|
73
|
+
for (let i = 0; i < n; i++) {
|
|
74
|
+
if (blob[start + i] !== bytes[from + i]) return false;
|
|
75
|
+
}
|
|
76
|
+
return true;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
_insertNew(bytes, from, to, slot) {
|
|
80
|
+
const n = to - from;
|
|
81
|
+
// Grow blob if needed
|
|
82
|
+
if (this._blobLen + n > this._blob.length) this._growBlob(n);
|
|
83
|
+
// Grow offsets if needed (leave room for +1 sentinel)
|
|
84
|
+
if (this._count + 1 >= this._offsets.length) this._growOffsets();
|
|
85
|
+
|
|
86
|
+
const idx = this._count;
|
|
87
|
+
this._offsets[idx] = this._blobLen;
|
|
88
|
+
// append bytes
|
|
89
|
+
const dst = this._blob;
|
|
90
|
+
let d = this._blobLen;
|
|
91
|
+
for (let s = from; s < to; s++) dst[d++] = bytes[s];
|
|
92
|
+
this._blobLen = d;
|
|
93
|
+
// update sentinel at idx+1
|
|
94
|
+
this._offsets[idx + 1] = this._blobLen;
|
|
95
|
+
this._count++;
|
|
96
|
+
|
|
97
|
+
// Insert into hash table
|
|
98
|
+
this._hashKeys[slot] = idx;
|
|
99
|
+
|
|
100
|
+
// Resize hash table if load factor exceeded
|
|
101
|
+
if (this._count * HASH_MAX_LOAD_DEN > this._hashCap * HASH_MAX_LOAD_NUM) {
|
|
102
|
+
this._resizeHash();
|
|
103
|
+
}
|
|
104
|
+
return idx;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
_growBlob(needed) {
|
|
108
|
+
let cap = this._blob.length;
|
|
109
|
+
while (cap < this._blobLen + needed) cap *= 2;
|
|
110
|
+
const nb = new Uint8Array(cap);
|
|
111
|
+
nb.set(this._blob);
|
|
112
|
+
this._blob = nb;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
_growOffsets() {
|
|
116
|
+
const nb = new Uint32Array(this._offsets.length * 2);
|
|
117
|
+
nb.set(this._offsets);
|
|
118
|
+
this._offsets = nb;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
_resizeHash() {
|
|
122
|
+
const newCap = this._hashCap * 2;
|
|
123
|
+
const newMask = newCap - 1;
|
|
124
|
+
const newKeys = new Uint32Array(newCap).fill(EMPTY_SLOT);
|
|
125
|
+
// Rehash every occupied slot by looking up its string in the blob.
|
|
126
|
+
const oldKeys = this._hashKeys;
|
|
127
|
+
for (let i = 0; i < oldKeys.length; i++) {
|
|
128
|
+
const idx = oldKeys[i];
|
|
129
|
+
if (idx === EMPTY_SLOT) continue;
|
|
130
|
+
const start = this._offsets[idx];
|
|
131
|
+
const end = this._offsets[idx + 1];
|
|
132
|
+
const h = fnv1a(this._blob, start, end);
|
|
133
|
+
let slot = h & newMask;
|
|
134
|
+
while (newKeys[slot] !== EMPTY_SLOT) slot = (slot + 1) & newMask;
|
|
135
|
+
newKeys[slot] = idx;
|
|
136
|
+
}
|
|
137
|
+
this._hashCap = newCap;
|
|
138
|
+
this._hashMask = newMask;
|
|
139
|
+
this._hashKeys = newKeys;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
get count() { return this._count; }
|
|
143
|
+
get blobLen() { return this._blobLen; }
|
|
144
|
+
|
|
145
|
+
// Zero-copy byte range for entry `idx`. Used by consumers that want the raw
|
|
146
|
+
// UTF-8 bytes without materializing a JS string (e.g. re-interning across
|
|
147
|
+
// string tables during sample drain).
|
|
148
|
+
bytesAt(idx) {
|
|
149
|
+
if (idx >= this._count) return undefined;
|
|
150
|
+
return this._blob.subarray(this._offsets[idx], this._offsets[idx + 1]);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// Reset in-place, keeping allocated arrays (per-shard reuse pattern).
|
|
154
|
+
reset() {
|
|
155
|
+
this._blobLen = 0;
|
|
156
|
+
this._count = 0;
|
|
157
|
+
// clear the sentinel/offsets that were in use to avoid stale data
|
|
158
|
+
// (we only touched [0..count], so the fill is bounded)
|
|
159
|
+
this._offsets[0] = 0;
|
|
160
|
+
this._hashKeys.fill(EMPTY_SLOT);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
// Serialize into the LBK1 per-shard string-table byte layout described at the
|
|
164
|
+
// top of this file. Returns { bytes: Uint8Array, byteLength: number }.
|
|
165
|
+
// byteLength includes 8-byte pad so the payload aligns cleanly.
|
|
166
|
+
serialize() {
|
|
167
|
+
const entryCount = this._count;
|
|
168
|
+
const blobLen = this._blobLen;
|
|
169
|
+
// header: 4 (entry_count) + 4 (blob_len) + 4*(entry_count+1) (offsets incl. sentinel) + blob
|
|
170
|
+
const bodyBytes = 8 + 4 * (entryCount + 1) + blobLen;
|
|
171
|
+
const padded = (bodyBytes + 7) & ~7;
|
|
172
|
+
const bytes = new Uint8Array(padded);
|
|
173
|
+
const dv = new DataView(bytes.buffer);
|
|
174
|
+
dv.setUint32(0, entryCount, true);
|
|
175
|
+
dv.setUint32(4, blobLen, true);
|
|
176
|
+
let off = 8;
|
|
177
|
+
for (let i = 0; i <= entryCount; i++) { // <=: include the sentinel
|
|
178
|
+
dv.setUint32(off, this._offsets[i], true);
|
|
179
|
+
off += 4;
|
|
180
|
+
}
|
|
181
|
+
bytes.set(this._blob.subarray(0, blobLen), off);
|
|
182
|
+
return { bytes, byteLength: padded };
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
// Static: parse a serialized table from a byte view. Returns a read-only
|
|
186
|
+
// accessor object (not a live StringTable — cheaper for the Reader path).
|
|
187
|
+
static parse(bytes, byteOffset) {
|
|
188
|
+
const dv = new DataView(bytes.buffer, bytes.byteOffset + byteOffset, bytes.byteLength - byteOffset);
|
|
189
|
+
const entryCount = dv.getUint32(0, true);
|
|
190
|
+
const blobLen = dv.getUint32(4, true);
|
|
191
|
+
// offsets live at bytes[byteOffset + 8 ..], length = (entryCount+1)*4
|
|
192
|
+
const offsetsByteOff = byteOffset + 8;
|
|
193
|
+
const offsets = new Uint32Array(bytes.buffer, bytes.byteOffset + offsetsByteOff, entryCount + 1);
|
|
194
|
+
const blobByteOff = offsetsByteOff + 4 * (entryCount + 1);
|
|
195
|
+
const blob = new Uint8Array(bytes.buffer, bytes.byteOffset + blobByteOff, blobLen);
|
|
196
|
+
return new StringTableView(entryCount, blob, offsets);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
// Read-only view over a serialized string table. Zero-copy over the underlying
|
|
201
|
+
// buffer; JS string materialized only on .get(i) call.
|
|
202
|
+
export class StringTableView {
|
|
203
|
+
constructor(count, blob, offsets) {
|
|
204
|
+
this._count = count;
|
|
205
|
+
this._blob = blob;
|
|
206
|
+
this._offsets = offsets;
|
|
207
|
+
this._decoder = null; // lazy — created on first get()
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
get count() { return this._count; }
|
|
211
|
+
|
|
212
|
+
get(idx) {
|
|
213
|
+
if (idx >= this._count) return undefined;
|
|
214
|
+
if (!this._decoder) this._decoder = new TextDecoder('utf-8');
|
|
215
|
+
const start = this._offsets[idx];
|
|
216
|
+
const end = this._offsets[idx + 1];
|
|
217
|
+
return this._decoder.decode(this._blob.subarray(start, end));
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// Raw byte range for callers that want to skip UTF-8 decode.
|
|
221
|
+
bytesAt(idx) {
|
|
222
|
+
if (idx >= this._count) return undefined;
|
|
223
|
+
return this._blob.subarray(this._offsets[idx], this._offsets[idx + 1]);
|
|
224
|
+
}
|
|
225
|
+
}
|