@zakkster/lite-bake-stream 1.6.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +165 -128
- package/README.md +293 -109
- package/SPEC.md +30 -30
- package/llms.txt +12 -14
- package/package.json +2 -2
- package/src/FileIngest.js +6 -2
- package/src/MultiReader.js +1 -1
- package/src/Opts.js +23 -0
- package/src/PreserveReader.js +3 -1
- package/src/PreserveTokenizer.js +12 -3
- package/src/PreserveWriter.js +8 -2
- package/src/RangeReader.js +111 -16
- package/src/Reader.js +3 -1
- package/src/Split.js +8 -4
- package/src/StringTable.js +7 -4
- package/src/Tokenizer.js +3 -2
- package/src/Views.js +3 -0
- package/src/Writer.js +6 -4
- package/src/index.js +5 -1
- package/types/MultiReader.d.ts +1 -1
- package/types/RangeReader.d.ts +21 -5
- package/types/Reader.d.ts +1 -1
- package/types/Split.d.ts +1 -1
- package/types/Tokenizer.d.ts +1 -1
- package/types/Writer.d.ts +2 -2
package/llms.txt
CHANGED
|
@@ -1,20 +1,20 @@
|
|
|
1
1
|
# @zakkster/lite-bake-stream
|
|
2
2
|
|
|
3
|
-
Streaming byte-level JSON to
|
|
3
|
+
Streaming byte-level JSON to LBK1 binary containers for JS runtimes.
|
|
4
4
|
|
|
5
5
|
## Purpose
|
|
6
6
|
|
|
7
|
-
Ingest gigabyte-scale JSON (top-level array or NDJSON) into the
|
|
7
|
+
Ingest gigabyte-scale JSON (top-level array or NDJSON) into the LBK1 binary container format without materializing the intermediate object graph. The container is flat, interleaved, and zero-GC to read; a Reader gives random access to any row without re-parsing JSON.
|
|
8
8
|
|
|
9
9
|
## Status
|
|
10
10
|
|
|
11
|
-
v1.
|
|
11
|
+
v1.7.0 -- stable. LBK1 format frozen at `format_version: 1`. Qualified on an 8 GB soak (M1 MacBook Pro): 98.37M rows, 4.89 GB container, zero major GC, zero minor GC, 499 KB total heap allocation, 590.21M cells verified byte-exact, zero mismatches. Tokenizer benches at 222-237 MB/s (~55% of JSON.parse, with no object graph allocated).
|
|
12
12
|
|
|
13
13
|
Public API follows semver from 1.0.0. Future additions (I64 lane, columnar payload mode, container-level string table) land via the format's forward-compat seams -- `min_reader_version` on ShardEntry, reserved FieldDescriptor flags, the `metadata_off` block wrapper -- without a format_version bump.
|
|
14
14
|
|
|
15
15
|
See SPEC.md for the LBK1 container format, section 3.6 for zone maps, section 4.3 for the reserved field flags.
|
|
16
16
|
|
|
17
|
-
## Public API (v1.
|
|
17
|
+
## Public API (v1.7.0)
|
|
18
18
|
|
|
19
19
|
Two ingest modes share one top-level API:
|
|
20
20
|
|
|
@@ -27,28 +27,25 @@ Schema-mode classes:
|
|
|
27
27
|
- `Tokenizer` (`/tokenizer`): chunk-safe UTF-8 JSON SAX scanner.
|
|
28
28
|
- `Writer` (`/writer`): LBK1 shard emitter, plugs into Tokenizer. F64/U32 lanes, per-shard string tables and zone maps.
|
|
29
29
|
- `Reader` (`/reader`): container parser, sync, zero-alloc `get(row, field)` accessors + zone-maps query APIs (`shardBounds`, `findShards`).
|
|
30
|
-
- `RangeReader` (`/range-reader`): HTTP Range lazy shard loading, synchronous query pruning after open.
|
|
30
|
+
- `RangeReader` (`/range-reader`): HTTP Range lazy shard loading, synchronous query pruning after open. Optional `{ signal }` cancels in-flight range I/O; the reader is dead for new I/O once it fires (BS-28).
|
|
31
31
|
- `MultiReader` (`/multi-reader`): logical union over N Readers sharing a schema.
|
|
32
32
|
- `splitNDJSON`, `compilePart`, `compileInParts`, `mergeContainers` (`/split`): worker-agnostic split/compile/merge.
|
|
33
33
|
|
|
34
34
|
Preserve-mode classes:
|
|
35
35
|
- `PreserveTokenizer` (`/preserve-tokenizer`): NDJSON record-boundary scanner, JSON-aware depth tracking, chunk-safe.
|
|
36
36
|
- `PreserveWriter` (`/preserve-writer`): opaque byte-blob sink, pre-allocated shard buffer, zero-GC record path.
|
|
37
|
-
- `PreserveReader` (`/preserve-reader`): tri-API
|
|
37
|
+
- `PreserveReader` (`/preserve-reader`): tri-API -- `getBytes(i)` (zero-alloc view), `getString(i)`, `getJSON(i)`. A returned view pins the container's ArrayBuffer (plain subarray semantics); copy the bytes out if you need the container to be collectable.
|
|
38
38
|
|
|
39
39
|
Shared:
|
|
40
40
|
- `StringTable` (`/string-table`): byte-level UTF-8 interning primitive.
|
|
41
|
-
- `ingestStream`, `ingestFile` (`/file-ingest`): browser helpers piping a `ReadableStream<Uint8Array>` through the pipeline. `preserve` option dispatches to the right writer.
|
|
41
|
+
- `ingestStream`, `ingestFile` (`/file-ingest`): browser helpers piping a `ReadableStream<Uint8Array>` (e.g. `File.stream()`) through the pipeline. `preserve` option dispatches to the right writer. Per-chunk `onProgress` callback; its state object is reused across calls (mutated in place, zero per-chunk allocation) -- copy it if retained past the callback.
|
|
42
42
|
|
|
43
43
|
Error classes with stable `code`: `TokenizerError`, `WriterError`, `ReaderError`, `RangeReaderError`, `MultiReaderError`, `SplitError`, `PreserveTokenizerError`, `PreserveWriterError`, `PreserveReaderError`.
|
|
44
|
-
- `ingestStream`, `ingestFile` (`/file-ingest`): browser helpers piping a `ReadableStream<Uint8Array>` (e.g. `File.stream()`) through the Tokenizer + Writer, returning a Reader. Per-chunk `onProgress` callback; its state object is reused across calls (mutated in place, zero per-chunk allocation) -- copy it if retained past the callback.
|
|
45
|
-
- `TokenizerError`, `WriterError`, `ReaderError`, `RangeReaderError`: thrown on parse/write/read errors with stable `code`.
|
|
46
|
-
- `VERSION` const per subpath.
|
|
47
44
|
|
|
48
45
|
## Schema forms
|
|
49
46
|
|
|
50
|
-
- `{ fields: ['id', 'x', 'y'] }`
|
|
51
|
-
- `{ fields: [{name:'id', laneKind:'f64'}, {name:'tag', laneKind:'u32'}] }`
|
|
47
|
+
- `{ fields: ['id', 'x', 'y'] }` -- all-F64 shorthand, back-compat with M2.
|
|
48
|
+
- `{ fields: [{name:'id', laneKind:'f64'}, {name:'tag', laneKind:'u32'}] }` -- mixed lanes (M3).
|
|
52
49
|
- Sample-and-infer (default when no schema is passed): the first `sampleBytes` of INPUT are observed, byte-true (BS-07) -- the window ends at the first record boundary at or after `sampleBytes` input bytes, read from the Tokenizer's `absOffset`, independent of chunking and record count (a hand-driven Writer with no wired source samples by record count). Fields with only numbers become F64, only strings become U32. null is lane-neutral (BS-20): it sets no kind, so null+string infers U32 (no longer "mixed") and null+number infers F64; post-freeze null reads 0 on an F64 lane and "" on a U32 lane. A field that saw both a real number and a real string raises `W_MIXED_LANE_TYPES` at freeze. Sample memory is O(`sampleBytes`): columnar staging plus a shared string table, independent of total input size.
|
|
53
50
|
|
|
54
51
|
## Contract
|
|
@@ -60,13 +57,14 @@ Error classes with stable `code`: `TokenizerError`, `WriterError`, `ReaderError`
|
|
|
60
57
|
|
|
61
58
|
## Preservation contract
|
|
62
59
|
|
|
63
|
-
Every declared field of every row round-trips. F64 lanes: bit-exact for numeric literals with
|
|
60
|
+
Every declared field of every row round-trips. F64 lanes: bit-exact for numeric literals with <=15 significant digits and |exponent| <= 22 (Clinger's fast path); <=1 ULP drift outside that domain (documented, pinned by NumericBoundary.test.js). U32 lanes: byte-exact UTF-8, unconditional. Missing fields -> documented defaults. Unknown keys -> silently dropped. Wrong-type value on post-freeze schema -> W_LANE_MISMATCH error, no corrupt container produced. See SPEC section 7 for the full table. Asserted by tests, property-based fuzz, AND torture-scale verification (~103,000 rows across 20 scenarios per fast-tier run).
|
|
64
61
|
|
|
65
62
|
## Row-index and refusal codes
|
|
66
63
|
|
|
67
64
|
- Random-access row surfaces fail closed on a bad `rowIdx` (negative, fractional, NaN, or >= totalRows): `Reader.get`, `RangeReader.get` and `PreserveReader.getBytes/getString/getJSON` throw `R_ROW_OUT_OF_RANGE`; `MultiReader.get` throws `M_ROW_OUT_OF_RANGE`. The SHARD-index escape hatches (`shardPayload`, `shardF64`, `shardStringTable`, `loadShard`) keep their unchecked raw-access contract; `shardBounds` returns null out-of-range.
|
|
68
65
|
- `R_OFFSET_TOO_LARGE`: a u64 header/schema/directory offset exceeds 2^53-1 (a Number() cast would lose precision). Raised by all three reader classes.
|
|
69
66
|
- `R_NOT_PREFETCHED`: `RangeReader.syncRange` over a shard `prefetchRange` has not cached (distinct from the `R_TRUNCATED` truncation door). `R_ADAPTER_SHORT_READ` also covers a non-Uint8Array or wrong-length adapter return.
|
|
67
|
+
- `R_ABORTED`: a reader-level `{ signal }` (constructor / `RangeReader.open`, forwarded into `HTTPRangeAdapter.open`) fired -- in-flight range I/O is cancelled. Fail-closed triangle: a pre-aborted signal requests zero bytes; a hanging adapter still rejects within the tick of the abort (the abort listener is removed on settle in both directions, so a long-lived signal retains nothing); resolved-after-abort bytes are discarded and nothing is cached. A foreign `AbortError` rejection while aborted maps here (never leaks); a non-abort rejection rethrows verbatim. The adapter contract goes additive `fetch(byteOffset, byteLength, signal?)` -- a 2-arg adapter degrades cancellation only, never correctness. See decisions/0013-range-abort.md.
|
|
70
68
|
- `ST_BLOB_OVERFLOW`: a per-shard string table would cross the u32 blob_length (4294967295) or entry_count (4294967294) ceiling; also raised by `PreserveWriter` before a shard payload_len would exceed u32. One shared code across `StringTable` and `PreserveWriter`. `StringTableError` is exported from the root and the `string-table` subpath.
|
|
71
69
|
|
|
72
70
|
## Shard budget
|
|
@@ -85,7 +83,7 @@ F64 only. Values exceeding IEEE 754 double range are rejected as `E_NUMBER_OVERF
|
|
|
85
83
|
|
|
86
84
|
## Tree-shaking
|
|
87
85
|
|
|
88
|
-
Subpath entries per SPEC section 6. `sideEffects: false`. Consumers import only the path they need; the browser reader never pulls the writer.
|
|
86
|
+
Subpath entries per SPEC section 6. `sideEffects: false`. Consumers import only the path they need; the browser reader never pulls the writer. Every subpath exports a `VERSION` const.
|
|
89
87
|
|
|
90
88
|
## Streaming emission and integrity (M6)
|
|
91
89
|
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zakkster/lite-bake-stream",
|
|
3
|
-
"version": "1.
|
|
4
|
-
"description": "Streaming byte-level JSON to
|
|
3
|
+
"version": "1.7.0",
|
|
4
|
+
"description": "Streaming byte-level JSON to LBK1 binary containers. Zero-GC, tree-shakeable, gigabyte-scale.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"sideEffects": false,
|
|
7
7
|
"exports": {
|
package/src/FileIngest.js
CHANGED
|
@@ -2,11 +2,15 @@
|
|
|
2
2
|
// Browser-side helper: File.stream() -> Tokenizer -> Writer -> Reader.
|
|
3
3
|
// Copyright (c) 2026 Zahary Shinikchiev. MIT.
|
|
4
4
|
//
|
|
5
|
+
// Error codes:
|
|
6
|
+
// E_OPTION_CONFLICT - preserve mode requested with a non-ndjson framing
|
|
7
|
+
//
|
|
5
8
|
// The natural browser ingest path is:
|
|
6
9
|
//
|
|
7
10
|
// const file = fileInputEl.files[0];
|
|
8
|
-
// const reader = await ingestStream(file.stream(),
|
|
11
|
+
// const reader = await ingestStream(file.stream(), {
|
|
9
12
|
// framing: 'ndjson',
|
|
13
|
+
// totalBytes: file.size,
|
|
10
14
|
// onProgress: ({bytesIngested, totalBytes, rowsWritten, shardsCommitted}) => {...}
|
|
11
15
|
// });
|
|
12
16
|
// // reader is a fully-materialized in-memory LBK1 Reader.
|
|
@@ -27,7 +31,7 @@ import { PreserveWriter } from './PreserveWriter.js';
|
|
|
27
31
|
import { PreserveReader } from './PreserveReader.js';
|
|
28
32
|
import { checkOpts } from './Opts.js';
|
|
29
33
|
|
|
30
|
-
export const VERSION = '1.
|
|
34
|
+
export const VERSION = '1.7.0';
|
|
31
35
|
|
|
32
36
|
const U32_MAX = 4294967295;
|
|
33
37
|
const INGEST_OPTS = {
|
package/src/MultiReader.js
CHANGED
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
// M_ROW_OUT_OF_RANGE - rowIdx >= totalRows
|
|
26
26
|
// M_TOO_MANY_ROWS - cumulative row count exceeds Number.MAX_SAFE_INTEGER
|
|
27
27
|
|
|
28
|
-
export const VERSION = '1.
|
|
28
|
+
export const VERSION = '1.7.0';
|
|
29
29
|
|
|
30
30
|
export class MultiReaderError extends Error {
|
|
31
31
|
constructor(code, msg) { super(msg); this.code = code; this.name = 'MultiReaderError'; }
|
package/src/Opts.js
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
// @zakkster/lite-bake-stream / Opts (internal, not exported from index.js)
|
|
2
2
|
// Copyright (c) 2026 Zahary Shinikchiev. MIT.
|
|
3
3
|
//
|
|
4
|
+
// Error codes:
|
|
5
|
+
// E_UNKNOWN_OPTION - an unknown option key (fails closed with a did-you-mean hint)
|
|
6
|
+
// E_OPTION_VALUE - an option value falls outside its declared domain
|
|
7
|
+
//
|
|
4
8
|
// Shared construction-time options validator. Every public constructor and
|
|
5
9
|
// entry function that takes an opts object runs checkOpts once, at construction
|
|
6
10
|
// or in a synchronous prologue -- never per byte, never per record. An unknown
|
|
@@ -14,6 +18,8 @@
|
|
|
14
18
|
// { t: 'int', min, max } - integer in [min, max] inclusive
|
|
15
19
|
// { t: 'bool' } - boolean
|
|
16
20
|
// { t: 'fn' } - function
|
|
21
|
+
// { t: 'signal' } - AbortSignal-shaped (aborted boolean +
|
|
22
|
+
// addEventListener/removeEventListener fns)
|
|
17
23
|
// { t: 'obj', nullable } - object (contents validated downstream)
|
|
18
24
|
// A value of `undefined` for any key means "use the default" and is skipped,
|
|
19
25
|
// matching each host's `opts.k !== undefined ? opts.k : DEFAULT` defaulting.
|
|
@@ -47,8 +53,10 @@
|
|
|
47
53
|
// totalBytes int [0, 9007199254740991] def -1 (ingest only)
|
|
48
54
|
// RangeReader(adapter,opts) / .open
|
|
49
55
|
// maxCachedShards int [0, 4294967295] def 8 (0 = retain no shard)
|
|
56
|
+
// signal AbortSignal-shaped def none (no cancellation)
|
|
50
57
|
// HTTPRangeAdapter.open(url,opts)
|
|
51
58
|
// fetch function def globalThis.fetch
|
|
59
|
+
// signal AbortSignal-shaped def none (no cancellation)
|
|
52
60
|
// splitNDJSON(bytes,opts) / compileInParts
|
|
53
61
|
// targetParts int [1, 4294967295] def 4 (0 forbidden)
|
|
54
62
|
// maxPartBytes int [1, 4294967295] or Infinity def Infinity (0 forbidden; Infinity = no cap)
|
|
@@ -115,6 +123,21 @@ function _checkValue(label, key, v, d, raise) {
|
|
|
115
123
|
}
|
|
116
124
|
return;
|
|
117
125
|
}
|
|
126
|
+
if (t === 'signal') {
|
|
127
|
+
// AbortSignal by DUCK shape, not instanceof (cross-realm false negatives
|
|
128
|
+
// against the exact scheduled consumer). removeEventListener is REQUIRED --
|
|
129
|
+
// a signal that cannot be un-listened cannot satisfy the settle-removal law
|
|
130
|
+
// (unverified state, refused at the door). See decisions/0013.
|
|
131
|
+
if (v === null || typeof v !== 'object' ||
|
|
132
|
+
typeof v.aborted !== 'boolean' ||
|
|
133
|
+
typeof v.addEventListener !== 'function' ||
|
|
134
|
+
typeof v.removeEventListener !== 'function') {
|
|
135
|
+
raise('E_OPTION_VALUE',
|
|
136
|
+
label + ": option '" + key + "' must be an AbortSignal (aborted boolean + " +
|
|
137
|
+
'addEventListener/removeEventListener); got ' + _show(v));
|
|
138
|
+
}
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
118
141
|
if (t === 'obj') {
|
|
119
142
|
if (v === null) {
|
|
120
143
|
if (!d.nullable) {
|
package/src/PreserveReader.js
CHANGED
|
@@ -29,13 +29,15 @@
|
|
|
29
29
|
// R_INVALID - a structure is internally inconsistent but in-bounds
|
|
30
30
|
// R_SHARD_VERSION_TOO_NEW - a shard's min_reader_version > this reader
|
|
31
31
|
// R_ROW_OUT_OF_RANGE - rowIdx negative, fractional, NaN, or >= totalRows (BS-32)
|
|
32
|
+
// R_BAD_CRC - CRC-32C verify requested and the stored checksum mismatched
|
|
33
|
+
// R_CRC_ABSENT - CRC verify requested but the container carries no checksum
|
|
32
34
|
// R_OFFSET_TOO_LARGE - a u64 header/directory offset exceeds 2^53-1
|
|
33
35
|
|
|
34
36
|
import { toContainerBuffer } from './Views.js';
|
|
35
37
|
import { checkOpts } from './Opts.js';
|
|
36
38
|
import { crc32cInit, crc32cUpdate, crc32cFinal } from './Crc32c.js';
|
|
37
39
|
|
|
38
|
-
export const VERSION = '1.
|
|
40
|
+
export const VERSION = '1.7.0';
|
|
39
41
|
|
|
40
42
|
const CONTAINER_HEADER_BYTES = 48;
|
|
41
43
|
const SHARD_ENTRY_BYTES = 40;
|
package/src/PreserveTokenizer.js
CHANGED
|
@@ -1,11 +1,20 @@
|
|
|
1
1
|
// @zakkster/lite-bake-stream / PreserveTokenizer
|
|
2
2
|
// NDJSON record-boundary scanner for preserve-mode. Copyright (c) 2026 Zahary Shinikchiev. MIT.
|
|
3
3
|
//
|
|
4
|
+
// Error codes:
|
|
5
|
+
// E_UNSUPPORTED_FRAMING - preserve mode fed a framing other than ndjson
|
|
6
|
+
// E_RECORD_TOO_LARGE - a single record exceeds maxRecordBytes
|
|
7
|
+
// E_UNBALANCED - a record's JSON structure never closes
|
|
8
|
+
// E_TRUNCATED - input ends mid-record
|
|
9
|
+
// E_ENDED - feed() called after end()
|
|
10
|
+
// E_POISONED - reuse after a prior throw
|
|
11
|
+
//
|
|
4
12
|
// The schema-mode Tokenizer walks the full JSON AST (numbers, strings, keys,
|
|
5
13
|
// object/array structure) so the Writer can pack values into typed lanes.
|
|
6
|
-
// Preserve-mode doesn't crack open records
|
|
14
|
+
// Preserve-mode doesn't crack open records -- it just needs to find where one
|
|
7
15
|
// record ends and the next begins, so the record's original bytes can be
|
|
8
16
|
// shoved into a shard intact.
|
|
17
|
+
// See decisions/0010-preserve-mode.md.
|
|
9
18
|
//
|
|
10
19
|
// Contract:
|
|
11
20
|
// - Input: NDJSON bytes. One JSON value per line, delimited by 0x0A.
|
|
@@ -32,7 +41,7 @@
|
|
|
32
41
|
|
|
33
42
|
import { checkOpts } from './Opts.js';
|
|
34
43
|
|
|
35
|
-
export const VERSION = '1.
|
|
44
|
+
export const VERSION = '1.7.0';
|
|
36
45
|
|
|
37
46
|
const U32_MAX = 4294967295;
|
|
38
47
|
const PRESERVE_TOKENIZER_OPTS = {
|
|
@@ -57,7 +66,7 @@ const BYTE_RBRACK = 0x5D;
|
|
|
57
66
|
const BYTE_SPACE = 0x20;
|
|
58
67
|
const BYTE_TAB = 0x09;
|
|
59
68
|
|
|
60
|
-
const INITIAL_BUF = 1 << 16; // 64 KiB
|
|
69
|
+
const INITIAL_BUF = 1 << 16; // 64 KiB -- grows on demand for larger records
|
|
61
70
|
|
|
62
71
|
export class PreserveTokenizer {
|
|
63
72
|
constructor(sink, opts) {
|
package/src/PreserveWriter.js
CHANGED
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
// @zakkster/lite-bake-stream / PreserveWriter
|
|
2
2
|
// Copyright (c) 2026 Zahary Shinikchiev. MIT.
|
|
3
3
|
//
|
|
4
|
+
// Error codes:
|
|
5
|
+
// W_EMPTY_INPUT - finalize with no records written
|
|
6
|
+
// W_BAD_SINK - sink is not an object / missing write / async return
|
|
7
|
+
// W_FINALIZED - write after the writer was finalized
|
|
8
|
+
// ST_BLOB_OVERFLOW - a shard payload_len would exceed the u32 ceiling
|
|
9
|
+
//
|
|
4
10
|
// Preserve-mode sink: opaque record blobs packed into shards, with a trailing
|
|
5
11
|
// u32 offset table per shard. No schema, no lanes, no string table, no zone
|
|
6
12
|
// maps. Bytes in, same bytes out.
|
|
@@ -20,7 +26,7 @@ import { checkOpts } from './Opts.js';
|
|
|
20
26
|
import { crc32cInit, crc32cUpdate, crc32cFinal, crc32cCombine } from './Crc32c.js';
|
|
21
27
|
import { validateSink, isThenable } from './Views.js';
|
|
22
28
|
|
|
23
|
-
export const VERSION = '1.
|
|
29
|
+
export const VERSION = '1.7.0';
|
|
24
30
|
|
|
25
31
|
export class PreserveWriterError extends Error {
|
|
26
32
|
constructor(code, msg) { super(msg); this.code = code; this.name = 'PreserveWriterError'; }
|
|
@@ -35,7 +41,7 @@ const FINALIZE_TO_SINK_OPTS = {
|
|
|
35
41
|
layout: { t: 'enum', values: ['prefix', 'stream'] },
|
|
36
42
|
crc: { t: 'bool' },
|
|
37
43
|
};
|
|
38
|
-
const DEFAULT_TARGET_SHARD_BYTES = 8 * 1024 * 1024; // 8 MiB
|
|
44
|
+
const DEFAULT_TARGET_SHARD_BYTES = 8 * 1024 * 1024; // 8 MiB -- smaller than schema mode
|
|
39
45
|
const INITIAL_OFFSETS_CAP = 4096;
|
|
40
46
|
const U32_MAX = 4294967295;
|
|
41
47
|
// maxRecordBytes is a declared, validated, deliberately-unused key here: the
|
package/src/RangeReader.js
CHANGED
|
@@ -19,9 +19,13 @@
|
|
|
19
19
|
//
|
|
20
20
|
// IO adapter contract:
|
|
21
21
|
// interface IOAdapter {
|
|
22
|
-
// size: number;
|
|
23
|
-
// fetch(byteOffset, byteLength): Promise<Uint8Array>; // MUST return exactly that many bytes
|
|
22
|
+
// size: number; // container byte length
|
|
23
|
+
// fetch(byteOffset, byteLength, signal?): Promise<Uint8Array>; // MUST return exactly that many bytes
|
|
24
24
|
// }
|
|
25
|
+
// The signal parameter is ADDITIVE (BS-28): a 2-arg adapter that ignores it
|
|
26
|
+
// degrades CANCELLATION only, never correctness -- the reader-side fail-closed
|
|
27
|
+
// triangle (pre-check / race / post-settle) guarantees the outcome regardless.
|
|
28
|
+
// See decisions/0013-range-abort.md.
|
|
25
29
|
//
|
|
26
30
|
// Two adapters ship in-box:
|
|
27
31
|
// HTTPRangeAdapter -- uses fetch() with Range headers.
|
|
@@ -33,6 +37,8 @@
|
|
|
33
37
|
// R_ADAPTER_SHORT_READ -- adapter returned fewer bytes than requested, OR a
|
|
34
38
|
// non-Uint8Array / wrong-length adapter return (T7c)
|
|
35
39
|
// R_ROW_OUT_OF_RANGE -- rowIdx negative, fractional, NaN, or >= totalRows (BS-32)
|
|
40
|
+
// R_BAD_CRC -- CRC-32C verify requested and the stored checksum mismatched
|
|
41
|
+
// R_CRC_ABSENT -- CRC verify requested but the container carries no checksum
|
|
36
42
|
// R_OFFSET_TOO_LARGE -- a u64 header/schema/directory offset exceeds 2^53-1
|
|
37
43
|
// R_NOT_PREFETCHED -- syncRange over a shard that prefetchRange has not cached
|
|
38
44
|
// R_WRONG_MODE -- preserve-mode container fed to the schema RangeReader
|
|
@@ -41,12 +47,15 @@
|
|
|
41
47
|
// R_BAD_FOOTER -- footer magic_end or footer_len is malformed
|
|
42
48
|
// R_BAD_METADATA -- metadata_off non-zero but the zone-map segment is unparseable
|
|
43
49
|
// R_INVALID -- a structure is internally inconsistent but in-bounds
|
|
50
|
+
// R_ABORTED -- a reader-level { signal } fired: in-flight range I/O is
|
|
51
|
+
// cancelled, resolved-after-abort bytes discarded, and a
|
|
52
|
+
// foreign AbortError rejection is mapped here (BS-28)
|
|
44
53
|
|
|
45
54
|
import { StringTable } from './StringTable.js';
|
|
46
55
|
import { checkOpts } from './Opts.js';
|
|
47
56
|
import { crc32cInit, crc32cUpdate, crc32cFinal } from './Crc32c.js';
|
|
48
57
|
|
|
49
|
-
export const VERSION = '1.
|
|
58
|
+
export const VERSION = '1.7.0';
|
|
50
59
|
|
|
51
60
|
const CONTAINER_HEADER_BYTES = 48;
|
|
52
61
|
const SHARD_ENTRY_BYTES = 40;
|
|
@@ -60,11 +69,13 @@ const U32_MAX = 4294967295;
|
|
|
60
69
|
const RANGE_READER_OPTS = {
|
|
61
70
|
maxCachedShards: { t: 'int', min: 0, max: U32_MAX },
|
|
62
71
|
verifyCrc: { t: 'bool' },
|
|
72
|
+
signal: { t: 'signal' },
|
|
63
73
|
};
|
|
64
|
-
const HTTP_ADAPTER_OPTS = { fetch: { t: 'fn' } };
|
|
74
|
+
const HTTP_ADAPTER_OPTS = { fetch: { t: 'fn' }, signal: { t: 'signal' } };
|
|
65
75
|
|
|
66
76
|
const CONTAINER_FOOTER_BYTES = 16;
|
|
67
77
|
const CRC_ABSENT = 0xFFFFFFFF;
|
|
78
|
+
const NOOP = function () {};
|
|
68
79
|
|
|
69
80
|
function laneBytesOf(k) { return k === LANE_F64 ? 8 : (k === LANE_U32 ? 4 : 0); }
|
|
70
81
|
|
|
@@ -73,6 +84,12 @@ export class RangeReaderError extends Error {
|
|
|
73
84
|
}
|
|
74
85
|
function raiseRange(code, msg) { throw new RangeReaderError(code, msg); }
|
|
75
86
|
|
|
87
|
+
// A reader-level signal fired: cancel in-flight range I/O. The one message for
|
|
88
|
+
// every R_ABORTED site (pre-check, post-settle, foreign-AbortError mapping).
|
|
89
|
+
function abortedError() {
|
|
90
|
+
return new RangeReaderError('R_ABORTED', 'range I/O aborted: the reader-level signal fired');
|
|
91
|
+
}
|
|
92
|
+
|
|
76
93
|
// Read a u64 field from `dv` (the header, schema, or directory view -- three
|
|
77
94
|
// different DataViews here) and narrow it to a JS number, failing closed past
|
|
78
95
|
// Number.MAX_SAFE_INTEGER. Past 2^53-1 a Number() cast loses precision and every
|
|
@@ -123,25 +140,42 @@ export class HTTPRangeAdapter {
|
|
|
123
140
|
if (typeof fetchImpl !== 'function') {
|
|
124
141
|
throw new RangeReaderError('R_TRUNCATED', 'no fetch() available in this environment');
|
|
125
142
|
}
|
|
143
|
+
// Reader-level cancellation (BS-28). The adapter itself stays signal-stateless
|
|
144
|
+
// (one adapter may serve N readers); this signal governs only its own open().
|
|
145
|
+
const signal = opts.signal !== undefined ? opts.signal : null;
|
|
146
|
+
if (signal !== null && signal.aborted) throw abortedError();
|
|
126
147
|
let size = -1;
|
|
127
|
-
// Try HEAD first.
|
|
148
|
+
// Try HEAD first (forward the signal so a real HEAD is cancellable).
|
|
128
149
|
try {
|
|
129
|
-
const head = await fetchImpl(url, { method: 'HEAD' });
|
|
150
|
+
const head = await fetchImpl(url, { method: 'HEAD', signal });
|
|
130
151
|
if (head.ok) {
|
|
131
152
|
const cl = head.headers.get('content-length');
|
|
132
153
|
if (cl) size = parseInt(cl, 10);
|
|
133
154
|
}
|
|
134
155
|
} catch { /* fall through */ }
|
|
156
|
+
// E1 (O-5): the HEAD's own rejection is swallowed by design. Re-check the
|
|
157
|
+
// signal after the catch and BEFORE the probe GET -- an abort during HEAD
|
|
158
|
+
// must not fire a second network request.
|
|
159
|
+
if (signal !== null && signal.aborted) throw abortedError();
|
|
135
160
|
// Fallback: single-byte range GET.
|
|
136
161
|
if (size < 0) {
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
162
|
+
try {
|
|
163
|
+
const probe = await fetchImpl(url, { headers: { Range: 'bytes=0-0' }, signal });
|
|
164
|
+
const cr = probe.headers.get('content-range');
|
|
165
|
+
if (cr) {
|
|
166
|
+
const m = cr.match(/\/(\d+)$/);
|
|
167
|
+
if (m) size = parseInt(m[1], 10);
|
|
168
|
+
}
|
|
169
|
+
// consume body so the connection doesn't hang
|
|
170
|
+
await probe.arrayBuffer();
|
|
171
|
+
} catch (err) {
|
|
172
|
+
// O-10: an abort landing mid-probe (fetch or drain) maps to R_ABORTED,
|
|
173
|
+
// symmetric with the E1-mapped HEAD path -- a foreign DOMException
|
|
174
|
+
// AbortError must not leak out of open(). A non-abort failure rethrows
|
|
175
|
+
// verbatim. Cold path; allocation here is fine.
|
|
176
|
+
if (signal !== null && signal.aborted) throw abortedError();
|
|
177
|
+
throw err;
|
|
142
178
|
}
|
|
143
|
-
// consume body so the connection doesn't hang
|
|
144
|
-
await probe.arrayBuffer();
|
|
145
179
|
}
|
|
146
180
|
if (size < 0) throw new RangeReaderError('R_TRUNCATED', 'could not determine container size for ' + url);
|
|
147
181
|
return new HTTPRangeAdapter(url, size, fetchImpl);
|
|
@@ -154,10 +188,11 @@ export class HTTPRangeAdapter {
|
|
|
154
188
|
this.stats = { requests: 0, bytesFetched: 0 };
|
|
155
189
|
}
|
|
156
190
|
|
|
157
|
-
async fetch(byteOffset, byteLength) {
|
|
191
|
+
async fetch(byteOffset, byteLength, signal) {
|
|
158
192
|
const rangeEnd = byteOffset + byteLength - 1;
|
|
159
193
|
const res = await this._fetch(this.url, {
|
|
160
194
|
headers: { Range: 'bytes=' + byteOffset + '-' + rangeEnd },
|
|
195
|
+
signal,
|
|
161
196
|
});
|
|
162
197
|
if (!res.ok && res.status !== 206) {
|
|
163
198
|
throw new RangeReaderError('R_TRUNCATED',
|
|
@@ -182,16 +217,23 @@ export class MockRangeAdapter {
|
|
|
182
217
|
this.bytes = bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes);
|
|
183
218
|
this.size = this.bytes.length;
|
|
184
219
|
this.log = []; // [[offset, length], ...] for test assertions
|
|
220
|
+
// OBSERVATION-ONLY (O-2, BS-28): `signals` is index-aligned with `log` (null
|
|
221
|
+
// when no signal was forwarded); `lastSignal` is the most recent one. Neither
|
|
222
|
+
// changes behaviour -- `log` stays byte-identical.
|
|
223
|
+
this.signals = [];
|
|
224
|
+
this.lastSignal = null;
|
|
185
225
|
this.stats = { requests: 0, bytesFetched: 0 };
|
|
186
226
|
}
|
|
187
227
|
|
|
188
|
-
async fetch(byteOffset, byteLength) {
|
|
228
|
+
async fetch(byteOffset, byteLength, signal) {
|
|
189
229
|
if (byteOffset + byteLength > this.size) {
|
|
190
230
|
throw new RangeReaderError('R_ADAPTER_SHORT_READ',
|
|
191
231
|
'mock adapter: request bytes=' + byteOffset + '-' + (byteOffset + byteLength - 1) +
|
|
192
232
|
' exceeds size ' + this.size);
|
|
193
233
|
}
|
|
194
234
|
this.log.push([byteOffset, byteLength]);
|
|
235
|
+
this.signals.push(signal !== undefined ? signal : null);
|
|
236
|
+
this.lastSignal = signal !== undefined ? signal : null;
|
|
195
237
|
this.stats.requests++;
|
|
196
238
|
this.stats.bytesFetched += byteLength;
|
|
197
239
|
// Return a COPY so mutations by the caller don't affect the "source of truth".
|
|
@@ -219,6 +261,10 @@ export class RangeReader {
|
|
|
219
261
|
opts = opts || {};
|
|
220
262
|
this.adapter = adapter;
|
|
221
263
|
this.maxCachedShards = opts.maxCachedShards !== undefined ? opts.maxCachedShards : 8;
|
|
264
|
+
// Reader-level cancellation (BS-28). null = no signal = the pre-MQ1 hot path
|
|
265
|
+
// (one property read in _fetchExact, zero race/listener allocation). A signal
|
|
266
|
+
// that has already fired makes this reader permanently dead for NEW I/O.
|
|
267
|
+
this._signal = opts.signal !== undefined ? opts.signal : null;
|
|
222
268
|
this._footerCrc = CRC_ABSENT;
|
|
223
269
|
// shard cache: shardIdx -> { payloadBytes, payloadDv, stringTable, lastAccess }
|
|
224
270
|
this._shardCache = new Map();
|
|
@@ -498,7 +544,56 @@ export class RangeReader {
|
|
|
498
544
|
// non-Uint8Array return would otherwise seed a DataView over the wrong extent
|
|
499
545
|
// and decode silent garbage. Fail closed with R_ADAPTER_SHORT_READ instead.
|
|
500
546
|
async _fetchExact(byteOffset, byteLength) {
|
|
501
|
-
|
|
547
|
+
// HOT PATH (no signal): one property read is the entire legacy-path cost.
|
|
548
|
+
// The abort triangle + race live in the cold _fetchAborting (O-8). Collapsing
|
|
549
|
+
// the two is forbidden -- a reader without a signal reaches no listener, no
|
|
550
|
+
// race, no extra allocation.
|
|
551
|
+
if (this._signal === null) {
|
|
552
|
+
const buf = await this.adapter.fetch(byteOffset, byteLength);
|
|
553
|
+
if (!(buf instanceof Uint8Array) || buf.byteLength !== byteLength) {
|
|
554
|
+
throw new RangeReaderError('R_ADAPTER_SHORT_READ',
|
|
555
|
+
'adapter returned ' +
|
|
556
|
+
(buf instanceof Uint8Array ? buf.byteLength + ' bytes' : Object.prototype.toString.call(buf)) +
|
|
557
|
+
' for a ' + byteLength + '-byte request at offset ' + byteOffset);
|
|
558
|
+
}
|
|
559
|
+
return buf;
|
|
560
|
+
}
|
|
561
|
+
return this._fetchAborting(byteOffset, byteLength, this._signal);
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
// COLD PATH (a signal is bound): the fail-closed triangle plus a race. See
|
|
565
|
+
// decisions/0013-range-abort.md and BS-28.
|
|
566
|
+
// (a) pre-check -- signal already fired -> R_ABORTED, ZERO adapter calls;
|
|
567
|
+
// (b) race -- adapter promise vs an abort promise, so a never-resolving
|
|
568
|
+
// fetch still rejects within the tick of the abort; the
|
|
569
|
+
// listener is removed on settle in BOTH directions;
|
|
570
|
+
// (c) post-settle -- a signal-ignoring adapter that resolved anyway still
|
|
571
|
+
// yields R_ABORTED (bytes discarded, nothing cached); an
|
|
572
|
+
// adapter rejection while aborted maps to R_ABORTED (a
|
|
573
|
+
// foreign DOMException AbortError never leaks); a non-abort
|
|
574
|
+
// rejection rethrows verbatim; the short-read shape check
|
|
575
|
+
// runs only on the clean-resolve path.
|
|
576
|
+
async _fetchAborting(byteOffset, byteLength, sig) {
|
|
577
|
+
if (sig.aborted) throw abortedError(); // (a)
|
|
578
|
+
const adapterP = this.adapter.fetch(byteOffset, byteLength, sig);
|
|
579
|
+
let onAbort = null;
|
|
580
|
+
const abortP = new Promise((_resolve, reject) => {
|
|
581
|
+
onAbort = function () { reject(abortedError()); };
|
|
582
|
+
sig.addEventListener('abort', onAbort, { once: true });
|
|
583
|
+
});
|
|
584
|
+
let buf;
|
|
585
|
+
try {
|
|
586
|
+
buf = await Promise.race([adapterP, abortP]); // (b)
|
|
587
|
+
} catch (err) {
|
|
588
|
+
if (sig.aborted) {
|
|
589
|
+
adapterP.catch(NOOP); // neutralize the loser: no late unhandledRejection
|
|
590
|
+
throw abortedError(); // foreign AbortError mapped; never leaks
|
|
591
|
+
}
|
|
592
|
+
throw err; // non-abort rejection rethrows verbatim
|
|
593
|
+
} finally {
|
|
594
|
+
sig.removeEventListener('abort', onAbort);
|
|
595
|
+
}
|
|
596
|
+
if (sig.aborted) throw abortedError(); // (c)
|
|
502
597
|
if (!(buf instanceof Uint8Array) || buf.byteLength !== byteLength) {
|
|
503
598
|
throw new RangeReaderError('R_ADAPTER_SHORT_READ',
|
|
504
599
|
'adapter returned ' +
|
package/src/Reader.js
CHANGED
|
@@ -25,6 +25,8 @@
|
|
|
25
25
|
// (Number() would lose precision; fail closed, BS-05)
|
|
26
26
|
// R_ROW_OUT_OF_RANGE - get(rowIdx) with rowIdx negative, fractional, NaN,
|
|
27
27
|
// or >= totalRows (BS-32)
|
|
28
|
+
// R_BAD_CRC - CRC-32C verify requested and the stored checksum mismatched
|
|
29
|
+
// R_CRC_ABSENT - CRC verify requested but the container carries no checksum
|
|
28
30
|
//
|
|
29
31
|
// Row-index policy (BS-32): get() range-checks rowIdx; the SHARD-index escape
|
|
30
32
|
// hatches (shardPayload, shardF64, shardStringTable) do NOT -- their contract is
|
|
@@ -36,7 +38,7 @@ import { toContainerBuffer } from './Views.js';
|
|
|
36
38
|
import { checkOpts } from './Opts.js';
|
|
37
39
|
import { crc32cInit, crc32cUpdate, crc32cFinal } from './Crc32c.js';
|
|
38
40
|
|
|
39
|
-
export const VERSION = '1.
|
|
41
|
+
export const VERSION = '1.7.0';
|
|
40
42
|
|
|
41
43
|
const CONTAINER_HEADER_BYTES = 48;
|
|
42
44
|
const SHARD_ENTRY_BYTES = 40;
|
package/src/Split.js
CHANGED
|
@@ -2,6 +2,10 @@
|
|
|
2
2
|
// Splitters + per-part compilation + container merge for parallel/checkpointed ingest.
|
|
3
3
|
// Copyright (c) 2026 Zahary Shinikchiev. MIT.
|
|
4
4
|
//
|
|
5
|
+
// Error codes:
|
|
6
|
+
// S_MERGE_EMPTY - mergeContainers called with no inputs
|
|
7
|
+
// S_SCHEMA_MISMATCH - parts carry incompatible schemas
|
|
8
|
+
//
|
|
5
9
|
// M6 covers three primitives that together enable worker-parallel or checkpoint-
|
|
6
10
|
// resumable ingest against LBK1:
|
|
7
11
|
//
|
|
@@ -38,7 +42,7 @@ import { StringTable } from './StringTable.js';
|
|
|
38
42
|
import { checkOpts } from './Opts.js';
|
|
39
43
|
import { crc32cInit, crc32cUpdate, crc32cFinal } from './Crc32c.js';
|
|
40
44
|
|
|
41
|
-
export const VERSION = '1.
|
|
45
|
+
export const VERSION = '1.7.0';
|
|
42
46
|
|
|
43
47
|
const LF = 0x0A;
|
|
44
48
|
const CONTAINER_HEADER_BYTES = 48;
|
|
@@ -143,7 +147,7 @@ export function compilePart(bytes, opts) {
|
|
|
143
147
|
// ---------- compileInParts ----------
|
|
144
148
|
|
|
145
149
|
// Sequential convenience: split then compile each part serially.
|
|
146
|
-
// Returns Uint8Array[]
|
|
150
|
+
// Returns Uint8Array[] -- one container per part. Equivalent output to running
|
|
147
151
|
// each part through a worker (or any parallel executor); use this when workers
|
|
148
152
|
// aren't available or for testing.
|
|
149
153
|
export function compileInParts(bytes, opts) {
|
|
@@ -237,7 +241,7 @@ export function mergeContainers(containers) {
|
|
|
237
241
|
const outShardDirBytes = totalShards * SHARD_ENTRY_BYTES;
|
|
238
242
|
|
|
239
243
|
// Zone maps: emit iff container 0 has them AND all others do too.
|
|
240
|
-
// (MultiReader uses the same "all-or-none" policy
|
|
244
|
+
// (MultiReader uses the same "all-or-none" policy -- see MultiReader.d.ts.)
|
|
241
245
|
const outHasZoneMaps = readers.every((r) => r.hasZoneMaps);
|
|
242
246
|
const zoneMapsRaw = readers.map((r) => r.zoneMapsRaw());
|
|
243
247
|
const T = outHasZoneMaps ? zoneMapsRaw[0].trackedFields.length : 0;
|
|
@@ -321,7 +325,7 @@ export function mergeContainers(containers) {
|
|
|
321
325
|
outDv.setUint32(outMetadataOff + 4, totalShards, true);
|
|
322
326
|
outDv.setUint32(outMetadataOff + 8, T, true);
|
|
323
327
|
outDv.setUint32(outMetadataOff + 12, 0, true);
|
|
324
|
-
// Field index table (copied from container 0
|
|
328
|
+
// Field index table (copied from container 0 -- all containers share the schema)
|
|
325
329
|
const fieldTableOut = outMetadataOff + zoneMapsHeaderLen;
|
|
326
330
|
const tracked0 = zoneMapsRaw[0].trackedFields;
|
|
327
331
|
for (let t = 0; t < T; t++) outDv.setUint16(fieldTableOut + t * 2, tracked0[t], true);
|
package/src/StringTable.js
CHANGED
|
@@ -2,6 +2,9 @@
|
|
|
2
2
|
// Byte-level UTF-8 string interning with zero-alloc hot path.
|
|
3
3
|
// Copyright (c) 2026 Zahary Shinikchiev. MIT.
|
|
4
4
|
//
|
|
5
|
+
// Error codes:
|
|
6
|
+
// ST_BLOB_OVERFLOW - the blob or entry count would cross its u32 ceiling
|
|
7
|
+
//
|
|
5
8
|
// intern(bytes, from, to) -> u32 index, deduplicating identical byte ranges.
|
|
6
9
|
// The hash path never allocates: bytes stay in the caller's buffer during
|
|
7
10
|
// lookup, and internal storage grows in doubling steps.
|
|
@@ -14,7 +17,7 @@
|
|
|
14
17
|
// u8 blob[blob_length_bytes]
|
|
15
18
|
//
|
|
16
19
|
// The trailing sentinel makes `len(i) = offsets[i+1] - offsets[i]` uniform for
|
|
17
|
-
// all i including the last
|
|
20
|
+
// all i including the last -- no branch in the reader hot path.
|
|
18
21
|
//
|
|
19
22
|
// Entry 0 is always the empty string: the table reserves it in the constructor
|
|
20
23
|
// and at every reset(), so an absent U32 row cell (which is 0) decodes as ""
|
|
@@ -38,7 +41,7 @@
|
|
|
38
41
|
// TEST-ONLY: not re-exported from index.js, absent from the .d.ts and docs,
|
|
39
42
|
// and carrying no semver guarantee.
|
|
40
43
|
|
|
41
|
-
export const VERSION = '1.
|
|
44
|
+
export const VERSION = '1.7.0';
|
|
42
45
|
|
|
43
46
|
const EMPTY_SLOT = 0xFFFFFFFF; // MUST be unsigned; typed-array reads are unsigned
|
|
44
47
|
const INITIAL_BLOB_BYTES = 64 * 1024;
|
|
@@ -256,7 +259,7 @@ export class StringTable {
|
|
|
256
259
|
}
|
|
257
260
|
|
|
258
261
|
// Static: parse a serialized table from a byte view. Returns a read-only
|
|
259
|
-
// accessor object (not a live StringTable
|
|
262
|
+
// accessor object (not a live StringTable -- cheaper for the Reader path).
|
|
260
263
|
static parse(bytes, byteOffset) {
|
|
261
264
|
const dv = new DataView(bytes.buffer, bytes.byteOffset + byteOffset, bytes.byteLength - byteOffset);
|
|
262
265
|
const entryCount = dv.getUint32(0, true);
|
|
@@ -277,7 +280,7 @@ export class StringTableView {
|
|
|
277
280
|
this._count = count;
|
|
278
281
|
this._blob = blob;
|
|
279
282
|
this._offsets = offsets;
|
|
280
|
-
this._decoder = null; // lazy
|
|
283
|
+
this._decoder = null; // lazy -- created on first get()
|
|
281
284
|
}
|
|
282
285
|
|
|
283
286
|
get count() { return this._count; }
|
package/src/Tokenizer.js
CHANGED
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
|
|
31
31
|
import { checkOpts } from './Opts.js';
|
|
32
32
|
|
|
33
|
-
export const VERSION = '1.
|
|
33
|
+
export const VERSION = '1.7.0';
|
|
34
34
|
|
|
35
35
|
const U32_MAX = 4294967295;
|
|
36
36
|
const TOKENIZER_OPTS = {
|
|
@@ -88,6 +88,7 @@ const NS_EXP = 7; // in exponent digits
|
|
|
88
88
|
// correctly-rounded F64 via a single multiplication (positive exp) or single
|
|
89
89
|
// division (negative exp) using this table. Outside that domain, correctly
|
|
90
90
|
// rounded parsing requires David Gay's strtod, deferred to M4+.
|
|
91
|
+
// See decisions/0012-clinger-fast-path.md.
|
|
91
92
|
const POW10 = new Float64Array(23);
|
|
92
93
|
POW10[0] = 1;
|
|
93
94
|
for (let _pi = 1; _pi < 23; _pi++) POW10[_pi] = POW10[_pi - 1] * 10;
|
|
@@ -691,7 +692,7 @@ export class Tokenizer {
|
|
|
691
692
|
const need = to - from;
|
|
692
693
|
if (this._strLen + need > this._maxStringBytes) this._err('E_STRING_TOO_LONG', 'string exceeds maxStringBytes cap (' + (this._strLen + need) + ' > ' + this._maxStringBytes + ')');
|
|
693
694
|
if (this._strLen + need > this._strBuf.length) this._growStrBuf(need);
|
|
694
|
-
// Manual copy loop
|
|
695
|
+
// Manual copy loop -- Uint8Array.set(source) via subarray allocates a small
|
|
695
696
|
// view header per call, which turns into MB-scale GC pressure across a
|
|
696
697
|
// string-heavy fixture. The loop is boring but keeps the hot path allocation-free.
|
|
697
698
|
const dst = this._strBuf;
|