@zakkster/lite-bake-stream 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +50 -0
- package/README.md +1 -1
- package/SPEC.md +2 -2
- package/llms.txt +2 -2
- package/package.json +7 -5
- package/src/FileIngest.js +25 -7
- package/src/MultiReader.js +1 -1
- package/src/Opts.js +189 -0
- package/src/PreserveReader.js +1 -1
- package/src/PreserveTokenizer.js +28 -3
- package/src/PreserveWriter.js +15 -2
- package/src/RangeReader.js +11 -3
- package/src/Reader.js +1 -1
- package/src/Split.js +28 -7
- package/src/StringTable.js +18 -1
- package/src/Tokenizer.js +61 -30
- package/src/Writer.js +70 -11
- package/src/index.js +28 -10
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,56 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
4
4
|
|
|
5
|
+
## [1.2.0] -- 2026-08-31
|
|
6
|
+
|
|
7
|
+
M1 -- the write path refuses what it cannot store. Eight findings closed by a set of doors that all run at construction, freeze, or terminal transition; the byte and record hot paths are unchanged (proven by the torture t6 gate, numbers within noise of M0). Suite: 359 tests / 358 pass / 0 fail / 1 todo. Torture: 37/37 fast, 39/39 full, arrayBuffers growth 0. Mutation check: all 63 door tests fail on source revert; all 13 behavior-unchanged pins hold.
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- `src/Opts.js` (internal, not exported): a shared construction-time options validator wired into `Tokenizer`, `Writer`, `PreserveTokenizer`, `PreserveWriter`, `serialize`, `ingestStream`/`ingestFile`, `RangeReader`, `HTTPRangeAdapter.open`, and the `./split` subpath (`splitNDJSON`, `compilePart`, `compileInParts`). Unknown keys throw with a did-you-mean hint (Levenshtein distance <= 2); out-of-domain values throw naming the key, the received value, and the domain.
|
|
12
|
+
- New stable error codes: `E_UNKNOWN_OPTION`, `E_OPTION_VALUE`, `E_OPTION_CONFLICT` (options); `W_DUPLICATE_FIELD`, `W_SCHEMA_TOO_WIDE`, `W_FIELD_NAME_INVALID` (schema freeze); `W_FINALIZED` on the schema `Writer` (already present on `PreserveWriter`); `E_ENDED`, `E_POISONED` (tokenizer terminal state); `E_STRING_TOO_LONG` (maxStringBytes cap).
|
|
13
|
+
|
|
14
|
+
### Changed
|
|
15
|
+
|
|
16
|
+
- Unknown or typo'd option keys now throw `E_UNKNOWN_OPTION` instead of being silently ignored.
|
|
17
|
+
- Explicit `0` now means 0 for `sampleBytes` (freeze from the first record), `maxCachedShards` (retain no shard), and `maxRecordBytes` (unlimited) instead of silently becoming the default. `null` is out-of-domain and throws (null is not zero).
|
|
18
|
+
- `Writer.finalize()` called twice now throws `W_FINALIZED` instead of returning the stale container.
|
|
19
|
+
- Over-cap strings now throw the dedicated `E_STRING_TOO_LONG` instead of the wrong `E_UNEXPECTED_BYTE`, and the cap is enforced exactly (no longer only when the 4096-byte buffer grows).
|
|
20
|
+
- `Tokenizer` no longer writes NOOP methods into the caller's sink object; a frozen sink now constructs and drives a full feed/end.
|
|
21
|
+
- `splitNDJSON`/`compileInParts`: an explicit `targetParts:0` or `maxPartBytes:0` now throws instead of silently becoming the default 4 / no-cap, and a negative `targetParts` now throws instead of clamping to 1.
|
|
22
|
+
- `compilePart`/`compileInParts`: `preserve` was never supported and was silently ignored; it is now an unknown-key error (use `serialize(..., {preserve:true})` for preserve mode).
|
|
23
|
+
|
|
24
|
+
### Fixed
|
|
25
|
+
|
|
26
|
+
- **BS-02:** an over-wide schema (a field offset past the u16 `offset_in_row` ceiling of 65535) now throws `W_SCHEMA_TOO_WIDE` at freeze instead of wrapping the offset and cross-writing fields. 8192 all-F64 fields freeze; 8193 throw.
|
|
27
|
+
- **BS-03:** duplicate schema field names now throw `W_DUPLICATE_FIELD` at freeze instead of writing a container the Writer reads from one lane and the Reader from another. See `decisions/0003-duplicate-field-names.md`.
|
|
28
|
+
- **BS-04:** a depth-0 non-object record (number, string, true, false, null) now throws `W_TOP_LEVEL_NOT_OBJECT` in schema mode instead of silently vanishing. Preserve mode is unchanged. See `decisions/0002-record-shape-policy.md`.
|
|
29
|
+
- **BS-11:** every construction-time options object is validated at every entry point that takes one; unknown keys and out-of-domain values fail closed.
|
|
30
|
+
- **BS-12:** `maxStringBytes` is enforced exactly against the running string length, so a cap below the initial 4096-byte buffer is honored, and over-cap strings throw `E_STRING_TOO_LONG`.
|
|
31
|
+
- **BS-13:** the schema `Writer` refuses any sink event or a second `finalize()` after `finalize()` with `W_FINALIZED`, matching `PreserveWriter`.
|
|
32
|
+
- **BS-14:** `Tokenizer` and `PreserveTokenizer` refuse `feed()`/`end()` after `end()` (`E_ENDED`) and after any thrown error (`E_POISONED`); a terminal instance never resurrects.
|
|
33
|
+
- **BS-19:** `Tokenizer` reads each sink method once at construction into a per-instance dispatch field (bound to the sink), so the caller's sink object is never mutated and a frozen sink works.
|
|
34
|
+
|
|
35
|
+
## [1.1.0] -- 2026-08-31
|
|
36
|
+
|
|
37
|
+
### Added
|
|
38
|
+
|
|
39
|
+
- Torture gate at the pipeline's canonical path: `node --expose-gc test/torture.mjs` prints `ok` on success, exit 0/1. Tiers: t5 (the 37-scenario stress suite, relocated from bench/ with behavior unchanged), t6 (zero-alloc gates with `maxArrayBuffersGrowth: 0` via measureOps `stabilize: 'deep'` over the tokenizer steady state, the StringTable intern hit path, and the Reader cell loop, each with structural buffer asserts), t7 (lite-leak retention soak: 4096 full-tier cycles, tracker size 0, arrayBuffers delta 0.00 MB), t9 (four in-process controls plus the `BAKE_TORTURE_BREAK=1` whole-suite control, proven to exit non-zero). Tiers t0-t4/t8 are registered reserved stubs. Preflight exits 2 with a remedy line when devDependencies are not installed.
|
|
40
|
+
- `decisions/` directory (repo-only, not shipped): 0001-reserved-empty-string.md records the BS-01 fix and its two rejected alternatives.
|
|
41
|
+
- Round-trip regression suite hardening: 284 tests (282 pass / 2 todo). Five permanent guards pin the BS-01 fix; the two remaining todos are registered known issues (BS-05 invalid UTF-8 acceptance, BS-03 duplicate schema field names). Absent-field fuzz generation, -0 sign, duplicate-key last-wins, null/bool lane-policy codes, and preserve-mode byte-exactness pins added.
|
|
42
|
+
- `npm run verify` (test + torture).
|
|
43
|
+
|
|
44
|
+
### Changed
|
|
45
|
+
|
|
46
|
+
- `npm test` now globs `test/*.test.js`; a new test file runs without editing package.json.
|
|
47
|
+
- `bench/torture.js` is a deprecation shim forwarding to `test/torture.mjs` (same exit codes, same `--tier` flag).
|
|
48
|
+
- devDependencies: added `@zakkster/lite-leak` `^1.10.0`; `@zakkster/lite-gc-profiler` `^1.0.0` -> `^1.16.0` (the `stabilize: 'deep'` option used by the t6 gate landed in 1.16.0).
|
|
49
|
+
- SPEC 3.3: string-table entry 0 is reserved as the empty string (writers MUST emit it); SPEC 7 absent-U32 row now reads `""`. See Fixed.
|
|
50
|
+
|
|
51
|
+
### Fixed
|
|
52
|
+
|
|
53
|
+
- **BS-01: absent string fields now decode as `""`.** A record that omits a U32-lane field leaves index 0 in that row slot. Index 0 of a shard's string table used to be whichever string interned first in that shard, so `{"s":"zebra"}` followed by `{}` read row 1 back as `"zebra"` -- silently, and differently depending on record order and where shard boundaries fell. `StringTable` now reserves index 0 as the empty string in its constructor and at every `reset()`, so absent U32 cells decode as `""` on every write path: explicit schema, sample-and-infer, and every shard of a multi-shard drain. Containers written before this change keep their old bytes and their old, data-dependent behavior on those bytes; the fix is not retroactive. Readers are unchanged. SPEC 3.3 and the SPEC 7 coercion table are amended to match.
|
|
54
|
+
|
|
5
55
|
## [1.0.0] — 2026-07-12
|
|
6
56
|
|
|
7
57
|
First stable release. The LBK1 container format is frozen at `format_version: 1`.
|
package/README.md
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
|
|
14
14
|
> Streaming byte-level JSON compiler for [`@zakkster/lite-bake`](https://github.com/PeshoVurtoleta/lite-bake). Zero-GC, tree-shakeable, gigabyte-scale.
|
|
15
15
|
|
|
16
|
-
**Status:** v1.
|
|
16
|
+
**Status:** v1.2.0. LBK1 format frozen at `format_version: 1`. Qualified on an 8 GB soak with zero GC and byte-exact preservation across 590 million cells.
|
|
17
17
|
|
|
18
18
|
## Two modes, one API
|
|
19
19
|
|
package/SPEC.md
CHANGED
|
@@ -95,7 +95,7 @@ u32 offsets[entry_count + 1] // byte offsets into blob, INCLUDING a trailing
|
|
|
95
95
|
u8 blob[blob_length_bytes]
|
|
96
96
|
```
|
|
97
97
|
|
|
98
|
-
Padded to 8. Strings are UTF-8, not null-terminated. Length of string `i` is `offsets[i+1] - offsets[i]`; the trailing sentinel makes this valid for `i = entry_count - 1` without a branch. Entry index 0 is
|
|
98
|
+
Padded to 8. Strings are UTF-8, not null-terminated. Length of string `i` is `offsets[i+1] - offsets[i]`; the trailing sentinel makes this valid for `i = entry_count - 1` without a branch. Entry index 0 is reserved: it is always the empty string. Writers MUST emit "" as entry 0 of every shard's local table, including tables that hold no other entries. Readers MAY rely on this for containers written at or after this rule; containers written before it may hold any string at index 0.
|
|
99
99
|
|
|
100
100
|
Per-shard tables are placed immediately after their shard payload in the container, so a single HTTP Range fetch covering `[payload_off, payload_off + payload_len + local_string_len)` retrieves both.
|
|
101
101
|
|
|
@@ -337,7 +337,7 @@ Given source NDJSON `S` and a schema `Σ` (either explicit or inferred), the LBK
|
|
|
337
337
|
| JSON string (any valid UTF-8) | U32 | **byte-exact UTF-8 sequence** |
|
|
338
338
|
| JSON `true` / `false` / `null` | F64 | `1` / `0` / `0` (documented coercion) |
|
|
339
339
|
| absent field (missing in source) | F64 | `0` (default) |
|
|
340
|
-
| absent field | U32 |
|
|
340
|
+
| absent field | U32 | `""` (index 0 is the reserved empty-string entry) |
|
|
341
341
|
| field in source but NOT in `Σ` | — | silently dropped (matches lite-bake core) |
|
|
342
342
|
| field in `Σ` with wrong value type (post-freeze) | — | `W_LANE_MISMATCH` error, container not produced |
|
|
343
343
|
|
package/llms.txt
CHANGED
|
@@ -8,13 +8,13 @@ Ingest gigabyte-scale JSON (top-level array or NDJSON) into the `lite-bake` LBK1
|
|
|
8
8
|
|
|
9
9
|
## Status
|
|
10
10
|
|
|
11
|
-
v1.
|
|
11
|
+
v1.2.0 — stable. LBK1 format frozen at `format_version: 1`. Qualified on an 8 GB soak (M1 MacBook Pro): 98.37M rows, 4.89 GB container, zero major GC, zero minor GC, 499 KB total heap allocation, 590.21M cells verified byte-exact, zero mismatches. Tokenizer benches at 222-237 MB/s (~55% of JSON.parse, with no object graph allocated).
|
|
12
12
|
|
|
13
13
|
Public API follows semver from 1.0.0. Future additions (I64 lane, columnar payload mode, container-level string table) land via the format's forward-compat seams -- `min_reader_version` on ShardEntry, reserved FieldDescriptor flags, the `metadata_off` block wrapper -- without a format_version bump.
|
|
14
14
|
|
|
15
15
|
See SPEC.md for the LBK1 container format, section 3.6 for zone maps, section 4.3 for the reserved field flags.
|
|
16
16
|
|
|
17
|
-
## Public API (v1.
|
|
17
|
+
## Public API (v1.2.0)
|
|
18
18
|
|
|
19
19
|
Two ingest modes share one top-level API:
|
|
20
20
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zakkster/lite-bake-stream",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.2.0",
|
|
4
4
|
"description": "Streaming byte-level JSON to lite-bake binary compiler. Zero-GC, tree-shakeable, gigabyte-scale.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"sideEffects": false,
|
|
@@ -76,10 +76,11 @@
|
|
|
76
76
|
"LICENSE"
|
|
77
77
|
],
|
|
78
78
|
"scripts": {
|
|
79
|
-
"test": "node --test test
|
|
79
|
+
"test": "node --test test/*.test.js",
|
|
80
80
|
"bench": "node --expose-gc bench/bench-tokenizer.js",
|
|
81
|
-
"torture": "node --expose-gc
|
|
82
|
-
"torture:full": "node --expose-gc
|
|
81
|
+
"torture": "node --expose-gc test/torture.mjs",
|
|
82
|
+
"torture:full": "node --expose-gc test/torture.mjs --tier=full",
|
|
83
|
+
"verify": "npm test && npm run torture",
|
|
83
84
|
"soak": "node --expose-gc bench/soak.js --mb=100 --verify",
|
|
84
85
|
"soak:500": "node --expose-gc bench/soak.js --mb=500 --verify",
|
|
85
86
|
"soak:1gb": "node --expose-gc bench/soak.js --gb=1 --verify",
|
|
@@ -112,6 +113,7 @@
|
|
|
112
113
|
"access": "public"
|
|
113
114
|
},
|
|
114
115
|
"devDependencies": {
|
|
115
|
-
"@zakkster/lite-gc-profiler": "^1.
|
|
116
|
+
"@zakkster/lite-gc-profiler": "^1.16.0",
|
|
117
|
+
"@zakkster/lite-leak": "^1.10.0"
|
|
116
118
|
}
|
|
117
119
|
}
|
package/src/FileIngest.js
CHANGED
|
@@ -20,13 +20,24 @@
|
|
|
20
20
|
// most engines), use server-side pre-compilation and RangeReader instead.
|
|
21
21
|
|
|
22
22
|
import { Tokenizer } from './Tokenizer.js';
|
|
23
|
-
import { Writer } from './Writer.js';
|
|
23
|
+
import { Writer, WriterError } from './Writer.js';
|
|
24
24
|
import { Reader } from './Reader.js';
|
|
25
25
|
import { PreserveTokenizer } from './PreserveTokenizer.js';
|
|
26
26
|
import { PreserveWriter } from './PreserveWriter.js';
|
|
27
27
|
import { PreserveReader } from './PreserveReader.js';
|
|
28
|
+
import { checkOpts } from './Opts.js';
|
|
28
29
|
|
|
29
|
-
export const VERSION = '1.
|
|
30
|
+
export const VERSION = '1.2.0';
|
|
31
|
+
|
|
32
|
+
const U32_MAX = 4294967295;
|
|
33
|
+
const INGEST_OPTS = {
|
|
34
|
+
preserve: { t: 'bool' },
|
|
35
|
+
framing: { t: 'enum', values: ['auto', 'array', 'ndjson'] },
|
|
36
|
+
writer: { t: 'obj' },
|
|
37
|
+
onProgress: { t: 'fn' },
|
|
38
|
+
totalBytes: { t: 'int', min: 0, max: Number.MAX_SAFE_INTEGER },
|
|
39
|
+
};
|
|
40
|
+
function raiseIngest(code, msg) { throw new WriterError(code, msg); }
|
|
30
41
|
|
|
31
42
|
// Ingest a ReadableStream<Uint8Array> and return a Reader over the resulting
|
|
32
43
|
// LBK1 container. Options:
|
|
@@ -42,16 +53,23 @@ export const VERSION = '1.0.0';
|
|
|
42
53
|
// totalBytes: optional total for progress percentage. Pass file.size
|
|
43
54
|
// when ingesting a File; omit for indeterminate streams.
|
|
44
55
|
export async function ingestStream(readableStream, opts) {
|
|
56
|
+
// Validated once, here, after ingestFile's totalBytes merge (R6): a single
|
|
57
|
+
// label, no double report.
|
|
58
|
+
checkOpts('ingestStream', opts, INGEST_OPTS, raiseIngest);
|
|
45
59
|
opts = opts || {};
|
|
46
60
|
const preserve = opts.preserve === true;
|
|
47
|
-
const framing = opts.framing
|
|
48
|
-
const writerOpts = opts.writer
|
|
49
|
-
const onProgress =
|
|
50
|
-
const totalBytes =
|
|
61
|
+
const framing = opts.framing !== undefined ? opts.framing : 'ndjson';
|
|
62
|
+
const writerOpts = opts.writer !== undefined ? opts.writer : {};
|
|
63
|
+
const onProgress = opts.onProgress !== undefined ? opts.onProgress : null;
|
|
64
|
+
const totalBytes = opts.totalBytes !== undefined ? opts.totalBytes : -1;
|
|
65
|
+
if (preserve && opts.framing !== undefined && opts.framing !== 'ndjson') {
|
|
66
|
+
throw new WriterError('E_OPTION_CONFLICT',
|
|
67
|
+
"ingestStream: preserve mode is NDJSON-only; remove framing:'" + opts.framing + "' or set preserve:false");
|
|
68
|
+
}
|
|
51
69
|
|
|
52
70
|
const writer = preserve ? new PreserveWriter(writerOpts) : new Writer(writerOpts);
|
|
53
71
|
const tokenizer = preserve
|
|
54
|
-
? new PreserveTokenizer(writer, { framing, maxRecordBytes: writerOpts.maxRecordBytes
|
|
72
|
+
? new PreserveTokenizer(writer, { framing, maxRecordBytes: writerOpts.maxRecordBytes !== undefined ? writerOpts.maxRecordBytes : 0 })
|
|
55
73
|
: new Tokenizer(writer, { framing });
|
|
56
74
|
|
|
57
75
|
const reader = readableStream.getReader();
|
package/src/MultiReader.js
CHANGED
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
// M_UNKNOWN_FIELD - .get() / .fieldIndex() called with unknown name
|
|
25
25
|
// M_ROW_OUT_OF_RANGE - rowIdx >= totalRows
|
|
26
26
|
|
|
27
|
-
export const VERSION = '1.
|
|
27
|
+
export const VERSION = '1.2.0';
|
|
28
28
|
|
|
29
29
|
export class MultiReaderError extends Error {
|
|
30
30
|
constructor(code, msg) { super(msg); this.code = code; this.name = 'MultiReaderError'; }
|
package/src/Opts.js
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
// @zakkster/lite-bake-stream / Opts (internal, not exported from index.js)
|
|
2
|
+
// Copyright (c) 2026 Zahary Shinikchiev. MIT.
|
|
3
|
+
//
|
|
4
|
+
// Shared construction-time options validator. Every public constructor and
|
|
5
|
+
// entry function that takes an opts object runs checkOpts once, at construction
|
|
6
|
+
// or in a synchronous prologue -- never per byte, never per record. An unknown
|
|
7
|
+
// key is an error with a did-you-mean hint (suite law: no silent ignore). An
|
|
8
|
+
// out-of-domain value is an error (suite law: null is not zero; explicit 0 is 0
|
|
9
|
+
// where 0 is in-domain). This file allocates only on the throw path.
|
|
10
|
+
//
|
|
11
|
+
// Each host declares a module-level frozen spec object literal mapping every
|
|
12
|
+
// known key to a domain descriptor:
|
|
13
|
+
// { t: 'enum', values: [...] } - string, must be a listed member
|
|
14
|
+
// { t: 'int', min, max } - integer in [min, max] inclusive
|
|
15
|
+
// { t: 'bool' } - boolean
|
|
16
|
+
// { t: 'fn' } - function
|
|
17
|
+
// { t: 'obj', nullable } - object (contents validated downstream)
|
|
18
|
+
// A value of `undefined` for any key means "use the default" and is skipped,
|
|
19
|
+
// matching each host's `opts.k !== undefined ? opts.k : DEFAULT` defaulting.
|
|
20
|
+
//
|
|
21
|
+
// OPTS DOMAIN TABLE (ships with the package; the M1 plan is the source of truth)
|
|
22
|
+
//
|
|
23
|
+
// Unknown-key rule (all entry points): throw E_UNKNOWN_OPTION. Levenshtein
|
|
24
|
+
// distance from the offending key to every known key; if the minimum is <= 2,
|
|
25
|
+
// message is "unknown option 'framming' -- did you mean 'framing'?"; else
|
|
26
|
+
// "unknown option 'zzz' (known: framing, maxStringBytes)". Ties break by spec
|
|
27
|
+
// declaration order.
|
|
28
|
+
//
|
|
29
|
+
// Tokenizer(sink,opts)
|
|
30
|
+
// framing enum 'auto'|'array'|'ndjson' def 'auto'
|
|
31
|
+
// maxStringBytes int [1, 4294967295] def 1048576 (0 forbidden)
|
|
32
|
+
// Writer(opts)
|
|
33
|
+
// schema object|null with array fields def null
|
|
34
|
+
// targetShardBytes int [1, 4294967295] def 33554432 (0 forbidden)
|
|
35
|
+
// sampleBytes int [0, 4294967295] def targetShardBytes (0 = freeze on record 1)
|
|
36
|
+
// PreserveTokenizer(sink,opts)
|
|
37
|
+
// framing enum 'ndjson' def 'ndjson'
|
|
38
|
+
// maxRecordBytes int [0, 4294967295] def 0 (0 = unlimited)
|
|
39
|
+
// PreserveWriter(opts)
|
|
40
|
+
// targetShardBytes int [1, 4294967295] def 8388608 (0 forbidden)
|
|
41
|
+
// maxRecordBytes int [0, 4294967295] declared, consumed by PreserveTokenizer
|
|
42
|
+
// serialize(input,opts) / ingestStream / ingestFile
|
|
43
|
+
// preserve bool def false
|
|
44
|
+
// framing enum 'auto'|'array'|'ndjson' def 'ndjson' (conflict if preserve && != 'ndjson')
|
|
45
|
+
// writer object def {}
|
|
46
|
+
// onProgress function (ingest only)
|
|
47
|
+
// totalBytes int [0, 9007199254740991] def -1 (ingest only)
|
|
48
|
+
// RangeReader(adapter,opts) / .open
|
|
49
|
+
// maxCachedShards int [0, 4294967295] def 8 (0 = retain no shard)
|
|
50
|
+
// HTTPRangeAdapter.open(url,opts)
|
|
51
|
+
// fetch function def globalThis.fetch
|
|
52
|
+
// splitNDJSON(bytes,opts) / compileInParts
|
|
53
|
+
// targetParts int [1, 4294967295] def 4 (0 forbidden)
|
|
54
|
+
// maxPartBytes int [1, 4294967295] or Infinity def Infinity (0 forbidden; Infinity = no cap)
|
|
55
|
+
// compilePart(bytes,opts) / compileInParts
|
|
56
|
+
// framing enum 'auto'|'array'|'ndjson' def 'ndjson'
|
|
57
|
+
// writer object def {}
|
|
58
|
+
//
|
|
59
|
+
// Non-integer, NaN, Infinity, negative, null, and wrong-typeof all land on
|
|
60
|
+
// E_OPTION_VALUE with a message naming key, received value, and the domain.
|
|
61
|
+
|
|
62
|
+
export function checkOpts(label, opts, spec, raise) {
|
|
63
|
+
if (opts === undefined || opts === null || typeof opts !== 'object') return;
|
|
64
|
+
for (const key in opts) {
|
|
65
|
+
if (!Object.prototype.hasOwnProperty.call(opts, key)) continue;
|
|
66
|
+
const d = spec[key];
|
|
67
|
+
if (d === undefined) {
|
|
68
|
+
const keys = Object.keys(spec);
|
|
69
|
+
const near = nearestKey(key, keys);
|
|
70
|
+
if (near !== null) {
|
|
71
|
+
raise('E_UNKNOWN_OPTION',
|
|
72
|
+
label + ": unknown option '" + key + "' -- did you mean '" + near + "'?");
|
|
73
|
+
} else {
|
|
74
|
+
raise('E_UNKNOWN_OPTION',
|
|
75
|
+
label + ": unknown option '" + key + "' (known: " + keys.join(', ') + ')');
|
|
76
|
+
}
|
|
77
|
+
continue;
|
|
78
|
+
}
|
|
79
|
+
const v = opts[key];
|
|
80
|
+
if (v === undefined) continue; // explicit undefined == use default
|
|
81
|
+
_checkValue(label, key, v, d, raise);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function _checkValue(label, key, v, d, raise) {
|
|
86
|
+
const t = d.t;
|
|
87
|
+
if (t === 'int') {
|
|
88
|
+
if (d.inf && v === Infinity) return; // Infinity is a legal explicit value for this key
|
|
89
|
+
if (typeof v !== 'number' || !Number.isInteger(v) || v < d.min || v > d.max) {
|
|
90
|
+
raise('E_OPTION_VALUE',
|
|
91
|
+
label + ": option '" + key + "' must be an integer in [" +
|
|
92
|
+
d.min + ', ' + d.max + ']' + (d.inf ? ' or Infinity' : '') + '; got ' + _show(v));
|
|
93
|
+
}
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
if (t === 'enum') {
|
|
97
|
+
if (typeof v !== 'string' || d.values.indexOf(v) < 0) {
|
|
98
|
+
raise('E_OPTION_VALUE',
|
|
99
|
+
label + ": option '" + key + "' must be one of " +
|
|
100
|
+
_quoteList(d.values) + '; got ' + _show(v));
|
|
101
|
+
}
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
if (t === 'bool') {
|
|
105
|
+
if (typeof v !== 'boolean') {
|
|
106
|
+
raise('E_OPTION_VALUE',
|
|
107
|
+
label + ": option '" + key + "' must be a boolean; got " + _show(v));
|
|
108
|
+
}
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
111
|
+
if (t === 'fn') {
|
|
112
|
+
if (typeof v !== 'function') {
|
|
113
|
+
raise('E_OPTION_VALUE',
|
|
114
|
+
label + ": option '" + key + "' must be a function; got " + _show(v));
|
|
115
|
+
}
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
if (t === 'obj') {
|
|
119
|
+
if (v === null) {
|
|
120
|
+
if (!d.nullable) {
|
|
121
|
+
raise('E_OPTION_VALUE',
|
|
122
|
+
label + ": option '" + key + "' must be an object; got null");
|
|
123
|
+
}
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
if (typeof v !== 'object' || Array.isArray(v)) {
|
|
127
|
+
raise('E_OPTION_VALUE',
|
|
128
|
+
label + ": option '" + key + "' must be an object; got " + _show(v));
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function _show(v) {
|
|
134
|
+
if (typeof v === 'string') return "'" + v + "'";
|
|
135
|
+
if (v === null) return 'null';
|
|
136
|
+
if (typeof v === 'object') return Array.isArray(v) ? 'array' : 'object';
|
|
137
|
+
return String(v);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function _quoteList(values) {
|
|
141
|
+
let s = '';
|
|
142
|
+
for (let i = 0; i < values.length; i++) {
|
|
143
|
+
if (i > 0) s += ', ';
|
|
144
|
+
s += "'" + values[i] + "'";
|
|
145
|
+
}
|
|
146
|
+
return s;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// Nearest known key by Levenshtein distance, returning it only when the minimum
|
|
150
|
+
// distance is <= 2. Ties break by declaration order (strict < keeps the first).
|
|
151
|
+
export function nearestKey(key, knownKeys) {
|
|
152
|
+
let best = null;
|
|
153
|
+
let bestDist = 3;
|
|
154
|
+
const klen = key.length;
|
|
155
|
+
for (let k = 0; k < knownKeys.length; k++) {
|
|
156
|
+
const cand = knownKeys[k];
|
|
157
|
+
const diff = klen - cand.length;
|
|
158
|
+
if (diff > 2 || diff < -2) continue; // length delta alone exceeds the cap
|
|
159
|
+
const dist = _lev(key, cand);
|
|
160
|
+
if (dist < bestDist) { bestDist = dist; best = cand; }
|
|
161
|
+
}
|
|
162
|
+
return bestDist <= 2 ? best : null;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// Two-row Levenshtein. Allocates two small arrays; only reached on the throw
|
|
166
|
+
// path (nearestKey is called only when an unknown key is being reported).
|
|
167
|
+
function _lev(a, b) {
|
|
168
|
+
const n = a.length, m = b.length;
|
|
169
|
+
if (n === 0) return m;
|
|
170
|
+
if (m === 0) return n;
|
|
171
|
+
let prev = new Array(m + 1);
|
|
172
|
+
let curr = new Array(m + 1);
|
|
173
|
+
for (let j = 0; j <= m; j++) prev[j] = j;
|
|
174
|
+
for (let i = 1; i <= n; i++) {
|
|
175
|
+
curr[0] = i;
|
|
176
|
+
const ca = a.charCodeAt(i - 1);
|
|
177
|
+
for (let j = 1; j <= m; j++) {
|
|
178
|
+
const cost = ca === b.charCodeAt(j - 1) ? 0 : 1;
|
|
179
|
+
let v = prev[j] + 1;
|
|
180
|
+
const del = curr[j - 1] + 1;
|
|
181
|
+
if (del < v) v = del;
|
|
182
|
+
const sub = prev[j - 1] + cost;
|
|
183
|
+
if (sub < v) v = sub;
|
|
184
|
+
curr[j] = v;
|
|
185
|
+
}
|
|
186
|
+
const tmp = prev; prev = curr; curr = tmp;
|
|
187
|
+
}
|
|
188
|
+
return prev[m];
|
|
189
|
+
}
|
package/src/PreserveReader.js
CHANGED
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
// buffer. It is valid for as long as the Reader instance holds the container.
|
|
18
18
|
// Do NOT mutate it, and do NOT retain it past the Reader's lifetime.
|
|
19
19
|
|
|
20
|
-
export const VERSION = '1.
|
|
20
|
+
export const VERSION = '1.2.0';
|
|
21
21
|
|
|
22
22
|
const CONTAINER_HEADER_BYTES = 48;
|
|
23
23
|
const SHARD_ENTRY_BYTES = 40;
|
package/src/PreserveTokenizer.js
CHANGED
|
@@ -25,12 +25,25 @@
|
|
|
25
25
|
// Errors (stable E_* codes):
|
|
26
26
|
// E_TRUNCATED - end of input reached mid-record (depth != 0 or in-string)
|
|
27
27
|
// E_RECORD_TOO_LARGE - single record exceeds maxRecordBytes (default: none)
|
|
28
|
+
// E_ENDED - feed()/end() after end()
|
|
29
|
+
// E_POISONED - feed()/end() after a thrown error
|
|
30
|
+
// E_UNKNOWN_OPTION - unknown constructor option key
|
|
31
|
+
// E_OPTION_VALUE - constructor option value out of domain
|
|
28
32
|
|
|
29
|
-
|
|
33
|
+
import { checkOpts } from './Opts.js';
|
|
34
|
+
|
|
35
|
+
export const VERSION = '1.2.0';
|
|
36
|
+
|
|
37
|
+
const U32_MAX = 4294967295;
|
|
38
|
+
const PRESERVE_TOKENIZER_OPTS = {
|
|
39
|
+
framing: { t: 'enum', values: ['ndjson', 'array', 'auto'] },
|
|
40
|
+
maxRecordBytes: { t: 'int', min: 0, max: U32_MAX },
|
|
41
|
+
};
|
|
30
42
|
|
|
31
43
|
export class PreserveTokenizerError extends Error {
|
|
32
44
|
constructor(code, msg) { super(msg); this.code = code; this.name = 'PreserveTokenizerError'; }
|
|
33
45
|
}
|
|
46
|
+
function raisePreserveTok(code, msg) { throw new PreserveTokenizerError(code, msg); }
|
|
34
47
|
|
|
35
48
|
// Byte values used by the scanner (named so the hot loop reads cleanly).
|
|
36
49
|
const BYTE_LF = 0x0A;
|
|
@@ -51,14 +64,18 @@ export class PreserveTokenizer {
|
|
|
51
64
|
if (!sink || typeof sink.onRecord !== 'function') {
|
|
52
65
|
throw new TypeError('PreserveTokenizer: sink must implement onRecord(bytes, from, to)');
|
|
53
66
|
}
|
|
67
|
+
checkOpts('PreserveTokenizer', opts, PRESERVE_TOKENIZER_OPTS, raisePreserveTok);
|
|
54
68
|
opts = opts || {};
|
|
55
|
-
if (opts.framing && opts.framing !== 'ndjson') {
|
|
69
|
+
if (opts.framing !== undefined && opts.framing !== 'ndjson') {
|
|
56
70
|
// 'array' framing (top-level `[a, b, c]`) is a future addition.
|
|
57
71
|
throw new PreserveTokenizerError('E_UNSUPPORTED_FRAMING',
|
|
58
72
|
'preserve-mode currently supports NDJSON only; got framing=' + opts.framing);
|
|
59
73
|
}
|
|
60
74
|
this._sink = sink;
|
|
61
|
-
this._maxRecordBytes =
|
|
75
|
+
this._maxRecordBytes = opts.maxRecordBytes !== undefined ? opts.maxRecordBytes : 0; // 0 = unlimited
|
|
76
|
+
// Terminal state (BS-14): once ended or poisoned, the instance is dead.
|
|
77
|
+
this._ended = false;
|
|
78
|
+
this._poisoned = false;
|
|
62
79
|
// Accumulator: bytes seen since the last emitted record's terminating LF.
|
|
63
80
|
this._buf = new Uint8Array(INITIAL_BUF);
|
|
64
81
|
this._bufLen = 0;
|
|
@@ -71,6 +88,8 @@ export class PreserveTokenizer {
|
|
|
71
88
|
}
|
|
72
89
|
|
|
73
90
|
feed(chunk) {
|
|
91
|
+
if (this._poisoned) throw new PreserveTokenizerError('E_POISONED', 'tokenizer poisoned by a previous error; construct a new instance');
|
|
92
|
+
if (this._ended) throw new PreserveTokenizerError('E_ENDED', 'tokenizer already ended; construct a new instance');
|
|
74
93
|
if (!(chunk instanceof Uint8Array)) {
|
|
75
94
|
throw new TypeError('PreserveTokenizer.feed: expected Uint8Array');
|
|
76
95
|
}
|
|
@@ -84,9 +103,12 @@ export class PreserveTokenizer {
|
|
|
84
103
|
}
|
|
85
104
|
|
|
86
105
|
end() {
|
|
106
|
+
if (this._poisoned) throw new PreserveTokenizerError('E_POISONED', 'tokenizer poisoned by a previous error; construct a new instance');
|
|
107
|
+
if (this._ended) throw new PreserveTokenizerError('E_ENDED', 'tokenizer already ended; construct a new instance');
|
|
87
108
|
if (this._bufLen > this._recordStart) {
|
|
88
109
|
// There's pending content that never terminated.
|
|
89
110
|
if (this._depth !== 0 || this._inString || this._escape) {
|
|
111
|
+
this._poisoned = true;
|
|
90
112
|
throw new PreserveTokenizerError('E_TRUNCATED',
|
|
91
113
|
'input ended mid-record at byte ' + (this._absOffset + this._bufLen) +
|
|
92
114
|
' (depth=' + this._depth + ', inString=' + this._inString + ')');
|
|
@@ -96,6 +118,7 @@ export class PreserveTokenizer {
|
|
|
96
118
|
this._emitRecord(this._recordStart, this._bufLen);
|
|
97
119
|
this._recordStart = this._bufLen;
|
|
98
120
|
}
|
|
121
|
+
this._ended = true;
|
|
99
122
|
}
|
|
100
123
|
|
|
101
124
|
_scan(from) {
|
|
@@ -119,6 +142,7 @@ export class PreserveTokenizer {
|
|
|
119
142
|
depth--;
|
|
120
143
|
if (depth < 0) {
|
|
121
144
|
this._depth = depth; this._inString = inString; this._escape = escape;
|
|
145
|
+
this._poisoned = true;
|
|
122
146
|
throw new PreserveTokenizerError('E_UNBALANCED',
|
|
123
147
|
'closing bracket without matching open at byte ' + (this._absOffset + i));
|
|
124
148
|
}
|
|
@@ -130,6 +154,7 @@ export class PreserveTokenizer {
|
|
|
130
154
|
}
|
|
131
155
|
}
|
|
132
156
|
if (this._maxRecordBytes > 0 && (i - this._recordStart) > this._maxRecordBytes) {
|
|
157
|
+
this._poisoned = true;
|
|
133
158
|
throw new PreserveTokenizerError('E_RECORD_TOO_LARGE',
|
|
134
159
|
'record starting at byte ' + (this._absOffset + this._recordStart) +
|
|
135
160
|
' exceeded maxRecordBytes=' + this._maxRecordBytes);
|
package/src/PreserveWriter.js
CHANGED
|
@@ -16,22 +16,35 @@
|
|
|
16
16
|
// Reader dispatch happens via the flag bit; a schema-mode Reader refuses this
|
|
17
17
|
// container with R_WRONG_MODE.
|
|
18
18
|
|
|
19
|
-
|
|
19
|
+
import { checkOpts } from './Opts.js';
|
|
20
|
+
|
|
21
|
+
export const VERSION = '1.2.0';
|
|
20
22
|
|
|
21
23
|
export class PreserveWriterError extends Error {
|
|
22
24
|
constructor(code, msg) { super(msg); this.code = code; this.name = 'PreserveWriterError'; }
|
|
23
25
|
}
|
|
26
|
+
function raisePreserveWriter(code, msg) { throw new PreserveWriterError(code, msg); }
|
|
24
27
|
|
|
25
28
|
const CONTAINER_HEADER_BYTES = 48;
|
|
26
29
|
const SHARD_ENTRY_BYTES = 40;
|
|
27
30
|
const FOOTER_BYTES = 16;
|
|
28
31
|
const DEFAULT_TARGET_SHARD_BYTES = 8 * 1024 * 1024; // 8 MiB — smaller than schema mode
|
|
29
32
|
const INITIAL_OFFSETS_CAP = 4096;
|
|
33
|
+
const U32_MAX = 4294967295;
|
|
34
|
+
// maxRecordBytes is a declared, validated, deliberately-unused key here: the
|
|
35
|
+
// documented call serialize(x, {preserve:true, writer:{maxRecordBytes:N}}) reads
|
|
36
|
+
// it out of opts.writer and hands it to PreserveTokenizer, which consumes it.
|
|
37
|
+
// PreserveWriter accepts it so that call shape does not trip E_UNKNOWN_OPTION.
|
|
38
|
+
const PRESERVE_WRITER_OPTS = {
|
|
39
|
+
targetShardBytes: { t: 'int', min: 1, max: U32_MAX },
|
|
40
|
+
maxRecordBytes: { t: 'int', min: 0, max: U32_MAX },
|
|
41
|
+
};
|
|
30
42
|
|
|
31
43
|
export class PreserveWriter {
|
|
32
44
|
constructor(opts) {
|
|
45
|
+
checkOpts('PreserveWriter', opts, PRESERVE_WRITER_OPTS, raisePreserveWriter);
|
|
33
46
|
opts = opts || {};
|
|
34
|
-
this._targetShardBytes = opts.targetShardBytes
|
|
47
|
+
this._targetShardBytes = opts.targetShardBytes !== undefined ? opts.targetShardBytes : DEFAULT_TARGET_SHARD_BYTES;
|
|
35
48
|
this._shards = [];
|
|
36
49
|
this._totalRows = 0;
|
|
37
50
|
this._finalized = false;
|
package/src/RangeReader.js
CHANGED
|
@@ -34,8 +34,9 @@
|
|
|
34
34
|
// R_ROW_OUT_OF_RANGE -- rowIdx >= totalRows
|
|
35
35
|
|
|
36
36
|
import { StringTable } from './StringTable.js';
|
|
37
|
+
import { checkOpts } from './Opts.js';
|
|
37
38
|
|
|
38
|
-
export const VERSION = '1.
|
|
39
|
+
export const VERSION = '1.2.0';
|
|
39
40
|
|
|
40
41
|
const CONTAINER_HEADER_BYTES = 48;
|
|
41
42
|
const SHARD_ENTRY_BYTES = 40;
|
|
@@ -44,10 +45,15 @@ const FIELD_DESCRIPTOR_BYTES = 24;
|
|
|
44
45
|
const LANE_F64 = 1;
|
|
45
46
|
const LANE_U32 = 3;
|
|
46
47
|
const READER_VERSION = 1;
|
|
48
|
+
const U32_MAX = 4294967295;
|
|
49
|
+
|
|
50
|
+
const RANGE_READER_OPTS = { maxCachedShards: { t: 'int', min: 0, max: U32_MAX } };
|
|
51
|
+
const HTTP_ADAPTER_OPTS = { fetch: { t: 'fn' } };
|
|
47
52
|
|
|
48
53
|
export class RangeReaderError extends Error {
|
|
49
54
|
constructor(code, msg) { super(msg); this.code = code; this.name = 'RangeReaderError'; }
|
|
50
55
|
}
|
|
56
|
+
function raiseRange(code, msg) { throw new RangeReaderError(code, msg); }
|
|
51
57
|
|
|
52
58
|
// ----- Adapters --------------------------------------------------------------
|
|
53
59
|
|
|
@@ -56,8 +62,9 @@ export class RangeReaderError extends Error {
|
|
|
56
62
|
// origin/CDN).
|
|
57
63
|
export class HTTPRangeAdapter {
|
|
58
64
|
static async open(url, opts) {
|
|
65
|
+
checkOpts('HTTPRangeAdapter', opts, HTTP_ADAPTER_OPTS, raiseRange);
|
|
59
66
|
opts = opts || {};
|
|
60
|
-
const fetchImpl = opts.fetch
|
|
67
|
+
const fetchImpl = opts.fetch !== undefined ? opts.fetch : globalThis.fetch;
|
|
61
68
|
if (typeof fetchImpl !== 'function') {
|
|
62
69
|
throw new RangeReaderError('R_TRUNCATED', 'no fetch() available in this environment');
|
|
63
70
|
}
|
|
@@ -148,9 +155,10 @@ export class RangeReader {
|
|
|
148
155
|
}
|
|
149
156
|
|
|
150
157
|
constructor(adapter, opts) {
|
|
158
|
+
checkOpts('RangeReader', opts, RANGE_READER_OPTS, raiseRange);
|
|
151
159
|
opts = opts || {};
|
|
152
160
|
this.adapter = adapter;
|
|
153
|
-
this.maxCachedShards = opts.maxCachedShards
|
|
161
|
+
this.maxCachedShards = opts.maxCachedShards !== undefined ? opts.maxCachedShards : 8;
|
|
154
162
|
// shard cache: shardIdx -> { payloadBytes, payloadDv, stringTable, lastAccess }
|
|
155
163
|
this._shardCache = new Map();
|
|
156
164
|
this._accessCounter = 0;
|
package/src/Reader.js
CHANGED
package/src/Split.js
CHANGED
|
@@ -35,8 +35,9 @@ import { Tokenizer, TokenizerError } from './Tokenizer.js';
|
|
|
35
35
|
import { Writer, WriterError } from './Writer.js';
|
|
36
36
|
import { Reader, ReaderError } from './Reader.js';
|
|
37
37
|
import { StringTable } from './StringTable.js';
|
|
38
|
+
import { checkOpts } from './Opts.js';
|
|
38
39
|
|
|
39
|
-
export const VERSION = '1.
|
|
40
|
+
export const VERSION = '1.2.0';
|
|
40
41
|
|
|
41
42
|
const LF = 0x0A;
|
|
42
43
|
const CONTAINER_HEADER_BYTES = 48;
|
|
@@ -44,10 +45,27 @@ const SHARD_ENTRY_BYTES = 40;
|
|
|
44
45
|
const FIELD_DESCRIPTOR_BYTES = 24;
|
|
45
46
|
const FOOTER_BYTES = 16;
|
|
46
47
|
const LANE_F64 = 1;
|
|
48
|
+
const U32_MAX = 4294967295;
|
|
49
|
+
|
|
50
|
+
const SPLIT_NDJSON_OPTS = {
|
|
51
|
+
targetParts: { t: 'int', min: 1, max: U32_MAX },
|
|
52
|
+
maxPartBytes: { t: 'int', min: 1, max: U32_MAX, inf: true },
|
|
53
|
+
};
|
|
54
|
+
const COMPILE_PART_OPTS = {
|
|
55
|
+
framing: { t: 'enum', values: ['auto', 'array', 'ndjson'] },
|
|
56
|
+
writer: { t: 'obj' },
|
|
57
|
+
};
|
|
58
|
+
const COMPILE_IN_PARTS_OPTS = {
|
|
59
|
+
targetParts: { t: 'int', min: 1, max: U32_MAX },
|
|
60
|
+
maxPartBytes: { t: 'int', min: 1, max: U32_MAX, inf: true },
|
|
61
|
+
framing: { t: 'enum', values: ['auto', 'array', 'ndjson'] },
|
|
62
|
+
writer: { t: 'obj' },
|
|
63
|
+
};
|
|
47
64
|
|
|
48
65
|
export class SplitError extends Error {
|
|
49
66
|
constructor(code, msg) { super(msg); this.code = code; this.name = 'SplitError'; }
|
|
50
67
|
}
|
|
68
|
+
function raiseSplit(code, msg) { throw new SplitError(code, msg); }
|
|
51
69
|
|
|
52
70
|
// ---------- splitNDJSON ----------
|
|
53
71
|
|
|
@@ -67,8 +85,9 @@ export function splitNDJSON(bytes, opts) {
|
|
|
67
85
|
throw new TypeError('splitNDJSON: expected Uint8Array');
|
|
68
86
|
}
|
|
69
87
|
opts = opts || {};
|
|
70
|
-
|
|
71
|
-
const
|
|
88
|
+
checkOpts('splitNDJSON', opts, SPLIT_NDJSON_OPTS, raiseSplit);
|
|
89
|
+
const targetParts = Math.max(1, opts.targetParts !== undefined ? opts.targetParts : 4);
|
|
90
|
+
const maxPartBytes = opts.maxPartBytes !== undefined ? opts.maxPartBytes : Infinity;
|
|
72
91
|
const total = bytes.length;
|
|
73
92
|
if (total === 0) return [];
|
|
74
93
|
|
|
@@ -110,8 +129,9 @@ export function compilePart(bytes, opts) {
|
|
|
110
129
|
throw new TypeError('compilePart: expected Uint8Array');
|
|
111
130
|
}
|
|
112
131
|
opts = opts || {};
|
|
113
|
-
|
|
114
|
-
const
|
|
132
|
+
checkOpts('compilePart', opts, COMPILE_PART_OPTS, raiseSplit);
|
|
133
|
+
const framing = opts.framing !== undefined ? opts.framing : 'ndjson';
|
|
134
|
+
const w = new Writer(opts.writer !== undefined ? opts.writer : {});
|
|
115
135
|
const t = new Tokenizer(w, { framing });
|
|
116
136
|
t.feed(bytes);
|
|
117
137
|
t.end();
|
|
@@ -126,13 +146,14 @@ export function compilePart(bytes, opts) {
|
|
|
126
146
|
// aren't available or for testing.
|
|
127
147
|
export function compileInParts(bytes, opts) {
|
|
128
148
|
opts = opts || {};
|
|
149
|
+
checkOpts('compileInParts', opts, COMPILE_IN_PARTS_OPTS, raiseSplit);
|
|
129
150
|
const splitOpts = {
|
|
130
151
|
targetParts: opts.targetParts,
|
|
131
152
|
maxPartBytes: opts.maxPartBytes,
|
|
132
153
|
};
|
|
133
154
|
const partOpts = {
|
|
134
|
-
framing: opts.framing
|
|
135
|
-
writer: opts.writer
|
|
155
|
+
framing: opts.framing !== undefined ? opts.framing : 'ndjson',
|
|
156
|
+
writer: opts.writer !== undefined ? opts.writer : {},
|
|
136
157
|
};
|
|
137
158
|
const ranges = splitNDJSON(bytes, splitOpts);
|
|
138
159
|
const containers = new Array(ranges.length);
|
package/src/StringTable.js
CHANGED
|
@@ -15,8 +15,12 @@
|
|
|
15
15
|
//
|
|
16
16
|
// The trailing sentinel makes `len(i) = offsets[i+1] - offsets[i]` uniform for
|
|
17
17
|
// all i including the last — no branch in the reader hot path.
|
|
18
|
+
//
|
|
19
|
+
// Entry 0 is always the empty string: the table reserves it in the constructor
|
|
20
|
+
// and at every reset(), so an absent U32 row cell (which is 0) decodes as ""
|
|
21
|
+
// rather than aliasing the shard's first-interned string (SPEC 3.3, SPEC 7).
|
|
18
22
|
|
|
19
|
-
export const VERSION = '1.
|
|
23
|
+
export const VERSION = '1.2.0';
|
|
20
24
|
|
|
21
25
|
const EMPTY_SLOT = 0xFFFFFFFF; // MUST be unsigned; typed-array reads are unsigned
|
|
22
26
|
const INITIAL_BLOB_BYTES = 64 * 1024;
|
|
@@ -25,6 +29,10 @@ const INITIAL_HASH_CAP = 2048; // load factor target 50%
|
|
|
25
29
|
const HASH_MAX_LOAD_NUM = 1; // 50% load factor: num/den = 1/2
|
|
26
30
|
const HASH_MAX_LOAD_DEN = 2;
|
|
27
31
|
|
|
32
|
+
// Reserved entry 0. A zero-length range: no blob bytes, no growth, one hash
|
|
33
|
+
// slot. Module-level so reset() reserves without allocating.
|
|
34
|
+
const EMPTY = new Uint8Array(0);
|
|
35
|
+
|
|
28
36
|
// FNV-1a 32-bit over a byte range.
|
|
29
37
|
function fnv1a(bytes, from, to) {
|
|
30
38
|
let h = 0x811c9dc5 | 0;
|
|
@@ -44,6 +52,7 @@ export class StringTable {
|
|
|
44
52
|
this._hashCap = INITIAL_HASH_CAP;
|
|
45
53
|
this._hashMask = INITIAL_HASH_CAP - 1;
|
|
46
54
|
this._hashKeys = new Uint32Array(INITIAL_HASH_CAP).fill(EMPTY_SLOT);
|
|
55
|
+
this._reserveEmpty();
|
|
47
56
|
}
|
|
48
57
|
|
|
49
58
|
// Returns u32 index of the string with these bytes, adding a new entry if
|
|
@@ -76,6 +85,13 @@ export class StringTable {
|
|
|
76
85
|
return true;
|
|
77
86
|
}
|
|
78
87
|
|
|
88
|
+
// Class invariant: index 0 is the empty string. Zero-alloc -- _insertNew with
|
|
89
|
+
// n === 0 appends no bytes, offsets[0]/[1] already exist, and the load-factor
|
|
90
|
+
// check cannot trip at count 1.
|
|
91
|
+
_reserveEmpty() {
|
|
92
|
+
this.intern(EMPTY, 0, 0);
|
|
93
|
+
}
|
|
94
|
+
|
|
79
95
|
_insertNew(bytes, from, to, slot) {
|
|
80
96
|
const n = to - from;
|
|
81
97
|
// Grow blob if needed
|
|
@@ -158,6 +174,7 @@ export class StringTable {
|
|
|
158
174
|
// (we only touched [0..count], so the fill is bounded)
|
|
159
175
|
this._offsets[0] = 0;
|
|
160
176
|
this._hashKeys.fill(EMPTY_SLOT);
|
|
177
|
+
this._reserveEmpty();
|
|
161
178
|
}
|
|
162
179
|
|
|
163
180
|
// Serialize into the LBK1 per-shard string-table byte layout described at the
|
package/src/Tokenizer.js
CHANGED
|
@@ -16,8 +16,22 @@
|
|
|
16
16
|
// E_KEYWORD_MISMATCH - true/false/null spelled wrong
|
|
17
17
|
// E_TRAILING_INPUT - non-whitespace bytes after top-level value(s) (array mode only)
|
|
18
18
|
// E_DEPTH_LIMIT - nesting depth exceeded MAX_DEPTH
|
|
19
|
+
// E_STRING_TOO_LONG - string exceeds maxStringBytes cap
|
|
20
|
+
// E_ENDED - feed()/end() after end()
|
|
21
|
+
// E_POISONED - feed()/end() after a thrown error
|
|
22
|
+
// E_UNKNOWN_OPTION - unknown constructor option key
|
|
23
|
+
// E_OPTION_VALUE - constructor option value out of domain
|
|
19
24
|
|
|
20
|
-
|
|
25
|
+
import { checkOpts } from './Opts.js';
|
|
26
|
+
|
|
27
|
+
export const VERSION = '1.2.0';
|
|
28
|
+
|
|
29
|
+
const U32_MAX = 4294967295;
|
|
30
|
+
const TOKENIZER_OPTS = {
|
|
31
|
+
framing: { t: 'enum', values: ['auto', 'array', 'ndjson'] },
|
|
32
|
+
maxStringBytes: { t: 'int', min: 1, max: U32_MAX },
|
|
33
|
+
};
|
|
34
|
+
function raiseTok(code, msg) { throw new TokenizerError(code, 0, msg); }
|
|
21
35
|
|
|
22
36
|
// ---------- byte constants ----------
|
|
23
37
|
const B_SPACE = 0x20, B_TAB = 0x09, B_LF = 0x0A, B_CR = 0x0D;
|
|
@@ -104,24 +118,33 @@ const NOOP = () => {};
|
|
|
104
118
|
export class Tokenizer {
|
|
105
119
|
constructor(sink, opts) {
|
|
106
120
|
if (!sink) throw new Error('sink required');
|
|
121
|
+
checkOpts('Tokenizer', opts, TOKENIZER_OPTS, raiseTok);
|
|
107
122
|
this.sink = sink;
|
|
108
123
|
|
|
109
124
|
// top-level framing: 'auto' (default), 'array', or 'ndjson'
|
|
110
|
-
this._framing = (opts && opts.framing)
|
|
111
|
-
this._maxStringBytes = (opts && opts.maxStringBytes)
|
|
112
|
-
|
|
113
|
-
//
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
+
this._framing = (opts && opts.framing !== undefined) ? opts.framing : 'auto';
|
|
126
|
+
this._maxStringBytes = (opts && opts.maxStringBytes !== undefined) ? opts.maxStringBytes : (1 << 20); // 1 MB default cap
|
|
127
|
+
|
|
128
|
+
// Per-instance dispatch fields: bind each present sink method to the sink
|
|
129
|
+
// once at construction (cold path), defaulting to NOOP. The caller's sink
|
|
130
|
+
// object is never mutated (BS-19), so a frozen sink constructs and drives
|
|
131
|
+
// fine. Binding preserves the sink as receiver while the hot call sites read
|
|
132
|
+
// a single monomorphic field (this._onX) instead of this.sink.onX.
|
|
133
|
+
this._onStartObject = sink.onStartObject ? sink.onStartObject.bind(sink) : NOOP;
|
|
134
|
+
this._onEndObject = sink.onEndObject ? sink.onEndObject.bind(sink) : NOOP;
|
|
135
|
+
this._onStartArray = sink.onStartArray ? sink.onStartArray.bind(sink) : NOOP;
|
|
136
|
+
this._onEndArray = sink.onEndArray ? sink.onEndArray.bind(sink) : NOOP;
|
|
137
|
+
this._onKey = sink.onKey ? sink.onKey.bind(sink) : NOOP;
|
|
138
|
+
this._onString = sink.onString ? sink.onString.bind(sink) : NOOP;
|
|
139
|
+
this._onNumber = sink.onNumber ? sink.onNumber.bind(sink) : NOOP;
|
|
140
|
+
this._onTrue = sink.onTrue ? sink.onTrue.bind(sink) : NOOP;
|
|
141
|
+
this._onFalse = sink.onFalse ? sink.onFalse.bind(sink) : NOOP;
|
|
142
|
+
this._onNull = sink.onNull ? sink.onNull.bind(sink) : NOOP;
|
|
143
|
+
this._onEnd = sink.onEnd ? sink.onEnd.bind(sink) : NOOP;
|
|
144
|
+
|
|
145
|
+
// Terminal state (BS-14): once ended or poisoned, the instance is dead.
|
|
146
|
+
this._ended = false;
|
|
147
|
+
this._poisoned = false;
|
|
125
148
|
|
|
126
149
|
// parser state
|
|
127
150
|
this._state = S_TOP;
|
|
@@ -158,6 +181,8 @@ export class Tokenizer {
|
|
|
158
181
|
|
|
159
182
|
// Feed a chunk. Bytes are consumed synchronously; sink events fire during this call.
|
|
160
183
|
feed(chunk) {
|
|
184
|
+
if (this._poisoned) throw new TokenizerError('E_POISONED', this._absOffset, 'tokenizer poisoned by a previous error; construct a new instance');
|
|
185
|
+
if (this._ended) throw new TokenizerError('E_ENDED', this._absOffset, 'tokenizer already ended; construct a new instance');
|
|
161
186
|
const len = chunk.length;
|
|
162
187
|
let i = 0;
|
|
163
188
|
while (i < len) {
|
|
@@ -306,9 +331,9 @@ export class Tokenizer {
|
|
|
306
331
|
this._kwPos++;
|
|
307
332
|
i++; this._absOffset++;
|
|
308
333
|
if (this._kwPos === tmpl.length) {
|
|
309
|
-
if (this._kwId === KW_ID_TRUE) this.
|
|
310
|
-
else if (this._kwId === KW_ID_FALSE) this.
|
|
311
|
-
else this.
|
|
334
|
+
if (this._kwId === KW_ID_TRUE) this._onTrue();
|
|
335
|
+
else if (this._kwId === KW_ID_FALSE) this._onFalse();
|
|
336
|
+
else this._onNull();
|
|
312
337
|
this._afterValue();
|
|
313
338
|
}
|
|
314
339
|
continue;
|
|
@@ -320,6 +345,8 @@ export class Tokenizer {
|
|
|
320
345
|
}
|
|
321
346
|
|
|
322
347
|
end() {
|
|
348
|
+
if (this._poisoned) throw new TokenizerError('E_POISONED', this._absOffset, 'tokenizer poisoned by a previous error; construct a new instance');
|
|
349
|
+
if (this._ended) throw new TokenizerError('E_ENDED', this._absOffset, 'tokenizer already ended; construct a new instance');
|
|
323
350
|
// A trailing number may still be pending
|
|
324
351
|
if (this._state === S_NUMBER) {
|
|
325
352
|
this._emitNumber();
|
|
@@ -328,7 +355,8 @@ export class Tokenizer {
|
|
|
328
355
|
if (this._arrayMode && this._arrayModeOuterOpen) {
|
|
329
356
|
this._err('E_UNEXPECTED_EOF', 'unclosed top-level array');
|
|
330
357
|
}
|
|
331
|
-
this.
|
|
358
|
+
this._ended = true;
|
|
359
|
+
this._onEnd();
|
|
332
360
|
return;
|
|
333
361
|
}
|
|
334
362
|
this._err('E_UNEXPECTED_EOF', 'input ended mid-token (state=' + this._state + ')');
|
|
@@ -378,7 +406,7 @@ export class Tokenizer {
|
|
|
378
406
|
|
|
379
407
|
_handleObjectStartByte(b) {
|
|
380
408
|
if (b === B_RBRACE) {
|
|
381
|
-
this.
|
|
409
|
+
this._onEndObject();
|
|
382
410
|
this._popContainer();
|
|
383
411
|
return;
|
|
384
412
|
}
|
|
@@ -389,7 +417,7 @@ export class Tokenizer {
|
|
|
389
417
|
_handleObjectValueEndByte(b) {
|
|
390
418
|
if (b === B_COMMA) { this._state = S_OBJECT_NEXT_KEY; }
|
|
391
419
|
else if (b === B_RBRACE) {
|
|
392
|
-
this.
|
|
420
|
+
this._onEndObject();
|
|
393
421
|
this._popContainer();
|
|
394
422
|
} else {
|
|
395
423
|
this._err('E_UNEXPECTED_BYTE', 'expected , or }');
|
|
@@ -404,7 +432,7 @@ export class Tokenizer {
|
|
|
404
432
|
this._state = S_TOP;
|
|
405
433
|
return;
|
|
406
434
|
}
|
|
407
|
-
this.
|
|
435
|
+
this._onEndArray();
|
|
408
436
|
this._popContainer();
|
|
409
437
|
return;
|
|
410
438
|
}
|
|
@@ -419,7 +447,7 @@ export class Tokenizer {
|
|
|
419
447
|
this._state = S_TOP;
|
|
420
448
|
return;
|
|
421
449
|
}
|
|
422
|
-
this.
|
|
450
|
+
this._onEndArray();
|
|
423
451
|
this._popContainer();
|
|
424
452
|
} else {
|
|
425
453
|
this._err('E_UNEXPECTED_BYTE', 'expected , or ]');
|
|
@@ -430,13 +458,13 @@ export class Tokenizer {
|
|
|
430
458
|
|
|
431
459
|
_beginValue(b) {
|
|
432
460
|
if (b === B_LBRACE) {
|
|
433
|
-
this.
|
|
461
|
+
this._onStartObject();
|
|
434
462
|
this._pushContainer(C_OBJECT);
|
|
435
463
|
this._state = S_OBJECT_START;
|
|
436
464
|
return;
|
|
437
465
|
}
|
|
438
466
|
if (b === B_LBRACKET) {
|
|
439
|
-
this.
|
|
467
|
+
this._onStartArray();
|
|
440
468
|
this._pushContainer(C_ARRAY);
|
|
441
469
|
this._state = S_ARRAY_START;
|
|
442
470
|
return;
|
|
@@ -578,16 +606,17 @@ export class Tokenizer {
|
|
|
578
606
|
v = this._numSign * v;
|
|
579
607
|
}
|
|
580
608
|
if (!Number.isFinite(v)) this._err('E_NUMBER_OVERFLOW', 'number exceeds F64 range');
|
|
581
|
-
this.
|
|
609
|
+
this._onNumber(v);
|
|
582
610
|
this._afterValue();
|
|
583
611
|
}
|
|
584
612
|
|
|
585
613
|
_emitString() {
|
|
614
|
+
if (this._strLen > this._maxStringBytes) this._err('E_STRING_TOO_LONG', 'string exceeds maxStringBytes cap (' + this._strLen + ' > ' + this._maxStringBytes + ')');
|
|
586
615
|
if (this._strIsKey) {
|
|
587
|
-
this.
|
|
616
|
+
this._onKey(this._strBuf, 0, this._strLen);
|
|
588
617
|
this._state = S_OBJECT_KEY_END;
|
|
589
618
|
} else {
|
|
590
|
-
this.
|
|
619
|
+
this._onString(this._strBuf, 0, this._strLen);
|
|
591
620
|
this._afterValue();
|
|
592
621
|
}
|
|
593
622
|
}
|
|
@@ -636,6 +665,7 @@ export class Tokenizer {
|
|
|
636
665
|
|
|
637
666
|
_appendStrRange(chunk, from, to) {
|
|
638
667
|
const need = to - from;
|
|
668
|
+
if (this._strLen + need > this._maxStringBytes) this._err('E_STRING_TOO_LONG', 'string exceeds maxStringBytes cap (' + (this._strLen + need) + ' > ' + this._maxStringBytes + ')');
|
|
639
669
|
if (this._strLen + need > this._strBuf.length) this._growStrBuf(need);
|
|
640
670
|
// Manual copy loop — Uint8Array.set(source) via subarray allocates a small
|
|
641
671
|
// view header per call, which turns into MB-scale GC pressure across a
|
|
@@ -668,9 +698,9 @@ export class Tokenizer {
|
|
|
668
698
|
}
|
|
669
699
|
|
|
670
700
|
_growStrBuf(need) {
|
|
701
|
+
if (this._strLen + need > this._maxStringBytes) this._err('E_STRING_TOO_LONG', 'string exceeds maxStringBytes cap (' + (this._strLen + need) + ' > ' + this._maxStringBytes + ')');
|
|
671
702
|
let cap = this._strBuf.length;
|
|
672
703
|
while (cap < this._strLen + need) cap *= 2;
|
|
673
|
-
if (cap > this._maxStringBytes) this._err('E_UNEXPECTED_BYTE', 'string exceeds maxStringBytes cap');
|
|
674
704
|
const nb = new Uint8Array(cap);
|
|
675
705
|
nb.set(this._strBuf);
|
|
676
706
|
this._strBuf = nb;
|
|
@@ -684,6 +714,7 @@ export class Tokenizer {
|
|
|
684
714
|
}
|
|
685
715
|
|
|
686
716
|
_err(code, msg) {
|
|
717
|
+
this._poisoned = true;
|
|
687
718
|
throw new TokenizerError(code, this._absOffset, msg);
|
|
688
719
|
}
|
|
689
720
|
}
|
package/src/Writer.js
CHANGED
|
@@ -26,10 +26,31 @@
|
|
|
26
26
|
// W_MIXED_LANE_TYPES - a field saw both number and string values
|
|
27
27
|
// W_LANE_MISMATCH - post-freeze: value type doesn't match schema lane
|
|
28
28
|
// W_UNKNOWN_LANE_KIND - explicit schema declares an unknown laneKind
|
|
29
|
+
// W_DUPLICATE_FIELD - schema declares the same field name twice
|
|
30
|
+
// W_SCHEMA_TOO_WIDE - a field offset exceeds the u16 offset_in_row ceiling
|
|
31
|
+
// W_FIELD_NAME_INVALID - field name empty, non-string, or > 255 UTF-8 bytes
|
|
32
|
+
// W_FINALIZED - a sink event or finalize() after finalize()
|
|
33
|
+
// E_UNKNOWN_OPTION - unknown constructor option key
|
|
34
|
+
// E_OPTION_VALUE - constructor option value out of domain
|
|
29
35
|
|
|
30
36
|
import { StringTable } from './StringTable.js';
|
|
31
|
-
|
|
32
|
-
|
|
37
|
+
import { checkOpts } from './Opts.js';
|
|
38
|
+
|
|
39
|
+
export const VERSION = '1.2.0';
|
|
40
|
+
|
|
41
|
+
const U32_MAX = 4294967295;
|
|
42
|
+
// Post-finalize sentinel for _recordDepth. Chosen = 2 so every post-finalize
|
|
43
|
+
// sink event lands in an existing cold arm (onNumber/onString depth != 1,
|
|
44
|
+
// onStartObject depth > 1) without adding a per-record compare on the hot path.
|
|
45
|
+
// It is NOT a real nesting depth: onEndObject is unreachable post-finalize
|
|
46
|
+
// because onStartObject throws W_FINALIZED first.
|
|
47
|
+
const FINALIZED_DEPTH = 2;
|
|
48
|
+
const WRITER_OPTS = {
|
|
49
|
+
schema: { t: 'obj', nullable: true },
|
|
50
|
+
targetShardBytes: { t: 'int', min: 1, max: U32_MAX },
|
|
51
|
+
sampleBytes: { t: 'int', min: 0, max: U32_MAX },
|
|
52
|
+
};
|
|
53
|
+
function raiseWriter(code, msg) { throw new WriterError(code, msg); }
|
|
33
54
|
|
|
34
55
|
const CONTAINER_HEADER_BYTES = 48;
|
|
35
56
|
const SHARD_ENTRY_BYTES = 40;
|
|
@@ -157,10 +178,11 @@ class SampleBuffer {
|
|
|
157
178
|
|
|
158
179
|
export class Writer {
|
|
159
180
|
constructor(opts) {
|
|
181
|
+
checkOpts('Writer', opts, WRITER_OPTS, raiseWriter);
|
|
160
182
|
opts = opts || {};
|
|
161
|
-
this._targetShardBytes = opts.targetShardBytes
|
|
162
|
-
this._sampleBytes = opts.sampleBytes
|
|
163
|
-
this._explicitSchema = opts.schema
|
|
183
|
+
this._targetShardBytes = opts.targetShardBytes !== undefined ? opts.targetShardBytes : DEFAULT_SHARD_BYTES;
|
|
184
|
+
this._sampleBytes = opts.sampleBytes !== undefined ? opts.sampleBytes : this._targetShardBytes;
|
|
185
|
+
this._explicitSchema = opts.schema !== undefined ? opts.schema : null;
|
|
164
186
|
|
|
165
187
|
// Frozen schema state
|
|
166
188
|
this._schema = null; // { fields: [{name, laneKind, offsetInRow}], rowStride }
|
|
@@ -201,7 +223,10 @@ export class Writer {
|
|
|
201
223
|
|
|
202
224
|
onStartObject() {
|
|
203
225
|
this._recordDepth++;
|
|
204
|
-
if (this._recordDepth > 1)
|
|
226
|
+
if (this._recordDepth > 1) {
|
|
227
|
+
if (this._finalized) throw new WriterError('W_FINALIZED', 'writer already finalized');
|
|
228
|
+
throw new WriterError('W_NESTED_UNSUPPORTED', 'nested object inside record');
|
|
229
|
+
}
|
|
205
230
|
if (this._schema) {
|
|
206
231
|
// clear scratch slots
|
|
207
232
|
const f = this._rowValueSlotsF64, u = this._rowValueSlotsU32;
|
|
@@ -216,7 +241,8 @@ export class Writer {
|
|
|
216
241
|
}
|
|
217
242
|
|
|
218
243
|
onStartArray() {
|
|
219
|
-
if (this.
|
|
244
|
+
if (this._finalized) throw new WriterError('W_FINALIZED', 'writer already finalized');
|
|
245
|
+
if (this._recordDepth === 0) throw new WriterError('W_TOP_LEVEL_NOT_OBJECT', 'top-level array is not an object; LBK1 schema mode stores objects only (SPEC 5.1)');
|
|
220
246
|
throw new WriterError('W_NESTED_UNSUPPORTED', 'nested array inside record');
|
|
221
247
|
}
|
|
222
248
|
|
|
@@ -236,7 +262,10 @@ export class Writer {
|
|
|
236
262
|
}
|
|
237
263
|
|
|
238
264
|
onNumber(v) {
|
|
239
|
-
if (this._recordDepth !== 1)
|
|
265
|
+
if (this._recordDepth !== 1) {
|
|
266
|
+
if (this._finalized) throw new WriterError('W_FINALIZED', 'writer already finalized');
|
|
267
|
+
throw new WriterError('W_TOP_LEVEL_NOT_OBJECT', 'top-level number is not an object; LBK1 schema mode stores objects only (SPEC 5.1)');
|
|
268
|
+
}
|
|
240
269
|
if (this._schema) {
|
|
241
270
|
const idx = this._currentFieldIdx;
|
|
242
271
|
if (idx < 0) return;
|
|
@@ -255,7 +284,10 @@ export class Writer {
|
|
|
255
284
|
onNull() { this.onNumber(0); }
|
|
256
285
|
|
|
257
286
|
onString(bytes, from, to) {
|
|
258
|
-
if (this._recordDepth
|
|
287
|
+
if (this._recordDepth !== 1) {
|
|
288
|
+
if (this._finalized) throw new WriterError('W_FINALIZED', 'writer already finalized');
|
|
289
|
+
throw new WriterError('W_TOP_LEVEL_NOT_OBJECT', 'top-level string is not an object; LBK1 schema mode stores objects only (SPEC 5.1)');
|
|
290
|
+
}
|
|
259
291
|
if (this._schema) {
|
|
260
292
|
const idx = this._currentFieldIdx;
|
|
261
293
|
if (idx < 0) return;
|
|
@@ -333,8 +365,33 @@ export class Writer {
|
|
|
333
365
|
this._fieldNameHashes = new Uint32Array(fields.length);
|
|
334
366
|
this._fieldLaneKinds = new Uint8Array(fields.length);
|
|
335
367
|
this._fieldOffsets = new Uint16Array(fields.length);
|
|
368
|
+
// Freeze-time schema validation (BS-02/BS-03): runs once per schema on both
|
|
369
|
+
// freeze paths (explicit + sample-drain funnel here), never per record.
|
|
370
|
+
const seenNames = new Map();
|
|
336
371
|
for (let i = 0; i < fields.length; i++) {
|
|
337
|
-
const
|
|
372
|
+
const name = fields[i].name;
|
|
373
|
+
if (typeof name !== 'string' || name.length === 0) {
|
|
374
|
+
throw new WriterError('W_FIELD_NAME_INVALID',
|
|
375
|
+
'field name at index ' + i + ' must be a non-empty string of at most 255 UTF-8 bytes; got ' +
|
|
376
|
+
(typeof name === 'string' ? '0 bytes' : typeof name));
|
|
377
|
+
}
|
|
378
|
+
const b = enc.encode(name);
|
|
379
|
+
if (b.length > 255) {
|
|
380
|
+
throw new WriterError('W_FIELD_NAME_INVALID',
|
|
381
|
+
'field name at index ' + i + ' must be a non-empty string of at most 255 UTF-8 bytes; got ' +
|
|
382
|
+
b.length + ' bytes');
|
|
383
|
+
}
|
|
384
|
+
const first = seenNames.get(name);
|
|
385
|
+
if (first !== undefined) {
|
|
386
|
+
throw new WriterError('W_DUPLICATE_FIELD',
|
|
387
|
+
"duplicate field name '" + name + "' at index " + i + ' (first declared at index ' + first + ')');
|
|
388
|
+
}
|
|
389
|
+
seenNames.set(name, i);
|
|
390
|
+
if (fields[i].offsetInRow > 65535) {
|
|
391
|
+
throw new WriterError('W_SCHEMA_TOO_WIDE',
|
|
392
|
+
"field '" + name + "' at offset " + fields[i].offsetInRow +
|
|
393
|
+
' exceeds the u16 offset_in_row ceiling (max 65535)');
|
|
394
|
+
}
|
|
338
395
|
this._fieldNamesUtf8[i] = b;
|
|
339
396
|
this._fieldNameHashes[i] = hashBytes(b, 0, b.length);
|
|
340
397
|
this._fieldLaneKinds[i] = fields[i].laneKind;
|
|
@@ -526,7 +583,7 @@ export class Writer {
|
|
|
526
583
|
}
|
|
527
584
|
|
|
528
585
|
finalize() {
|
|
529
|
-
if (this._finalized)
|
|
586
|
+
if (this._finalized) throw new WriterError('W_FINALIZED', 'writer already finalized');
|
|
530
587
|
if (!this._schema) {
|
|
531
588
|
if (this._sample.rowCount === 0) throw new WriterError('W_EMPTY_INPUT', 'no records to write');
|
|
532
589
|
this._freezeSchemaFromSample();
|
|
@@ -536,6 +593,8 @@ export class Writer {
|
|
|
536
593
|
if (this._shards.length === 0) throw new WriterError('W_EMPTY_INPUT', 'no records to write');
|
|
537
594
|
this._container = this._assembleContainer();
|
|
538
595
|
this._finalized = true;
|
|
596
|
+
// Post-finalize sentinel: route every later sink event into a cold arm.
|
|
597
|
+
this._recordDepth = FINALIZED_DEPTH;
|
|
539
598
|
return this._container;
|
|
540
599
|
}
|
|
541
600
|
|
package/src/index.js
CHANGED
|
@@ -16,6 +16,7 @@ import { StringTable } from './StringTable.js';
|
|
|
16
16
|
import { PreserveTokenizer, PreserveTokenizerError } from './PreserveTokenizer.js';
|
|
17
17
|
import { PreserveWriter, PreserveWriterError } from './PreserveWriter.js';
|
|
18
18
|
import { PreserveReader, PreserveReaderError } from './PreserveReader.js';
|
|
19
|
+
import { checkOpts } from './Opts.js';
|
|
19
20
|
|
|
20
21
|
export {
|
|
21
22
|
Tokenizer, TokenizerError,
|
|
@@ -26,10 +27,17 @@ export {
|
|
|
26
27
|
PreserveWriter, PreserveWriterError,
|
|
27
28
|
PreserveReader, PreserveReaderError,
|
|
28
29
|
};
|
|
29
|
-
export const VERSION = '1.
|
|
30
|
+
export const VERSION = '1.2.0';
|
|
30
31
|
|
|
31
32
|
const encoder = new TextEncoder();
|
|
32
33
|
|
|
34
|
+
const SERIALIZE_OPTS = {
|
|
35
|
+
preserve: { t: 'bool' },
|
|
36
|
+
framing: { t: 'enum', values: ['auto', 'array', 'ndjson'] },
|
|
37
|
+
writer: { t: 'obj' },
|
|
38
|
+
};
|
|
39
|
+
function raiseSerialize(code, msg) { throw new WriterError(code, msg); }
|
|
40
|
+
|
|
33
41
|
// Serialize input data into an LBK1 container. Input may be:
|
|
34
42
|
// - Uint8Array (raw NDJSON bytes)
|
|
35
43
|
// - string (NDJSON text)
|
|
@@ -43,22 +51,32 @@ const encoder = new TextEncoder();
|
|
|
43
51
|
// writer: writer options. Schema mode: { schema, targetShardBytes, sampleBytes }.
|
|
44
52
|
// Preserve mode: { targetShardBytes, maxRecordBytes }.
|
|
45
53
|
export function serialize(input, opts) {
|
|
54
|
+
// Validate synchronously in the prologue so a typo throws before any async
|
|
55
|
+
// dispatch (R7): E_UNKNOWN_OPTION must surface as a throw, never a rejected
|
|
56
|
+
// promise, for ReadableStream / AsyncIterable inputs too.
|
|
57
|
+
checkOpts('serialize', opts, SERIALIZE_OPTS, raiseSerialize);
|
|
46
58
|
opts = opts || {};
|
|
47
59
|
const preserve = opts.preserve === true;
|
|
60
|
+
const framing = opts.framing !== undefined ? opts.framing : 'ndjson';
|
|
61
|
+
const writerOpts = opts.writer !== undefined ? opts.writer : {};
|
|
62
|
+
if (preserve && opts.framing !== undefined && opts.framing !== 'ndjson') {
|
|
63
|
+
throw new WriterError('E_OPTION_CONFLICT',
|
|
64
|
+
"serialize: preserve mode is NDJSON-only; remove framing:'" + opts.framing + "' or set preserve:false");
|
|
65
|
+
}
|
|
48
66
|
|
|
49
67
|
if (input && typeof input.getReader === 'function') {
|
|
50
68
|
return preserve
|
|
51
69
|
? _serializeReadableStreamPreserve(input, opts)
|
|
52
|
-
: _serializeReadableStream(input,
|
|
70
|
+
: _serializeReadableStream(input, writerOpts, framing);
|
|
53
71
|
}
|
|
54
72
|
if (input && typeof input[Symbol.asyncIterator] === 'function') {
|
|
55
73
|
return preserve
|
|
56
74
|
? _serializeAsyncIterablePreserve(input, opts)
|
|
57
|
-
: _serializeAsyncIterable(input,
|
|
75
|
+
: _serializeAsyncIterable(input, writerOpts, framing);
|
|
58
76
|
}
|
|
59
77
|
|
|
60
78
|
if (preserve) return _serializeSyncPreserve(input, opts);
|
|
61
|
-
return _serializeSyncSchema(input,
|
|
79
|
+
return _serializeSyncSchema(input, writerOpts, framing);
|
|
62
80
|
}
|
|
63
81
|
|
|
64
82
|
function _serializeSyncSchema(input, writerOpts, framing) {
|
|
@@ -83,10 +101,10 @@ function _serializeSyncSchema(input, writerOpts, framing) {
|
|
|
83
101
|
}
|
|
84
102
|
|
|
85
103
|
function _serializeSyncPreserve(input, opts) {
|
|
86
|
-
const w = new PreserveWriter(opts.writer
|
|
104
|
+
const w = new PreserveWriter(opts.writer !== undefined ? opts.writer : {});
|
|
87
105
|
const t = new PreserveTokenizer(w, {
|
|
88
106
|
framing: 'ndjson',
|
|
89
|
-
maxRecordBytes: (opts.writer && opts.writer.maxRecordBytes)
|
|
107
|
+
maxRecordBytes: (opts.writer !== undefined && opts.writer.maxRecordBytes !== undefined) ? opts.writer.maxRecordBytes : 0,
|
|
90
108
|
});
|
|
91
109
|
if (input instanceof Uint8Array) {
|
|
92
110
|
t.feed(input);
|
|
@@ -124,10 +142,10 @@ async function _serializeReadableStream(stream, writerOpts, framing) {
|
|
|
124
142
|
}
|
|
125
143
|
|
|
126
144
|
async function _serializeReadableStreamPreserve(stream, opts) {
|
|
127
|
-
const w = new PreserveWriter(opts.writer
|
|
145
|
+
const w = new PreserveWriter(opts.writer !== undefined ? opts.writer : {});
|
|
128
146
|
const t = new PreserveTokenizer(w, {
|
|
129
147
|
framing: 'ndjson',
|
|
130
|
-
maxRecordBytes: (opts.writer && opts.writer.maxRecordBytes)
|
|
148
|
+
maxRecordBytes: (opts.writer !== undefined && opts.writer.maxRecordBytes !== undefined) ? opts.writer.maxRecordBytes : 0,
|
|
131
149
|
});
|
|
132
150
|
const reader = stream.getReader();
|
|
133
151
|
try {
|
|
@@ -154,10 +172,10 @@ async function _serializeAsyncIterable(iterable, writerOpts, framing) {
|
|
|
154
172
|
}
|
|
155
173
|
|
|
156
174
|
async function _serializeAsyncIterablePreserve(iterable, opts) {
|
|
157
|
-
const w = new PreserveWriter(opts.writer
|
|
175
|
+
const w = new PreserveWriter(opts.writer !== undefined ? opts.writer : {});
|
|
158
176
|
const t = new PreserveTokenizer(w, {
|
|
159
177
|
framing: 'ndjson',
|
|
160
|
-
maxRecordBytes: (opts.writer && opts.writer.maxRecordBytes)
|
|
178
|
+
maxRecordBytes: (opts.writer !== undefined && opts.writer.maxRecordBytes !== undefined) ? opts.writer.maxRecordBytes : 0,
|
|
161
179
|
});
|
|
162
180
|
for await (const chunk of iterable) {
|
|
163
181
|
if (!(chunk instanceof Uint8Array)) throw new TypeError('serialize: async iterable must yield Uint8Array');
|