@zakkster/lite-bake-stream 1.6.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +165 -128
- package/README.md +293 -109
- package/SPEC.md +30 -30
- package/llms.txt +12 -14
- package/package.json +2 -2
- package/src/FileIngest.js +6 -2
- package/src/MultiReader.js +1 -1
- package/src/Opts.js +23 -0
- package/src/PreserveReader.js +3 -1
- package/src/PreserveTokenizer.js +12 -3
- package/src/PreserveWriter.js +8 -2
- package/src/RangeReader.js +111 -16
- package/src/Reader.js +3 -1
- package/src/Split.js +8 -4
- package/src/StringTable.js +7 -4
- package/src/Tokenizer.js +3 -2
- package/src/Views.js +3 -0
- package/src/Writer.js +6 -4
- package/src/index.js +5 -1
- package/types/MultiReader.d.ts +1 -1
- package/types/RangeReader.d.ts +21 -5
- package/types/Reader.d.ts +1 -1
- package/types/Split.d.ts +1 -1
- package/types/Tokenizer.d.ts +1 -1
- package/types/Writer.d.ts +2 -2
package/src/Views.js
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
// @zakkster/lite-bake-stream / Views (internal)
|
|
2
2
|
// Copyright (c) 2026 Zahary Shinikchiev. MIT.
|
|
3
3
|
//
|
|
4
|
+
// Error codes:
|
|
5
|
+
// W_BAD_SINK - shared finalizeToSink guard: sink missing write/writeAt or async
|
|
6
|
+
//
|
|
4
7
|
// One shared helper so every container entry point (deserialize,
|
|
5
8
|
// Reader.fromBuffer, PreserveReader.fromBuffer) resolves caller-supplied bytes
|
|
6
9
|
// to an ArrayBuffer by exactly the SAME rule (BS-09). A Uint8Array that is a
|
package/src/Writer.js
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
// setInputSource; hand-driven Writers with no source sample by record
|
|
12
12
|
// count) buffered in a columnar staging area, per-field lane kind inferred
|
|
13
13
|
// from observed value types (number->F64, string->U32-into-string-table).
|
|
14
|
+
// See decisions/0011-sample-drain-reintern.md.
|
|
14
15
|
// null is lane-neutral (BS-20): it marks the field but sets no kind, so
|
|
15
16
|
// null+string infers U32 and null+number infers F64. A field that saw both
|
|
16
17
|
// a real number and a real string raises W_MIXED_LANE_TYPES at freeze.
|
|
@@ -52,6 +53,7 @@
|
|
|
52
53
|
// W_SCHEMA_TOO_WIDE - a field offset exceeds the u16 offset_in_row ceiling
|
|
53
54
|
// W_FIELD_NAME_INVALID - field name empty, non-string, or > 255 UTF-8 bytes
|
|
54
55
|
// W_FINALIZED - a sink event or finalize() after finalize()
|
|
56
|
+
// W_BAD_SINK - finalizeToSink given a sink missing write/writeAt or async
|
|
55
57
|
// E_UNKNOWN_OPTION - unknown constructor option key
|
|
56
58
|
// E_OPTION_VALUE - constructor option value out of domain
|
|
57
59
|
|
|
@@ -60,7 +62,7 @@ import { checkOpts } from './Opts.js';
|
|
|
60
62
|
import { crc32cInit, crc32cUpdate, crc32cFinal, crc32cCombine } from './Crc32c.js';
|
|
61
63
|
import { validateSink, isThenable } from './Views.js';
|
|
62
64
|
|
|
63
|
-
export const VERSION = '1.
|
|
65
|
+
export const VERSION = '1.7.0';
|
|
64
66
|
|
|
65
67
|
const U32_MAX = 4294967295;
|
|
66
68
|
// Post-finalize sentinel for _recordDepth. Chosen = 2 so every post-finalize
|
|
@@ -253,8 +255,8 @@ export class Writer {
|
|
|
253
255
|
this._schema = null; // { fields: [{name, laneKind, offsetInRow}], rowStride }
|
|
254
256
|
this._fieldNamesUtf8 = null;
|
|
255
257
|
this._fieldNameHashes = null;
|
|
256
|
-
this._fieldLaneKinds = null; // Uint8Array
|
|
257
|
-
this._fieldOffsets = null; // Uint16Array
|
|
258
|
+
this._fieldLaneKinds = null; // Uint8Array -- per-field LANE_* (fast dispatch)
|
|
259
|
+
this._fieldOffsets = null; // Uint16Array -- per-field byte offset in row
|
|
258
260
|
this._rowValueSlotsF64 = null; // Float64Array<fieldCount> scratch
|
|
259
261
|
this._rowValueSlotsU32 = null; // Uint32Array<fieldCount> scratch
|
|
260
262
|
|
|
@@ -336,7 +338,7 @@ export class Writer {
|
|
|
336
338
|
} else {
|
|
337
339
|
// Sample window: decode to a JS string NOW. The tokenizer will reuse this
|
|
338
340
|
// buffer for the value bytes before onNumber/onString fires, so we cannot
|
|
339
|
-
// defer decoding. String allocation here is expected
|
|
341
|
+
// defer decoding. String allocation here is expected -- bounded to the
|
|
340
342
|
// sample window; steady-state post-freeze remains zero-alloc.
|
|
341
343
|
this._currentKeyName = KEY_DECODER.decode(bytes.subarray(from, to));
|
|
342
344
|
}
|
package/src/index.js
CHANGED
|
@@ -8,6 +8,10 @@
|
|
|
8
8
|
// Set opts.preserve = true. Bytes in, same bytes out.
|
|
9
9
|
// deserialize() auto-detects the mode via the container's flag bit and
|
|
10
10
|
// returns the appropriate Reader.
|
|
11
|
+
//
|
|
12
|
+
// Error codes:
|
|
13
|
+
// E_OPTION_CONFLICT - serialize() opts combine preserve with an incompatible framing
|
|
14
|
+
// R_TRUNCATED - deserialize() container too small to inspect header flags
|
|
11
15
|
|
|
12
16
|
import { Tokenizer, TokenizerError } from './Tokenizer.js';
|
|
13
17
|
import { Writer, WriterError } from './Writer.js';
|
|
@@ -28,7 +32,7 @@ export {
|
|
|
28
32
|
PreserveWriter, PreserveWriterError,
|
|
29
33
|
PreserveReader, PreserveReaderError,
|
|
30
34
|
};
|
|
31
|
-
export const VERSION = '1.
|
|
35
|
+
export const VERSION = '1.7.0';
|
|
32
36
|
|
|
33
37
|
const encoder = new TextEncoder();
|
|
34
38
|
|
package/types/MultiReader.d.ts
CHANGED
|
@@ -24,7 +24,7 @@ export class MultiReader {
|
|
|
24
24
|
/** Which sub-Reader owns a given global shard? */
|
|
25
25
|
readerForShard(globalShardIdx: number): { readerIdx: number; localShard: number } | null;
|
|
26
26
|
|
|
27
|
-
// Zone maps
|
|
27
|
+
// Zone maps -- global shard indices merged across sub-readers
|
|
28
28
|
shardBounds(globalShardIdx: number, fieldName: string): Bounds | null;
|
|
29
29
|
findShards(fieldName: string, opts?: FindShardsOptions): number[];
|
|
30
30
|
}
|
package/types/RangeReader.d.ts
CHANGED
|
@@ -8,7 +8,12 @@ export const VERSION: string;
|
|
|
8
8
|
|
|
9
9
|
export interface IOAdapter {
|
|
10
10
|
size: number;
|
|
11
|
-
|
|
11
|
+
/**
|
|
12
|
+
* `signal` is ADDITIVE (BS-28): a 2-arg adapter that ignores it degrades
|
|
13
|
+
* CANCELLATION only, never correctness -- the reader-side fail-closed triangle
|
|
14
|
+
* guarantees the outcome regardless of whether the adapter honoured the signal.
|
|
15
|
+
*/
|
|
16
|
+
fetch(byteOffset: number, byteLength: number, signal?: AbortSignal): Promise<Uint8Array>;
|
|
12
17
|
}
|
|
13
18
|
|
|
14
19
|
export interface AdapterStats {
|
|
@@ -17,11 +22,11 @@ export interface AdapterStats {
|
|
|
17
22
|
}
|
|
18
23
|
|
|
19
24
|
export class HTTPRangeAdapter implements IOAdapter {
|
|
20
|
-
static open(url: string, opts?: { fetch?: typeof fetch }): Promise<HTTPRangeAdapter>;
|
|
25
|
+
static open(url: string, opts?: { fetch?: typeof fetch; signal?: AbortSignal }): Promise<HTTPRangeAdapter>;
|
|
21
26
|
readonly url: string;
|
|
22
27
|
size: number;
|
|
23
28
|
stats: AdapterStats;
|
|
24
|
-
fetch(byteOffset: number, byteLength: number): Promise<Uint8Array>;
|
|
29
|
+
fetch(byteOffset: number, byteLength: number, signal?: AbortSignal): Promise<Uint8Array>;
|
|
25
30
|
}
|
|
26
31
|
|
|
27
32
|
export class MockRangeAdapter implements IOAdapter {
|
|
@@ -29,8 +34,12 @@ export class MockRangeAdapter implements IOAdapter {
|
|
|
29
34
|
size: number;
|
|
30
35
|
/** Fetch log: [[offset, length], ...] for test assertions. */
|
|
31
36
|
log: [number, number][];
|
|
37
|
+
/** Observation only (BS-28): index-aligned with `log`, null when no signal forwarded. */
|
|
38
|
+
signals: (AbortSignal | null)[];
|
|
39
|
+
/** Observation only (BS-28): the most recently forwarded signal (or null). */
|
|
40
|
+
lastSignal: AbortSignal | null;
|
|
32
41
|
stats: AdapterStats;
|
|
33
|
-
fetch(byteOffset: number, byteLength: number): Promise<Uint8Array>;
|
|
42
|
+
fetch(byteOffset: number, byteLength: number, signal?: AbortSignal): Promise<Uint8Array>;
|
|
34
43
|
}
|
|
35
44
|
|
|
36
45
|
export interface RangeReaderOptions {
|
|
@@ -38,6 +47,13 @@ export interface RangeReaderOptions {
|
|
|
38
47
|
maxCachedShards?: number;
|
|
39
48
|
/** Verify the footer CRC-32C at open; rejects R_BAD_CRC on mismatch, R_CRC_ABSENT if absent. */
|
|
40
49
|
verifyCrc?: boolean;
|
|
50
|
+
/**
|
|
51
|
+
* Reader-level cancellation (BS-28). One signal governs the reader's whole
|
|
52
|
+
* lifetime. Once it fires the reader is permanently DEAD for NEW I/O -- every
|
|
53
|
+
* fetch throws R_ABORTED. Already-cached shards stay readable through
|
|
54
|
+
* syncRange. A remount makes a new reader with a new signal (design A).
|
|
55
|
+
*/
|
|
56
|
+
signal?: AbortSignal;
|
|
41
57
|
}
|
|
42
58
|
|
|
43
59
|
export interface RangeShardHandle {
|
|
@@ -86,7 +102,7 @@ export class RangeReader {
|
|
|
86
102
|
get(rowIdx: number, fieldName: string): number | string | undefined;
|
|
87
103
|
};
|
|
88
104
|
|
|
89
|
-
// Zone maps (M7)
|
|
105
|
+
// Zone maps (M7) -- synchronous, populated at open()
|
|
90
106
|
shardBounds(shardIdx: number, fieldName: string): Bounds | null;
|
|
91
107
|
findShards(fieldName: string, opts?: FindShardsOptions): number[];
|
|
92
108
|
}
|
package/types/Reader.d.ts
CHANGED
|
@@ -63,7 +63,7 @@ export class Reader {
|
|
|
63
63
|
fieldIndex(name: string): number;
|
|
64
64
|
|
|
65
65
|
/**
|
|
66
|
-
* Get value at (rowIdx, fieldName). F64 lane
|
|
66
|
+
* Get value at (rowIdx, fieldName). F64 lane -> number; U32 lane -> string
|
|
67
67
|
* (resolved via the shard's local string table).
|
|
68
68
|
*/
|
|
69
69
|
get(rowIdx: number, fieldName: string): number | string | undefined;
|
package/types/Split.d.ts
CHANGED
|
@@ -28,7 +28,7 @@ export interface CompileInPartsOptions extends SplitOptions, CompilePartOptions
|
|
|
28
28
|
|
|
29
29
|
/**
|
|
30
30
|
* Divide NDJSON bytes into N ranges at safe line boundaries. Every returned
|
|
31
|
-
* range contains complete records
|
|
31
|
+
* range contains complete records -- no split mid-line. The union of ranges
|
|
32
32
|
* equals the original bytes (no gaps, no overlaps).
|
|
33
33
|
*/
|
|
34
34
|
export function splitNDJSON(bytes: Uint8Array, opts?: SplitOptions): SplitRange[];
|
package/types/Tokenizer.d.ts
CHANGED
|
@@ -5,7 +5,7 @@ export const VERSION: string;
|
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
7
|
* Sink protocol the Tokenizer emits into. Byte ranges (bytes[from, to)) are
|
|
8
|
-
* ephemeral
|
|
8
|
+
* ephemeral -- valid only for the duration of the call. Consumers that need
|
|
9
9
|
* to retain content must copy or decode inside the sink method.
|
|
10
10
|
*/
|
|
11
11
|
export interface TokenizerSink {
|
package/types/Writer.d.ts
CHANGED
|
@@ -78,7 +78,7 @@ export interface Container {
|
|
|
78
78
|
export class Writer implements TokenizerSinkForWriter {
|
|
79
79
|
constructor(opts?: WriterOptions);
|
|
80
80
|
/**
|
|
81
|
-
* Sink protocol
|
|
81
|
+
* Sink protocol -- the Tokenizer calls these. Byte ranges are ephemeral.
|
|
82
82
|
*/
|
|
83
83
|
onStartObject(): void;
|
|
84
84
|
onEndObject(): void;
|
|
@@ -127,7 +127,7 @@ export class Writer implements TokenizerSinkForWriter {
|
|
|
127
127
|
readonly shardCount: number;
|
|
128
128
|
}
|
|
129
129
|
|
|
130
|
-
// Structural type
|
|
130
|
+
// Structural type -- the Writer conforms to the Tokenizer's sink protocol.
|
|
131
131
|
// (Redeclared here so the /writer subpath is self-contained.)
|
|
132
132
|
interface TokenizerSinkForWriter {
|
|
133
133
|
onStartObject(): void;
|