@elaraai/east 1.0.69 → 1.0.71

package/dist/src/serialization/beast2/v5/range.js

@@ -0,0 +1,252 @@
+/**
+ * Copyright (c) 2025 Elara AI Pty Ltd
+ * Dual-licensed under AGPL-3.0 and commercial license. See LICENSE for details.
+ */
+/**
+ * Beast2 v5 ranged access — reading a segmented, indexed blob's geometry
+ * through positioned byte reads instead of a whole buffer.
+ *
+ * A v5 collection blob is `header · tag frame · segment frames · terminator
+ * frame · index · footer`, so its whole geometry is recoverable from two
+ * ranged reads: the tail (footer + index) and the head (header sections +
+ * tag frame). Every segment frame's byte range is then known up front, and a
+ * paging reader fetches only the frames it decodes: an Array index read or
+ * a Set/Dict keyed read touches one frame, iteration touches each frame
+ * once. The exception is a Set/Dict root's FIRST keyed read, which verifies
+ * the segment fences by probing every frame once (as east-c does) — a
+ * bounded prefix of each frame, read and inflated as far as it goes, never
+ * the frame unless its first key is wider than the probe. I/O is O(header +
+ * index + touched frames) in every case; peak memory is O(segment).
+ *
+ * Two reader shapes serve two consumers: {@link Beast2RangeReader} (async —
+ * an HTTP range request, an S3 ranged GET, e3's dataset paging) and
+ * {@link Beast2SyncRangeReader} (sync — a file descriptor with positioned
+ * reads, a memory mapping, a plain `Uint8Array`), which is what a compiled
+ * East body drives through `Beast2Pages` and `openBeast2LazyFor`. Core stays
+ * free of any file system: the reader is injected.
+ */
+import {} from "../../../type_of_type.js";
+import { BufferReader } from "../../binary-utils.js";
+import { readTypeSection } from "./type-section.js";
+import { MAGIC_BYTES_V5, FOOTER_MAGIC_V5, INDEX_FLAG_SELF_CONTAINED, readSourceMapSectionV5, isSegmentedRoot, } from "./codec.js";
+/** The exact bytes of the root NEW tag frame and of the terminator frame:
+ *  codec `none`, one logical byte, payload `0x00`. */
+export const TAG_OR_TERMINATOR_FRAME = new Uint8Array([0x00, 0x01, 0x01, 0x00]);
+/** Whether `source` is a {@link Beast2SyncRangeReader} rather than a blob —
+ *  judged by shape, since a typed array from another realm fails
+ *  `instanceof Uint8Array`. */
+export function isBeast2SyncRangeReader(source) {
+    const r = source;
+    return r !== null && typeof r === "object" && typeof r.read === "function" && typeof r.size === "number";
+}
+/**
+ * Reads exactly `length` bytes at `offset` through a sync reader, refusing a
+ * short result up front: a reader that returns fewer bytes (a bare
+ * `readSync` without the retry loop) would otherwise be reported as a
+ * malformed blob.
+ *
+ * @param reader - the reader
+ * @param offset - absolute byte offset
+ * @param length - number of bytes
+ * @returns exactly the requested bytes
+ * @throws {Error} When the reader returns a different number of bytes.
+ */
+export function readExact(reader, offset, length) {
+    const bytes = reader.read(offset, length);
+    if (bytes.length !== length) {
+        throw new Error(`beast2 v5: reader returned ${bytes.length} bytes for a ${length}-byte range at offset ${offset}`);
+    }
+    return bytes;
+}
+/** Whether the 4 bytes at `offset` are a tag/terminator frame. */
+export function isTagOrTerminatorFrame(data, offset) {
+    if (offset + 4 > data.length)
+        return false;
+    for (let i = 0; i < 4; i++) {
+        if (data[offset + i] !== TAG_OR_TERMINATOR_FRAME[i])
+            return false;
+    }
+    return true;
+}
+/** Parses the u64 little-endian index offset from 8 bytes at `at`. */
+export function readU64LE(data, at) {
+    let value = 0n;
+    for (let i = 7; i >= 0; i--) {
+        value = (value << 8n) | BigInt(data[at + i]);
+    }
+    return Number(value);
+}
+/** The size of the initial tail read: the footer plus a probe of the index. */
+function tailProbeSize(size, options) {
+    return Math.min(size, Math.max(options?.tailProbeBytes ?? 64 * 1024, 16));
+}
+/** Reads the footer from a tail read holding `[tailStart, size)`: the index
+ *  offset, validated against the blob size. */
+function footerIndexOffset(tail, tailStart, size) {
+    const footerStart = size - 16;
+    for (let i = 0; i < 8; i++) {
+        if (tail[footerStart + 8 + i - tailStart] !== FOOTER_MAGIC_V5[i]) {
+            throw new Error(`beast2 v5: blob carries no index — ranged reads need one (write with the index enabled, the default)`);
+        }
+    }
+    const indexOffset = readU64LE(tail, footerStart - tailStart);
+    if (indexOffset < MAGIC_BYTES_V5.length || indexOffset >= footerStart) {
+        throw new Error(`beast2 v5: footer index offset ${indexOffset} out of range`);
+    }
+    return indexOffset;
+}
+/** Parses the index section and checks the terminator frame from a tail
+ *  read holding `[tailStart, size)` that reaches back to the terminator. */
+function parseIndexTail(tail, tailStart, size, indexOffset) {
+    const footerStart = size - 16;
+    const segmentsEnd = indexOffset - TAG_OR_TERMINATOR_FRAME.length;
+    const indexReader = new BufferReader(tail, indexOffset - tailStart);
+    const flags = indexReader.readVarint();
+    if ((flags & ~INDEX_FLAG_SELF_CONTAINED) !== 0) {
+        throw new Error(`beast2 v5: unknown index flags 0x${flags.toString(16)}`);
+    }
+    const segmentCount = indexReader.readVarint();
+    const offsets = new Array(segmentCount);
+    const counts = new Array(segmentCount);
+    let prev = 0;
+    let elementCount = 0;
+    for (let i = 0; i < segmentCount; i++) {
+        prev += indexReader.readVarint();
+        offsets[i] = prev;
+        counts[i] = indexReader.readVarint();
+        elementCount += counts[i];
+        if (prev >= indexOffset) {
+            throw new Error(`beast2 v5: index segment offset ${prev} overlaps the index section`);
+        }
+    }
+    if (indexReader.offset !== footerStart - tailStart) {
+        throw new Error(`beast2 v5: index section size mismatch (ends at ${indexReader.offset + tailStart}, footer at ${footerStart})`);
+    }
+    if (!isTagOrTerminatorFrame(tail, segmentsEnd - tailStart)) {
+        throw new Error(`beast2 v5: terminator frame not found where expected (offset ${segmentsEnd})`);
+    }
+    return { segmentsEnd, offsets, counts, elementCount, selfContained: (flags & INDEX_FLAG_SELF_CONTAINED) !== 0 };
+}
+/** Parses the header sections + root tag frame from a head read holding
+ *  `[0, prefixEnd)`: the v5 magic, a collection root type, the source-map
+ *  section, and the tag frame ending exactly at `prefixEnd`. */
+function parseHead(head, prefixEnd) {
+    for (let i = 0; i < 8; i++) {
+        if (head[i] !== MAGIC_BYTES_V5[i]) {
+            if (i === 7 && head[i] === 0x04) {
+                throw new Error(`beast2 v5: ranged reads need v5 blobs; this is a v4 container (re-encode with version 5)`);
+            }
+            throw new Error(`beast2 v5: not a beast2 v5 container`);
+        }
+    }
+    const headReader = new BufferReader(head, MAGIC_BYTES_V5.length);
+    const { rootType } = readTypeSection(headReader);
+    if (!isSegmentedRoot(rootType)) {
+        throw new Error(`beast2 v5: ranged reads address Array, Set or Dict roots, not ${rootType.type}`);
+    }
+    const sourceMap = readSourceMapSectionV5(headReader);
+    const frameOffset = headReader.offset;
+    if (frameOffset + 4 !== prefixEnd || !isTagOrTerminatorFrame(head, frameOffset)) {
+        throw new Error(`beast2 v5: root tag frame not found where expected (offset ${frameOffset})`);
+    }
+    return { rootType, sourceMapEmpty: sourceMap.size <= 1n };
+}
+/** Assembles the extents from the parsed tail and head. */
+function assembleExtents(size, indexOffset, index, prefixEnd, head) {
+    const { rootType, sourceMapEmpty } = parseHead(head, prefixEnd);
+    return {
+        prefixEnd,
+        segmentsEnd: index.segmentsEnd,
+        indexOffset,
+        offsets: index.offsets,
+        counts: index.counts,
+        elementCount: index.elementCount,
+        selfContained: index.selfContained,
+        sourceMapEmpty,
+        typeValue: rootType,
+        size,
+        head,
+    };
+}
+/**
+ * Reads the byte geometry of a segmented, indexed v5 collection blob through
+ * ranged access — the footer and index from a tail read, the header sections
+ * from a head read — without ever buffering the blob whole.
+ *
+ * The result carries everything `carveBeast2Ranged` needs to assemble a
+ * standalone blob for any segment span whose frame bytes are then the only
+ * further reads: total I/O for a window is O(header + index + window), not
+ * O(blob).
+ *
+ * @param reader - ranged access to the blob
+ * @param options - tail-probe tuning
+ * @returns the blob's {@link Beast2RangedExtents}
+ * @throws {Error} When the blob is not a v5 container, its root is not a
+ *   collection, it carries no index, or the frame geometry is malformed —
+ *   the same conditions `readBeast2Extents` rejects.
+ */
+export async function readBeast2ExtentsRanged(reader, options) {
+    const size = reader.size;
+    if (size < MAGIC_BYTES_V5.length + 16) {
+        throw new Error(`Data too short for Beast2 format: ${size} bytes`);
+    }
+    // Tail: footer first, then the index section (one further read when the
+    // probe missed its start).
+    let tailStart = size - tailProbeSize(size, options);
+    let tail = checkedLength(await reader.read(tailStart, size - tailStart), tailStart, size - tailStart);
+    const indexOffset = footerIndexOffset(tail, tailStart, size);
+    const segmentsEnd = indexOffset - TAG_OR_TERMINATOR_FRAME.length;
+    if (segmentsEnd < tailStart) {
+        tailStart = segmentsEnd;
+        tail = checkedLength(await reader.read(tailStart, size - tailStart), tailStart, size - tailStart);
+    }
+    const index = parseIndexTail(tail, tailStart, size, indexOffset);
+    // Head: header sections + root tag frame, ending exactly where the first
+    // segment starts (or at the terminator, for an empty blob).
+    const prefixEnd = index.offsets.length > 0 ? index.offsets[0] : segmentsEnd;
+    const head = checkedLength(await reader.read(0, prefixEnd), 0, prefixEnd);
+    return assembleExtents(size, indexOffset, index, prefixEnd, head);
+}
+/** The exact-length check {@link readExact} applies, for an awaited read. */
+function checkedLength(bytes, offset, length) {
+    if (bytes.length !== length) {
+        throw new Error(`beast2 v5: reader returned ${bytes.length} bytes for a ${length}-byte range at offset ${offset}`);
+    }
+    return bytes;
+}
+/**
+ * The synchronous twin of {@link readBeast2ExtentsRanged}: the same two
+ * ranged reads (tail, then head) through a {@link Beast2SyncRangeReader},
+ * for the paging readers a compiled body drives without awaiting.
+ *
+ * @param reader - synchronous ranged access to the blob
+ * @param options - tail-probe tuning
+ * @returns the blob's {@link Beast2RangedExtents}
+ * @throws {Error} When the blob is not a v5 container, its root is not a
+ *   collection, it carries no index, or the frame geometry is malformed.
+ */
+export function readBeast2ExtentsSync(reader, options) {
+    const size = reader.size;
+    if (size < MAGIC_BYTES_V5.length + 16) {
+        throw new Error(`Data too short for Beast2 format: ${size} bytes`);
+    }
+    let tailStart = size - tailProbeSize(size, options);
+    let tail = readExact(reader, tailStart, size - tailStart);
+    const indexOffset = footerIndexOffset(tail, tailStart, size);
+    const segmentsEnd = indexOffset - TAG_OR_TERMINATOR_FRAME.length;
+    if (segmentsEnd < tailStart) {
+        tailStart = segmentsEnd;
+        tail = readExact(reader, tailStart, size - tailStart);
+    }
+    const index = parseIndexTail(tail, tailStart, size, indexOffset);
+    const prefixEnd = index.offsets.length > 0 ? index.offsets[0] : segmentsEnd;
+    const head = readExact(reader, 0, prefixEnd);
+    return assembleExtents(size, indexOffset, index, prefixEnd, head);
+}
+/** A {@link Beast2SyncRangeReader} over a whole in-memory blob: reads are
+ *  zero-copy views. What the `Uint8Array` entry points wrap, so every paging
+ *  reader has one source shape. */
+export function bytesReader(data) {
+    return { size: data.length, read: (offset, length) => data.subarray(offset, offset + length) };
+}
+//# sourceMappingURL=range.js.map

package/dist/src/serialization/beast2/v5/range.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"range.js","sourceRoot":"","sources":["../../../../../src/serialization/beast2/v5/range.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAsB,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EACL,cAAc,EACd,eAAe,EACf,yBAAyB,EACzB,sBAAsB,EACtB,eAAe,GAChB,MAAM,YAAY,CAAC;AAEpB;sDACsD;AACtD,MAAM,CAAC,MAAM,uBAAuB,GAAG,IAAI,UAAU,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;AAyHhF;;+BAE+B;AAC/B,MAAM,UAAU,uBAAuB,CAAC,MAAe;IACrD,MAAM,CAAC,GAAG,MAAmD,CAAC;IAC9D,OAAO,CAAC,KAAK,IAAI,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,UAAU,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC;AAC3G,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,SAAS,CAAC,MAA6B,EAAE,MAAc,EAAE,MAAc;IACrF,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1C,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,8BAA8B,KAAK,CAAC,MAAM,gBAAgB,MAAM,yBAAyB,MAAM,EAAE,CAAC,CAAC;IACrH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,sBAAsB,CAAC,IAAgB,EAAE,MAAc;IACrE,IAAI,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,uBAAuB,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;IACpE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,sEAAsE;AACtE,MAAM,UAAU,SAAS,CAAC,IAAgB,EAAE,EAAU;IACpD,IAAI,KAAK,GAAG,EAAE,CAAC;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,KAAK,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAE,CAAC,CAAC;IAChD,CAAC;IACD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;AACvB,CAAC;AAED,+EAA+E;AAC/E,SAAS,aAAa,CAAC,IAAY,EAAE,OAAmD;IACtF,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,cAAc,IAAI,EAAE,GAAG,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;AAC5E,CAAC;AAED;+CAC+C;AAC/C,SAAS,iBAAiB,CAAC,IAAgB,EAAE,SAAiB,EAAE,IAAY;IAC1E,MAAM,WAAW,GAAG,IAAI,GAAG,EAAE,CAAC;IAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,IAAI,IAAI,CAAC,WAAW,GAAG,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,KAAK,eAAe,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,sGAAsG,CAAC,CAAC;QAC1H,CAAC;IACH,CAAC;IACD,MAAM,WAAW,GAAG,SAAS,CAAC,IAAI,EAAE,WAAW,GAAG,SAAS,CAAC,CAAC;IAC7D,IAAI,WAAW,GAAG,cAAc,CAAC,MAAM,IAAI,WAAW,IAAI,WAAW,EAAE,CAAC;QACtE,MAAM,IAAI,KAAK,CAAC,kCAAkC,WAAW,eAAe,CAAC,CAAC;IAChF,CAAC;IACD,OAAO,WAAW,CAAC;AACrB,CAAC;AAYD;4EAC4E;AAC5E,SAAS,cAAc,CAAC,IAAgB,EAAE,SAAiB,EAAE,IAAY,EAAE,WAAmB;IAC5F,MAAM,WAAW,GAAG,IAAI,GAAG,EAAE,CAAC;IAC9B,MAAM,WAAW,GAAG,WAAW,GAAG,uBAAuB,CAAC,MAAM,CAAC;IACjE,MAAM,WAAW,GAAG,IAAI,YAAY,CAAC,IAAI,EAAE,WAAW,GAAG,SAAS,CAAC,CAAC;IACpE,MAAM,KAAK,GAAG,WAAW,CAAC,UAAU,EAAE,CAAC;IACvC,IAAI,CAAC,KAAK,GAAG,CAAC,yBAAyB,CAAC,KAAK,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CAAC,oCAAoC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IAC5E,CAAC;IACD,MAAM,YAAY,GAAG,WAAW,CAAC,UAAU,EAAE,CAAC;IAC9C,MAAM,OAAO,GAAa,IAAI,KAAK,CAAC,YAAY,CAAC,CAAC;IAClD,MAAM,MAAM,GAAa,IAAI,KAAK,CAAC,YAAY,CAAC,CAAC;IACjD,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,IAAI,IAAI,WAAW,CAAC,UAAU,EAAE,CAAC;QACjC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;QAClB,MAAM,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,UAAU,EAAE,CAAC;QACrC,YAAY,IAAI,MAAM,CAAC,CAAC,CAAE,CAAC;QAC3B,IAAI,IAAI,IAAI,WAAW,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,mCAAmC,IAAI,6BAA6B,CAAC,CAAC;QACxF,CAAC;IACH,CAAC;IACD,IAAI,WAAW,CAAC,MAAM,KAAK,WAAW,GAAG,SAAS,EAAE,CAAC;QACnD,MAAM,IAAI,KAAK,CAAC,mDAAmD,WAAW,CAAC,MAAM,GAAG,SAAS,eAAe,WAAW,GAAG,CAAC,CAAC;IAClI,CAAC;IACD,IAAI,CAAC,sBAAsB,CAAC,IAAI,EAAE,WAAW,GAAG,SAAS,CAAC,EAAE,CAAC;QAC3D,MAAM,IAAI,KAAK,CAAC,gEAAgE,WAAW,GAAG,CAAC,CAAC;IAClG,CAAC;IACD,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,CAAC,KAAK,GAAG,yBAAyB,CAAC,KAAK,CAAC,EAAE,CAAC;AAClH,CAAC;AAED;;gEAEgE;AAChE,SAAS,SAAS,CAAC,IAAgB,EAAE,SAAiB;IACpD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC;YAClC,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;gBAChC,MAAM,IAAI,KAAK,CAAC,0FAA0F,CAAC,CAAC;YAC9G,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,YAAY,CAAC,IAAI,EAAE,cAAc,CAAC,MAAM,CAAC,CAAC;IACjE,MAAM,EAAE,QAAQ,EAAE,GAAG,eAAe,CAAC,UAAU,CAAC,CAAC;IACjD,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,iEAAiE,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IACpG,CAAC;IACD,MAAM,SAAS,GAAG,sBAAsB,CAAC,UAAU,CAAC,CAAC;IACrD,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC;IACtC,IAAI,WAAW,GAAG,CAAC,KAAK,SAAS,IAAI,CAAC,sBAAsB,CAAC,IAAI,EAAE,WAAW,CAAC,EAAE,CAAC;QAChF,MAAM,IAAI,KAAK,CAAC,8DAA8D,WAAW,GAAG,CAAC,CAAC;IAChG,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,SAAS,CAAC,IAAI,IAAI,EAAE,EAAE,CAAC;AAC5D,CAAC;AAED,2DAA2D;AAC3D,SAAS,eAAe,CAAC,IAAY,EAAE,WAAmB,EAAE,KAAkB,EAAE,SAAiB,EAAE,IAAgB;IACjH,MAAM,EAAE,QAAQ,EAAE,cAAc,EAAE,GAAG,SAAS,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IAChE,OAAO;QACL,SAAS;QACT,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,WAAW;QACX,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,aAAa,EAAE,KAAK,CAAC,aAAa;QAClC,cAAc;QACd,SAAS,EAAE,QAAQ;QACnB,IAAI;QACJ,IAAI;KACL,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,MAAyB,EAAE,OAAwC;IAC/G,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;IACzB,IAAI,IAAI,GAAG,cAAc,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,qCAAqC,IAAI,QAAQ,CAAC,CAAC;IACrE,CAAC;IACD,wEAAwE;IACxE,2BAA2B;IAC3B,IAAI,SAAS,GAAG,IAAI,GAAG,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACpD,IAAI,IAAI,GAAG,aAAa,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,GAAG,SAAS,CAAC,EAAE,SAAS,EAAE,IAAI,GAAG,SAAS,CAAC,CAAC;IACtG,MAAM,WAAW,GAAG,iBAAiB,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;IAC7D,MAAM,WAAW,GAAG,WAAW,GAAG,uBAAuB,CAAC,MAAM,CAAC;IACjE,IAAI,WAAW,GAAG,SAAS,EAAE,CAAC;QAC5B,SAAS,GAAG,WAAW,CAAC;QACxB,IAAI,GAAG,aAAa,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,GAAG,SAAS,CAAC,EAAE,SAAS,EAAE,IAAI,GAAG,SAAS,CAAC,CAAC;IACpG,CAAC;IACD,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;IACjE,yEAAyE;IACzE,4DAA4D;IAC5D,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,WAAW,CAAC;IAC7E,MAAM,IAAI,GAAG,aAAa,CAAC,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,SAAS,CAAC,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC;IAC1E,OAAO,eAAe,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;AACpE,CAAC;AAED,6EAA6E;AAC7E,SAAS,aAAa,CAAC,KAAiB,EAAE,MAAc,EAAE,MAAc;IACtE,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,8BAA8B,KAAK,CAAC,MAAM,gBAAgB,MAAM,yBAAyB,MAAM,EAAE,CAAC,CAAC;IACrH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,qBAAqB,CAAC,MAA6B,EAAE,OAAwC;IAC3G,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;IACzB,IAAI,IAAI,GAAG,cAAc,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,qCAAqC,IAAI,QAAQ,CAAC,CAAC;IACrE,CAAC;IACD,IAAI,SAAS,GAAG,IAAI,GAAG,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACpD,IAAI,IAAI,GAAG,SAAS,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,GAAG,SAAS,CAAC,CAAC;IAC1D,MAAM,WAAW,GAAG,iBAAiB,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;IAC7D,MAAM,WAAW,GAAG,WAAW,GAAG,uBAAuB,CAAC,MAAM,CAAC;IACjE,IAAI,WAAW,GAAG,SAAS,EAAE,CAAC;QAC5B,SAAS,GAAG,WAAW,CAAC;QACxB,IAAI,GAAG,SAAS,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,GAAG,SAAS,CAAC,CAAC;IACxD,CAAC;IACD,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;IACjE,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,WAAW,CAAC;IAC7E,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC;IAC7C,OAAO,eAAe,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;AACpE,CAAC;AAED;;mCAEmC;AACnC,MAAM,UAAU,WAAW,CAAC,IAAgB;IAC1C,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC;AACjG,CAAC"}

package/dist/src/serialization/beast2/v5/stream.d.ts

@@ -24,6 +24,7 @@ import type { EastType, ValueTypeOf } from "../../../types.js";
 import { SourceMap } from "../../../location.js";
 import { type Beast2DecodeOptions } from "../shared.js";
 import { type Beast2Codec } from "./frames.js";
+import { type Beast2SyncRangeReader } from "./range.js";
 /** Options accepted by {@link Beast2Writer} and {@link encodeBeast2SegmentsFor}. */
 export type Beast2WriterOptions = {
     /** Per-frame codec. Defaults to `"deflate"`. */
@@ -191,6 +192,15 @@ export declare function iterBeast2SegmentsFor<T extends EastType>(type: T | East
  * small LRU, so a read loop over neighbouring rows decodes each segment once
  * rather than once per element; {@link segment} itself always decodes fresh.
  *
+ * The blob is either a whole `Uint8Array` or a {@link Beast2SyncRangeReader}:
+ * through a reader the open reads only the tail (footer + index) and the
+ * head (header sections), and every segment read fetches exactly that
+ * segment's frame — a file-backed reader keeps the wire bytes in the page
+ * cache rather than on the heap. A Set/Dict root's first keyed read verifies
+ * the fences, probing a bounded prefix of every frame once (never the frame,
+ * unless its first key is wider than the probe); later keyed reads fetch one
+ * frame.
+ *
  * Set/Dict blobs page like Arrays: the wire holds the canonical value split
  * at segment boundaries (strictly ascending, disjoint segments), so row
  * windows ({@link slice}) and key lookups ({@link get}) address the sorted
@@ -209,7 +219,9 @@ export declare class Beast2Pages<T extends EastType = EastType> {
     readonly elementCount: number;
     /** Whether segments are independently decodable. */
     readonly selfContained: boolean;
-    private readonly data;
+    private readonly source;
+    /** Wire offset of the terminator frame — where the last segment's frame ends. */
+    private readonly segmentsEnd;
     private readonly indexData;
     private readonly kind;
     private readonly typeValue;
@@ -230,9 +242,14 @@ export declare class Beast2Pages<T extends EastType = EastType> {
      *  container walk. */
     private readonly segmentCache;
     /** @internal Use {@link openBeast2PagesFor}. */
-    constructor(data: Uint8Array, typeValue: EastTypeValue, options?: Beast2DecodeOptions);
+    constructor(source: Uint8Array | Beast2SyncRangeReader, typeValue: EastTypeValue, options?: Beast2DecodeOptions);
     /** Number of segments in the blob. */
     get segmentCount(): number;
+    /** Reads segment `i`'s frame — exactly its wire bytes, from its index
+     *  offset to the next segment's (or the terminator) — and opens its
+     *  logical chunk. The only place segment bytes are fetched, so a ranged
+     *  source touches one frame per decode. */
+    private frameReader;
     /**
      * Decodes one segment by index.
      *
@@ -256,7 +273,10 @@ export declare class Beast2Pages<T extends EastType = EastType> {
      *  only — see {@link segmentCache}. */
     private segmentCached;
     /** Decodes just the first key/element of segment `i` — a bounded probe
-     *  (one frame inflate, one element decode), not a whole-segment decode. */
+     *  (a prefix of the frame read and inflated, one element decode), not a
+     *  whole-segment decode. A first key wider than the probe grows it; the
+     *  final attempt reads the frame whole, so corruption is still reported
+     *  with the frame's own error. */
     private firstKey;
     /**
      * Probes segment `i`'s fence: its first Dict key, Set element, or Array
@@ -326,13 +346,17 @@ export declare class Beast2Pages<T extends EastType = EastType> {
     get(key: ValueTypeOf<T> extends Map<infer K, any> ? K : ValueTypeOf<T> extends Set<infer E> ? E : never): (ValueTypeOf<T> extends Map<any, infer V> ? V : ValueTypeOf<T> extends Set<infer E> ? E : never) | undefined;
 }
 /**
- * Builds a curried pages opener: `open(data)` parses the header, footer and
+ * Builds a curried pages opener: `open(source)` parses the header, footer and
  * index once and returns a {@link Beast2Pages} for random access.
  *
+ * `source` is the whole blob, or a {@link Beast2SyncRangeReader} over it —
+ * then only the tail, the head and the segments actually read are ever
+ * fetched.
+ *
  * @param type - the collection type (Array/Set/Dict)
  * @param options - decode options (platform functions for decoded functions)
  * @returns a function opening a blob for paged reads
  * @throws {TypeError} When `type` is not an Array, Set or Dict type.
  */
-export declare function openBeast2PagesFor<T extends EastType>(type: T | EastTypeValue, options?: Beast2DecodeOptions): (data: Uint8Array) => Beast2Pages<T>;
+export declare function openBeast2PagesFor<T extends EastType>(type: T | EastTypeValue, options?: Beast2DecodeOptions): (source: Uint8Array | Beast2SyncRangeReader) => Beast2Pages<T>;
 //# sourceMappingURL=stream.d.ts.map

package/dist/src/serialization/beast2/v5/stream.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stream.d.ts","sourceRoot":"","sources":["../../../../../src/serialization/beast2/v5/stream.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,KAAK,aAAa,EAAuC,MAAM,0BAA0B,CAAC;AACnG,OAAO,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAG/D,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AAIjD,OAAO,EAAE,KAAK,mBAAmB,EAAwB,MAAM,cAAc,CAAC;AAE9E,OAAO,EAAE,KAAK,WAAW,EAA2B,MAAM,aAAa,CAAC;AAsCxE,oFAAoF;AACpF,MAAM,MAAM,mBAAmB,GAAG;IAChC,gDAAgD;IAChD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB;6CACyC;IACzC,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;8BAC0B;IAC1B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,2EAA2E;IAC3E,SAAS,CAAC,EAAE,SAAS,GAAG,IAAI,CAAC;IAC7B;;;;6EAIyE;IACzE,YAAY,CAAC,EAAE,UAAU,CAAC;CAC3B,CAAC;AAMF;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,qBAAa,YAAY,CAAC,CAAC,SAAS,QAAQ,GAAG,QAAQ;IACrD,yCAAyC;IACzC,QAAQ,SAAK;IACb,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAgB;IACrC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA8B;IACnD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAc;IACpC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAU;IACxC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAU;IACpC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAkB;IACtC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA8C;IAC1E,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA2C;IACjE,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsC;IAC/D,OAAO,CAAC,OAAO,CAAM;IACrB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,QAAQ,CAAS;IAEzB;;;;;;OAMG;gBACS,IAAI,EAAE,CAAC,GAAG,aAAa,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,IAAI,EAAE,OAAO,CAAC,EAAE,mBAAmB;IA4DrG;;;;;;;;;;;;;;;OAeG;IACH,KAAK,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC,GAAG,IAAI;IAsBlC;;;OAGG;IACH,MAAM,IAAI,IAAI;IAcd;iFAC6E;IAC7E,OAAO,CAAC,WAAW;IAenB,OAAO,CAAC,IAAI;CAIb;AAuBD;;;;;;;;;GASG;AACH,wBAAgB,uBAAuB,CAAC,CAAC,SAAS,QAAQ,EAAE,IAAI,EAAE,CAAC,GAAG,aAAa,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK,UAAU,CAcrK;AAMD;;;kCAGkC;AAClC,eAAO,MAAM,0BAA0B,OAAQ,CAAC;AAEhD;;;4EAG4E;AAC5E,eAAO,MAAM,iCAAiC,QAAkB,CAAC;AAMjE,wDAAwD;AACxD,MAAM,MAAM,wBAAwB,GAAG;IACrC;8CAC0C;IAC1C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;qDAEiD;IACjD,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,gDAAgD;IAChD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,2EAA2E;IAC3E,SAAS,CAAC,EAAE,SAAS,GAAG,IAAI,CAAC;CAC9B,CAAC;AAEF;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,oBAAoB,CAAC,CAAC,SAAS,QAAQ,EAAE,IAAI,EAAE,CAAC,GAAG,aAAa,EAAE,OAAO,CAAC,EAAE,wBAAwB,GAAG,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC,KAAK,UAAU,CAwF3J;AAyFD;;;;;;;;;GASG;AACH,wBAAgB,qBAAqB,CAAC,CAAC,SAAS,QAAQ,EAAE,IAAI,EAAE,CAAC,GAAG,aAAa,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,CAAC,IAAI,EAAE,UAAU,KAAK,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAoCjK;AAaD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,WAAW,CAAC,CAAC,SAAS,QAAQ,GAAG,QAAQ;IACpD,wEAAwE;IACxE,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC;;0CAEsC;IACtC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,oDAAoD;IACpD,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAa;IAClC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAc;IACxC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAgB;IACrC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgB;IAC1C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAY;IACtC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAuF;IACrH,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAkC;IAC3D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAW;IACtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsC;IAC/D,2EAA2E;IAC3E,OAAO,CAAC,MAAM,CAAsB;IACpC,OAAO,CAAC,QAAQ,CAAsE;IACtF;;;;;;0BAMsB;IACtB,OAAO,CAAC,QAAQ,CAAC,YAAY,CAA0D;IAEvF,gDAAgD;gBACpC,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,OAAO,CAAC,EAAE,mBAAmB;IA0BrF,sCAAsC;IACtC,IAAI,YAAY,IAAI,MAAM,CAEzB;IAED;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC;IAKlC,4EAA4E;IAC5E,OAAO,CAAC,iBAAiB;IAqBzB;;;;2CAIuC;IACvC,OAAO,CAAC,aAAa;IA8BrB;+EAC2E;IAC3E,OAAO,CAAC,QAAQ;IAYhB;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK;IAUzJ;4DACwD;IACxD,OAAO,CAAC,YAAY;IAiBpB;;iFAE6E;IAC7E,OAAO,CAAC,cAAc;IAQtB;uEACmE;IACnE,OAAO,CAAC,UAAU;IAUlB;;;;;;;OAOG;IACH,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK;IAYpE;;;;;;;;;;;;;;;;;;OAkBG;IACH,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC;IAuDpD;;;;;;;;;;OAUG;IACH,GAAG,CACD,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,KAAK,GAClG,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,SAAS;CA4BhH;AAED;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,CAAC,SAAS,QAAQ,EAAE,IAAI,EAAE,CAAC,GAAG,aAAa,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,CAAC,IAAI,EAAE,UAAU,KAAK,WAAW,CAAC,CAAC,CAAC,CAInJ"}
+{"version":3,"file":"stream.d.ts","sourceRoot":"","sources":["../../../../../src/serialization/beast2/v5/stream.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,KAAK,aAAa,EAAuC,MAAM,0BAA0B,CAAC;AACnG,OAAO,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAG/D,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AAIjD,OAAO,EAAE,KAAK,mBAAmB,EAAwB,MAAM,cAAc,CAAC;AAE9E,OAAO,EAAE,KAAK,WAAW,EAA4C,MAAM,aAAa,CAAC;AAgBzF,OAAO,EAAE,KAAK,qBAAqB,EAA8G,MAAM,YAAY,CAAC;AAuBpK,oFAAoF;AACpF,MAAM,MAAM,mBAAmB,GAAG;IAChC,gDAAgD;IAChD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB;6CACyC;IACzC,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;8BAC0B;IAC1B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,2EAA2E;IAC3E,SAAS,CAAC,EAAE,SAAS,GAAG,IAAI,CAAC;IAC7B;;;;6EAIyE;IACzE,YAAY,CAAC,EAAE,UAAU,CAAC;CAC3B,CAAC;AAMF;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,qBAAa,YAAY,CAAC,CAAC,SAAS,QAAQ,GAAG,QAAQ;IACrD,yCAAyC;IACzC,QAAQ,SAAK;IACb,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAgB;IACrC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA8B;IACnD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAc;IACpC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAU;IACxC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAU;IACpC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAkB;IACtC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA8C;IAC1E,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA2C;IACjE,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsC;IAC/D,OAAO,CAAC,OAAO,CAAM;IACrB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,QAAQ,CAAS;IAEzB;;;;;;OAMG;gBACS,IAAI,EAAE,CAAC,GAAG,aAAa,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,IAAI,EAAE,OAAO,CAAC,EAAE,mBAAmB;IA4DrG;;;;;;;;;;;;;;;OAeG;IACH,KAAK,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC,GAAG,IAAI;IAsBlC;;;OAGG;IACH,MAAM,IAAI,IAAI;IAcd;iFAC6E;IAC7E,OAAO,CAAC,WAAW;IAenB,OAAO,CAAC,IAAI;CAIb;AAuBD;;;;;;;;;GASG;AACH,wBAAgB,uBAAuB,CAAC,CAAC,SAAS,QAAQ,EAAE,IAAI,EAAE,CAAC,GAAG,aAAa,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK,UAAU,CAcrK;AAMD;;;kCAGkC;AAClC,eAAO,MAAM,0BAA0B,OAAQ,CAAC;AAEhD;;;4EAG4E;AAC5E,eAAO,MAAM,iCAAiC,QAAkB,CAAC;AAMjE,wDAAwD;AACxD,MAAM,MAAM,wBAAwB,GAAG;IACrC;8CAC0C;IAC1C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;qDAEiD;IACjD,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,gDAAgD;IAChD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,2EAA2E;IAC3E,SAAS,CAAC,EAAE,SAAS,GAAG,IAAI,CAAC;CAC9B,CAAC;AAEF;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,oBAAoB,CAAC,CAAC,SAAS,QAAQ,EAAE,IAAI,EAAE,CAAC,GAAG,aAAa,EAAE,OAAO,CAAC,EAAE,wBAAwB,GAAG,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC,KAAK,UAAU,CAwF3J;AAyFD;;;;;;;;;GASG;AACH,wBAAgB,qBAAqB,CAAC,CAAC,SAAS,QAAQ,EAAE,IAAI,EAAE,CAAC,GAAG,aAAa,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,CAAC,IAAI,EAAE,UAAU,KAAK,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAoCjK;AAmBD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,qBAAa,WAAW,CAAC,CAAC,SAAS,QAAQ,GAAG,QAAQ;IACpD,wEAAwE;IACxE,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC;;0CAEsC;IACtC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,oDAAoD;IACpD,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAwB;IAC/C,iFAAiF;IACjF,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAc;IACxC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAgB;IACrC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAgB;IAC1C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAY;IACtC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAuF;IACrH,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAkC;IAC3D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAW;IACtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsC;IAC/D,2EAA2E;IAC3E,OAAO,CAAC,MAAM,CAAsB;IACpC,OAAO,CAAC,QAAQ,CAAsE;IACtF;;;;;;0BAMsB;IACtB,OAAO,CAAC,QAAQ,CAAC,YAAY,CAA0D;IAEvF,gDAAgD;gBACpC,MAAM,EAAE,UAAU,GAAG,qBAAqB,EAAE,SAAS,EAAE,aAAa,EAAE,OAAO,CAAC,EAAE,mBAAmB;IA0C/G,sCAAsC;IACtC,IAAI,YAAY,IAAI,MAAM,CAEzB;IAED;;;+CAG2C;IAC3C,OAAO,CAAC,WAAW;IAMnB;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC;IAKlC,4EAA4E;IAC5E,OAAO,CAAC,iBAAiB;IAoBzB;;;;2CAIuC;IACvC,OAAO,CAAC,aAAa;IA8BrB;;;;sCAIkC;IAClC,OAAO,CAAC,QAAQ;IA6BhB;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK;IAUzJ;4DACwD;IACxD,OAAO,CAAC,YAAY;IAiBpB;;iFAE6E;IAC7E,OAAO,CAAC,cAAc;IAQtB;uEACmE;IACnE,OAAO,CAAC,UAAU;IAUlB;;;;;;;OAOG;IACH,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK;IAYpE;;;;;;;;;;;;;;;;;;OAkBG;IACH,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC;IAuDpD;;;;;;;;;;OAUG;IACH,GAAG,CACD,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,KAAK,GAClG,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,SAAS;CA4BhH;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,kBAAkB,CAAC,CAAC,SAAS,QAAQ,EAAE,IAAI,EAAE,CAAC,GAAG,aAAa,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,CAAC,MAAM,EAAE,UAAU,GAAG,qBAAqB,KAAK,WAAW,CAAC,CAAC,CAAC,CAI7K"}

package/dist/src/serialization/beast2/v5/stream.js

@@ -28,8 +28,9 @@ import { SortedSet } from "../../../containers/sortedset.js";
 import { SortedMap } from "../../../containers/sortedmap.js";
 import { buildPlatformContext } from "../shared.js";
 import { writeTypeSection, readTypeSection, asTypeValue } from "./type-section.js";
-import { FrameReader, writeFrame } from "./frames.js";
+import { FrameReader, openFramePrefix, writeFrame } from "./frames.js";
 import { MAGIC_BYTES_V5, TAG_NEW, createV5EncodeContext, buildV5Encoder, buildV5Decoder, writeSourceMapSectionV5, readSourceMapSectionV5, writeIndexAndFooter, readIndex, isSegmentedRoot, } from "./codec.js";
+import { TAG_OR_TERMINATOR_FRAME, bytesReader, isBeast2SyncRangeReader, readExact, readU64LE, readBeast2ExtentsSync } from "./range.js";
 /** Validates that a root type is streamable and returns its kind. */
 function checkSegmented(typeValue) {
     if (!isSegmentedRoot(typeValue)) {
@@ -537,6 +538,11 @@ export function iterBeast2SegmentsFor(type, options) {
  *  of once per element. Bounded — at most this many decoded segments live
  *  per reader. */
 const SEGMENT_CACHE_CAPACITY = 4;
+/** The bytes a fence probe reads first: the frame header plus enough of the
+ *  payload for the first key of any ordinary row. A key that does not fit
+ *  grows the probe fourfold until it does, the final attempt reading the
+ *  frame whole (mirrors east-c's probe). */
+const FENCE_PROBE_BYTES = 4096;
 /**
  * Random access over an indexed, self-contained v5 collection blob.
  *
@@ -548,6 +554,15 @@ const SEGMENT_CACHE_CAPACITY = 4;
  * small LRU, so a read loop over neighbouring rows decodes each segment once
  * rather than once per element; {@link segment} itself always decodes fresh.
  *
+ * The blob is either a whole `Uint8Array` or a {@link Beast2SyncRangeReader}:
+ * through a reader the open reads only the tail (footer + index) and the
+ * head (header sections), and every segment read fetches exactly that
+ * segment's frame — a file-backed reader keeps the wire bytes in the page
+ * cache rather than on the heap. A Set/Dict root's first keyed read verifies
+ * the fences, probing a bounded prefix of every frame once (never the frame,
+ * unless its first key is wider than the probe); later keyed reads fetch one
+ * frame.
+ *
  * Set/Dict blobs page like Arrays: the wire holds the canonical value split
  * at segment boundaries (strictly ascending, disjoint segments), so row
  * windows ({@link slice}) and key lookups ({@link get}) address the sorted
@@ -566,7 +581,9 @@ export class Beast2Pages {
     elementCount;
     /** Whether segments are independently decodable. */
     selfContained;
-    data;
+    source;
+    /** Wire offset of the terminator frame — where the last segment's frame ends. */
+    segmentsEnd;
     indexData;
     kind;
     typeValue;
@@ -587,14 +604,31 @@ export class Beast2Pages {
      *  container walk. */
     segmentCache = new Map();
     /** @internal Use {@link openBeast2PagesFor}. */
-    constructor(data, typeValue, options) {
-        const { kind, sourceMap, frameOffset } = openSegmented(data, typeValue);
-        void frameOffset;
-        const index = readIndex(data);
-        if (!index) {
-            throw new Error(`beast2 v5: blob has no index/footer — paging needs a writer with index enabled`);
+    constructor(source, typeValue, options) {
+        let kind;
+        let sourceMap;
+        let index;
+        if (!isBeast2SyncRangeReader(source)) {
+            ({ kind, sourceMap } = openSegmented(source, typeValue));
+            const whole = readIndex(source);
+            if (!whole) {
+                throw new Error(`beast2 v5: blob has no index/footer — paging needs a writer with index enabled`);
+            }
+            index = whole;
+            // readIndex validated the footer, so the u64 before its magic is the
+            // index offset; the terminator frame sits directly ahead of it.
+            this.segmentsEnd = readU64LE(source, source.length - 16) - TAG_OR_TERMINATOR_FRAME.length;
+            this.source = bytesReader(source);
+        }
+        else {
+            // Two positioned reads — the tail, then the head — give the whole
+            // geometry; the segment frames are read one at a time from here on.
+            const extents = readBeast2ExtentsSync(source);
+            ({ kind, sourceMap } = openSegmented(extents.head, typeValue));
+            index = { selfContained: extents.selfContained, offsets: [...extents.offsets], counts: [...extents.counts], totalCount: extents.elementCount };
+            this.segmentsEnd = extents.segmentsEnd;
+            this.source = source;
         }
-        this.data = data;
         this.indexData = index;
         this.kind = kind;
         this.typeValue = typeValue;
@@ -616,6 +650,15 @@ export class Beast2Pages {
     get segmentCount() {
         return this.indexData.offsets.length;
     }
+    /** Reads segment `i`'s frame — exactly its wire bytes, from its index
+     *  offset to the next segment's (or the terminator) — and opens its
+     *  logical chunk. The only place segment bytes are fetched, so a ranged
+     *  source touches one frame per decode. */
+    frameReader(i) {
+        const start = this.indexData.offsets[i];
+        const end = i + 1 < this.indexData.offsets.length ? this.indexData.offsets[i + 1] : this.segmentsEnd;
+        return new FrameReader(readExact(this.source, start, end - start), 0).next();
+    }
     /**
      * Decodes one segment by index.
      *
@@ -641,8 +684,7 @@ export class Beast2Pages {
         if (i < 0 || i >= this.indexData.offsets.length) {
             throw new Error(`beast2 v5: segment ${i} out of range (${this.indexData.offsets.length} segments)`);
         }
-        const cursor = new FrameReader(this.data, this.indexData.offsets[i]);
-        const reader = cursor.next();
+        const reader = this.frameReader(i);
         const n = reader.readVarint();
         if (n !== this.indexData.counts[i]) {
             throw new Error(`beast2 v5: segment ${i} declares ${n} elements, index says ${this.indexData.counts[i]}`);
@@ -689,17 +731,38 @@ export class Beast2Pages {
         return seg;
     }
     /** Decodes just the first key/element of segment `i` — a bounded probe
-     *  (one frame inflate, one element decode), not a whole-segment decode. */
+     *  (a prefix of the frame read and inflated, one element decode), not a
+     *  whole-segment decode. A first key wider than the probe grows it; the
+     *  final attempt reads the frame whole, so corruption is still reported
+     *  with the frame's own error. */
     firstKey(i) {
         if (!this.fenceDec) {
             const keyType = this.kind === "Dict" ? this.typeValue.value.key : this.typeValue.value;
             this.fenceDec = buildV5Decoder(keyType);
         }
-        const cursor = new FrameReader(this.data, this.indexData.offsets[i]);
-        const reader = cursor.next();
-        reader.readVarint(); // element count — segments are never empty
-        const ctx = { containers: [], sourceMap: this.sourceMap, frozen: this.platform?.frozen ?? false, ...buildPlatformContext(this.platform) };
-        return this.fenceDec(reader, ctx);
+        const start = this.indexData.offsets[i];
+        const end = i + 1 < this.indexData.offsets.length ? this.indexData.offsets[i + 1] : this.segmentsEnd;
+        const frameLen = end - start;
+        for (let probe = Math.min(frameLen, FENCE_PROBE_BYTES);; probe = Math.min(frameLen, probe * 4)) {
+            const whole = probe === frameLen;
+            const bytes = readExact(this.source, start, probe);
+            const ctx = { containers: [], sourceMap: this.sourceMap, frozen: this.platform?.frozen ?? false, ...buildPlatformContext(this.platform) };
+            if (whole) {
+                const reader = new FrameReader(bytes, 0).next();
+                reader.readVarint(); // element count — segments are never empty
+                return this.fenceDec(reader, ctx);
+            }
+            try {
+                const reader = openFramePrefix(bytes);
+                if (reader !== null) {
+                    reader.readVarint();
+                    return this.fenceDec(reader, ctx);
+                }
+            }
+            catch {
+                // Short, or corrupt: more of the frame decides which.
+            }
+        }
     }
     /**
      * Probes segment `i`'s fence: its first Dict key, Set element, or Array
@@ -914,9 +977,13 @@ export class Beast2Pages {
     }
 }
 /**
- * Builds a curried pages opener: `open(data)` parses the header, footer and
+ * Builds a curried pages opener: `open(source)` parses the header, footer and
  * index once and returns a {@link Beast2Pages} for random access.
  *
+ * `source` is the whole blob, or a {@link Beast2SyncRangeReader} over it —
+ * then only the tail, the head and the segments actually read are ever
+ * fetched.
+ *
  * @param type - the collection type (Array/Set/Dict)
  * @param options - decode options (platform functions for decoded functions)
  * @returns a function opening a blob for paged reads
@@ -925,6 +992,6 @@ export class Beast2Pages {
 export function openBeast2PagesFor(type, options) {
     const typeValue = asTypeValue(type);
     checkSegmented(typeValue);
-    return (data) => new Beast2Pages(data, typeValue, options);
+    return (source) => new Beast2Pages(source, typeValue, options);
 }
 //# sourceMappingURL=stream.js.map