@woosh/meep-engine 3.19.0 → 3.21.0

package/src/shade/renderer/geometry/virtual/format/read/VGeoContainerReader.js

@@ -0,0 +1,572 @@
+import { assert } from "../../../../../../core/assert.js";
+import { BinaryBuffer } from "../../../../../../core/binary/BinaryBuffer.js";
+import { vgeo_compute_frame_checksum } from "../checksum/vgeo_compute_frame_checksum.js";
+import { VGEO_FRAME_OFFSET } from "../frame/VGEO_FRAME_OFFSET.js";
+import { vgeo_read_frame_extent } from "../frame/vgeo_read_frame_extent.js";
+import { vgeo_validate_container } from "../vgeo_validate_container.js";
+import { VGeoReadOptions } from "./VGeoReadOptions.js";
+import { vgeo_read_directory } from "./vgeo_read_directory.js";
+import { vgeo_read_header, vgeo_resident_head_size, VGEO_MIN_HEAD_BYTES } from "./vgeo_read_header.js";
+import { vgeo_read_page } from "./vgeo_read_page.js";
+
+/**
+ * Everything in the container is little-endian (`VGEO_FORMAT.md` §2).
+ * @type {boolean}
+ */
+const LE = true;
+
+/**
+ * A `.vgeo` container, read a page at a time.
+ *
+ * **Nothing here ever holds the whole file.** Opening it reads a head prefix — the header, the level
+ * table, and the root frame, which §10 calls the always-resident set — and after that a page arrives
+ * only when something asks for it, from exactly the byte range the page pointing at it wrote down.
+ * That is not an optimization bolted onto a whole-file reader; it is the shape the format was
+ * designed for, and a reader built the other way cannot be made into this one.
+ *
+ * What that buys, concretely: a 27-million-triangle asset opens in the bytes of its root page, and a
+ * view of it costs the pages that view actually selects. What it costs is that a traversal has to
+ * cope with a page it wants and does not have — which the format also already answers, because the
+ * clamped cut is watertight and correct at every arrival order (§10, activation).
+ *
+ * Residency is this object. {@link page} answers for what is here, {@link install_want} brings one
+ * more, {@link evict} drops one, and a {@link VGeoPage} owns its own bytes and tables so that
+ * dropping it drops everything it cost.
+ *
+ * **Validation is not on this path.** The mandatory reject rules of §4 and §10 are applied
+ * unconditionally by {@link vgeo_read_header} and {@link vgeo_read_page} — a file whose magic,
+ * versions, encoding or codec this build does not understand is refused rather than misread. The
+ * whole-file audit is {@link validate}, it reads every byte of the file, and it is off by default;
+ * see {@link VGeoReadOptions.validate}.
+ *
+ * @author Alex Goldring
+ * @copyright Company Named Limited (c) 2026
+ */
+export class VGeoContainerReader {
+    /**
+     * @type {VGeoContainerHeader}
+     */
+    header = undefined;
+
+    /**
+     * The page directory, once {@link read_directory} has been asked for it. A traversal never needs
+     * it; a tool looking at every page does.
+     * @type {VGeoDirectory|undefined}
+     */
+    directory = undefined;
+
+    /**
+     * @type {VGeoByteSource}
+     */
+    #source;
+
+    /**
+     * @type {VGeoReadOptions}
+     */
+    #options;
+
+    /**
+     * Resident pages, by page index.
+     * @type {Map<number, VGeoPage>}
+     */
+    #pages = new Map();
+
+    /**
+     * Fetches in flight, by page index — so that two clusters wanting the same page produce one
+     * request. Wants dedup on page index across clusters, groups and instances (§10), and this is
+     * where that happens for the ones already on the wire.
+     * @type {Map<number, Promise<VGeoPage>>}
+     */
+    #pending = new Map();
+
+    /**
+     * The speculative head prefix, kept.
+     *
+     * Coarse-first layout is what makes this worth keeping rather than parsing and dropping: a
+     * prefix that overshot the root frame overshot into the pages the traversal is most likely to
+     * want next, and serving those from here costs no request at all.
+     *
+     * @type {Uint8Array}
+     */
+    #head = new Uint8Array(0);
+
+    /**
+     * @type {number}
+     */
+    #resident_bytes = 0;
+
+    /**
+     * @param {VGeoByteSource} source
+     * @param {VGeoReadOptions} [options]
+     */
+    constructor(source, options = new VGeoReadOptions()) {
+        assert.defined(source, 'source');
+
+        this.#source = source;
+        this.#options = options;
+    }
+
+    /**
+     * @returns {VGeoByteSource}
+     */
+    get source() {
+        return this.#source;
+    }
+
+    /**
+     * @returns {VGeoReadOptions}
+     */
+    get options() {
+        return this.#options;
+    }
+
+    /**
+     * @returns {number}
+     */
+    get resident_page_count() {
+        return this.#pages.size;
+    }
+
+    /**
+     * Decoded bytes currently held. The number a residency budget is spent against.
+     * @returns {number}
+     */
+    get resident_bytes() {
+        return this.#resident_bytes;
+    }
+
+    /**
+     * @returns {number}
+     */
+    get pending_page_count() {
+        return this.#pending.size;
+    }
+
+    /**
+     * The root page, which is pinned for the life of the reader — §10 makes it the one page that is
+     * never evictable, because every traversal starts by walking its parentless groups.
+     * @returns {VGeoPage}
+     */
+    get root() {
+        return this.#pages.get(0);
+    }
+
+    /**
+     * Every resident page, in page-index order.
+     * @returns {VGeoPage[]}
+     */
+    get pages() {
+        return Array.from(this.#pages.values()).sort((a, b) => a.index - b.index);
+    }
+
+    /**
+     * Open a container: header, level table, root page.
+     *
+     * @param {VGeoByteSource} source
+     * @param {VGeoReadOptions} [options]
+     * @returns {Promise<VGeoContainerReader>}
+     * @throws {Error} when the bytes are not a `.vgeo` this build reads, or the root frame does not
+     *     match what the header says about it
+     */
+    static async open(source, options = new VGeoReadOptions()) {
+        const reader = new VGeoContainerReader(source, options);
+
+        await reader.#open();
+
+        return reader;
+    }
+
+    /**
+     * @returns {Promise<void>}
+     */
+    async #open() {
+        const source = this.#source;
+
+        if (source.byte_length < VGEO_MIN_HEAD_BYTES) {
+            throw new Error(
+                `${source.name} is ${source.byte_length} bytes, shorter than a .vgeo header`
+            );
+        }
+
+        const prefix = Math.min(
+            source.byte_length,
+            Math.max(this.#options.head_prefix_bytes, VGEO_MIN_HEAD_BYTES)
+        );
+
+        this.#head = await source.read(0, prefix);
+
+        /*
+         §10's always-resident set: the header, the level table, and the root frame — in that order,
+         contiguously, and bounded by two fields of the fixed header. Whatever the prefix already
+         covered, one range brings the rest, so a cold start is two requests at worst and one
+         whenever the prefix was generous. Nothing is fetched twice: the head only grows at its tail.
+        */
+        await this.#grow_head(vgeo_resident_head_size(this.#head));
+
+        this.header = vgeo_read_header(this.#head);
+
+        await this.#install_root();
+
+        if (this.#options.validate) {
+            await this.validate();
+        }
+    }
+
+    /**
+     * Install page 0, checking it against what the header promised — §10's cold-start rule.
+     *
+     * @returns {Promise<void>}
+     */
+    async #install_root() {
+        const header = this.header;
+
+        const root = await this.install(
+            0, header.root_page_offset, header.root_page_fetch_size
+        );
+
+        /*
+         Both of these are §10 reject rules rather than validation: the header states the root's
+         extent and decoded size so that a reader can admit RAM and size a fetch before it has seen
+         the frame, and a root frame that disagrees means one of the two numbers is wrong. Which one
+         is unknowable from here, and reading on under either is how a reader ends up trusting a
+         number it has already caught lying.
+         */
+        if (root.decoded_size !== header.root_page_decoded_size) {
+            throw new Error(
+                `the root frame decodes to ${root.decoded_size} bytes and the header says`
+                + ` ${header.root_page_decoded_size}`
+            );
+        }
+
+        if (root.fetch_size !== header.root_page_fetch_size) {
+            throw new Error(
+                `the root frame occupies ${root.fetch_size} bytes and the header says`
+                + ` ${header.root_page_fetch_size}`
+            );
+        }
+    }
+
+    /**
+     * The page, if it is here.
+     *
+     * Synchronous and total: a traversal asks this per cluster and must not await inside itself, so
+     * `undefined` is an answer it acts on — clamp, and record a want — rather than a case to handle.
+     *
+     * @param {number} page_index
+     * @returns {VGeoPage|undefined}
+     */
+    page(page_index) {
+        return this.#pages.get(page_index);
+    }
+
+    /**
+     * @param {number} page_index
+     * @returns {boolean}
+     */
+    is_resident(page_index) {
+        return this.#pages.has(page_index);
+    }
+
+    /**
+     * Fetch and install one page, from the exact range something already resident named.
+     *
+     * Concurrent calls for the same page share one fetch; a page already resident is returned
+     * without one.
+     *
+     * @param {number} page_index
+     * @param {number} frame_offset absolute
+     * @param {number} fetch_size
+     * @returns {Promise<VGeoPage>}
+     */
+    install(page_index, frame_offset, fetch_size) {
+        const resident = this.#pages.get(page_index);
+
+        if (resident !== undefined) {
+            return Promise.resolve(resident);
+        }
+
+        const in_flight = this.#pending.get(page_index);
+
+        if (in_flight !== undefined) {
+            return in_flight;
+        }
+
+        const fetch = this.#fetch_page(page_index, frame_offset, fetch_size)
+            .finally(() => this.#pending.delete(page_index));
+
+        this.#pending.set(page_index, fetch);
+
+        return fetch;
+    }
+
+    /**
+     * @param {number} page_index
+     * @param {number} frame_offset
+     * @param {number} fetch_size
+     * @returns {Promise<VGeoPage>}
+     */
+    async #fetch_page(page_index, frame_offset, fetch_size) {
+        const bytes = await this.#read(frame_offset, fetch_size);
+
+        if (this.#options.verify_frames) {
+            this.#verify_frame(bytes, page_index);
+        }
+
+        const page = vgeo_read_page(bytes, page_index);
+
+        page.frame_offset = frame_offset;
+
+        // Two callers can reach here for one page only if the second started before the first
+        // resolved, which `#pending` already prevents — but residency is the accounting, so it is
+        // read here rather than assumed.
+        if (!this.#pages.has(page_index)) {
+            this.#pages.set(page_index, page);
+
+            this.#resident_bytes += page.decoded_size;
+        }
+
+        return this.#pages.get(page_index);
+    }
+
+    /**
+     * Install the page a want names.
+     *
+     * @param {VGeoPageWant} want
+     * @returns {Promise<VGeoPage>}
+     */
+    install_want(want) {
+        return this.install(want.page_index, want.frame_offset, want.fetch_size);
+    }
+
+    /**
+     * Install a batch of wants, most wanted first, with a bounded number of fetches in flight.
+     *
+     * Ordered by {@link VGeoPageWant.priority}, which is how far past the pixel threshold the
+     * clamped cluster projected — so a caller that stops early has fetched the pages that would fix
+     * the most visible coarseness, not an arbitrary prefix of the list.
+     *
+     * @param {VGeoPageWant[]} wants
+     * @param {number} [limit] most pages to install; the rest are left for the next traversal to
+     *     want again, which it will, from a cut taken with everything this call did install
+     * @returns {Promise<VGeoPage[]>} the pages installed, in completion order
+     */
+    async install_wants(wants, limit = wants.length) {
+        // Sorted before it is cut, not after: a limit applied to the caller's order would install an
+        // arbitrary prefix and the ranking would have bought nothing.
+        const queue = wants.slice().sort((a, b) => b.priority - a.priority).slice(0, limit);
+
+        const installed = [];
+
+        let next = 0;
+
+        const worker = async () => {
+            while (next < queue.length) {
+                const want = queue[next++];
+
+                installed.push(await this.install_want(want));
+            }
+        };
+
+        const workers = [];
+
+        const width = Math.min(this.#options.max_concurrent_fetches, queue.length);
+
+        for (let i = 0; i < width; i++) {
+            workers.push(worker());
+        }
+
+        await Promise.all(workers);
+
+        return installed;
+    }
+
+    /**
+     * Drop a page.
+     *
+     * The eviction *rule* — a page is evictable only while it has no resident child pages, and the
+     * root is pinned (§10) — is a residency policy and belongs to whatever is managing the budget,
+     * not here. What this does is the dropping.
+     *
+     * @param {number} page_index
+     * @returns {boolean} whether a page was there to drop
+     */
+    evict(page_index) {
+        const page = this.#pages.get(page_index);
+
+        if (page === undefined) {
+            return false;
+        }
+
+        this.#pages.delete(page_index);
+
+        this.#resident_bytes -= page.decoded_size;
+
+        return true;
+    }
+
+    /**
+     * Drop every page but the root, which is pinned.
+     *
+     * @returns {number} pages dropped
+     */
+    evict_all() {
+        let dropped = 0;
+
+        for (const index of Array.from(this.#pages.keys())) {
+            if (index !== 0 && this.evict(index)) {
+                dropped++;
+            }
+        }
+
+        return dropped;
+    }
+
+    /**
+     * Read the page directory, and keep it on {@link directory}.
+     *
+     * One fetch of a region that is 32 bytes per page — a hundred kilobytes for a file with three
+     * thousand pages. Auxiliary by design: nothing a traversal does needs it, and asking for it says
+     * the caller wants to reach pages the DAG does not lead it to.
+     *
+     * @returns {Promise<VGeoDirectory>}
+     */
+    async read_directory() {
+        if (this.directory !== undefined) {
+            return this.directory;
+        }
+
+        const header = this.header;
+
+        const end = header.dir_block_checksums_offset > header.directory_offset
+            ? header.dir_block_checksums_offset
+            : this.#source.byte_length;
+
+        const bytes = await this.#read(header.directory_offset, end - header.directory_offset);
+
+        this.directory = vgeo_read_directory(bytes, header.total_pages);
+
+        return this.directory;
+    }
+
+    /**
+     * Install every page the directory lists.
+     *
+     * The opposite of what this reader is for, and exactly what a tool wants when the question is
+     * about the file rather than about a view of it — the pages no cut reaches are the ones a bug
+     * hides in.
+     *
+     * @param {function(number, number): void} [on_progress] called with pages installed and total
+     * @returns {Promise<void>}
+     */
+    async install_all_pages(on_progress = undefined) {
+        const directory = await this.read_directory();
+
+        let done = 0;
+
+        for (let index = 0; index < directory.page_count; index++) {
+            await this.install(index, directory.frame_offset[index], directory.fetch_size[index]);
+
+            done++;
+
+            if (on_progress !== undefined) {
+                on_progress(done, directory.page_count);
+            }
+        }
+    }
+
+    /**
+     * The whole-file audit — every build invariant of §11, every checksum, and the directory against
+     * what a frame scan reconstructs.
+     *
+     * **This reads the entire file**, which is the one thing the rest of this class exists to avoid,
+     * and it is why it is a separate call and off by default. It answers a question about whoever
+     * produced the file; nothing about reading one depends on it having been asked.
+     *
+     * @returns {Promise<VGeoDefect[]>} empty is the only good answer
+     */
+    async validate() {
+        const bytes = await this.#source.read(0, this.#source.byte_length);
+
+        const whole = bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength
+            ? bytes.buffer
+            : bytes.slice().buffer;
+
+        return vgeo_validate_container(BinaryBuffer.fromArrayBuffer(whole));
+    }
+
+    /**
+     * A byte range, from the kept head prefix where it lies inside it and from the source otherwise.
+     *
+     * @param {number} offset
+     * @param {number} length
+     * @returns {Promise<Uint8Array>}
+     */
+    #read(offset, length) {
+        if (offset + length <= this.#head.byteLength) {
+            return Promise.resolve(this.#head.subarray(offset, offset + length));
+        }
+
+        return this.#source.read(offset, length);
+    }
+
+    /**
+     * Extend the kept head prefix to `to` bytes, fetching only the part that is missing.
+     *
+     * Only ever at the tail, which is what keeps a cold start from paying for the same bytes twice:
+     * the header is read once even when the prefix that carried it stopped short of the level table.
+     *
+     * @param {number} to
+     * @returns {Promise<void>}
+     */
+    async #grow_head(to) {
+        const target = Math.min(to, this.#source.byte_length);
+
+        const have = this.#head.byteLength;
+
+        if (target <= have) {
+            return;
+        }
+
+        const tail = await this.#source.read(have, target - have);
+
+        const grown = new Uint8Array(target);
+
+        grown.set(this.#head, 0);
+        grown.set(tail, have);
+
+        this.#head = grown;
+    }
+
+    /**
+     * Check a frame against its identity-seeded checksum before anything decodes it.
+     *
+     * The seed is `asset_id_low64 XOR page_index`, so this catches a well-formed frame of the wrong
+     * page or the wrong asset as well as corruption — which is the case a traversal can actually
+     * produce, by following a child-table entry whose offset is wrong.
+     *
+     * @param {Uint8Array} bytes the fetched frame
+     * @param {number} page_index
+     */
+    #verify_frame(bytes, page_index) {
+        const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+
+        const frame = vgeo_read_frame_extent(view, 0, page_index);
+
+        const stored = view.getBigUint64(VGEO_FRAME_OFFSET.CHECKSUM, LE);
+
+        const copy = bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength
+            ? bytes.buffer
+            : bytes.slice().buffer;
+
+        const computed = vgeo_compute_frame_checksum(
+            BinaryBuffer.fromArrayBuffer(copy), frame, this.header.asset_id_low64
+        );
+
+        if (computed !== stored) {
+            throw new Error(
+                `page ${page_index}: its frame checksums to 0x${computed.toString(16)}`
+                + ` and stores 0x${stored.toString(16)}`
+            );
+        }
+    }
+}

package/src/shade/renderer/geometry/virtual/format/read/VGeoDirectory.d.ts

@@ -0,0 +1,64 @@
+/**
+ * The page directory, decoded — one row per page, in page-index order.
+ *
+ * Auxiliary and fully derivable from the frames (`VGEO_FORMAT.md` §3, §9): a traversal never touches
+ * it, because every range it needs is already in the child table of a page it holds. What it is for
+ * is the tasks that are not a traversal — RAM budgeting before a fetch, resuming from a disk cache,
+ * and reaching a page the DAG does not lead to, which is exactly what a tool looking at a file wants
+ * and a runtime never does.
+ *
+ * A structure of arrays because a large asset has tens of thousands of pages and every use of this
+ * is a scan or a single indexed lookup.
+ *
+ * @author Alex Goldring
+ * @copyright Company Named Limited (c) 2026
+ */
+export class VGeoDirectory {
+    /**
+     * @type {number}
+     */
+    page_count: number;
+    /**
+     * Absolute frame offsets. A double rather than a BigInt: every offset past 2^53 names a file
+     * nothing here could hold, and the ones below it are exact.
+     * @type {Float64Array}
+     */
+    frame_offset: Float64Array;
+    /**
+     * The exact byte range to fetch each page: `[frame_offset, frame_offset + fetch_size)`.
+     * @type {Uint32Array}
+     */
+    fetch_size: Uint32Array;
+    /**
+     * @type {Uint32Array}
+     */
+    decoded_size: Uint32Array;
+    /**
+     * @type {Uint8Array}
+     */
+    min_level: Uint8Array;
+    /**
+     * @type {Uint8Array}
+     */
+    max_level: Uint8Array;
+    /**
+     * Bit 0 is `VGEO_DIRECTORY_FLAG_ROOT_PAGE`.
+     * @type {Uint16Array}
+     */
+    flags: Uint16Array;
+    /**
+     * @type {Uint16Array}
+     */
+    group_count: Uint16Array;
+    /**
+     * @type {Uint16Array}
+     */
+    cluster_count: Uint16Array;
+    /**
+     * What every page of this asset costs decoded, summed — the whole resident footprint, known
+     * without decoding one.
+     * @returns {number}
+     */
+    get total_decoded_bytes(): number;
+}
+//# sourceMappingURL=VGeoDirectory.d.ts.map

package/src/shade/renderer/geometry/virtual/format/read/VGeoDirectory.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"VGeoDirectory.d.ts","sourceRoot":"","sources":["../../../../../../../../src/shade/renderer/geometry/virtual/format/read/VGeoDirectory.js"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AACH;IACI;;OAEG;IACH,YAFU,MAAM,CAED;IAEf;;;;OAIG;IACH,cAFU,YAAY,CAEa;IAEnC;;;OAGG;IACH,YAFU,WAAW,CAEW;IAEhC;;OAEG;IACH,cAFU,WAAW,CAEa;IAElC;;OAEG;IACH,WAFU,UAAU,CAEU;IAE9B;;OAEG;IACH,WAFU,UAAU,CAEU;IAE9B;;;OAGG;IACH,OAFU,WAAW,CAEM;IAE3B;;OAEG;IACH,aAFU,WAAW,CAEY;IAEjC;;OAEG;IACH,eAFU,WAAW,CAEc;IAEnC;;;;OAIG;IACH,kCAQC;CACJ"}