@bldrs-ai/conway 1.389.1221 → 1.394.1504

package/compiled/src/step/parsing/prefix_type_index.test.js

@@ -0,0 +1,298 @@
+/* eslint-disable no-magic-numbers */
+// M2 (#393): the type index over the records parsed so far.
+//
+// Two things are pinned here, and they are the two the event-fed
+// IncrementalTypeIndex this replaces got wrong:
+//   1. final membership equals the resident model's type index — the exit
+//      criterion "consumers produce output identical to today's end-of-parse
+//      construction";
+//   2. complex (multi-mapped) records are attributed to their mapped classes,
+//      which is invisible to a consumer reading the record event's typeID
+//      (complex records arrive there as 0).
+import * as fs from "fs";
+import { beforeAll, describe, expect, test } from "@jest/globals";
+import ParsingBuffer from "../../parsing/parsing_buffer.js";
+import IfcStepParser from "../../ifc/ifc_step_parser.js";
+import { EntityTypesIfcCount } from "../../ifc/ifc4_gen/entity_types_ifc.gen.js";
+import { IfcRoot, IfcProduct, IfcProject, IfcWall, IfcPropertySet, } from "../../ifc/ifc4_gen/index.js";
+import { MultiIndexSetCursorOr } from "../../indexing/multi_index_set_cursor_or.js";
+import { SingleIndexSetCursor } from "../../indexing/single_index_set_cursor.js";
+import { StepTypeIndexer } from "../indexing/step_type_indexer.js";
+import { BufferByteSource } from "./byte_source.js";
+import { ColumnarIndexSink } from "./columnar_index.js";
+import { PrefixTypeIndex } from "./prefix_type_index.js";
+import { buildIndexStreaming } from "./streaming_index_builder.js";
+import { ParseResult } from "./step_parser.js";
+let bytes;
+let model;
+// The resident parse's top-level entries, in parse order — the independent
+// ground truth the property test below reconstructs prefixes from. It never
+// goes near the streaming path, so it cannot agree with a bug by sharing one.
+let residentEntries;
+beforeAll(() => {
+    bytes = new Uint8Array(fs.readFileSync("data/index.ifc"));
+    const input = new ParsingBuffer(bytes);
+    IfcStepParser.Instance.parseHeader(input);
+    model = IfcStepParser.Instance.parseDataToModel(input)[1];
+    const groundTruthInput = new ParsingBuffer(bytes);
+    IfcStepParser.Instance.parseHeader(groundTruthInput);
+    residentEntries = IfcStepParser.Instance.parseDataBlock(groundTruthInput)[0].elements;
+});
+/**
+ * The express IDs a correct index would hold over the first `records`
+ * top-level records, computed straight from the resident parse.
+ *
+ * @param records How many records the view covers.
+ * @param closure The queried subtype closure, as raw type IDs.
+ * @return {Set<number>} The expected express IDs.
+ */
+function expectedOverPrefix(records, closure) {
+    const wanted = new Set(closure);
+    const result = new Set();
+    for (let localID = 0; localID < records; ++localID) {
+        const entry = residentEntries[localID];
+        if (entry === void 0) {
+            break;
+        }
+        if (entry.typeID !== void 0 && wanted.has(entry.typeID)) {
+            result.add(entry.expressID);
+            continue;
+        }
+        // Complex records carry their real classes in the multi-mapping.
+        for (const mapped of entry.multiMapping ?? []) {
+            if (mapped.typeID !== void 0 && wanted.has(mapped.typeID)) {
+                result.add(entry.expressID);
+                break;
+            }
+        }
+    }
+    return result;
+}
+/**
+ * Stream index.ifc into a sink, optionally pausing at `stopAfter` records to
+ * hand the caller a mid-parse view, and return the index over that sink.
+ *
+ * @param onRecord Optional per-record hook, so a test can query the index
+ * while the parse is still filling the sink behind it.
+ * @return {PrefixTypeIndex} The index over the (fully parsed) sink.
+ */
+function streamed(onRecord) {
+    const sink = new ColumnarIndexSink();
+    const index = new PrefixTypeIndex(sink, new StepTypeIndexer(EntityTypesIfcCount));
+    const result = buildIndexStreaming(new BufferByteSource(bytes), IfcStepParser.Instance, 4 * 1024, onRecord === void 0 ? void 0 : (localID) => onRecord(index, localID), sink);
+    expect(result.result).toBe(ParseResult.COMPLETE);
+    return index;
+}
+describe("PrefixTypeIndex", () => {
+    test.each([
+        ["IfcRoot", IfcRoot],
+        ["IfcProduct", IfcProduct],
+        ["IfcWall", IfcWall],
+        ["IfcPropertySet", IfcPropertySet],
+    ])("final membership matches the resident type index for %s", (_name, ctor) => {
+        const index = streamed();
+        const prefix = new Set(index.expressIDsOfTypes(ctor));
+        const resident = new Set(model.expressIDsOfTypes(ctor));
+        expect(prefix).toEqual(resident);
+    });
+    test("a multi-type query unions the subtype closures", () => {
+        const index = streamed();
+        const prefix = new Set(index.expressIDsOfTypes(IfcWall, IfcPropertySet));
+        const resident = new Set(model.expressIDsOfTypes(IfcWall, IfcPropertySet));
+        expect(prefix).toEqual(resident);
+    });
+    test("nothing is built until something is queried", () => {
+        const sink = new ColumnarIndexSink();
+        const index = new PrefixTypeIndex(sink, new StepTypeIndexer(EntityTypesIfcCount));
+        expect(index.generation).toBe(0);
+        expect(index.recordCount).toBe(0);
+        // The whole point of deriving rather than pushing: a consumer that never
+        // asks pays nothing, however long the parse ran.
+        buildIndexStreaming(new BufferByteSource(bytes), IfcStepParser.Instance, 4 * 1024, void 0, sink);
+        expect(index.generation).toBe(0);
+    });
+    test("a mid-parse query answers over the prefix, and grows with it", () => {
+        // Rebuild on every record so the view tracks the parse exactly; the
+        // default growth pacing is about cost, not correctness.
+        const sink = new ColumnarIndexSink();
+        const index = new PrefixTypeIndex(sink, new StepTypeIndexer(EntityTypesIfcCount), { growthFactor: 1, minimumRecords: 0 });
+        const counts = [];
+        buildIndexStreaming(new BufferByteSource(bytes), IfcStepParser.Instance, 4 * 1024, (localID) => {
+            if (localID === 64 || localID === 128) {
+                counts.push([...index.expressIDsOfTypes(IfcRoot)].length);
+            }
+        }, sink);
+        const final = [...index.expressIDsOfTypes(IfcRoot)].length;
+        const resident = [...model.expressIDsOfTypes(IfcRoot)].length;
+        expect(counts).toHaveLength(2);
+        expect(counts[0]).toBeLessThanOrEqual(counts[1]);
+        expect(counts[1]).toBeLessThanOrEqual(final);
+        expect(final).toBe(resident);
+    });
+    test("a record is indexed before its own event fires", () => {
+        // The event announces record N; a consumer querying the index from that
+        // event must see N, not 0..N-1. One record makes the off-by-one total:
+        // the only record in the file would be invisible to the only event.
+        const source = [
+            "ISO-10303-21;",
+            "HEADER;",
+            "FILE_DESCRIPTION((''),'');",
+            "FILE_NAME('','',(''),(''),'','','');",
+            "FILE_SCHEMA(('IFC4'));",
+            "ENDSEC;",
+            "DATA;",
+            "#1= IFCPROJECT('p',$,'Only Record',$,$,$,$,$,$);",
+            "ENDSEC;",
+            "END-ISO-10303-21;",
+            "",
+        ].join("\n");
+        const sink = new ColumnarIndexSink();
+        const index = new PrefixTypeIndex(sink, new StepTypeIndexer(EntityTypesIfcCount), { growthFactor: 1, minimumRecords: 0 });
+        const seenInEvent = [];
+        const result = buildIndexStreaming(new BufferByteSource(new TextEncoder().encode(source)), IfcStepParser.Instance, 4 * 1024, () => {
+            seenInEvent.push([...index.expressIDsOfTypes(IfcProject)]);
+        }, sink);
+        expect(result.result).toBe(ParseResult.COMPLETE);
+        expect(seenInEvent).toEqual([[1]]);
+    });
+    test("an early query on a small file does not freeze the view", () => {
+        // index.ifc has fewer records than the default minimumRecords, so it sits
+        // entirely in the "rebuild is trivial, stay exact" regime. Querying from
+        // the very first event must not pin the view to that one-record prefix
+        // for the rest of the parse — least of all after it finishes.
+        const sink = new ColumnarIndexSink();
+        const index = new PrefixTypeIndex(sink, new StepTypeIndexer(EntityTypesIfcCount));
+        let queriedEarly = false;
+        buildIndexStreaming(new BufferByteSource(bytes), IfcStepParser.Instance, 4 * 1024, (localID) => {
+            if (localID === 0) {
+                expect([...index.expressIDsOfTypes(IfcRoot)]).toHaveLength(0);
+                queriedEarly = true;
+            }
+        }, sink);
+        expect(queriedEarly).toBe(true);
+        expect(sink.topLevelCount).toBeLessThan(1024);
+        const final = new Set(index.expressIDsOfTypes(IfcRoot));
+        const resident = new Set(model.expressIDsOfTypes(IfcRoot));
+        expect(final).toEqual(resident);
+    });
+    test("a sink reset invalidates the view instead of pacing past it", () => {
+        // The streaming builder's grow-and-restart resets the sink. Growth pacing
+        // alone would never notice: the count drops, and a restarted parse can
+        // finish below the threshold the abandoned pass already built at.
+        const sink = new ColumnarIndexSink();
+        const index = new PrefixTypeIndex(sink, new StepTypeIndexer(16));
+        for (let record = 0; record < 8; ++record) {
+            sink.pushTopLevel({ address: record, length: 1, typeID: 5, expressID: record + 1 });
+        }
+        expect([...index.expressIDsOfTypeIDs(5)]).toHaveLength(8);
+        expect(index.generation).toBe(1);
+        sink.reset();
+        sink.pushTopLevel({ address: 0, length: 1, typeID: 5, expressID: 99 });
+        expect([...index.expressIDsOfTypeIDs(5)]).toEqual([99]);
+        expect(index.generation).toBe(2);
+    });
+    // Property sweep: whatever the query cadence and pacing, two invariants must
+    // hold at every query, and between them they pin the whole contract —
+    //   consistency: the view equals a resident parse of exactly the records it
+    //     claims to cover (`recordCount`), so it is never a mix of prefixes;
+    //   freshness:   `recordCount` is either current, or within `growthFactor`
+    //     of current, so a view can go stale but can never be abandoned.
+    // Each of the mid-parse bugs found in review violated one of these; asserting
+    // only the final state (as the first tests here did) passes through all of
+    // them, because everything is exact once the parse returns.
+    describe.each([
+        ["default pacing", { growthFactor: 2, minimumRecords: 1024 }],
+        ["paced from the start", { growthFactor: 2, minimumRecords: 0 }],
+        ["coarse pacing", { growthFactor: 4, minimumRecords: 8 }],
+        ["fine pacing", { growthFactor: 1.25, minimumRecords: 32 }],
+        ["never paced", { growthFactor: 1, minimumRecords: 0 }],
+    ])("invariants under %s", (_name, options) => {
+        test.each([1, 7, 97, 0])("hold when queried every %i records (0 = only at the end)", (cadence) => {
+            const closure = IfcRoot.query;
+            const sink = new ColumnarIndexSink();
+            const index = new PrefixTypeIndex(sink, new StepTypeIndexer(EntityTypesIfcCount), options);
+            let queries = 0;
+            /** Query once and assert both invariants against the ground truth. */
+            const queryAndCheck = () => {
+                const parsed = sink.topLevelCount;
+                const view = new Set(index.expressIDsOfTypes(IfcRoot));
+                ++queries;
+                expect(view).toEqual(expectedOverPrefix(index.recordCount, closure));
+                const fresh = index.recordCount === parsed ||
+                    parsed < index.recordCount * options.growthFactor;
+                expect(fresh).toBe(true);
+            };
+            const result = buildIndexStreaming(new BufferByteSource(bytes), IfcStepParser.Instance, 4 * 1024, cadence === 0 ?
+                void 0 :
+                (localID) => {
+                    if (localID % cadence === 0) {
+                        queryAndCheck();
+                    }
+                }, sink);
+            expect(result.result).toBe(ParseResult.COMPLETE);
+            queryAndCheck();
+            expect(queries).toBeGreaterThan(0);
+            // And an explicit refresh always lands on the whole file, whatever
+            // the pacing left behind.
+            index.refresh();
+            expect(new Set(index.expressIDsOfTypes(IfcRoot)))
+                .toEqual(new Set(model.expressIDsOfTypes(IfcRoot)));
+        });
+    });
+    // Cursors come from LIFO pools, so "was it returned?" is observable by
+    // identity: park one in the pool, run a query, and see whether the next
+    // allocation hands back the same instance. A query that leaks its cursor
+    // leaves the pool empty and the next allocation is a fresh object.
+    //
+    // Both cursor kinds are covered because which one a query allocates depends
+    // on the size of the subtype closure — IfcProject's is a single type
+    // (SingleIndexSetCursor), a two-type query is a union
+    // (MultiIndexSetCursorOr). Testing only a subtyped query would park a single
+    // cursor that the query never touches, and pass no matter what.
+    describe.each([
+        ["single-type", () => SingleIndexSetCursor.allocate(new Uint32Array(2)), (index) => index.expressIDsOfTypes(IfcProject)],
+        ["multi-type", () => MultiIndexSetCursorOr.allocate(new Uint32Array(2)), (index) => index.expressIDsOfTypes(IfcProject, IfcPropertySet)],
+    ])("the %s cursor pool", (_name, park, query) => {
+        test.each([
+            ["a fully consumed query", (ids) => {
+                    for (const _id of ids) {
+                        // drain
+                    }
+                }],
+            ["a query abandoned partway", (ids) => {
+                    for (const _id of ids) {
+                        break;
+                    }
+                }],
+        ])("gets its cursor back after %s", (_consumeName, consume) => {
+            // The abandoned case is the one that needs try/finally rather than a
+            // free() after the loop — a consumer breaking out of a query is
+            // ordinary, not exceptional.
+            const index = streamed();
+            const parked = park();
+            parked.free();
+            consume(query(index));
+            const reused = park();
+            reused.free();
+            expect(reused).toBe(parked);
+        });
+    });
+    test("complex records are attributed to their mapped classes", () => {
+        // Built by hand rather than parsed: the point is precisely the shape the
+        // record event cannot express — one address, typeID 0, several mapped
+        // classes hanging off it.
+        const sink = new ColumnarIndexSink();
+        sink.pushTopLevel({ address: 0, length: 10, typeID: 5, expressID: 1 });
+        sink.pushTopLevel({
+            address: 10,
+            length: 20,
+            typeID: 0,
+            expressID: 2,
+            multiMapping: [{ address: 12, length: 5, typeID: 7 }],
+        });
+        const index = new PrefixTypeIndex(sink, new StepTypeIndexer(16));
+        expect([...index.expressIDsOfTypeIDs(5)]).toEqual([1]);
+        expect([...index.expressIDsOfTypeIDs(7)]).toEqual([2]);
+        expect([...index.types()]).toEqual(expect.arrayContaining([5, 7]));
+    });
+});

package/compiled/src/step/parsing/range_byte_source.d.ts

@@ -0,0 +1,69 @@
+import { StepExternalByteStore } from "../step_buffer_provider.js";
+/**
+ * Fetch accounting for a {@link RangeByteSource}: how many range requests were
+ * issued and how many bytes actually crossed the wire, versus how many the
+ * caller asked for. The gap between `bytesFetched` and `bytesServed` is the
+ * over-read imposed by block alignment (a real HTTP-Range / OPFS-block store
+ * can only fetch whole blocks); a small gap means the access pattern has good
+ * locality — the property an index-first open (M4) leans on.
+ */
+export interface RangeFetchStats {
+    /** Number of `read` calls that reached the backing store. */
+    requestCount: number;
+    /** Total bytes actually fetched from the backing store (aligned spans). */
+    bytesFetched: number;
+    /** Total bytes handed back to callers (the sum of requested lengths). */
+    bytesServed: number;
+}
+/**
+ * An asynchronous {@link StepExternalByteStore} that models a **range-fetched**
+ * source — an HTTP server honouring `Range:` requests, or an OPFS/blob store
+ * read block-by-block — so the index-first open path (M4) can be exercised and
+ * measured without a network. It wraps a fully-resident buffer (the "server")
+ * but never assumes the client holds it: every `read` is counted as a fetch,
+ * and with a `blockBytes` grid the fetch is rounded out to whole blocks, so
+ * {@link stats} reports the real transfer an aligned store would incur.
+ *
+ * This is the source a sidecar-driven open pairs with: reconstruct the entity
+ * index from the sidecar (no scan), then pull only the byte ranges demand asks
+ * for through a source like this — and read back from `stats` how little of the
+ * file that touched.
+ */
+export declare class RangeByteSource implements StepExternalByteStore {
+    private readonly bytes_;
+    private readonly blockBytes_;
+    private requestCount_;
+    private bytesFetched_;
+    private bytesServed_;
+    /**
+     * @param bytes_ The full backing bytes (the notional server-side file). Not
+     * copied; treated as read-only.
+     * @param blockBytes Optional fetch granularity: reads are rounded out to a
+     * multiple of this so `bytesFetched` reflects whole-block transfer. Omit (or
+     * pass 0) for exact, unaligned fetches.
+     */
+    constructor(bytes_: Uint8Array, blockBytes?: number);
+    /**
+     * Total size of the stored byte sequence.
+     *
+     * @return {number} The byte length.
+     */
+    get byteLength(): number;
+    /**
+     * A snapshot of the fetch counters.
+     *
+     * @return {RangeFetchStats} The current stats.
+     */
+    get stats(): RangeFetchStats;
+    /**
+     * Read a range, accounting for the (block-aligned) fetch it would incur.
+     * Returns exactly the requested `length` bytes as a standalone array, even
+     * though the underlying fetch may have pulled a wider aligned span.
+     *
+     * @param offset Absolute offset of the first byte to read.
+     * @param length Number of bytes to read.
+     * @return {Promise< Uint8Array >} The requested bytes (byteOffset 0).
+     */
+    read(offset: number, length: number): Promise<Uint8Array>;
+}
+//# sourceMappingURL=range_byte_source.d.ts.map

package/compiled/src/step/parsing/range_byte_source.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"range_byte_source.d.ts","sourceRoot":"","sources":["../../../../src/step/parsing/range_byte_source.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAA;AAG/D;;;;;;;GAOG;AACH,MAAM,WAAW,eAAe;IAE9B,6DAA6D;IAC7D,YAAY,EAAE,MAAM,CAAA;IAEpB,2EAA2E;IAC3E,YAAY,EAAE,MAAM,CAAA;IAEpB,yEAAyE;IACzE,WAAW,EAAE,MAAM,CAAA;CACpB;AAQD;;;;;;;;;;;;;GAaG;AACH,qBAAa,eAAgB,YAAW,qBAAqB;IAkBvD,OAAO,CAAC,QAAQ,CAAC,MAAM;IAhB3B,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAQ;IAEpC,OAAO,CAAC,aAAa,CAAI;IAEzB,OAAO,CAAC,aAAa,CAAI;IAEzB,OAAO,CAAC,YAAY,CAAI;IAExB;;;;;;OAMG;gBAEkB,MAAM,EAAE,UAAU,EACnC,UAAU,GAAE,MAAU;IAS1B;;;;OAIG;IACH,IAAW,UAAU,IAAI,MAAM,CAE9B;IAED;;;;OAIG;IACH,IAAW,KAAK,IAAI,eAAe,CAMlC;IAED;;;;;;;;OAQG;IACI,IAAI,CAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAI,OAAO,CAAE,UAAU,CAAE;CAkBrE"}

package/compiled/src/step/parsing/range_byte_source.js

@@ -0,0 +1,78 @@
+// Default block granularity when none is given: reads are unaligned, so
+// `bytesFetched` equals `bytesServed` (each read fetches exactly its range).
+const NO_ALIGNMENT = 1;
+/**
+ * An asynchronous {@link StepExternalByteStore} that models a **range-fetched**
+ * source — an HTTP server honouring `Range:` requests, or an OPFS/blob store
+ * read block-by-block — so the index-first open path (M4) can be exercised and
+ * measured without a network. It wraps a fully-resident buffer (the "server")
+ * but never assumes the client holds it: every `read` is counted as a fetch,
+ * and with a `blockBytes` grid the fetch is rounded out to whole blocks, so
+ * {@link stats} reports the real transfer an aligned store would incur.
+ *
+ * This is the source a sidecar-driven open pairs with: reconstruct the entity
+ * index from the sidecar (no scan), then pull only the byte ranges demand asks
+ * for through a source like this — and read back from `stats` how little of the
+ * file that touched.
+ */
+export class RangeByteSource {
+    /**
+     * @param bytes_ The full backing bytes (the notional server-side file). Not
+     * copied; treated as read-only.
+     * @param blockBytes Optional fetch granularity: reads are rounded out to a
+     * multiple of this so `bytesFetched` reflects whole-block transfer. Omit (or
+     * pass 0) for exact, unaligned fetches.
+     */
+    constructor(bytes_, blockBytes = 0) {
+        this.bytes_ = bytes_;
+        this.requestCount_ = 0;
+        this.bytesFetched_ = 0;
+        this.bytesServed_ = 0;
+        if (blockBytes < 0) {
+            throw new Error(`Invalid blockBytes ${blockBytes}`);
+        }
+        this.blockBytes_ = blockBytes > 0 ? blockBytes : NO_ALIGNMENT;
+    }
+    /**
+     * Total size of the stored byte sequence.
+     *
+     * @return {number} The byte length.
+     */
+    get byteLength() {
+        return this.bytes_.byteLength;
+    }
+    /**
+     * A snapshot of the fetch counters.
+     *
+     * @return {RangeFetchStats} The current stats.
+     */
+    get stats() {
+        return {
+            requestCount: this.requestCount_,
+            bytesFetched: this.bytesFetched_,
+            bytesServed: this.bytesServed_,
+        };
+    }
+    /**
+     * Read a range, accounting for the (block-aligned) fetch it would incur.
+     * Returns exactly the requested `length` bytes as a standalone array, even
+     * though the underlying fetch may have pulled a wider aligned span.
+     *
+     * @param offset Absolute offset of the first byte to read.
+     * @param length Number of bytes to read.
+     * @return {Promise< Uint8Array >} The requested bytes (byteOffset 0).
+     */
+    read(offset, length) {
+        const total = this.bytes_.byteLength;
+        const start = Math.max(0, Math.min(offset, total));
+        const end = Math.max(start, Math.min(offset + length, total));
+        // The aligned span a block-granular store would actually transfer.
+        const block = this.blockBytes_;
+        const fetchStart = start - (start % block);
+        const fetchEnd = Math.min(total, Math.ceil(end / block) * block);
+        this.requestCount_ += 1;
+        this.bytesFetched_ += fetchEnd - fetchStart;
+        this.bytesServed_ += end - start;
+        return Promise.resolve(this.bytes_.slice(start, end));
+    }
+}

package/compiled/src/step/parsing/range_byte_source.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=range_byte_source.test.d.ts.map

package/compiled/src/step/parsing/range_byte_source.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"range_byte_source.test.d.ts","sourceRoot":"","sources":["../../../../src/step/parsing/range_byte_source.test.ts"],"names":[],"mappings":""}

package/compiled/src/step/parsing/range_byte_source.test.js

@@ -0,0 +1,70 @@
+/* eslint-disable no-magic-numbers */
+// M4: the range-fetched byte source models an HTTP-Range / block store — it
+// returns exactly the requested bytes while accounting for the (possibly
+// wider, block-aligned) fetch it would really incur. Index-first open reads
+// back from those stats to prove it touched only a fraction of the file.
+import { describe, expect, test } from "@jest/globals";
+import { RangeByteSource } from "./range_byte_source.js";
+/**
+ * A deterministic ramp buffer: byte i === i mod 256.
+ *
+ * @param n Length.
+ * @return {Uint8Array} The buffer.
+ */
+function ramp(n) {
+    const bytes = new Uint8Array(n);
+    for (let where = 0; where < n; ++where) {
+        bytes[where] = where & 255;
+    }
+    return bytes;
+}
+describe("RangeByteSource", () => {
+    test("returns exactly the requested range", async () => {
+        const source = new RangeByteSource(ramp(1000));
+        const got = await source.read(100, 16);
+        expect(got.byteLength).toBe(16);
+        expect(Array.from(got)).toEqual(Array.from({ length: 16 }, (_v, i) => (100 + i) & 255));
+    });
+    test("unaligned reads fetch exactly what is served", async () => {
+        const source = new RangeByteSource(ramp(1000));
+        await source.read(10, 20);
+        await source.read(500, 4);
+        const stats = source.stats;
+        expect(stats.requestCount).toBe(2);
+        expect(stats.bytesServed).toBe(24);
+        expect(stats.bytesFetched).toBe(24);
+    });
+    test("block alignment over-reads to whole blocks", async () => {
+        const source = new RangeByteSource(ramp(10000), 4096);
+        // A 10-byte read at offset 100 sits inside the first 4096-byte block.
+        await source.read(100, 10);
+        expect(source.stats.bytesFetched).toBe(4096);
+        expect(source.stats.bytesServed).toBe(10);
+        // A read straddling the first/second block boundary pulls two blocks.
+        await source.read(4090, 20);
+        expect(source.stats.bytesFetched).toBe(4096 + 8192);
+        expect(source.stats.bytesServed).toBe(30);
+    });
+    test("clamps reads at end of source", async () => {
+        const source = new RangeByteSource(ramp(100));
+        const got = await source.read(90, 50);
+        expect(got.byteLength).toBe(10);
+        expect(source.stats.bytesServed).toBe(10);
+    });
+    test("a read fully past the end is empty and fetches nothing", async () => {
+        const source = new RangeByteSource(ramp(100));
+        const got = await source.read(200, 10);
+        expect(got.byteLength).toBe(0);
+        expect(source.stats.bytesFetched).toBe(0);
+    });
+    test("block-aligned fetch never exceeds the source length", async () => {
+        const source = new RangeByteSource(ramp(5000), 4096);
+        // Last block is partial (5000 = 4096 + 904); fetching into it must clamp.
+        await source.read(4990, 10);
+        expect(source.stats.bytesFetched).toBe(5000 - 4096);
+        expect(source.stats.bytesFetched).toBeLessThanOrEqual(5000);
+    });
+    test("reports byteLength of the backing source", () => {
+        expect(new RangeByteSource(ramp(777)).byteLength).toBe(777);
+    });
+});

package/compiled/src/step/parsing/record_event.d.ts

@@ -0,0 +1,23 @@
+/**
+ * The streaming parse's per-record event (M2, issue #393): fired as each
+ * top-level record is indexed, before the record's bytes leave the window.
+ *
+ * The record's raw bytes are handed over as `(buffer, byteOffset, byteLength)`
+ * rather than as a `Uint8Array` slice, deliberately: a slice would be a fresh
+ * object per record — 9.4 M allocations on a PSB-class file, measured at ~5.7 %
+ * of parse wall-clock and ~46 MB of retained heap for handlers that never look
+ * at the bytes (the spike in `scripts/m2_consumer_spike.mjs`). With the view
+ * un-materialised, a handler that only wants ids pays nothing, and one that
+ * wants bytes calls `buffer.subarray( byteOffset, byteOffset + byteLength )`
+ * itself.
+ *
+ * `buffer` is the parse window, so the bytes are valid ONLY for the duration
+ * of the call — the window slides and is overwritten as the parse advances.
+ * A handler that needs them later must copy.
+ *
+ * Handlers run synchronously inside the parse loop and must stay cheap
+ * (copy ids into compact structures). Anything expensive belongs on the
+ * demand queue — see design/new/streaming-federated-loader.md § 2.
+ */
+export type RecordEventHandler<TypeIDType> = (localID: number, expressID: number, typeID: TypeIDType | undefined, buffer?: Uint8Array, byteOffset?: number, byteLength?: number) => void;
+//# sourceMappingURL=record_event.d.ts.map

package/compiled/src/step/parsing/record_event.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"record_event.d.ts","sourceRoot":"","sources":["../../../../src/step/parsing/record_event.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,MAAM,kBAAkB,CAAC,UAAU,IAAI,CAC3C,OAAO,EAAE,MAAM,EACf,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,UAAU,GAAG,SAAS,EAC9B,MAAM,CAAC,EAAE,UAAU,EACnB,UAAU,CAAC,EAAE,MAAM,EACnB,UAAU,CAAC,EAAE,MAAM,KAAM,IAAI,CAAA"}

package/compiled/src/step/parsing/record_event.js

@@ -0,0 +1 @@
+export {};

package/compiled/src/step/parsing/record_field_cursor.d.ts

@@ -0,0 +1,94 @@
+import StepParser from "./step_parser.js";
+/**
+ * Reads individual attributes out of a record's raw bytes, without a model and
+ * without materialising an entity (M2, issue #393).
+ *
+ * This is what makes a parse-path consumer viable at all: the streaming
+ * parse's record event hands over the record's attribute list as a window
+ * range, and a consumer that wants two strings out of it should not have to
+ * wait for the model, page the record back in, and construct an entity to get
+ * them. Field positions come from the production tokenizer
+ * ({@link StepParser.extractDataEntry}), so a record this reads and a record
+ * the model reads agree by construction — including on the awkward cases
+ * (quoted commas, nested parens, escaped quotes) a hand-rolled splitter gets
+ * wrong.
+ *
+ * One instance is reused across records: {@link open} clears and refills a
+ * single vtable buffer, so a consumer running over millions of records
+ * allocates nothing per record. Attribute indices are the same ones the
+ * generated accessors use (`extractString( 2, … )` for `IfcRoot.Name`, and so
+ * on), read against the *record's own* attribute list — inherited attributes
+ * included, since STEP serialises the flattened list.
+ */
+export declare class RecordFieldCursor<TypeIDType> {
+    private readonly parser_;
+    private readonly vtable_;
+    private buffer_;
+    private indexMark_;
+    private fieldCount_;
+    private endCursor_;
+    /**
+     * @param parser_ The schema's parser — used only for its record tokenizer.
+     */
+    constructor(parser_: StepParser<TypeIDType>);
+    /**
+     * Point this at one record's attribute list.
+     *
+     * @param buffer The buffer holding the record (the live parse window, for an
+     * event consumer — so everything read here must be read before returning).
+     * @param byteOffset Offset of the record's first attribute byte.
+     * @param byteLength Length of the attribute list in bytes.
+     * @return {number} The number of top-level attributes found; 0 if the record
+     * could not be tokenized (a truncated or malformed record — the caller
+     * should skip it rather than guess).
+     */
+    open(buffer: Uint8Array, byteOffset: number, byteLength: number): number;
+    /**
+     * The byte range of one attribute, as `[start, end)`.
+     *
+     * @param index The attribute index.
+     * @return {[number, number] | undefined} The range, or undefined when the
+     * record has no such attribute.
+     */
+    private range;
+    /**
+     * Read a string attribute.
+     *
+     * @param index The attribute index.
+     * @return {string | undefined} The decoded string, or undefined when the
+     * attribute is absent, `$`, or not a string.
+     */
+    string(index: number): string | undefined;
+    /**
+     * Read a single express reference attribute (`#123`).
+     *
+     * @param index The attribute index.
+     * @return {number | undefined} The express ID, or undefined when the
+     * attribute is absent or is not a reference.
+     */
+    reference(index: number): number | undefined;
+    /**
+     * Visit every express reference in an attribute, which is how a reference
+     * list (`(#1,#2,#3)`) is read — the enclosing parens and separators need no
+     * interpretation, since a reference is the only thing a `#` can start.
+     *
+     * @param index The attribute index.
+     * @param visit Called with each express ID, in order.
+     */
+    forEachReference(index: number, visit: (expressID: number) => void): void;
+    /**
+     * Read the digits of a reference whose `#` sits at `cursor`.
+     *
+     * @param cursor Offset of the `#`.
+     * @param end Exclusive end of the attribute.
+     * @return {number | undefined} The express ID, or undefined if `cursor` is
+     * not a `#` followed by at least one digit.
+     */
+    private readReferenceAt;
+    /**
+     * @param code A byte.
+     * @return {boolean} Whether it is an ASCII digit.
+     */
+    private isDigit;
+}
+//# sourceMappingURL=record_field_cursor.d.ts.map

package/compiled/src/step/parsing/record_field_cursor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"record_field_cursor.d.ts","sourceRoot":"","sources":["../../../../src/step/parsing/record_field_cursor.ts"],"names":[],"mappings":"AACA,OAAO,UAAU,MAAM,eAAe,CAAA;AAkBtC;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,qBAAa,iBAAiB,CAAC,UAAU;IAe1B,OAAO,CAAC,QAAQ,CAAC,OAAO;IAbrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA0B;IAElD,OAAO,CAAC,OAAO,CAAiC;IAEhD,OAAO,CAAC,UAAU,CAAI;IAEtB,OAAO,CAAC,WAAW,CAAI;IAEvB,OAAO,CAAC,UAAU,CAAI;IAEtB;;OAEG;gBAC2B,OAAO,EAAE,UAAU,CAAC,UAAU,CAAC;IAE7D;;;;;;;;;;OAUG;IACI,IAAI,CAAE,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAI,MAAM;IAsBjF;;;;;;OAMG;IACH,OAAO,CAAC,KAAK;IAiBb;;;;;;OAMG;IACI,MAAM,CAAE,KAAK,EAAE,MAAM,GAAI,MAAM,GAAG,SAAS;IAUlD;;;;;;OAMG;IACI,SAAS,CAAE,KAAK,EAAE,MAAM,GAAI,MAAM,GAAG,SAAS;IAwBrD;;;;;;;OAOG;IACI,gBAAgB,CAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAE,SAAS,EAAE,MAAM,KAAM,IAAI,GAAI,IAAI;IA8BpF;;;;;;;OAOG;IACH,OAAO,CAAC,eAAe;IAoBvB;;;OAGG;IACH,OAAO,CAAC,OAAO;CAGhB"}