@bldrs-ai/conway 1.389.1221 → 1.394.1504

package/compiled/src/step/parsing/streaming_index_builder.js

@@ -0,0 +1,280 @@
+import ParsingBuffer from "../../parsing/parsing_buffer.js";
+import { ColumnarIndexSink } from "./columnar_index.js";
+import { ParseResult, } from "./step_parser.js";
+/**
+ * Build the entity index by streaming a `ByteSource` through a fixed-size
+ * moving window, instead of parsing one resident buffer. The parse loop is
+ * byte-for-byte the resident one (`StepParser.parseDataBlockStreamed` drives
+ * the same generator); this coordinator only owns the window: it fills it,
+ * parses the header from the first fill, then slides the window forward at
+ * top-level record boundaries as the parser advances.
+ *
+ * Because the parser records file-absolute addresses and its rewind stack is
+ * empty at every top-level boundary, sliding there is transparent: the
+ * unconsumed tail is copied to the front, fresh bytes are appended, and the
+ * buffer is rebased so `address` keeps its file-absolute value. Peak JS heap
+ * for the index build is therefore `window + index columns`, independent of
+ * file size.
+ *
+ * The window slides only once the cursor passes `pool / 2`, bounding the
+ * memmove frequency; a record up to `pool / 2` bytes always fits after a
+ * slide. If a single record exceeds that (never on the current corpus, whose
+ * largest STEP record is ~25 KB), the whole parse restarts from the
+ * beginning with the window doubled, and repeats until every record fits —
+ * correctness over the pathological case, at the cost of a re-scan. (M1's
+ * production path will instead grow in place / restart from the last
+ * boundary; from-scratch keeps the spike simple.)
+ *
+ * @param source The byte source.
+ * @param parser The STEP parser (typed to the schema).
+ * @param pool Target window size in bytes.
+ * @param onRecordIndexed Optional per-record event, invoked live as each
+ * top-level record is indexed with its (localID, expressID, typeID) — the
+ * seam for incremental semantic consumers (M2). localIDs are dense and
+ * assigned in parse order from 0. On the rare grow-and-restart (a single
+ * record larger than the window — never on real files, whose largest STEP
+ * record is ~25 KB), the parse re-runs from the start and records re-fire
+ * from localID 0; consumers must therefore be idempotent by localID /
+ * expressID (the standard consumers — type index keyed by localID, roots
+ * registry keyed by expressID — are). Must be synchronous and cheap;
+ * expensive work belongs on a demand queue, not the parse path.
+ * @return {StreamingIndexResult} The index, header, result and diagnostics.
+ */
+export function buildIndexStreaming(source, parser, pool, onRecordIndexed, sink) {
+    const fileSize = source.byteLength;
+    // Grow-and-restart loop: the body runs a full streamed parse at the current
+    // window size. It only re-runs if a single record couldn't fit the window
+    // (false end-of-file before true EOF), doubling the window each time.
+    let windowBytes = Math.max(pool, MIN_WINDOW);
+    for (;;) {
+        const window = new Uint8Array(windowBytes);
+        let windowStartFile = 0;
+        let windowLen = source.read(0, windowBytes, window, 0);
+        let bytesRead = windowLen;
+        const input = new ParsingBuffer(window, 0, windowLen);
+        const [header, headerResult] = parser.parseHeader(input);
+        if (headerResult !== ParseResult.COMPLETE) {
+            // Header didn't complete in the first window: either malformed, or a
+            // header larger than the pool. Report it as-is — the resident parse
+            // would surface the same header result.
+            return {
+                header,
+                elements: [],
+                result: headerResult,
+                stats: { pool, windowBytes, slides: 0, maxRecordLen: 0, bytesRead },
+            };
+        }
+        // Slide once the cursor is past half the window; a record up to this much
+        // headroom always fits without a false EOF.
+        const slideThreshold = windowBytes >> 1;
+        let slides = 0;
+        let maxRecordLen = 0;
+        let prevBoundaryFile = windowStartFile + input.cursor;
+        const onRecordBoundary = (buffer) => {
+            const cursor = buffer.cursor;
+            const recordFileStart = windowStartFile + cursor;
+            const recordLen = recordFileStart - prevBoundaryFile;
+            if (recordLen > maxRecordLen) {
+                maxRecordLen = recordLen;
+            }
+            prevBoundaryFile = recordFileStart;
+            // Window already spans to EOF — nothing left to slide in.
+            if (windowStartFile + windowLen >= fileSize) {
+                return;
+            }
+            // Enough headroom remains; defer the slide to bound memmove frequency.
+            if (cursor < slideThreshold) {
+                return;
+            }
+            // Slide: move the unconsumed tail to the front, refill behind it.
+            const tail = windowLen - cursor;
+            window.copyWithin(0, cursor, windowLen);
+            const want = windowBytes - tail;
+            const got = source.read(windowStartFile + windowLen, want, window, tail);
+            bytesRead += got;
+            windowLen = tail + got;
+            windowStartFile = recordFileStart;
+            buffer.rebaseWindow(window, 0, windowLen, windowStartFile);
+            ++slides;
+        };
+        const [index, result] = parser.parseDataBlockStreamed(input, onRecordBoundary, onRecordIndexed, void 0, sink);
+        // A parse that stopped short of true EOF, with the window not spanning to
+        // EOF, means a single record overflowed the window: grow and retry.
+        const stoppedShort = result !== ParseResult.COMPLETE;
+        const notAtEof = windowStartFile + windowLen < fileSize;
+        if (stoppedShort && notAtEof) {
+            windowBytes *= 2;
+            sink?.reset();
+            continue;
+        }
+        // Capture the final record's length (no boundary fires after the last).
+        const lastRecordLen = (windowStartFile + input.cursor) - prevBoundaryFile;
+        if (lastRecordLen > maxRecordLen) {
+            maxRecordLen = lastRecordLen;
+        }
+        return {
+            header,
+            elements: index.elements,
+            result,
+            stats: { pool, windowBytes, slides, maxRecordLen, bytesRead },
+        };
+    }
+}
+// Floor on the window so a pathologically tiny pool still holds the header
+// and a record with slide headroom. Tests drive pools down to this to force
+// many slides on a small fixture; the pool sweep uses ≥ 128 KB anyway.
+// eslint-disable-next-line no-magic-numbers
+const MIN_WINDOW = 4 * 1024;
+/**
+ * Fill `into` from either a sync or async byte source. A sync `read`
+ * resolves immediately; this is what lets the cooperative builder
+ * parse from an OPFS `File.slice()` store without a worker.
+ *
+ * @param source Sync or async positioned source.
+ * @param offset Absolute source offset.
+ * @param length Maximum bytes to copy.
+ * @param into Destination buffer.
+ * @param intoOffset Offset within `into`.
+ * @return {Promise<number>} Bytes copied.
+ */
+async function readSource(source, offset, length, into, intoOffset) {
+    return await source.read(offset, length, into, intoOffset);
+}
+/**
+ * Cooperative twin of {@link buildIndexStreaming}: identical parse, window
+ * and grow-and-restart behaviour (mirrored the same way the parser mirrors
+ * parseDataBlock/parseDataBlockAsync), but the parse periodically yields to
+ * the event loop so browsers repaint progress UI mid-parse instead of
+ * flagging the tab as stalled (issue #301 §2 for the streamed path).
+ *
+ * @param source The byte source.
+ * @param parser The STEP parser (typed to the schema).
+ * @param pool Target window size in bytes.
+ * @param onRecordIndexed Optional per-record event (see buildIndexStreaming).
+ * @param sink Optional index sink (columnar builds).
+ * @param onProgress Optional progress callback with the ABSOLUTE source byte
+ * cursor (unlike the parser's window-relative cursor), so callers can report
+ * `cursor / source.byteLength` directly.
+ * @param yieldIntervalMs Minimum ms between event-loop yields.
+ * @return {Promise<StreamingIndexResult>} The index, header, result and
+ * diagnostics.
+ */
+export async function buildIndexStreamingAsync(source, parser, pool, onRecordIndexed, sink, onProgress, yieldIntervalMs) {
+    const fileSize = source.byteLength;
+    let windowBytes = Math.max(pool, MIN_WINDOW);
+    for (;;) {
+        const window = new Uint8Array(windowBytes);
+        let windowStartFile = 0;
+        let windowLen = await readSource(source, 0, windowBytes, window, 0);
+        let bytesRead = windowLen;
+        const input = new ParsingBuffer(window, 0, windowLen);
+        const [header, headerResult] = parser.parseHeader(input);
+        if (headerResult !== ParseResult.COMPLETE) {
+            return {
+                header,
+                elements: [],
+                result: headerResult,
+                stats: { pool, windowBytes, slides: 0, maxRecordLen: 0, bytesRead },
+            };
+        }
+        const slideThreshold = windowBytes >> 1;
+        let slides = 0;
+        let maxRecordLen = 0;
+        let prevBoundaryFile = windowStartFile + input.cursor;
+        const onRecordBoundary = (buffer) => {
+            const cursor = buffer.cursor;
+            const recordFileStart = windowStartFile + cursor;
+            const recordLen = recordFileStart - prevBoundaryFile;
+            if (recordLen > maxRecordLen) {
+                maxRecordLen = recordLen;
+            }
+            prevBoundaryFile = recordFileStart;
+            if (windowStartFile + windowLen >= fileSize) {
+                return;
+            }
+            if (cursor < slideThreshold) {
+                return;
+            }
+            const tail = windowLen - cursor;
+            window.copyWithin(0, cursor, windowLen);
+            const want = windowBytes - tail;
+            const fillAt = windowStartFile + windowLen;
+            // Only return a Promise when this boundary actually slides —
+            // an `async` callback would force an await at every record
+            // (millions of hops on a PSB-class file).
+            return readSource(source, fillAt, want, window, tail).then((got) => {
+                bytesRead += got;
+                windowLen = tail + got;
+                windowStartFile = recordFileStart;
+                buffer.rebaseWindow(window, 0, windowLen, windowStartFile);
+                ++slides;
+            });
+        };
+        // Translate the parser's window-relative cursor to an absolute source
+        // cursor (windowStartFile advances as the window slides).
+        const progressTick = onProgress !== void 0 ?
+            (cursor) => onProgress(windowStartFile + cursor) : void 0;
+        const [index, result] = await parser.parseDataBlockStreamedAsync(input, onRecordBoundary, onRecordIndexed, progressTick, sink, yieldIntervalMs);
+        const stoppedShort = result !== ParseResult.COMPLETE;
+        const notAtEof = windowStartFile + windowLen < fileSize;
+        if (stoppedShort && notAtEof) {
+            windowBytes *= 2;
+            sink?.reset();
+            continue;
+        }
+        const lastRecordLen = (windowStartFile + input.cursor) - prevBoundaryFile;
+        if (lastRecordLen > maxRecordLen) {
+            maxRecordLen = lastRecordLen;
+        }
+        return {
+            header,
+            elements: index.elements,
+            result,
+            stats: { pool, windowBytes, slides, maxRecordLen, bytesRead },
+        };
+    }
+}
+/**
+ * Build the entity index by streaming, **directly into typed-array columns**
+ * (M7): identical parse and window behaviour to {@link buildIndexStreaming},
+ * but completed records are encoded straight into a {@link ColumnarIndexSink}
+ * so the per-record object index never materialises. Peak JS heap for the
+ * index build becomes `window + columns` — the object phase (the dominant
+ * heap cost on large models) is gone.
+ *
+ * @param source The byte source.
+ * @param parser The STEP parser (typed to the schema).
+ * @param pool Target window size in bytes.
+ * @param onRecordIndexed Optional per-record event (see buildIndexStreaming).
+ * @param sink Optional caller-owned sink. Pass one to hold the *live* columns
+ * while the parse runs — that is what a prefix-derived consumer
+ * ({@link import('./prefix_type_index').PrefixTypeIndex}) reads. Omit it and
+ * the columns only exist once this returns.
+ * @return {StreamingColumnarIndexResult} Columns, header, result, stats.
+ */
+export function buildColumnarIndexStreaming(source, parser, pool, onRecordIndexed, sink = new ColumnarIndexSink()) {
+    const { header, result, stats } = buildIndexStreaming(source, parser, pool, onRecordIndexed, sink);
+    return { header, columns: sink.finalize(), result, stats };
+}
+/**
+ * Cooperative twin of {@link buildColumnarIndexStreaming}: identical
+ * columnar build, but the parse periodically yields to the event loop (see
+ * {@link buildIndexStreamingAsync}) and reports absolute byte-cursor
+ * progress — the browser-facing variant for large models.
+ *
+ * @param source The byte source.
+ * @param parser The STEP parser (typed to the schema).
+ * @param pool Target window size in bytes.
+ * @param onRecordIndexed Optional per-record event (see buildIndexStreaming).
+ * @param onProgress Optional absolute byte-cursor progress callback.
+ * @param yieldIntervalMs Minimum ms between event-loop yields.
+ * @param sink Optional caller-owned sink (see
+ * {@link buildColumnarIndexStreaming}) — the seam for reading the columns
+ * while the parse is still running.
+ * @return {Promise<StreamingColumnarIndexResult>} Columns, header, result,
+ * stats.
+ */
+export async function buildColumnarIndexStreamingAsync(source, parser, pool, onRecordIndexed, onProgress, yieldIntervalMs, sink = new ColumnarIndexSink()) {
+    const { header, result, stats } = await buildIndexStreamingAsync(source, parser, pool, onRecordIndexed, sink, onProgress, yieldIntervalMs);
+    return { header, columns: sink.finalize(), result, stats };
+}

package/compiled/src/step/parsing/streaming_index_builder.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=streaming_index_builder.test.d.ts.map

package/compiled/src/step/parsing/streaming_index_builder.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"streaming_index_builder.test.d.ts","sourceRoot":"","sources":["../../../../src/step/parsing/streaming_index_builder.test.ts"],"names":[],"mappings":""}

package/compiled/src/step/parsing/streaming_index_builder.test.js

@@ -0,0 +1,107 @@
+/* eslint-disable no-magic-numbers */
+// M0 spike: streaming index build must be byte-identical to the resident
+// parse. The parser reads file-absolute addresses and only rewinds within a
+// single top-level record, so feeding it from a moving window that slides at
+// record boundaries must produce the exact same index columns — at every
+// pool size, including one small enough to force many slides + straddles.
+import * as fs from "fs";
+import { beforeAll, describe, expect, test } from "@jest/globals";
+import ParsingBuffer from "../../parsing/parsing_buffer.js";
+import IfcStepParser from "../../ifc/ifc_step_parser.js";
+import { BufferByteSource } from "./byte_source.js";
+import { buildIndexStreaming } from "./streaming_index_builder.js";
+import { ParseResult } from "./step_parser.js";
+/**
+ * Flatten the index (top-level records + their inline / multi-mapping
+ * children) into a stable, comparable column dump: every field the streaming
+ * parse must reproduce identically.
+ *
+ * @param elements The index elements.
+ * @return {object[]} A flat list of column tuples in traversal order.
+ */
+function columns(elements) {
+    const out = [];
+    const visit = (e) => {
+        out.push({
+            address: e.address,
+            length: e.length,
+            typeID: e.typeID,
+            expressID: e.expressID ?? null,
+        });
+        for (const child of e.inlineEntities ?? []) {
+            visit(child);
+        }
+        for (const child of e.multiMapping ?? []) {
+            visit(child);
+        }
+    };
+    for (const e of elements) {
+        visit(e);
+    }
+    return out;
+}
+let bytes;
+let residentColumns;
+let residentCount;
+beforeAll(() => {
+    bytes = new Uint8Array(fs.readFileSync("data/index.ifc"));
+    const input = new ParsingBuffer(bytes);
+    IfcStepParser.Instance.parseHeader(input);
+    const [index, result] = IfcStepParser.Instance.parseDataBlock(input);
+    expect(result).toBe(ParseResult.COMPLETE);
+    residentColumns = columns(index.elements);
+    residentCount = index.elements.length;
+});
+describe("buildIndexStreaming", () => {
+    // A tiny pool relative to the 18 KB fixture forces repeated slides and at
+    // least one record straddling a window boundary; a large one takes the
+    // whole file in a single window (no slide). Both must match the resident
+    // parse exactly.
+    const POOLS = [4 * 1024, 8 * 1024, 16 * 1024, 128 * 1024];
+    test.each(POOLS)("produces the resident index at pool=%d bytes", (pool) => {
+        const source = new BufferByteSource(bytes);
+        const result = buildIndexStreaming(source, IfcStepParser.Instance, pool);
+        expect(result.result).toBe(ParseResult.COMPLETE);
+        expect(result.elements.length).toBe(residentCount);
+        expect(columns(result.elements)).toEqual(residentColumns);
+    });
+    test("the small pool actually slides the window (straddle exercised)", () => {
+        const source = new BufferByteSource(bytes);
+        const result = buildIndexStreaming(source, IfcStepParser.Instance, 4 * 1024);
+        // 18 KB fixture through a 4 KB window must slide several times.
+        expect(result.stats.slides).toBeGreaterThan(1);
+        // Physical window never grew past the pool (no record exceeds it).
+        expect(result.stats.windowBytes).toBe(4 * 1024);
+        // Every byte read once (no growth re-reads) — bounded to file size.
+        expect(result.stats.bytesRead).toBe(bytes.length);
+    });
+    test("the large pool takes the whole file in one window (no slide)", () => {
+        const source = new BufferByteSource(bytes);
+        const result = buildIndexStreaming(source, IfcStepParser.Instance, 128 * 1024);
+        expect(result.stats.slides).toBe(0);
+        expect(result.stats.maxRecordLen).toBeGreaterThan(0);
+    });
+    test("grows the window and restarts when a single record exceeds the pool", () => {
+        // Synthetic file with one record far larger than a tiny pool — the
+        // corpus never hits this (largest STEP record ~25 KB), so exercise the
+        // grow-and-restart safety valve directly. A long text literal makes one
+        // record ~12 KB; a 4 KB pool must grow past it and still index it.
+        const bigValue = "X".repeat(12000);
+        const text = "ISO-10303-21;\nHEADER;\nENDSEC;\nDATA;\n" +
+            "#1=IFCWALL('a');\n" +
+            `#2=IFCPROPERTYSINGLEVALUE('${bigValue}');\n` +
+            "#3=IFCWALL('c');\nENDSEC;\n";
+        const synthetic = new TextEncoder().encode(text);
+        const resident = new ParsingBuffer(synthetic);
+        IfcStepParser.Instance.parseHeader(resident);
+        const [residentIndex, residentResult] = IfcStepParser.Instance.parseDataBlock(resident);
+        expect(residentResult).toBe(ParseResult.COMPLETE);
+        const streamed = buildIndexStreaming(new BufferByteSource(synthetic), IfcStepParser.Instance, 4 * 1024);
+        expect(streamed.result).toBe(ParseResult.COMPLETE);
+        // Window grew past the 4 KB pool to fit the ~12 KB record.
+        expect(streamed.stats.windowBytes).toBeGreaterThan(4 * 1024);
+        expect(streamed.stats.maxRecordLen).toBeGreaterThan(12000);
+        // ...and the index is still byte-identical to the resident parse.
+        expect(columns(streamed.elements)).toEqual(columns(residentIndex.elements));
+    });
+});

package/compiled/src/step/parsing/streaming_index_builder_async.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=streaming_index_builder_async.test.d.ts.map

package/compiled/src/step/parsing/streaming_index_builder_async.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"streaming_index_builder_async.test.d.ts","sourceRoot":"","sources":["../../../../src/step/parsing/streaming_index_builder_async.test.ts"],"names":[],"mappings":""}

package/compiled/src/step/parsing/streaming_index_builder_async.test.js

@@ -0,0 +1,109 @@
+/* eslint-disable no-magic-numbers */
+// The cooperative streamed columnar build must be byte-identical to the
+// synchronous build (same columns, same window mechanics) AND actually
+// yield to the event loop mid-parse — the repaint/no-stall property
+// (#301 §2) the streamed open path relies on. The parse generator only
+// suspends every PARSE_PROGRESS_ELEMENT_MASK+1 (4096) records, so the
+// yield/progress assertions use a synthesized >4096-record model; the
+// column-parity assertions use the real fixture.
+import fs from "fs";
+import { describe, expect, test } from "@jest/globals";
+import IfcStepParser from "../../ifc/ifc_step_parser.js";
+import { BufferByteSource, StoreByteSource } from "./byte_source.js";
+import { InMemoryStepByteStore } from "../step_buffer_provider.js";
+import { buildColumnarIndexStreaming, buildColumnarIndexStreamingAsync, } from "./streaming_index_builder.js";
+import { ParseResult } from "./step_parser.js";
+const POOL = 8 * 1024; // small window → forces slides on both inputs
+/**
+ * Synthesize a valid IFC4 file with `recordCount` simple records — enough
+ * to cross the parser's progress-suspension mask several times.
+ *
+ * @param recordCount Number of data records.
+ * @return {Uint8Array} The encoded file.
+ */
+function syntheticIfc(recordCount) {
+    const parts = [
+        "ISO-10303-21;\nHEADER;\n" +
+            "FILE_DESCRIPTION((''),'2;1');\n" +
+            "FILE_NAME('','',(''),(''),'','','');\n" +
+            "FILE_SCHEMA(('IFC4'));\nENDSEC;\nDATA;\n",
+    ];
+    for (let where = 1; where <= recordCount; ++where) {
+        parts.push(`#${where}=IFCCARTESIANPOINT((${where}.,0.,0.));\n`);
+    }
+    parts.push("ENDSEC;\nEND-ISO-10303-21;\n");
+    return new TextEncoder().encode(parts.join(""));
+}
+describe("buildColumnarIndexStreamingAsync", () => {
+    test("produces identical columns to the sync build (real fixture)", async () => {
+        const bytes = new Uint8Array(fs.readFileSync("data/index.ifc"));
+        const parser = IfcStepParser.Instance;
+        const sync = buildColumnarIndexStreaming(new BufferByteSource(bytes), parser, POOL);
+        const cooperative = await buildColumnarIndexStreamingAsync(new BufferByteSource(bytes), parser, POOL, void 0, void 0, 0);
+        expect(sync.result).toBe(ParseResult.COMPLETE);
+        expect(cooperative.result).toBe(ParseResult.COMPLETE);
+        const a = sync.columns;
+        const b = cooperative.columns;
+        expect(b.count).toBe(a.count);
+        expect(b.firstInlineElement).toBe(a.firstInlineElement);
+        expect(Array.from(b.address)).toEqual(Array.from(a.address));
+        expect(Array.from(b.length)).toEqual(Array.from(a.length));
+        expect(Array.from(b.typeID)).toEqual(Array.from(a.typeID));
+        expect(Array.from(b.expressID)).toEqual(Array.from(a.expressID));
+        // Same window mechanics (slides prove the moving window really moved).
+        expect(cooperative.stats.slides).toBe(sync.stats.slides);
+        expect(cooperative.stats.slides).toBeGreaterThan(0);
+    }, 60000);
+    test("StoreByteSource fills produce the same columns as a buffer source", async () => {
+        const bytes = new Uint8Array(fs.readFileSync("data/index.ifc"));
+        const parser = IfcStepParser.Instance;
+        const fromBuffer = await buildColumnarIndexStreamingAsync(new BufferByteSource(bytes), parser, POOL, void 0, void 0, 0);
+        const fromStore = await buildColumnarIndexStreamingAsync(new StoreByteSource(new InMemoryStepByteStore(bytes)), parser, POOL, void 0, void 0, 0);
+        expect(fromStore.result).toBe(ParseResult.COMPLETE);
+        expect(Array.from(fromStore.columns.address))
+            .toEqual(Array.from(fromBuffer.columns.address));
+        expect(Array.from(fromStore.columns.expressID))
+            .toEqual(Array.from(fromBuffer.columns.expressID));
+    }, 60000);
+    test("yields to the event loop and reports absolute progress mid-parse", async () => {
+        const bytes = syntheticIfc(10000);
+        const parser = IfcStepParser.Instance;
+        // Count macrotask turns that run while the build is in flight — with a
+        // zero yield interval the parse must give the event loop real turns.
+        let turns = 0;
+        let pumping = true;
+        const pump = () => {
+            if (!pumping) {
+                return;
+            }
+            ++turns;
+            setTimeout(pump, 0);
+        };
+        setTimeout(pump, 0);
+        const progressCursors = [];
+        const cooperative = await buildColumnarIndexStreamingAsync(new BufferByteSource(bytes), parser, POOL, void 0, (cursor) => progressCursors.push(cursor), 0);
+        pumping = false;
+        expect(cooperative.result).toBe(ParseResult.COMPLETE);
+        expect(cooperative.columns.count).toBe(10000);
+        expect(turns).toBeGreaterThan(0);
+        // Progress reports absolute source cursors: monotonically
+        // non-decreasing, bounded by the file size, and past the first window —
+        // the window-relative → absolute translation must hold across slides.
+        expect(progressCursors.length).toBeGreaterThan(1);
+        for (let where = 1; where < progressCursors.length; ++where) {
+            expect(progressCursors[where])
+                .toBeGreaterThanOrEqual(progressCursors[where - 1]);
+        }
+        expect(progressCursors[progressCursors.length - 1])
+            .toBeLessThanOrEqual(bytes.byteLength);
+        expect(progressCursors[progressCursors.length - 1])
+            .toBeGreaterThan(POOL);
+        // And the synthetic build matches its own sync twin exactly.
+        const sync = buildColumnarIndexStreaming(new BufferByteSource(bytes), parser, POOL);
+        expect(Array.from(cooperative.columns.address))
+            .toEqual(Array.from(sync.columns.address));
+        expect(Array.from(cooperative.columns.expressID))
+            .toEqual(Array.from(sync.columns.expressID));
+        expect(cooperative.stats.slides).toBe(sync.stats.slides);
+    }, 60000);
+});

package/compiled/src/step/parsing/streaming_record_dispatcher.d.ts

@@ -0,0 +1,71 @@
+import { StepEntityConstructorAbstract } from "../step_entity_constructor.js";
+import { RecordEventHandler } from "./record_event.js";
+/**
+ * A per-record event handler: a record's dense `localID` (assigned in parse
+ * order from 0), its `expressID`, its `typeID` (0 for external-mapping
+ * records), and the record's raw bytes as an un-materialised
+ * `(buffer, byteOffset, byteLength)` view over the live parse window.
+ * See {@link RecordEventHandler} — in particular, the bytes are only valid
+ * for the duration of the call.
+ */
+export type RecordHandler<TypeIDType> = RecordEventHandler<TypeIDType>;
+/**
+ * Routes the streaming parse's per-record events (M2) to consumers that
+ * subscribe by **type set**. A subscription's type set is the subtype closure
+ * of the entity constructors it names — the same `query` closure the type
+ * index and `expressIDsOfTypes` use (conway #383) — so `on([IfcRoot], …)`
+ * matches every product, relationship, property set and quantity, and any
+ * future IfcRoot subtype, with no whitelist to maintain.
+ *
+ * Feed `dispatcher.onRecordIndexed` to `buildIndexStreaming` /
+ * `parseDataBlockStreamed`. Handlers run synchronously in the parse path, so
+ * they must be cheap (copy ids into a compact structure); anything expensive
+ * belongs on a demand queue, not here.
+ *
+ * Because subscriptions match on the raw parse `typeID`, external-mapping /
+ * complex records (typeID 0) are only delivered via `onAnyRecord`. Resolving
+ * their concrete types is not this class's job and never was: the columns
+ * carry the multi-mapping, so
+ * {@link import('./prefix_type_index').PrefixTypeIndex} attributes them
+ * correctly without the event stream. For the overwhelming majority of records
+ * (simple, one type each) type-set delivery is exact.
+ */
+export declare class StreamingRecordDispatcher<TypeIDType extends number> {
+    /**
+     * Concrete type → the handlers subscribed to it, unioned across
+     * subscriptions. Dispatch is one lookup per record no matter how many
+     * consumers are attached: the per-record cost of a subscription is the only
+     * cost this class has at 9.4 M records, so it must not scale with the
+     * subscriber count (issue #393).
+     */
+    private readonly byType;
+    private readonly any;
+    /**
+     * Subscribe to records of the given entity types (including their
+     * subtypes).
+     *
+     * @param types The entity constructors whose subtype closures to match.
+     * @param handler Called for each matching record.
+     */
+    on(types: StepEntityConstructorAbstract<TypeIDType>[], handler: RecordHandler<TypeIDType>): void;
+    /**
+     * Subscribe to every record regardless of type (the firehose — including
+     * external-mapping records).
+     *
+     * @param handler Called for every record.
+     */
+    onAnyRecord(handler: RecordHandler<TypeIDType>): void;
+    /**
+     * The per-record callback to hand to the streaming parse. Dispatches each
+     * record to the matching subscribers. Bound so it can be passed directly.
+     *
+     * @param localID The record's dense local ID.
+     * @param expressID The record's express ID.
+     * @param typeID The record's type ID (0 for external-mapping records).
+     * @param buffer The live parse window holding the record's bytes.
+     * @param byteOffset Offset of the record's first byte within `buffer`.
+     * @param byteLength Length of the record in bytes.
+     */
+    readonly onRecordIndexed: RecordHandler<TypeIDType>;
+}
+//# sourceMappingURL=streaming_record_dispatcher.d.ts.map

package/compiled/src/step/parsing/streaming_record_dispatcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"streaming_record_dispatcher.d.ts","sourceRoot":"","sources":["../../../../src/step/parsing/streaming_record_dispatcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,6BAA6B,EAAE,MAAM,4BAA4B,CAAA;AAC1E,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAA;AAGnD;;;;;;;GAOG;AACH,MAAM,MAAM,aAAa,CAAC,UAAU,IAAI,kBAAkB,CAAC,UAAU,CAAC,CAAA;AAGtE;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,qBAAa,yBAAyB,CAAC,UAAU,SAAS,MAAM;IAE9D;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqD;IAE5E,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAkC;IAEtD;;;;;;OAMG;IACI,EAAE,CACL,KAAK,EAAE,6BAA6B,CAAC,UAAU,CAAC,EAAE,EAClD,OAAO,EAAE,aAAa,CAAC,UAAU,CAAC,GAAI,IAAI;IAkB9C;;;;;OAKG;IACI,WAAW,CAAE,OAAO,EAAE,aAAa,CAAC,UAAU,CAAC,GAAI,IAAI;IAI9D;;;;;;;;;;OAUG;IACH,SAAgB,eAAe,EAAE,aAAa,CAAC,UAAU,CAAC,CAsBvD;CACJ"}