@bldrs-ai/conway 1.418.1264 → 1.419.1268

package/compiled/src/step/parsing/streaming_index_builder.js

@@ -125,6 +125,95 @@ export function buildIndexStreaming(source, parser, pool, onRecordIndexed, sink)
 // many slides on a small fixture; the pool sweep uses ≥ 128 KB anyway.
 // eslint-disable-next-line no-magic-numbers
 const MIN_WINDOW = 4 * 1024;
+/**
+ * Cooperative twin of {@link buildIndexStreaming}: identical parse, window
+ * and grow-and-restart behaviour (mirrored the same way the parser mirrors
+ * parseDataBlock/parseDataBlockAsync), but the parse periodically yields to
+ * the event loop so browsers repaint progress UI mid-parse instead of
+ * flagging the tab as stalled (issue #301 §2 for the streamed path).
+ *
+ * @param source The byte source.
+ * @param parser The STEP parser (typed to the schema).
+ * @param pool Target window size in bytes.
+ * @param onRecordIndexed Optional per-record event (see buildIndexStreaming).
+ * @param sink Optional index sink (columnar builds).
+ * @param onProgress Optional progress callback with the ABSOLUTE source byte
+ * cursor (unlike the parser's window-relative cursor), so callers can report
+ * `cursor / source.byteLength` directly.
+ * @param yieldIntervalMs Minimum ms between event-loop yields.
+ * @return {Promise<StreamingIndexResult>} The index, header, result and
+ * diagnostics.
+ */
+export async function buildIndexStreamingAsync(source, parser, pool, onRecordIndexed, sink, onProgress, yieldIntervalMs) {
+    const fileSize = source.byteLength;
+    let windowBytes = Math.max(pool, MIN_WINDOW);
+    for (;;) {
+        const window = new Uint8Array(windowBytes);
+        let windowStartFile = 0;
+        let windowLen = source.read(0, windowBytes, window, 0);
+        let bytesRead = windowLen;
+        const input = new ParsingBuffer(window, 0, windowLen);
+        const [header, headerResult] = parser.parseHeader(input);
+        if (headerResult !== ParseResult.COMPLETE) {
+            return {
+                header,
+                elements: [],
+                result: headerResult,
+                stats: { pool, windowBytes, slides: 0, maxRecordLen: 0, bytesRead },
+            };
+        }
+        const slideThreshold = windowBytes >> 1;
+        let slides = 0;
+        let maxRecordLen = 0;
+        let prevBoundaryFile = windowStartFile + input.cursor;
+        const onRecordBoundary = (buffer) => {
+            const cursor = buffer.cursor;
+            const recordFileStart = windowStartFile + cursor;
+            const recordLen = recordFileStart - prevBoundaryFile;
+            if (recordLen > maxRecordLen) {
+                maxRecordLen = recordLen;
+            }
+            prevBoundaryFile = recordFileStart;
+            if (windowStartFile + windowLen >= fileSize) {
+                return;
+            }
+            if (cursor < slideThreshold) {
+                return;
+            }
+            const tail = windowLen - cursor;
+            window.copyWithin(0, cursor, windowLen);
+            const want = windowBytes - tail;
+            const got = source.read(windowStartFile + windowLen, want, window, tail);
+            bytesRead += got;
+            windowLen = tail + got;
+            windowStartFile = recordFileStart;
+            buffer.rebaseWindow(window, 0, windowLen, windowStartFile);
+            ++slides;
+        };
+        // Translate the parser's window-relative cursor to an absolute source
+        // cursor (windowStartFile advances as the window slides).
+        const progressTick = onProgress !== void 0 ?
+            (cursor) => onProgress(windowStartFile + cursor) : void 0;
+        const [index, result] = await parser.parseDataBlockStreamedAsync(input, onRecordBoundary, onRecordIndexed, progressTick, sink, yieldIntervalMs);
+        const stoppedShort = result !== ParseResult.COMPLETE;
+        const notAtEof = windowStartFile + windowLen < fileSize;
+        if (stoppedShort && notAtEof) {
+            windowBytes *= 2;
+            sink?.reset();
+            continue;
+        }
+        const lastRecordLen = (windowStartFile + input.cursor) - prevBoundaryFile;
+        if (lastRecordLen > maxRecordLen) {
+            maxRecordLen = lastRecordLen;
+        }
+        return {
+            header,
+            elements: index.elements,
+            result,
+            stats: { pool, windowBytes, slides, maxRecordLen, bytesRead },
+        };
+    }
+}
 /**
  * Build the entity index by streaming, **directly into typed-array columns**
  * (M7): identical parse and window behaviour to {@link buildIndexStreaming},
@@ -144,3 +233,23 @@ export function buildColumnarIndexStreaming(source, parser, pool, onRecordIndexe
     const { header, result, stats } = buildIndexStreaming(source, parser, pool, onRecordIndexed, sink);
     return { header, columns: sink.finalize(), result, stats };
 }
+/**
+ * Cooperative twin of {@link buildColumnarIndexStreaming}: identical
+ * columnar build, but the parse periodically yields to the event loop (see
+ * {@link buildIndexStreamingAsync}) and reports absolute byte-cursor
+ * progress — the browser-facing variant for large models.
+ *
+ * @param source The byte source.
+ * @param parser The STEP parser (typed to the schema).
+ * @param pool Target window size in bytes.
+ * @param onRecordIndexed Optional per-record event (see buildIndexStreaming).
+ * @param onProgress Optional absolute byte-cursor progress callback.
+ * @param yieldIntervalMs Minimum ms between event-loop yields.
+ * @return {Promise<StreamingColumnarIndexResult>} Columns, header, result,
+ * stats.
+ */
+export async function buildColumnarIndexStreamingAsync(source, parser, pool, onRecordIndexed, onProgress, yieldIntervalMs) {
+    const sink = new ColumnarIndexSink();
+    const { header, result, stats } = await buildIndexStreamingAsync(source, parser, pool, onRecordIndexed, sink, onProgress, yieldIntervalMs);
+    return { header, columns: sink.finalize(), result, stats };
+}

package/compiled/src/step/parsing/streaming_index_builder_async.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=streaming_index_builder_async.test.d.ts.map

package/compiled/src/step/parsing/streaming_index_builder_async.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"streaming_index_builder_async.test.d.ts","sourceRoot":"","sources":["../../../../src/step/parsing/streaming_index_builder_async.test.ts"],"names":[],"mappings":""}

package/compiled/src/step/parsing/streaming_index_builder_async.test.js

@@ -0,0 +1,97 @@
+/* eslint-disable no-magic-numbers */
+// The cooperative streamed columnar build must be byte-identical to the
+// synchronous build (same columns, same window mechanics) AND actually
+// yield to the event loop mid-parse — the repaint/no-stall property
+// (#301 §2) the streamed open path relies on. The parse generator only
+// suspends every PARSE_PROGRESS_ELEMENT_MASK+1 (4096) records, so the
+// yield/progress assertions use a synthesized >4096-record model; the
+// column-parity assertions use the real fixture.
+import fs from "fs";
+import { describe, expect, test } from "@jest/globals";
+import IfcStepParser from "../../ifc/ifc_step_parser.js";
+import { BufferByteSource } from "./byte_source.js";
+import { buildColumnarIndexStreaming, buildColumnarIndexStreamingAsync, } from "./streaming_index_builder.js";
+import { ParseResult } from "./step_parser.js";
+const POOL = 8 * 1024; // small window → forces slides on both inputs
+/**
+ * Synthesize a valid IFC4 file with `recordCount` simple records — enough
+ * to cross the parser's progress-suspension mask several times.
+ *
+ * @param recordCount Number of data records.
+ * @return {Uint8Array} The encoded file.
+ */
+function syntheticIfc(recordCount) {
+    const parts = [
+        "ISO-10303-21;\nHEADER;\n" +
+            "FILE_DESCRIPTION((''),'2;1');\n" +
+            "FILE_NAME('','',(''),(''),'','','');\n" +
+            "FILE_SCHEMA(('IFC4'));\nENDSEC;\nDATA;\n",
+    ];
+    for (let where = 1; where <= recordCount; ++where) {
+        parts.push(`#${where}=IFCCARTESIANPOINT((${where}.,0.,0.));\n`);
+    }
+    parts.push("ENDSEC;\nEND-ISO-10303-21;\n");
+    return new TextEncoder().encode(parts.join(""));
+}
+describe("buildColumnarIndexStreamingAsync", () => {
+    test("produces identical columns to the sync build (real fixture)", async () => {
+        const bytes = new Uint8Array(fs.readFileSync("data/index.ifc"));
+        const parser = IfcStepParser.Instance;
+        const sync = buildColumnarIndexStreaming(new BufferByteSource(bytes), parser, POOL);
+        const cooperative = await buildColumnarIndexStreamingAsync(new BufferByteSource(bytes), parser, POOL, void 0, void 0, 0);
+        expect(sync.result).toBe(ParseResult.COMPLETE);
+        expect(cooperative.result).toBe(ParseResult.COMPLETE);
+        const a = sync.columns;
+        const b = cooperative.columns;
+        expect(b.count).toBe(a.count);
+        expect(b.firstInlineElement).toBe(a.firstInlineElement);
+        expect(Array.from(b.address)).toEqual(Array.from(a.address));
+        expect(Array.from(b.length)).toEqual(Array.from(a.length));
+        expect(Array.from(b.typeID)).toEqual(Array.from(a.typeID));
+        expect(Array.from(b.expressID)).toEqual(Array.from(a.expressID));
+        // Same window mechanics (slides prove the moving window really moved).
+        expect(cooperative.stats.slides).toBe(sync.stats.slides);
+        expect(cooperative.stats.slides).toBeGreaterThan(0);
+    }, 60000);
+    test("yields to the event loop and reports absolute progress mid-parse", async () => {
+        const bytes = syntheticIfc(10000);
+        const parser = IfcStepParser.Instance;
+        // Count macrotask turns that run while the build is in flight — with a
+        // zero yield interval the parse must give the event loop real turns.
+        let turns = 0;
+        let pumping = true;
+        const pump = () => {
+            if (!pumping) {
+                return;
+            }
+            ++turns;
+            setTimeout(pump, 0);
+        };
+        setTimeout(pump, 0);
+        const progressCursors = [];
+        const cooperative = await buildColumnarIndexStreamingAsync(new BufferByteSource(bytes), parser, POOL, void 0, (cursor) => progressCursors.push(cursor), 0);
+        pumping = false;
+        expect(cooperative.result).toBe(ParseResult.COMPLETE);
+        expect(cooperative.columns.count).toBe(10000);
+        expect(turns).toBeGreaterThan(0);
+        // Progress reports absolute source cursors: monotonically
+        // non-decreasing, bounded by the file size, and past the first window —
+        // the window-relative → absolute translation must hold across slides.
+        expect(progressCursors.length).toBeGreaterThan(1);
+        for (let where = 1; where < progressCursors.length; ++where) {
+            expect(progressCursors[where])
+                .toBeGreaterThanOrEqual(progressCursors[where - 1]);
+        }
+        expect(progressCursors[progressCursors.length - 1])
+            .toBeLessThanOrEqual(bytes.byteLength);
+        expect(progressCursors[progressCursors.length - 1])
+            .toBeGreaterThan(POOL);
+        // And the synthetic build matches its own sync twin exactly.
+        const sync = buildColumnarIndexStreaming(new BufferByteSource(bytes), parser, POOL);
+        expect(Array.from(cooperative.columns.address))
+            .toEqual(Array.from(sync.columns.address));
+        expect(Array.from(cooperative.columns.expressID))
+            .toEqual(Array.from(sync.columns.expressID));
+        expect(cooperative.stats.slides).toBe(sync.stats.slides);
+    }, 60000);
+});

package/compiled/src/stream/index.d.ts

@@ -0,0 +1,26 @@
+/**
+ * `@bldrs-ai/conway/stream` — the streaming parse/index plane (epic #390;
+ * design/new/streaming-federated-loader.md).
+ *
+ * Fixed-memory opens: a model parses through a bounded moving window over
+ * a {@link ByteSource} into a columnar record index ({@link StepIndexColumns}
+ * — no per-record object phase), reads its source bytes on demand through a
+ * windowed provider over a {@link StepExternalByteStore} (e.g. OPFS), and
+ * serializes its index to a revisit sidecar whose payload IS the columns.
+ * Incremental consumers (type index, cross-refs) subscribe to per-record
+ * events via {@link StreamingRecordDispatcher} while the parse runs.
+ *
+ * This is the conway-native namespace for new-era consumers. The web-ifc
+ * compat shim (`@bldrs-ai/conway/web-ifc`) adapts parts of this surface to
+ * web-ifc's API (e.g. `OpenModelStreamed`) and is headed for retirement;
+ * new integrations should build against this module directly.
+ */
+export { openStreamedIfcModel, StreamedIfcOpen, StreamedIfcOpenOptions, } from "../ifc/ifc_stream_open.js";
+export { buildColumnarIndexStreaming, buildColumnarIndexStreamingAsync, StreamingColumnarIndexResult, StreamingIndexStats, } from "../step/parsing/streaming_index_builder.js";
+export { ByteSource, BufferByteSource } from "../step/parsing/byte_source.js";
+export { StepExternalByteStore, InMemoryStepByteStore, StepBufferProvider, WindowedStepBufferProvider, } from "../step/step_buffer_provider.js";
+export { StepIndexColumns } from "../step/parsing/columnar_index.js";
+export { serializeIndexSidecarFromColumns, deserializeIndexSidecarToColumns, sidecarMatchesSource, hashSource, } from "../step/parsing/index_sidecar.js";
+export { StreamingRecordDispatcher, RecordHandler, } from "../step/parsing/streaming_record_dispatcher.js";
+export { IncrementalTypeIndex } from "../step/parsing/incremental_type_index.js";
+//# sourceMappingURL=index.d.ts.map

package/compiled/src/stream/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/stream/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AACH,OAAO,EACL,oBAAoB,EACpB,eAAe,EACf,sBAAsB,GACvB,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EACL,2BAA2B,EAC3B,gCAAgC,EAChC,4BAA4B,EAC5B,mBAAmB,GACpB,MAAM,yCAAyC,CAAA;AAChD,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAA;AAC1E,OAAO,EACL,qBAAqB,EACrB,qBAAqB,EACrB,kBAAkB,EAClB,0BAA0B,GAC3B,MAAM,8BAA8B,CAAA;AACrC,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAA;AACjE,OAAO,EACL,gCAAgC,EAChC,gCAAgC,EAChC,oBAAoB,EACpB,UAAU,GACX,MAAM,+BAA+B,CAAA;AACtC,OAAO,EACL,yBAAyB,EACzB,aAAa,GACd,MAAM,6CAA6C,CAAA;AACpD,OAAO,EAAE,oBAAoB,EAAE,MAAM,wCAAwC,CAAA"}

package/compiled/src/stream/index.js

@@ -0,0 +1,24 @@
+/**
+ * `@bldrs-ai/conway/stream` — the streaming parse/index plane (epic #390;
+ * design/new/streaming-federated-loader.md).
+ *
+ * Fixed-memory opens: a model parses through a bounded moving window over
+ * a {@link ByteSource} into a columnar record index ({@link StepIndexColumns}
+ * — no per-record object phase), reads its source bytes on demand through a
+ * windowed provider over a {@link StepExternalByteStore} (e.g. OPFS), and
+ * serializes its index to a revisit sidecar whose payload IS the columns.
+ * Incremental consumers (type index, cross-refs) subscribe to per-record
+ * events via {@link StreamingRecordDispatcher} while the parse runs.
+ *
+ * This is the conway-native namespace for new-era consumers. The web-ifc
+ * compat shim (`@bldrs-ai/conway/web-ifc`) adapts parts of this surface to
+ * web-ifc's API (e.g. `OpenModelStreamed`) and is headed for retirement;
+ * new integrations should build against this module directly.
+ */
+export { openStreamedIfcModel, } from "../ifc/ifc_stream_open.js";
+export { buildColumnarIndexStreaming, buildColumnarIndexStreamingAsync, } from "../step/parsing/streaming_index_builder.js";
+export { BufferByteSource } from "../step/parsing/byte_source.js";
+export { InMemoryStepByteStore, WindowedStepBufferProvider, } from "../step/step_buffer_provider.js";
+export { serializeIndexSidecarFromColumns, deserializeIndexSidecarToColumns, sidecarMatchesSource, hashSource, } from "../step/parsing/index_sidecar.js";
+export { StreamingRecordDispatcher, } from "../step/parsing/streaming_record_dispatcher.js";
+export { IncrementalTypeIndex } from "../step/parsing/incremental_type_index.js";

package/compiled/src/version/version.js

@@ -5,5 +5,5 @@
 // only the first segment (major) is meaningful and is the one CI carries forward.
 // Must stay in `vN.N.N` shape: the CI stamp regex, scripts/updateVersion.mjs, and
 // statistics.ts all match `v\d+\.\d+\.\d+`.
-const versionString = 'Conway v1.418.1264';
+const versionString = 'Conway v1.419.1268';
 export { versionString };