@bldrs-ai/conway 1.399.1223 → 1.401.1237

package/compiled/src/step/parsing/streaming_index_builder.js

@@ -0,0 +1,125 @@
+import ParsingBuffer from "../../parsing/parsing_buffer.js";
+import { ParseResult, } from "./step_parser.js";
+/**
+ * Build the entity index by streaming a `ByteSource` through a fixed-size
+ * moving window, instead of parsing one resident buffer. The parse loop is
+ * byte-for-byte the resident one (`StepParser.parseDataBlockStreamed` drives
+ * the same generator); this coordinator only owns the window: it fills it,
+ * parses the header from the first fill, then slides the window forward at
+ * top-level record boundaries as the parser advances.
+ *
+ * Because the parser records file-absolute addresses and its rewind stack is
+ * empty at every top-level boundary, sliding there is transparent: the
+ * unconsumed tail is copied to the front, fresh bytes are appended, and the
+ * buffer is rebased so `address` keeps its file-absolute value. Peak JS heap
+ * for the index build is therefore `window + index columns`, independent of
+ * file size.
+ *
+ * The window slides only once the cursor passes `pool / 2`, bounding the
+ * memmove frequency; a record up to `pool / 2` bytes always fits after a
+ * slide. If a single record exceeds that (never on the current corpus, whose
+ * largest STEP record is ~25 KB), the whole parse restarts from the
+ * beginning with the window doubled, and repeats until every record fits —
+ * correctness over the pathological case, at the cost of a re-scan. (M1's
+ * production path will instead grow in place / restart from the last
+ * boundary; from-scratch keeps the spike simple.)
+ *
+ * @param source The byte source.
+ * @param parser The STEP parser (typed to the schema).
+ * @param pool Target window size in bytes.
+ * @param onRecordIndexed Optional per-record event, invoked live as each
+ * top-level record is indexed with its (localID, expressID, typeID) — the
+ * seam for incremental semantic consumers (M2). localIDs are dense and
+ * assigned in parse order from 0. On the rare grow-and-restart (a single
+ * record larger than the window — never on real files, whose largest STEP
+ * record is ~25 KB), the parse re-runs from the start and records re-fire
+ * from localID 0; consumers must therefore be idempotent by localID /
+ * expressID (the standard consumers — type index keyed by localID, roots
+ * registry keyed by expressID — are). Must be synchronous and cheap;
+ * expensive work belongs on a demand queue, not the parse path.
+ * @return {StreamingIndexResult} The index, header, result and diagnostics.
+ */
+export function buildIndexStreaming(source, parser, pool, onRecordIndexed) {
+    const fileSize = source.byteLength;
+    // Grow-and-restart loop: the body runs a full streamed parse at the current
+    // window size. It only re-runs if a single record couldn't fit the window
+    // (false end-of-file before true EOF), doubling the window each time.
+    let windowBytes = Math.max(pool, MIN_WINDOW);
+    for (;;) {
+        const window = new Uint8Array(windowBytes);
+        let windowStartFile = 0;
+        let windowLen = source.read(0, windowBytes, window, 0);
+        let bytesRead = windowLen;
+        const input = new ParsingBuffer(window, 0, windowLen);
+        const [header, headerResult] = parser.parseHeader(input);
+        if (headerResult !== ParseResult.COMPLETE) {
+            // Header didn't complete in the first window: either malformed, or a
+            // header larger than the pool. Report it as-is — the resident parse
+            // would surface the same header result.
+            return {
+                header,
+                elements: [],
+                result: headerResult,
+                stats: { pool, windowBytes, slides: 0, maxRecordLen: 0, bytesRead },
+            };
+        }
+        // Slide once the cursor is past half the window; a record up to this much
+        // headroom always fits without a false EOF.
+        const slideThreshold = windowBytes >> 1;
+        let slides = 0;
+        let maxRecordLen = 0;
+        let prevBoundaryFile = windowStartFile + input.cursor;
+        const onRecordBoundary = (buffer) => {
+            const cursor = buffer.cursor;
+            const recordFileStart = windowStartFile + cursor;
+            const recordLen = recordFileStart - prevBoundaryFile;
+            if (recordLen > maxRecordLen) {
+                maxRecordLen = recordLen;
+            }
+            prevBoundaryFile = recordFileStart;
+            // Window already spans to EOF — nothing left to slide in.
+            if (windowStartFile + windowLen >= fileSize) {
+                return;
+            }
+            // Enough headroom remains; defer the slide to bound memmove frequency.
+            if (cursor < slideThreshold) {
+                return;
+            }
+            // Slide: move the unconsumed tail to the front, refill behind it.
+            const tail = windowLen - cursor;
+            window.copyWithin(0, cursor, windowLen);
+            const want = windowBytes - tail;
+            const got = source.read(windowStartFile + windowLen, want, window, tail);
+            bytesRead += got;
+            windowLen = tail + got;
+            windowStartFile = recordFileStart;
+            buffer.rebaseWindow(window, 0, windowLen, windowStartFile);
+            ++slides;
+        };
+        const [index, result] = parser.parseDataBlockStreamed(input, onRecordBoundary, onRecordIndexed);
+        // A parse that stopped short of true EOF, with the window not spanning to
+        // EOF, means a single record overflowed the window: grow and retry.
+        const stoppedShort = result !== ParseResult.COMPLETE;
+        const notAtEof = windowStartFile + windowLen < fileSize;
+        if (stoppedShort && notAtEof) {
+            windowBytes *= 2;
+            continue;
+        }
+        // Capture the final record's length (no boundary fires after the last).
+        const lastRecordLen = (windowStartFile + input.cursor) - prevBoundaryFile;
+        if (lastRecordLen > maxRecordLen) {
+            maxRecordLen = lastRecordLen;
+        }
+        return {
+            header,
+            elements: index.elements,
+            result,
+            stats: { pool, windowBytes, slides, maxRecordLen, bytesRead },
+        };
+    }
+}
+// Floor on the window so a pathologically tiny pool still holds the header
+// and a record with slide headroom. Tests drive pools down to this to force
+// many slides on a small fixture; the pool sweep uses ≥ 128 KB anyway.
+// eslint-disable-next-line no-magic-numbers
+const MIN_WINDOW = 4 * 1024;

package/compiled/src/step/parsing/streaming_index_builder.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=streaming_index_builder.test.d.ts.map

package/compiled/src/step/parsing/streaming_index_builder.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"streaming_index_builder.test.d.ts","sourceRoot":"","sources":["../../../../src/step/parsing/streaming_index_builder.test.ts"],"names":[],"mappings":""}

package/compiled/src/step/parsing/streaming_index_builder.test.js

@@ -0,0 +1,107 @@
+/* eslint-disable no-magic-numbers */
+// M0 spike: streaming index build must be byte-identical to the resident
+// parse. The parser reads file-absolute addresses and only rewinds within a
+// single top-level record, so feeding it from a moving window that slides at
+// record boundaries must produce the exact same index columns — at every
+// pool size, including one small enough to force many slides + straddles.
+import * as fs from "fs";
+import { beforeAll, describe, expect, test } from "@jest/globals";
+import ParsingBuffer from "../../parsing/parsing_buffer.js";
+import IfcStepParser from "../../ifc/ifc_step_parser.js";
+import { BufferByteSource } from "./byte_source.js";
+import { buildIndexStreaming } from "./streaming_index_builder.js";
+import { ParseResult } from "./step_parser.js";
+/**
+ * Flatten the index (top-level records + their inline / multi-mapping
+ * children) into a stable, comparable column dump: every field the streaming
+ * parse must reproduce identically.
+ *
+ * @param elements The index elements.
+ * @return {object[]} A flat list of column tuples in traversal order.
+ */
+function columns(elements) {
+    const out = [];
+    const visit = (e) => {
+        out.push({
+            address: e.address,
+            length: e.length,
+            typeID: e.typeID,
+            expressID: e.expressID ?? null,
+        });
+        for (const child of e.inlineEntities ?? []) {
+            visit(child);
+        }
+        for (const child of e.multiMapping ?? []) {
+            visit(child);
+        }
+    };
+    for (const e of elements) {
+        visit(e);
+    }
+    return out;
+}
+let bytes;
+let residentColumns;
+let residentCount;
+beforeAll(() => {
+    bytes = new Uint8Array(fs.readFileSync("data/index.ifc"));
+    const input = new ParsingBuffer(bytes);
+    IfcStepParser.Instance.parseHeader(input);
+    const [index, result] = IfcStepParser.Instance.parseDataBlock(input);
+    expect(result).toBe(ParseResult.COMPLETE);
+    residentColumns = columns(index.elements);
+    residentCount = index.elements.length;
+});
+describe("buildIndexStreaming", () => {
+    // A tiny pool relative to the 18 KB fixture forces repeated slides and at
+    // least one record straddling a window boundary; a large one takes the
+    // whole file in a single window (no slide). Both must match the resident
+    // parse exactly.
+    const POOLS = [4 * 1024, 8 * 1024, 16 * 1024, 128 * 1024];
+    test.each(POOLS)("produces the resident index at pool=%d bytes", (pool) => {
+        const source = new BufferByteSource(bytes);
+        const result = buildIndexStreaming(source, IfcStepParser.Instance, pool);
+        expect(result.result).toBe(ParseResult.COMPLETE);
+        expect(result.elements.length).toBe(residentCount);
+        expect(columns(result.elements)).toEqual(residentColumns);
+    });
+    test("the small pool actually slides the window (straddle exercised)", () => {
+        const source = new BufferByteSource(bytes);
+        const result = buildIndexStreaming(source, IfcStepParser.Instance, 4 * 1024);
+        // 18 KB fixture through a 4 KB window must slide several times.
+        expect(result.stats.slides).toBeGreaterThan(1);
+        // Physical window never grew past the pool (no record exceeds it).
+        expect(result.stats.windowBytes).toBe(4 * 1024);
+        // Every byte read once (no growth re-reads) — bounded to file size.
+        expect(result.stats.bytesRead).toBe(bytes.length);
+    });
+    test("the large pool takes the whole file in one window (no slide)", () => {
+        const source = new BufferByteSource(bytes);
+        const result = buildIndexStreaming(source, IfcStepParser.Instance, 128 * 1024);
+        expect(result.stats.slides).toBe(0);
+        expect(result.stats.maxRecordLen).toBeGreaterThan(0);
+    });
+    test("grows the window and restarts when a single record exceeds the pool", () => {
+        // Synthetic file with one record far larger than a tiny pool — the
+        // corpus never hits this (largest STEP record ~25 KB), so exercise the
+        // grow-and-restart safety valve directly. A long text literal makes one
+        // record ~12 KB; a 4 KB pool must grow past it and still index it.
+        const bigValue = "X".repeat(12000);
+        const text = "ISO-10303-21;\nHEADER;\nENDSEC;\nDATA;\n" +
+            "#1=IFCWALL('a');\n" +
+            `#2=IFCPROPERTYSINGLEVALUE('${bigValue}');\n` +
+            "#3=IFCWALL('c');\nENDSEC;\n";
+        const synthetic = new TextEncoder().encode(text);
+        const resident = new ParsingBuffer(synthetic);
+        IfcStepParser.Instance.parseHeader(resident);
+        const [residentIndex, residentResult] = IfcStepParser.Instance.parseDataBlock(resident);
+        expect(residentResult).toBe(ParseResult.COMPLETE);
+        const streamed = buildIndexStreaming(new BufferByteSource(synthetic), IfcStepParser.Instance, 4 * 1024);
+        expect(streamed.result).toBe(ParseResult.COMPLETE);
+        // Window grew past the 4 KB pool to fit the ~12 KB record.
+        expect(streamed.stats.windowBytes).toBeGreaterThan(4 * 1024);
+        expect(streamed.stats.maxRecordLen).toBeGreaterThan(12000);
+        // ...and the index is still byte-identical to the resident parse.
+        expect(columns(streamed.elements)).toEqual(columns(residentIndex.elements));
+    });
+});

package/compiled/src/step/parsing/streaming_record_dispatcher.d.ts

@@ -0,0 +1,55 @@
+import { StepEntityConstructorAbstract } from "../step_entity_constructor.js";
+/**
+ * A per-record event handler. Receives a record's dense `localID` (assigned
+ * in parse order from 0), its `expressID`, and its `typeID` (0 for
+ * external-mapping records).
+ */
+export type RecordHandler<TypeIDType> = (localID: number, expressID: number, typeID: TypeIDType | undefined) => void;
+/**
+ * Routes the streaming parse's per-record events (M2) to consumers that
+ * subscribe by **type set**. A subscription's type set is the subtype closure
+ * of the entity constructors it names — the same `query` closure the type
+ * index and `expressIDsOfTypes` use (conway #383) — so `on([IfcRoot], …)`
+ * matches every product, relationship, property set and quantity, and any
+ * future IfcRoot subtype, with no whitelist to maintain.
+ *
+ * Feed `dispatcher.onRecordIndexed` to `buildIndexStreaming` /
+ * `parseDataBlockStreamed`. Handlers run synchronously in the parse path, so
+ * they must be cheap (copy ids into a compact structure); anything expensive
+ * belongs on a demand queue, not here.
+ *
+ * Because subscriptions match on the raw parse `typeID`, external-mapping /
+ * complex records (typeID 0) are only delivered via `onAnyRecord` — resolving
+ * their concrete type is the incremental-type-index consumer's job (it reads
+ * `multiMapping`), a follow-on. For the overwhelming majority of records
+ * (simple, one type each) type-set delivery is exact.
+ */
+export declare class StreamingRecordDispatcher<TypeIDType extends number> {
+    private readonly typed;
+    private readonly any;
+    /**
+     * Subscribe to records of the given entity types (including their
+     * subtypes).
+     *
+     * @param types The entity constructors whose subtype closures to match.
+     * @param handler Called for each matching record.
+     */
+    on(types: StepEntityConstructorAbstract<TypeIDType>[], handler: RecordHandler<TypeIDType>): void;
+    /**
+     * Subscribe to every record regardless of type (the firehose — including
+     * external-mapping records).
+     *
+     * @param handler Called for every record.
+     */
+    onAnyRecord(handler: RecordHandler<TypeIDType>): void;
+    /**
+     * The per-record callback to hand to the streaming parse. Dispatches each
+     * record to the matching subscribers. Bound so it can be passed directly.
+     *
+     * @param localID The record's dense local ID.
+     * @param expressID The record's express ID.
+     * @param typeID The record's type ID (0 for external-mapping records).
+     */
+    readonly onRecordIndexed: RecordHandler<TypeIDType>;
+}
+//# sourceMappingURL=streaming_record_dispatcher.d.ts.map

package/compiled/src/step/parsing/streaming_record_dispatcher.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"streaming_record_dispatcher.d.ts","sourceRoot":"","sources":["../../../../src/step/parsing/streaming_record_dispatcher.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,6BAA6B,EAAE,MAAM,4BAA4B,CAAA;AAG1E;;;;GAIG;AACH,MAAM,MAAM,aAAa,CAAC,UAAU,IAClC,CAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,GAAG,SAAS,KAAM,IAAI,CAAA;AAGhF;;;;;;;;;;;;;;;;;;GAkBG;AACH,qBAAa,yBAAyB,CAAC,UAAU,SAAS,MAAM;IAE9D,OAAO,CAAC,QAAQ,CAAC,KAAK,CAGf;IAEP,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAkC;IAEtD;;;;;;OAMG;IACI,EAAE,CACL,KAAK,EAAE,6BAA6B,CAAC,UAAU,CAAC,EAAE,EAClD,OAAO,EAAE,aAAa,CAAC,UAAU,CAAC,GAAI,IAAI;IAQ9C;;;;;OAKG;IACI,WAAW,CAAE,OAAO,EAAE,aAAa,CAAC,UAAU,CAAC,GAAI,IAAI;IAI9D;;;;;;;OAOG;IACH,SAAgB,eAAe,EAAE,aAAa,CAAC,UAAU,CAAC,CAcvD;CACJ"}

package/compiled/src/step/parsing/streaming_record_dispatcher.js

@@ -0,0 +1,67 @@
+/**
+ * Routes the streaming parse's per-record events (M2) to consumers that
+ * subscribe by **type set**. A subscription's type set is the subtype closure
+ * of the entity constructors it names — the same `query` closure the type
+ * index and `expressIDsOfTypes` use (conway #383) — so `on([IfcRoot], …)`
+ * matches every product, relationship, property set and quantity, and any
+ * future IfcRoot subtype, with no whitelist to maintain.
+ *
+ * Feed `dispatcher.onRecordIndexed` to `buildIndexStreaming` /
+ * `parseDataBlockStreamed`. Handlers run synchronously in the parse path, so
+ * they must be cheap (copy ids into a compact structure); anything expensive
+ * belongs on a demand queue, not here.
+ *
+ * Because subscriptions match on the raw parse `typeID`, external-mapping /
+ * complex records (typeID 0) are only delivered via `onAnyRecord` — resolving
+ * their concrete type is the incremental-type-index consumer's job (it reads
+ * `multiMapping`), a follow-on. For the overwhelming majority of records
+ * (simple, one type each) type-set delivery is exact.
+ */
+export class StreamingRecordDispatcher {
+    constructor() {
+        this.typed = [];
+        this.any = [];
+        /**
+         * The per-record callback to hand to the streaming parse. Dispatches each
+         * record to the matching subscribers. Bound so it can be passed directly.
+         *
+         * @param localID The record's dense local ID.
+         * @param expressID The record's express ID.
+         * @param typeID The record's type ID (0 for external-mapping records).
+         */
+        this.onRecordIndexed = (localID, expressID, typeID) => {
+            for (const handler of this.any) {
+                handler(localID, expressID, typeID);
+            }
+            if (typeID !== void 0) {
+                for (const { set, handler } of this.typed) {
+                    if (set.has(typeID)) {
+                        handler(localID, expressID, typeID);
+                    }
+                }
+            }
+        };
+    }
+    /**
+     * Subscribe to records of the given entity types (including their
+     * subtypes).
+     *
+     * @param types The entity constructors whose subtype closures to match.
+     * @param handler Called for each matching record.
+     */
+    on(types, handler) {
+        this.typed.push({
+            set: new Set(types.flatMap((type) => type.query)),
+            handler,
+        });
+    }
+    /**
+     * Subscribe to every record regardless of type (the firehose — including
+     * external-mapping records).
+     *
+     * @param handler Called for every record.
+     */
+    onAnyRecord(handler) {
+        this.any.push(handler);
+    }
+}

package/compiled/src/step/parsing/streaming_record_dispatcher.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=streaming_record_dispatcher.test.d.ts.map

package/compiled/src/step/parsing/streaming_record_dispatcher.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"streaming_record_dispatcher.test.d.ts","sourceRoot":"","sources":["../../../../src/step/parsing/streaming_record_dispatcher.test.ts"],"names":[],"mappings":""}

package/compiled/src/step/parsing/streaming_record_dispatcher.test.js

@@ -0,0 +1,66 @@
+/* eslint-disable no-magic-numbers */
+// M2 core: record events emitted live from the streaming parse, routed by a
+// type-set subscription, let a consumer build a semantic structure
+// incrementally — and it must match what the finished model's type index
+// yields. Here: a roots registry collected from the event stream equals
+// `expressIDsOfTypes(IfcRoot)` on the resident model.
+import * as fs from "fs";
+import { beforeAll, describe, expect, test } from "@jest/globals";
+import ParsingBuffer from "../../parsing/parsing_buffer.js";
+import IfcStepParser from "../../ifc/ifc_step_parser.js";
+import { BufferByteSource } from "./byte_source.js";
+import { buildIndexStreaming } from "./streaming_index_builder.js";
+import { StreamingRecordDispatcher } from "./streaming_record_dispatcher.js";
+import { ParseResult } from "./step_parser.js";
+import { IfcRoot, IfcProduct } from "../../ifc/ifc4_gen/index.js";
+let bytes;
+let residentRoots;
+let residentProducts;
+beforeAll(() => {
+    bytes = new Uint8Array(fs.readFileSync("data/index.ifc"));
+    const input = new ParsingBuffer(bytes);
+    IfcStepParser.Instance.parseHeader(input);
+    const [, model] = IfcStepParser.Instance.parseDataToModel(input);
+    residentRoots = new Set(model.expressIDsOfTypes(IfcRoot));
+    residentProducts = new Set(model.expressIDsOfTypes(IfcProduct));
+});
+describe("StreamingRecordDispatcher", () => {
+    test("a roots registry built live from the event stream matches the type index", () => {
+        const roots = [];
+        const products = new Set();
+        const dispatcher = new StreamingRecordDispatcher();
+        dispatcher.on([IfcRoot], (_localID, expressID) => roots.push(expressID));
+        dispatcher.on([IfcProduct], (_localID, expressID) => products.add(expressID));
+        const result = buildIndexStreaming(new BufferByteSource(bytes), IfcStepParser.Instance, 4 * 1024, // tiny pool → the parse slides; events still fire per record
+        dispatcher.onRecordIndexed);
+        expect(result.result).toBe(ParseResult.COMPLETE);
+        // index.ifc has no external-mapping records, so raw-typeID delivery is
+        // exact vs. the type index.
+        expect(new Set(roots)).toEqual(residentRoots);
+        expect(products).toEqual(residentProducts);
+        // Products are a strict subset of roots (IfcProduct ⊂ IfcRoot).
+        expect(residentProducts.size).toBeLessThan(residentRoots.size);
+        for (const p of products) {
+            expect(residentRoots.has(p)).toBe(true);
+        }
+    });
+    test("localIDs arrive dense and in ascending parse order", () => {
+        const localIDs = [];
+        const dispatcher = new StreamingRecordDispatcher();
+        dispatcher.onAnyRecord((localID) => localIDs.push(localID));
+        buildIndexStreaming(new BufferByteSource(bytes), IfcStepParser.Instance, 128 * 1024, dispatcher.onRecordIndexed);
+        expect(localIDs.length).toBeGreaterThan(10);
+        // 0,1,2,…,n-1 in order.
+        expect(localIDs).toEqual(localIDs.map((_v, i) => i));
+    });
+    test("onAnyRecord sees every record; a type filter sees a subset", () => {
+        let all = 0;
+        let rootsCount = 0;
+        const dispatcher = new StreamingRecordDispatcher();
+        dispatcher.onAnyRecord(() => ++all);
+        dispatcher.on([IfcRoot], () => ++rootsCount);
+        buildIndexStreaming(new BufferByteSource(bytes), IfcStepParser.Instance, 128 * 1024, dispatcher.onRecordIndexed);
+        expect(all).toBeGreaterThan(rootsCount);
+        expect(rootsCount).toBe(residentRoots.size);
+    });
+});

package/compiled/src/step/step_model_base.d.ts

@@ -1,5 +1,5 @@
 import { StepIndexEntry } from "./parsing/step_parser.js";
-import { StepExternalByteStore } from "./step_buffer_provider.js";
+import { StepBufferProvider, StepExternalByteStore } from "./step_buffer_provider.js";
 import StepEntityBase from "./step_entity_base.js";
 import StepEntitySchema from "./step_entity_schema.js";
 import StepEntityInternalReference from "./step_entity_internal_reference.js";
@@ -52,11 +52,17 @@ export default abstract class StepModelBase<EntityTypeIDs extends number, BaseEn
      * Construct this step model with its matching schema, a buffer to read from and an element index.
      *
      * @param schema The Step schema this is based on.
-     * @param buffer_ The buffer to read this from.
+     * @param buffer_ The buffer to read this from. Pass `undefined` together
+     * with `provider` to build a model whose source is windowed from
+     * construction (e.g. a streaming open — see buildModelStreaming); the
+     * synchronous read paths (geometry extraction) then require the relevant
+     * ranges to be resident, exactly as after `spillSourceToExternalStore`.
      * @param elementIndex The element index for this, parsed or deserialized - note this takes
      * ownership of this array in the sense it will modify values/unfold inline elements etc.
+     * @param provider Optional pre-built buffer provider. When omitted, a
+     * resident provider over `buffer_` is used (the classic path).
      */
-    constructor(schema: StepEntitySchema<EntityTypeIDs, BaseEntity>, buffer_: Uint8Array | undefined, elementIndex: StepIndexEntry<EntityTypeIDs>[]);
+    constructor(schema: StepEntitySchema<EntityTypeIDs, BaseEntity>, buffer_: Uint8Array | undefined, elementIndex: StepIndexEntry<EntityTypeIDs>[], provider?: StepBufferProvider);
     /**
      * Materialise the descriptor object for a local ID from the columns. Only the
      * persistent scalars are set; lazy fields (vtable, buffer, entity) are filled

package/compiled/src/step/step_model_base.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"step_model_base.d.ts","sourceRoot":"","sources":["../../../src/step/step_model_base.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAA;AAEtD,OAAO,EAGL,qBAAqB,EAEtB,MAAM,wBAAwB,CAAA;AAE/B,OAAO,cAAc,MAAM,oBAAoB,CAAA;AAC/C,OAAO,gBAAgB,MAAM,sBAAsB,CAAA;AACnD,OAAO,2BAAmE,MAAM,kCAAkC,CAAA;AAClH,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAE5D,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAA;AAC3D,OAAO,qBAAqB,EAAE,EAAE,6BAA6B,EAAE,MAAM,2BAA2B,CAAA;AAChG,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AACpD,OAAO,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAA;AAGlE,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAC9D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAA;AAGxD;;GAEG;AACH,MAAM,CAAC,OAAO,CAAC,QAAQ,OAAO,aAAa,CACzC,aAAa,SAAS,MAAM,EAC5B,UAAU,SAAS,cAAc,CAAC,aAAa,CAAC,GAAG,cAAc,CAAC,aAAa,CAAC,CAClF,YAAW,QAAQ,CAAC,UAAU,CAAC,EAAE,KAAK;aAyElB,MAAM,EAAE,gBAAgB,CAAE,aAAa,EAAE,UAAU,CAAE;IACrE,OAAO,CAAC,OAAO;IAzEjB,kBAAyB,SAAS,EAAE,aAAa,CAAC,aAAa,CAAC,CAAA;IAChE,kBAAyB,mBAAmB,EAAE,qBAAqB,CAAE,aAAa,EAAE,UAAU,CAAE,CAAA;IAEhG,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA6C;IAC5E,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA4B;IAC1D,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAA4B;IAW9D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAa;IACtC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAa;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAY;IACpC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAQ;IAM5C,OAAO,CAAC,gBAAgB,CACgE;IAIxF,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAC+C;IAQhF,OAAO,CAAC,eAAe,CAAoB;IAE3C;;;;;;;OAOG;IACI,kBAAkB,EAAE,OAAO,CAAO;IAEzC;;;OAGG;IACI,YAAY,EAAE,OAAO,CAAO;IAEnC,kBAAyB,SAAS,CAAC,EAAE,cAAc,CAAA;IAEnD,kBAAyB,QAAQ,CAAC,EAAE,aAAa,CAAA;IAEjD;;;;;;;OAOG;gBAEe,MAAM,EAAE,gBAAgB,CAAE,aAAa,EAAE,UAAU,CAAE,EAC7D,OAAO,EAAE,UAAU,GAAG,SAAS,EAAE,YAAY,EAAE,cAAc,CAAC,aAAa,CAAC,EAAE;IA+GxF;;;;;;;OAOG;IACH,OAAO,CAAC,cAAc;IActB;;;;;;;OAOG;IACH,OAAO,CAAC,KAAK;IAsBb;;;;;OAKG;IACI,UAAU,CAAE,UAAU,GAAE,OAAe,GAAI,IAAI;IAsEtD;;;;;OAKG;IACI,sBAAsB,CAC3B,OAAO,EAAE,2BAA2B,CAAE,aAAa,CAAE,GAAI,OAAO;IA+BlE;;;;;;;OAOG;IACI,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO;IAWpD;;;;;OAKG;IACI,mBAAmB,CAAE,OAAO,EAAE,MAAM,GAAI,IAAI;IAYnD;;;;OAIG;IACH,IAAW,cAAc,IAAI,MAAM,CAElC;IAED;;;;;OAKG;IACH,IAAW,gBAAgB,IAAI,OAAO,CAErC;IAED;;;;;OAKG;IACH,IAAW,mBAAmB,IAAI,MAAM,CAEvC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACI,0BAA0B,CAC7B,KAAK,EAAE,qBAAqB,EAC5B,UAAU,CAAC,EAAE,MAAM,EACnB,iBAAiB,CAAC,EAAE,MAAM,GAAI,IAAI;IActC;;;;;;;;;;OAUG;IACU,uBAAuB,CAAE,OAAO,EAAE,MAAM,GAAI,OAAO,CAAE,IAAI,CAAE;IAuBxE;;;;;;;;OAQG;IACU,yBAAyB,CAAE,SAAS,EAAE,MAAM,GAAI,OAAO,CAAE,IAAI,CAAE;IAgB5E;;;;OAIG;IACH,IAAW,IAAI,IAAI,MAAM,CAExB;IAED;;;;;OAKG;IACI,yBAAyB,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,GAAG,UAAU,GAAG,SAAS;IAcrF;;;;;;OAMG;IACI,8BAA8B,CAAC,CAAC,SAAS,6BAA6B,CAAE,aAAa,CAAE,EAAE,CAAC,SAAS,YAAY,CAAE,CAAC,CAAE,GAAG,UAAU,EAAG,OAAO,EAAE,MAAM,GAAG,SAAS,EAAE,IAAI,EAAE,CAAC,GAAI,CAAC,GAAG,SAAS;IAchM;;;;;;OAMG;IACI,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS;IAUvE;;;;;;;;OAQG;IACI,0BAA0B,CAAE,CAAC,SAAS,6BAA6B,CAAE,aAAa,CAAE,EAAE,CAAC,SAAS,YAAY,CAAE,CAAC,CAAE,GAAG,UAAU,EACnI,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,CAAC,GACP,CAAC,GAAG,SAAS;IAWf;;;;;;;;;OASG;IACI,wBAAwB,CAAE,CAAC,SAAS,6BAA6B,CAAE,aAAa,CAAE,EAAE,CAAC,SAAS,YAAY,CAAE,CAAC,CAAE,GAAG,UAAU,EACjI,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,6BAA6B,CAAE,aAAa,CAAE,GACpD,CAAC,GAAG,SAAS;IAuFf;;;;;OAKG;IACI,uBAAuB,CAAE,IAAI,EAAE,mBAAmB,GAAI,WAAW;IASxE;;;;;;OAMG;IACI,qBAAqB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAKjE;;;;;;;OAOG;IACI,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS;IAsFnE;;;;;;;OAOG;IACI,KAAK,CAAC,CAAC,SAAS,6BAA6B,CAAC,aAAa,CAAC,EAAE,EAAE,GAAG,KAAK,EAAE,CAAC,GAChF,gBAAgB,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IAS3C;;;;;;;;;OASG;IACI,SAAS,CAAC,CAAC,SAAS,6BAA6B,CAAC,aAAa,CAAC,EAAE,EACrE,GAAG,KAAK,EAAE,CAAC,GAAI,MAAM;IAOzB;;;;;;;;;;;;OAYG;IACK,iBAAiB,CAAC,CAAC,SAAS,6BAA6B,CAAC,aAAa,CAAC,EAAE,EAC9E,GAAG,KAAK,EAAE,CAAC,GAAI,gBAAgB,CAAC,MAAM,CAAC;IA8B3C;;;;OAIG;IACI,eAAe,IAAK,GAAG,CAAE,aAAa,CAAE;IAO/C;;;;OAIG;IACI,wBAAwB,IAAK,gBAAgB,CAAE,aAAa,CAAE;IAKrE;;;;;;;OAOG;IACI,OAAO,CAAC,GAAG,KAAK,EAAE,aAAa,EAAE,GAAG,gBAAgB,CAAC,UAAU,CAAC;IAOvE;;;;;;;OAOG;IACI,kBAAkB,CAAC,GAAG,KAAK,EAAE,aAAa,EAAE,GAAG,gBAAgB,CAAC,UAAU,CAAC;IAIlF;;;;;;;;OAQG;IACK,IAAI,CAAC,MAAM,EAAE,eAAe,EAAE,UAAU,GAAE,OAAe,GAAG,gBAAgB,CAAC,UAAU,CAAC;IAyBhG;;;;;;;OAOG;IACK,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,gBAAgB,CAAC,UAAU,CAAC;IAUrE;;;;;;OAMG;IACK,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,gBAAgB,CAAC,UAAU,CAAC;IAWzD;;;;;;;OAOG;IACI,2BAA2B,CAAE,IAAI,EAAE,iBAAiB,GACzD,iBAAiB,GAAG,SAAS;IAK/B;;;;;;;OAOG;IACI,uBAAuB,CAAE,IAAI,EAAE,iBAAiB,GAAI,aAAa,GAAG,SAAS;CAIrF"}
+{"version":3,"file":"step_model_base.d.ts","sourceRoot":"","sources":["../../../src/step/step_model_base.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAA;AAEtD,OAAO,EAEL,kBAAkB,EAClB,qBAAqB,EAEtB,MAAM,wBAAwB,CAAA;AAE/B,OAAO,cAAc,MAAM,oBAAoB,CAAA;AAC/C,OAAO,gBAAgB,MAAM,sBAAsB,CAAA;AACnD,OAAO,2BAAmE,MAAM,kCAAkC,CAAA;AAClH,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAE5D,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAA;AAC3D,OAAO,qBAAqB,EAAE,EAAE,6BAA6B,EAAE,MAAM,2BAA2B,CAAA;AAChG,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AACpD,OAAO,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAA;AAGlE,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAC9D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAA;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAA;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAA;AAGxD;;GAEG;AACH,MAAM,CAAC,OAAO,CAAC,QAAQ,OAAO,aAAa,CACzC,aAAa,SAAS,MAAM,EAC5B,UAAU,SAAS,cAAc,CAAC,aAAa,CAAC,GAAG,cAAc,CAAC,aAAa,CAAC,CAClF,YAAW,QAAQ,CAAC,UAAU,CAAC,EAAE,KAAK;aA+ElB,MAAM,EAAE,gBAAgB,CAAE,aAAa,EAAE,UAAU,CAAE;IACrE,OAAO,CAAC,OAAO;IA/EjB,kBAAyB,SAAS,EAAE,aAAa,CAAC,aAAa,CAAC,CAAA;IAChE,kBAAyB,mBAAmB,EAAE,qBAAqB,CAAE,aAAa,EAAE,UAAU,CAAE,CAAA;IAEhG,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA6C;IAC5E,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA4B;IAC1D,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAA4B;IAW9D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAa;IACtC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAa;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAY;IACpC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAQ;IAM5C,OAAO,CAAC,gBAAgB,CACgE;IAIxF,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAC+C;IAQhF,OAAO,CAAC,eAAe,CAAoB;IAE3C;;;;;;;OAOG;IACI,kBAAkB,EAAE,OAAO,CAAO;IAEzC;;;OAGG;IACI,YAAY,EAAE,OAAO,CAAO;IAEnC,kBAAyB,SAAS,CAAC,EAAE,cAAc,CAAA;IAEnD,kBAAyB,QAAQ,CAAC,EAAE,aAAa,CAAA;IAEjD;;;;;;;;;;;;;OAaG;gBAEe,MAAM,EAAE,gBAAgB,CAAE,aAAa,EAAE,UAAU,CAAE,EAC7D,OAAO,EAAE,UAAU,GAAG,SAAS,EAAE,YAAY,EAAE,cAAc,CAAC,aAAa,CAAC,EAAE,EACtF,QAAQ,CAAC,EAAE,kBAAkB;IA+G/B;;;;;;;OAOG;IACH,OAAO,CAAC,cAAc;IActB;;;;;;;OAOG;IACH,OAAO,CAAC,KAAK;IAsBb;;;;;OAKG;IACI,UAAU,CAAE,UAAU,GAAE,OAAe,GAAI,IAAI;IAsEtD;;;;;OAKG;IACI,sBAAsB,CAC3B,OAAO,EAAE,2BAA2B,CAAE,aAAa,CAAE,GAAI,OAAO;IA+BlE;;;;;;;OAOG;IACI,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO;IAWpD;;;;;OAKG;IACI,mBAAmB,CAAE,OAAO,EAAE,MAAM,GAAI,IAAI;IAYnD;;;;OAIG;IACH,IAAW,cAAc,IAAI,MAAM,CAElC;IAED;;;;;OAKG;IACH,IAAW,gBAAgB,IAAI,OAAO,CAErC;IAED;;;;;OAKG;IACH,IAAW,mBAAmB,IAAI,MAAM,CAEvC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACI,0BAA0B,CAC7B,KAAK,EAAE,qBAAqB,EAC5B,UAAU,CAAC,EAAE,MAAM,EACnB,iBAAiB,CAAC,EAAE,MAAM,GAAI,IAAI;IActC;;;;;;;;;;OAUG;IACU,uBAAuB,CAAE,OAAO,EAAE,MAAM,GAAI,OAAO,CAAE,IAAI,CAAE;IAuBxE;;;;;;;;OAQG;IACU,yBAAyB,CAAE,SAAS,EAAE,MAAM,GAAI,OAAO,CAAE,IAAI,CAAE;IAgB5E;;;;OAIG;IACH,IAAW,IAAI,IAAI,MAAM,CAExB;IAED;;;;;OAKG;IACI,yBAAyB,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,GAAG,UAAU,GAAG,SAAS;IAcrF;;;;;;OAMG;IACI,8BAA8B,CAAC,CAAC,SAAS,6BAA6B,CAAE,aAAa,CAAE,EAAE,CAAC,SAAS,YAAY,CAAE,CAAC,CAAE,GAAG,UAAU,EAAG,OAAO,EAAE,MAAM,GAAG,SAAS,EAAE,IAAI,EAAE,CAAC,GAAI,CAAC,GAAG,SAAS;IAchM;;;;;;OAMG;IACI,qBAAqB,CAAC,SAAS,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS;IAUvE;;;;;;;;OAQG;IACI,0BAA0B,CAAE,CAAC,SAAS,6BAA6B,CAAE,aAAa,CAAE,EAAE,CAAC,SAAS,YAAY,CAAE,CAAC,CAAE,GAAG,UAAU,EACnI,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,CAAC,GACP,CAAC,GAAG,SAAS;IAWf;;;;;;;;;OASG;IACI,wBAAwB,CAAE,CAAC,SAAS,6BAA6B,CAAE,aAAa,CAAE,EAAE,CAAC,SAAS,YAAY,CAAE,CAAC,CAAE,GAAG,UAAU,EACjI,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,6BAA6B,CAAE,aAAa,CAAE,GACpD,CAAC,GAAG,SAAS;IAuFf;;;;;OAKG;IACI,uBAAuB,CAAE,IAAI,EAAE,mBAAmB,GAAI,WAAW;IASxE;;;;;;OAMG;IACI,qBAAqB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAKjE;;;;;;;OAOG;IACI,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,UAAU,GAAG,SAAS;IAsFnE;;;;;;;OAOG;IACI,KAAK,CAAC,CAAC,SAAS,6BAA6B,CAAC,aAAa,CAAC,EAAE,EAAE,GAAG,KAAK,EAAE,CAAC,GAChF,gBAAgB,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IAS3C;;;;;;;;;OASG;IACI,SAAS,CAAC,CAAC,SAAS,6BAA6B,CAAC,aAAa,CAAC,EAAE,EACrE,GAAG,KAAK,EAAE,CAAC,GAAI,MAAM;IAOzB;;;;;;;;;;;;OAYG;IACK,iBAAiB,CAAC,CAAC,SAAS,6BAA6B,CAAC,aAAa,CAAC,EAAE,EAC9E,GAAG,KAAK,EAAE,CAAC,GAAI,gBAAgB,CAAC,MAAM,CAAC;IA8B3C;;;;OAIG;IACI,eAAe,IAAK,GAAG,CAAE,aAAa,CAAE;IAO/C;;;;OAIG;IACI,wBAAwB,IAAK,gBAAgB,CAAE,aAAa,CAAE;IAKrE;;;;;;;OAOG;IACI,OAAO,CAAC,GAAG,KAAK,EAAE,aAAa,EAAE,GAAG,gBAAgB,CAAC,UAAU,CAAC;IAOvE;;;;;;;OAOG;IACI,kBAAkB,CAAC,GAAG,KAAK,EAAE,aAAa,EAAE,GAAG,gBAAgB,CAAC,UAAU,CAAC;IAIlF;;;;;;;;OAQG;IACK,IAAI,CAAC,MAAM,EAAE,eAAe,EAAE,UAAU,GAAE,OAAe,GAAG,gBAAgB,CAAC,UAAU,CAAC;IAyBhG;;;;;;;OAOG;IACK,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,gBAAgB,CAAC,UAAU,CAAC;IAUrE;;;;;;OAMG;IACK,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,gBAAgB,CAAC,UAAU,CAAC;IAWzD;;;;;;;OAOG;IACI,2BAA2B,CAAE,IAAI,EAAE,iBAAiB,GACzD,iBAAiB,GAAG,SAAS;IAK/B;;;;;;;OAOG;IACI,uBAAuB,CAAE,IAAI,EAAE,iBAAiB,GAAI,aAAa,GAAG,SAAS;CAIrF"}

package/compiled/src/step/step_model_base.js

@@ -12,11 +12,17 @@ export default class StepModelBase {
      * Construct this step model with its matching schema, a buffer to read from and an element index.
      *
      * @param schema The Step schema this is based on.
-     * @param buffer_ The buffer to read this from.
+     * @param buffer_ The buffer to read this from. Pass `undefined` together
+     * with `provider` to build a model whose source is windowed from
+     * construction (e.g. a streaming open — see buildModelStreaming); the
+     * synchronous read paths (geometry extraction) then require the relevant
+     * ranges to be resident, exactly as after `spillSourceToExternalStore`.
      * @param elementIndex The element index for this, parsed or deserialized - note this takes
      * ownership of this array in the sense it will modify values/unfold inline elements etc.
+     * @param provider Optional pre-built buffer provider. When omitted, a
+     * resident provider over `buffer_` is used (the classic path).
      */
-    constructor(schema, buffer_, elementIndex) {
+    constructor(schema, buffer_, elementIndex, provider) {
         this.schema = schema;
         this.buffer_ = buffer_;
         this.vtableBuilder_ = new StepVtableBuilder();
@@ -39,7 +45,7 @@ export default class StepModelBase {
          * error, and the field is optional, return null instead of throwing an exception.
          */
         this.nullOnErrors = true;
-        this.bufferProvider_ = new ResidentStepBufferProvider(buffer_);
+        this.bufferProvider_ = provider ?? new ResidentStepBufferProvider(buffer_);
         const localElementIndex = elementIndex;
         let where = 0;
         const firstInlineElement = localElementIndex.length;

package/compiled/src/version/version.js

@@ -5,5 +5,5 @@
 // only the first segment (major) is meaningful and is the one CI carries forward.
 // Must stay in `vN.N.N` shape: the CI stamp regex, scripts/updateVersion.mjs, and
 // statistics.ts all match `v\d+\.\d+\.\d+`.
-const versionString = 'Conway v1.399.1223';
+const versionString = 'Conway v1.401.1237';
 export { versionString };