@bldrs-ai/conway 1.399.1223 → 1.401.1237

package/compiled/src/step/parsing/byte_source.d.ts

@@ -0,0 +1,55 @@
+/**
+ * A random-access, forward-readable byte source for the streaming parser.
+ *
+ * The streaming index builder (see streaming_index_builder.ts) reads the
+ * source sequentially into a small moving window, so a `ByteSource` only has
+ * to satisfy positioned reads — it never needs the whole file resident. The
+ * in-memory `BufferByteSource` here is for tests and the resident case; a
+ * file-descriptor source (node) or an HTTP-Range source (M4) implements the
+ * same shape without holding the file in the JS heap.
+ *
+ * M0 keeps reads synchronous to reuse the existing synchronous parse loop
+ * unchanged. The asynchronous variant (network pull-parser) is M4.
+ */
+export interface ByteSource {
+    /** Total length of the source in bytes. */
+    readonly byteLength: number;
+    /**
+     * Read up to `length` bytes starting at `offset` into `into` at
+     * `intoOffset`, returning the number of bytes actually copied (fewer than
+     * `length` only at end of source).
+     *
+     * @param offset Absolute source offset to read from.
+     * @param length Maximum number of bytes to read.
+     * @param into Destination buffer.
+     * @param intoOffset Offset within `into` to write at.
+     * @return {number} The number of bytes copied.
+     */
+    read(offset: number, length: number, into: Uint8Array, intoOffset: number): number;
+}
+/**
+ * A `ByteSource` backed by an in-memory `Uint8Array`. Reads are `subarray`
+ * copies into the destination window. Used by tests and the resident path;
+ * note it does hold the whole buffer (that residency is the source's, not the
+ * parser's — the parser still only touches a window).
+ */
+export declare class BufferByteSource implements ByteSource {
+    private readonly buffer;
+    /**
+     * @param buffer The backing bytes.
+     */
+    constructor(buffer: Uint8Array);
+    /**
+     * @return {number} The buffer length.
+     */
+    get byteLength(): number;
+    /**
+     * @param offset Absolute source offset to read from.
+     * @param length Maximum number of bytes to read.
+     * @param into Destination buffer.
+     * @param intoOffset Offset within `into` to write at.
+     * @return {number} The number of bytes copied.
+     */
+    read(offset: number, length: number, into: Uint8Array, intoOffset: number): number;
+}
+//# sourceMappingURL=byte_source.d.ts.map

package/compiled/src/step/parsing/byte_source.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"byte_source.d.ts","sourceRoot":"","sources":["../../../../src/step/parsing/byte_source.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,UAAU;IAEzB,2CAA2C;IAC3C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAE3B;;;;;;;;;;OAUG;IACH,IAAI,CAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,GAAI,MAAM,CAAA;CACrF;AAED;;;;;GAKG;AACH,qBAAa,gBAAiB,YAAW,UAAU;IAKpC,OAAO,CAAC,QAAQ,CAAC,MAAM;IAHpC;;OAEG;gBAC2B,MAAM,EAAE,UAAU;IAEhD;;OAEG;IACH,IAAW,UAAU,IAAI,MAAM,CAE9B;IAED;;;;;;OAMG;IACI,IAAI,CACP,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,GAAI,MAAM;CAYnF"}

package/compiled/src/step/parsing/byte_source.js

@@ -0,0 +1,35 @@
+/**
+ * A `ByteSource` backed by an in-memory `Uint8Array`. Reads are `subarray`
+ * copies into the destination window. Used by tests and the resident path;
+ * note it does hold the whole buffer (that residency is the source's, not the
+ * parser's — the parser still only touches a window).
+ */
+export class BufferByteSource {
+    /**
+     * @param buffer The backing bytes.
+     */
+    constructor(buffer) {
+        this.buffer = buffer;
+    }
+    /**
+     * @return {number} The buffer length.
+     */
+    get byteLength() {
+        return this.buffer.length;
+    }
+    /**
+     * @param offset Absolute source offset to read from.
+     * @param length Maximum number of bytes to read.
+     * @param into Destination buffer.
+     * @param intoOffset Offset within `into` to write at.
+     * @return {number} The number of bytes copied.
+     */
+    read(offset, length, into, intoOffset) {
+        const end = Math.min(offset + length, this.buffer.length);
+        if (end <= offset) {
+            return 0;
+        }
+        into.set(this.buffer.subarray(offset, end), intoOffset);
+        return end - offset;
+    }
+}

package/compiled/src/step/parsing/incremental_type_index.d.ts

@@ -0,0 +1,64 @@
+import { StepEntityConstructorAbstract } from "../step_entity_constructor.js";
+import { RecordHandler } from "./streaming_record_dispatcher.js";
+/**
+ * A type index built **incrementally** from the streaming parse's per-record
+ * events (M2), rather than from the finished element array. Feed its
+ * {@link handler} to a {@link StreamingRecordDispatcher} (or directly to
+ * `buildIndexStreaming`'s `onRecordIndexed`) and it accumulates, per concrete
+ * type, the set of express IDs seen — so the moment parsing reaches a record
+ * it is queryable, without waiting for end-of-parse.
+ *
+ * Queries take entity constructors and expand to their subtype closure via
+ * the generated `query` (conway #383), so `expressIDsOfTypes(IfcProduct)`
+ * unions every product subtype exactly as the resident model's type index
+ * does.
+ *
+ * Express IDs are held in per-type `Set`s, so the consumer is idempotent
+ * under the streaming builder's rare grow-and-restart (records re-fire from
+ * localID 0): re-adding an express ID is a no-op. Memory is O(records) — the
+ * same order as the element index it mirrors.
+ *
+ * Scope: membership is keyed on the raw parse `typeID`. External-mapping /
+ * complex records (typeID 0) are therefore not attributed to their mapped
+ * classes here — resolving those needs the record's `multiMapping`, which the
+ * event stream doesn't carry; that's the model's construction-time index. For
+ * the overwhelming majority of records (simple, one type each) this index is
+ * exact, which the parity test pins.
+ */
+export declare class IncrementalTypeIndex<TypeIDType extends number> {
+    /** Concrete typeID → express IDs of records of exactly that type. */
+    private readonly byType_;
+    /**
+     * Record one indexed entity. Bound as {@link handler} for direct use as a
+     * dispatcher subscription / `onRecordIndexed` callback.
+     *
+     * @param localID Unused (kept for the RecordHandler shape).
+     * @param expressID The record's express ID.
+     * @param typeID The record's concrete type ID (undefined / 0 records are
+     * ignored — they carry no concrete queryable type here).
+     */
+    readonly handler: RecordHandler<TypeIDType>;
+    /**
+     * The distinct concrete type IDs seen so far.
+     *
+     * @return {IterableIterator<TypeIDType>} The concrete types.
+     */
+    concreteTypes(): IterableIterator<TypeIDType>;
+    /**
+     * Lazily iterate the express IDs of all records of the given types
+     * (including subtypes).
+     *
+     * @param types The entity constructors to query (subtype closures unioned).
+     * @return {IterableIterator<number>} Express IDs of matching records.
+     * @yields {number} Each matching express ID.
+     */
+    expressIDsOfTypes(...types: StepEntityConstructorAbstract<TypeIDType>[]): IterableIterator<number>;
+    /**
+     * Count records of the given types (including subtypes) seen so far.
+     *
+     * @param types The entity constructors to count.
+     * @return {number} The number of matching records.
+     */
+    count(...types: StepEntityConstructorAbstract<TypeIDType>[]): number;
+}
+//# sourceMappingURL=incremental_type_index.d.ts.map

package/compiled/src/step/parsing/incremental_type_index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"incremental_type_index.d.ts","sourceRoot":"","sources":["../../../../src/step/parsing/incremental_type_index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,6BAA6B,EAAE,MAAM,4BAA4B,CAAA;AAC1E,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAG7D;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,qBAAa,oBAAoB,CAAC,UAAU,SAAS,MAAM;IAEzD,qEAAqE;IACrE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqC;IAE7D;;;;;;;;OAQG;IACH,SAAgB,OAAO,EAAE,aAAa,CAAC,UAAU,CAAC,CAe/C;IAEH;;;;OAIG;IACI,aAAa,IAAI,gBAAgB,CAAC,UAAU,CAAC;IAIpD;;;;;;;OAOG;IACK,iBAAiB,CACrB,GAAG,KAAK,EAAE,6BAA6B,CAAC,UAAU,CAAC,EAAE,GACrD,gBAAgB,CAAC,MAAM,CAAC;IAc5B;;;;;OAKG;IACI,KAAK,CAAE,GAAG,KAAK,EAAE,6BAA6B,CAAC,UAAU,CAAC,EAAE,GAAI,MAAM;CAa9E"}

package/compiled/src/step/parsing/incremental_type_index.js

@@ -0,0 +1,92 @@
+/**
+ * A type index built **incrementally** from the streaming parse's per-record
+ * events (M2), rather than from the finished element array. Feed its
+ * {@link handler} to a {@link StreamingRecordDispatcher} (or directly to
+ * `buildIndexStreaming`'s `onRecordIndexed`) and it accumulates, per concrete
+ * type, the set of express IDs seen — so the moment parsing reaches a record
+ * it is queryable, without waiting for end-of-parse.
+ *
+ * Queries take entity constructors and expand to their subtype closure via
+ * the generated `query` (conway #383), so `expressIDsOfTypes(IfcProduct)`
+ * unions every product subtype exactly as the resident model's type index
+ * does.
+ *
+ * Express IDs are held in per-type `Set`s, so the consumer is idempotent
+ * under the streaming builder's rare grow-and-restart (records re-fire from
+ * localID 0): re-adding an express ID is a no-op. Memory is O(records) — the
+ * same order as the element index it mirrors.
+ *
+ * Scope: membership is keyed on the raw parse `typeID`. External-mapping /
+ * complex records (typeID 0) are therefore not attributed to their mapped
+ * classes here — resolving those needs the record's `multiMapping`, which the
+ * event stream doesn't carry; that's the model's construction-time index. For
+ * the overwhelming majority of records (simple, one type each) this index is
+ * exact, which the parity test pins.
+ */
+export class IncrementalTypeIndex {
+    constructor() {
+        /** Concrete typeID → express IDs of records of exactly that type. */
+        this.byType_ = new Map();
+        /**
+         * Record one indexed entity. Bound as {@link handler} for direct use as a
+         * dispatcher subscription / `onRecordIndexed` callback.
+         *
+         * @param localID Unused (kept for the RecordHandler shape).
+         * @param expressID The record's express ID.
+         * @param typeID The record's concrete type ID (undefined / 0 records are
+         * ignored — they carry no concrete queryable type here).
+         */
+        this.handler = (localID, expressID, typeID) => {
+            if (typeID === void 0) {
+                return;
+            }
+            let ids = this.byType_.get(typeID);
+            if (ids === void 0) {
+                ids = new Set();
+                this.byType_.set(typeID, ids);
+            }
+            ids.add(expressID);
+        };
+    }
+    /**
+     * The distinct concrete type IDs seen so far.
+     *
+     * @return {IterableIterator<TypeIDType>} The concrete types.
+     */
+    concreteTypes() {
+        return this.byType_.keys();
+    }
+    /**
+     * Lazily iterate the express IDs of all records of the given types
+     * (including subtypes).
+     *
+     * @param types The entity constructors to query (subtype closures unioned).
+     * @return {IterableIterator<number>} Express IDs of matching records.
+     * @yields {number} Each matching express ID.
+     */
+    *expressIDsOfTypes(...types) {
+        const typeSet = types.length === 1 ? types[0].query :
+            new Set(types.flatMap((type) => type.query));
+        for (const typeID of typeSet) {
+            const ids = this.byType_.get(typeID);
+            if (ids !== void 0) {
+                yield* ids;
+            }
+        }
+    }
+    /**
+     * Count records of the given types (including subtypes) seen so far.
+     *
+     * @param types The entity constructors to count.
+     * @return {number} The number of matching records.
+     */
+    count(...types) {
+        const typeSet = types.length === 1 ? types[0].query :
+            new Set(types.flatMap((type) => type.query));
+        let total = 0;
+        for (const typeID of typeSet) {
+            total += this.byType_.get(typeID)?.size ?? 0;
+        }
+        return total;
+    }
+}

package/compiled/src/step/parsing/incremental_type_index.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=incremental_type_index.test.d.ts.map

package/compiled/src/step/parsing/incremental_type_index.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"incremental_type_index.test.d.ts","sourceRoot":"","sources":["../../../../src/step/parsing/incremental_type_index.test.ts"],"names":[],"mappings":""}

package/compiled/src/step/parsing/incremental_type_index.test.js

@@ -0,0 +1,55 @@
+/* eslint-disable no-magic-numbers */
+// M2b: a type index built incrementally from the streaming parse's record
+// events must, once parsing finishes, hold the same membership as the
+// resident model's type index — for every queried type (and its subtypes).
+import * as fs from "fs";
+import { beforeAll, describe, expect, test } from "@jest/globals";
+import ParsingBuffer from "../../parsing/parsing_buffer.js";
+import IfcStepParser from "../../ifc/ifc_step_parser.js";
+import { BufferByteSource } from "./byte_source.js";
+import { buildIndexStreaming } from "./streaming_index_builder.js";
+import { IncrementalTypeIndex } from "./incremental_type_index.js";
+import { StreamingRecordDispatcher } from "./streaming_record_dispatcher.js";
+import { ParseResult } from "./step_parser.js";
+import { IfcRoot, IfcProduct, IfcWall, IfcPropertySet } from "../../ifc/ifc4_gen/index.js";
+let bytes;
+let model;
+beforeAll(() => {
+    bytes = new Uint8Array(fs.readFileSync("data/index.ifc"));
+    const input = new ParsingBuffer(bytes);
+    IfcStepParser.Instance.parseHeader(input);
+    model = IfcStepParser.Instance.parseDataToModel(input)[1];
+});
+describe("IncrementalTypeIndex", () => {
+    /**
+     * Stream index.ifc, feeding an IncrementalTypeIndex, and return it.
+     *
+     * @return {IncrementalTypeIndex} The populated index.
+     */
+    function streamed() {
+        const index = new IncrementalTypeIndex();
+        const dispatcher = new StreamingRecordDispatcher();
+        dispatcher.onAnyRecord(index.handler);
+        const r = buildIndexStreaming(new BufferByteSource(bytes), IfcStepParser.Instance, 4 * 1024, dispatcher.onRecordIndexed);
+        expect(r.result).toBe(ParseResult.COMPLETE);
+        return index;
+    }
+    test.each([
+        ["IfcRoot", IfcRoot],
+        ["IfcProduct", IfcProduct],
+        ["IfcWall", IfcWall],
+        ["IfcPropertySet", IfcPropertySet],
+    ])("membership matches the resident type index for %s", (_name, ctor) => {
+        const index = streamed();
+        const incremental = new Set(index.expressIDsOfTypes(ctor));
+        const resident = new Set(model.expressIDsOfTypes(ctor));
+        expect(incremental).toEqual(resident);
+        expect(index.count(ctor)).toBe(resident.size);
+    });
+    test("a multi-type query unions the subtype closures", () => {
+        const index = streamed();
+        const incremental = new Set(index.expressIDsOfTypes(IfcWall, IfcPropertySet));
+        const resident = new Set(model.expressIDsOfTypes(IfcWall, IfcPropertySet));
+        expect(incremental).toEqual(resident);
+    });
+});

package/compiled/src/step/parsing/step_parser.d.ts

@@ -116,6 +116,25 @@ export default class StepParser<TypeIDType> extends StepHeaderParser {
      * @return {BlockParseResult} The parsing result, including the index and result enum.
      */
     parseDataBlock(input: ParsingBuffer, onProgress?: ParseProgressCallback): BlockParseResult<TypeIDType>;
+    /**
+     * Streaming driver over parseDataBlockIncremental: identical parse (same
+     * generator body) but invokes `onRecordBoundary` at every top-level record
+     * boundary so a caller feeding the parser from a moving window can slide
+     * that window forward while the rewind stack is empty. See
+     * streaming_index_builder.ts for the coordinator that owns the window.
+     *
+     * @param input The input parsing buffer, positioned at the data section.
+     * @param onRecordBoundary Called at each top-level record boundary with the
+     * buffer; the callback may rebase the buffer's window in place.
+     * @param onRecordIndexed Called as each top-level record is indexed, with
+     * its localID, expressID and typeID (0 for external-mapping records) — the
+     * seam for incremental semantic consumers (type index, roots registry,
+     * names skeleton). Must be synchronous and cheap; expensive work belongs on
+     * a demand queue, not the parse path.
+     * @param onProgress Optional byte-cursor progress callback.
+     * @return {BlockParseResult} The parsing result, including the index and result enum.
+     */
+    parseDataBlockStreamed(input: ParsingBuffer, onRecordBoundary: (input: ParsingBuffer) => void, onRecordIndexed?: (localID: number, expressID: number, typeID: TypeIDType | undefined) => void, onProgress?: ParseProgressCallback): BlockParseResult<TypeIDType>;
     /**
      * Cooperative variant of parseDataBlock: identical parse (same generator
      * body), but periodically awaits a macrotask so the event loop can run —

package/compiled/src/step/parsing/step_parser.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"step_parser.d.ts","sourceRoot":"","sources":["../../../../src/step/parsing/step_parser.ts"],"names":[],"mappings":"AACA,OAAO,SAAS,MAAM,2BAA2B,CAAA;AAEjD,OAAO,aAAa,MAAM,8BAA8B,CAAA;AAcxD,OAAO,iBAAiB,EAAE,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAA;AAEpE,MAAM,WAAW,kBAAkB,CAAC,UAAU;IAE5C,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,UAAU,CAAA;IACnB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,YAAY,CAAC,EAAE,kBAAkB,CAAC,UAAU,CAAC,EAAE,CAAA;IAE/C,cAAc,CAAC,EAAE,oBAAoB,CAAC,UAAU,CAAC,EAAE,CAAA;CACpD;AAED,MAAM,WAAW,oBAAoB,CAAC,UAAU,CAAE,SAAQ,kBAAkB,CAAC,UAAU,CAAC;IAEtF,SAAS,CAAC,EAAE,SAAS,CAAA;CACtB;AAED,MAAM,WAAW,cAAc,CAAC,UAAU,CAAE,SAAQ,kBAAkB,CAAC,UAAU,CAAC;IAEhF,SAAS,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,SAAS,CAAC,UAAU;IAEnC,QAAQ,EAAE,cAAc,CAAC,UAAU,CAAC,EAAE,CAAA;CAEvC;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,GAAG,CAAA;CACX;AAID;;GAEG;AAGH,oBAAY,WAAW;IAErB,QAAQ,IAAI;IACZ,UAAU,IAAI;IACd,YAAY,IAAI;IAChB,YAAY,IAAI;IAChB,YAAY,IAAI;CACjB;AAyBD,MAAM,MAAM,gBAAgB,CAAC,UAAU,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,WAAW,CAAC,CAAA;AAC/E,MAAM,MAAM,uBAAuB,CAAC,UAAU,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,WAAW,CAAC,CAAA;AAEtF;;;;;GAKG;AACH,MAAM,MAAM,qBAAqB,GAAG,CAAE,WAAW,EAAE,MAAM,KAAM,IAAI,CAAA;AAWnE,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAC7B;AAED,MAAM,MAAM,iBAAiB,GAAG,CAAC,UAAU,EAAE,WAAW,CAAC,CAAA;AAGzD;;GAEG;AACH,qBAAa,gBAAgB;IAE3B,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAkB;IAE3C;;;;OAIG;IACH,WAAkB,QAAQ,IAAI,gBAAgB,CAQ7C;IAED;;;;;OAKG;IACI,WAAW,CAAC,KAAK,EAAE,aAAa,GAAG,iBAAiB;CA0L5D;AACD;;GAEG;AACH,MAAM,CAAC,OAAO,OAAO,UAAU,CAAC,UAAU,CAAE,SAAQ,gBAAgB;IAQtD,OAAO,CAAC,QAAQ,CAAC,MAAM;IANnC;;;;;OAKG;gBAC0B,MAAM,EAAE,QAAQ,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IAKpE;;;;;;;;;;;;;OAaG;IACI,gBAAgB,CACnB,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,iBAAiB,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS;IA+D9E;;;;;;;;;;OAUG;IACI,cAAc,CACjB,KAAK,EAAE,aAAa,EACpB,UAAU,CAAC,EAAE,qBAAqB,GAAI,gBAAgB,CAAC,UAAU,CAAC;IAgBtE;;;;;;;;;;;OAWG;IACU,mBAAmB,CAC5B,KAAK,EAAE,aAAa,EACpB,UAAU,CAAC,EAAE,qBAAqB,EAClC,eAAe,GAAE,MAAwC,GACzD,OAAO,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;IAuBzC;;;;;;;;;OASG;IACH,OAAO,CAAE,yBAAyB;IAyZlC;;;;;;;OAOG;IACI,gBAAgB,CAAC,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,GAAG,CAAC,GAAG,EAAE,WAAW,CAAC;CAiarF"}
+{"version":3,"file":"step_parser.d.ts","sourceRoot":"","sources":["../../../../src/step/parsing/step_parser.ts"],"names":[],"mappings":"AACA,OAAO,SAAS,MAAM,2BAA2B,CAAA;AAEjD,OAAO,aAAa,MAAM,8BAA8B,CAAA;AAcxD,OAAO,iBAAiB,EAAE,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAA;AAEpE,MAAM,WAAW,kBAAkB,CAAC,UAAU;IAE5C,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,UAAU,CAAA;IACnB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,YAAY,CAAC,EAAE,kBAAkB,CAAC,UAAU,CAAC,EAAE,CAAA;IAE/C,cAAc,CAAC,EAAE,oBAAoB,CAAC,UAAU,CAAC,EAAE,CAAA;CACpD;AAED,MAAM,WAAW,oBAAoB,CAAC,UAAU,CAAE,SAAQ,kBAAkB,CAAC,UAAU,CAAC;IAEtF,SAAS,CAAC,EAAE,SAAS,CAAA;CACtB;AAED,MAAM,WAAW,cAAc,CAAC,UAAU,CAAE,SAAQ,kBAAkB,CAAC,UAAU,CAAC;IAEhF,SAAS,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,SAAS,CAAC,UAAU;IAEnC,QAAQ,EAAE,cAAc,CAAC,UAAU,CAAC,EAAE,CAAA;CAEvC;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,GAAG,CAAA;CACX;AAID;;GAEG;AAGH,oBAAY,WAAW;IAErB,QAAQ,IAAI;IACZ,UAAU,IAAI;IACd,YAAY,IAAI;IAChB,YAAY,IAAI;IAChB,YAAY,IAAI;CACjB;AAyBD,MAAM,MAAM,gBAAgB,CAAC,UAAU,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,WAAW,CAAC,CAAA;AAC/E,MAAM,MAAM,uBAAuB,CAAC,UAAU,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,WAAW,CAAC,CAAA;AAEtF;;;;;GAKG;AACH,MAAM,MAAM,qBAAqB,GAAG,CAAE,WAAW,EAAE,MAAM,KAAM,IAAI,CAAA;AAWnE,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAC7B;AAED,MAAM,MAAM,iBAAiB,GAAG,CAAC,UAAU,EAAE,WAAW,CAAC,CAAA;AAGzD;;GAEG;AACH,qBAAa,gBAAgB;IAE3B,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAkB;IAE3C;;;;OAIG;IACH,WAAkB,QAAQ,IAAI,gBAAgB,CAQ7C;IAED;;;;;OAKG;IACI,WAAW,CAAC,KAAK,EAAE,aAAa,GAAG,iBAAiB;CA0L5D;AACD;;GAEG;AACH,MAAM,CAAC,OAAO,OAAO,UAAU,CAAC,UAAU,CAAE,SAAQ,gBAAgB;IAQtD,OAAO,CAAC,QAAQ,CAAC,MAAM;IANnC;;;;;OAKG;gBAC0B,MAAM,EAAE,QAAQ,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IAKpE;;;;;;;;;;;;;OAaG;IACI,gBAAgB,CACnB,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,iBAAiB,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS;IA+D9E;;;;;;;;;;OAUG;IACI,cAAc,CACjB,KAAK,EAAE,aAAa,EACpB,UAAU,CAAC,EAAE,qBAAqB,GAAI,gBAAgB,CAAC,UAAU,CAAC;IAgBtE;;;;;;;;;;;;;;;;;OAiBG;IACI,sBAAsB,CACzB,KAAK,EAAE,aAAa,EACpB,gBAAgB,EAAE,CAAE,KAAK,EAAE,aAAa,KAAM,IAAI,EAClD,eAAe,CAAC,EAAE,CAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,GAAG,SAAS,KAAM,IAAI,EAChG,UAAU,CAAC,EAAE,qBAAqB,GAAI,gBAAgB,CAAC,UAAU,CAAC;IAgBtE;;;;;;;;;;;OAWG;IACU,mBAAmB,CAC5B,KAAK,EAAE,aAAa,EACpB,UAAU,CAAC,EAAE,qBAAqB,EAClC,eAAe,GAAE,MAAwC,GACzD,OAAO,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;IAuBzC;;;;;;;;;OASG;IACH,OAAO,CAAE,yBAAyB;IA+alC;;;;;;;OAOG;IACI,gBAAgB,CAAC,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,GAAG,CAAC,GAAG,EAAE,WAAW,CAAC;CAiarF"}

package/compiled/src/step/parsing/step_parser.js

@@ -310,6 +310,34 @@ export default class StepParser extends StepHeaderParser {
             onProgress?.(next.value);
         }
     }
+    /**
+     * Streaming driver over parseDataBlockIncremental: identical parse (same
+     * generator body) but invokes `onRecordBoundary` at every top-level record
+     * boundary so a caller feeding the parser from a moving window can slide
+     * that window forward while the rewind stack is empty. See
+     * streaming_index_builder.ts for the coordinator that owns the window.
+     *
+     * @param input The input parsing buffer, positioned at the data section.
+     * @param onRecordBoundary Called at each top-level record boundary with the
+     * buffer; the callback may rebase the buffer's window in place.
+     * @param onRecordIndexed Called as each top-level record is indexed, with
+     * its localID, expressID and typeID (0 for external-mapping records) — the
+     * seam for incremental semantic consumers (type index, roots registry,
+     * names skeleton). Must be synchronous and cheap; expensive work belongs on
+     * a demand queue, not the parse path.
+     * @param onProgress Optional byte-cursor progress callback.
+     * @return {BlockParseResult} The parsing result, including the index and result enum.
+     */
+    parseDataBlockStreamed(input, onRecordBoundary, onRecordIndexed, onProgress) {
+        const parser = this.parseDataBlockIncremental(input, onRecordBoundary, onRecordIndexed);
+        while (true) {
+            const next = parser.next();
+            if (next.done === true) {
+                return next.value;
+            }
+            onProgress?.(next.value);
+        }
+    }
     /**
      * Cooperative variant of parseDataBlock: identical parse (same generator
      * body), but periodically awaits a macrotask so the event loop can run —
@@ -347,7 +375,7 @@ export default class StepParser extends StepHeaderParser {
      * @yields {number} The current byte cursor within the input buffer.
      * @return {BlockParseResult} The parsing result, including the index and result enum.
      */
-    *parseDataBlockIncremental(input) {
+    *parseDataBlockIncremental(input, onRecordBoundary, onRecordIndexed) {
         const indexResult = { elements: [] };
         const match = input.match;
         const comment = () => match(commentParser);
@@ -394,7 +422,10 @@ export default class StepParser extends StepHeaderParser {
                 return syntaxError();
             }
             whitespace();
-            const startElement = input.cursor;
+            // File-absolute (== input.cursor when initialOffset is 0, i.e. every
+            // resident parse; file-absolute under a streaming moving window so the
+            // recorded address/length stay stable as the window slides).
+            const startElement = input.address;
             let stackDepth = 1;
             const savedInlineElements = inlineElements;
             inlineElements = void 0;
@@ -492,6 +523,11 @@ export default class StepParser extends StepHeaderParser {
             if ((++parsedElementCount & PARSE_PROGRESS_ELEMENT_MASK) === 0) {
                 yield input.cursor;
             }
+            // Top-level record boundary: the rewind stack is empty and the cursor
+            // sits at the next record's leading whitespace/`#`. A streaming driver
+            // uses this to slide its moving window forward (see
+            // streaming_index_builder.ts). No-op on the resident path.
+            onRecordBoundary?.(input);
             if (!charws(HASH)) {
                 if (tokenws(END_SECTION)) {
                     return parseResult(ParseResult.COMPLETE);
@@ -516,7 +552,10 @@ export default class StepParser extends StepHeaderParser {
                 }
                 whitespace();
                 inlineElements = void 0;
-                const startElement = input.cursor;
+                // File-absolute (== input.cursor when initialOffset is 0, i.e. every
+                // resident parse; file-absolute under a streaming moving window so the
+                // recorded address/length stay stable as the window slides).
+                const startElement = input.address;
                 // todo, loop and read inline then add them to a special "External Mapping" node.
                 while (!charws(CLOSE_PAREN)) {
                     input.begin();
@@ -530,6 +569,7 @@ export default class StepParser extends StepHeaderParser {
                 if (!charws(SEMICOLON)) {
                     return syntaxError();
                 }
+                onRecordIndexed?.(indexResult.elements.length, expressID, 0);
                 indexResult.elements.push({
                     address: startElement,
                     length: input.address - startElement,
@@ -547,7 +587,10 @@ export default class StepParser extends StepHeaderParser {
                 return syntaxError();
             }
             whitespace();
-            const startElement = input.cursor;
+            // File-absolute (== input.cursor when initialOffset is 0, i.e. every
+            // resident parse; file-absolute under a streaming moving window so the
+            // recorded address/length stay stable as the window slides).
+            const startElement = input.address;
             let stackDepth = 1;
             inlineElements = void 0;
             while (stackDepth > 0) {
@@ -620,6 +663,7 @@ export default class StepParser extends StepHeaderParser {
             if (!charws(SEMICOLON)) {
                 return syntaxError();
             }
+            onRecordIndexed?.(indexResult.elements.length, expressID, foundItem);
             indexResult.elements.push({
                 address: startElement,
                 length: input.address - startElement,

package/compiled/src/step/parsing/streaming_index_builder.d.ts

@@ -0,0 +1,70 @@
+import { ByteSource } from "./byte_source.js";
+import StepParser, { ParseResult, StepHeader, StepIndexEntry } from "./step_parser.js";
+/**
+ * Diagnostics from a streaming index build — how hard the moving window had
+ * to work, so a pool-size sweep can compare cost against a resident parse.
+ */
+export interface StreamingIndexStats {
+    /** Target window size in bytes (the pool). */
+    pool: number;
+    /** Physical window buffer size actually used (grows past `pool` only if a
+     * single record exceeds the pool). */
+    windowBytes: number;
+    /** Number of times the window slid forward. */
+    slides: number;
+    /** Largest single top-level record observed, in bytes. */
+    maxRecordLen: number;
+    /** Total bytes read from the source (≈ file size + re-reads on growth). */
+    bytesRead: number;
+}
+/**
+ * The output of a streaming index build: the same element index a resident
+ * `parseDataBlock` produces, plus the header and window diagnostics.
+ */
+export interface StreamingIndexResult<TypeIDType> {
+    header: StepHeader;
+    elements: StepIndexEntry<TypeIDType>[];
+    result: ParseResult;
+    stats: StreamingIndexStats;
+}
+/**
+ * Build the entity index by streaming a `ByteSource` through a fixed-size
+ * moving window, instead of parsing one resident buffer. The parse loop is
+ * byte-for-byte the resident one (`StepParser.parseDataBlockStreamed` drives
+ * the same generator); this coordinator only owns the window: it fills it,
+ * parses the header from the first fill, then slides the window forward at
+ * top-level record boundaries as the parser advances.
+ *
+ * Because the parser records file-absolute addresses and its rewind stack is
+ * empty at every top-level boundary, sliding there is transparent: the
+ * unconsumed tail is copied to the front, fresh bytes are appended, and the
+ * buffer is rebased so `address` keeps its file-absolute value. Peak JS heap
+ * for the index build is therefore `window + index columns`, independent of
+ * file size.
+ *
+ * The window slides only once the cursor passes `pool / 2`, bounding the
+ * memmove frequency; a record up to `pool / 2` bytes always fits after a
+ * slide. If a single record exceeds that (never on the current corpus, whose
+ * largest STEP record is ~25 KB), the whole parse restarts from the
+ * beginning with the window doubled, and repeats until every record fits —
+ * correctness over the pathological case, at the cost of a re-scan. (M1's
+ * production path will instead grow in place / restart from the last
+ * boundary; from-scratch keeps the spike simple.)
+ *
+ * @param source The byte source.
+ * @param parser The STEP parser (typed to the schema).
+ * @param pool Target window size in bytes.
+ * @param onRecordIndexed Optional per-record event, invoked live as each
+ * top-level record is indexed with its (localID, expressID, typeID) — the
+ * seam for incremental semantic consumers (M2). localIDs are dense and
+ * assigned in parse order from 0. On the rare grow-and-restart (a single
+ * record larger than the window — never on real files, whose largest STEP
+ * record is ~25 KB), the parse re-runs from the start and records re-fire
+ * from localID 0; consumers must therefore be idempotent by localID /
+ * expressID (the standard consumers — type index keyed by localID, roots
+ * registry keyed by expressID — are). Must be synchronous and cheap;
+ * expensive work belongs on a demand queue, not the parse path.
+ * @return {StreamingIndexResult} The index, header, result and diagnostics.
+ */
+export declare function buildIndexStreaming<TypeIDType>(source: ByteSource, parser: StepParser<TypeIDType>, pool: number, onRecordIndexed?: (localID: number, expressID: number, typeID: TypeIDType | undefined) => void): StreamingIndexResult<TypeIDType>;
+//# sourceMappingURL=streaming_index_builder.d.ts.map

package/compiled/src/step/parsing/streaming_index_builder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"streaming_index_builder.d.ts","sourceRoot":"","sources":["../../../../src/step/parsing/streaming_index_builder.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC1C,OAAO,UAAU,EAAE,EACjB,WAAW,EACX,UAAU,EACV,cAAc,EACf,MAAM,eAAe,CAAA;AAGtB;;;GAGG;AACH,MAAM,WAAW,mBAAmB;IAElC,8CAA8C;IAC9C,IAAI,EAAE,MAAM,CAAA;IAEZ;0CACsC;IACtC,WAAW,EAAE,MAAM,CAAA;IAEnB,+CAA+C;IAC/C,MAAM,EAAE,MAAM,CAAA;IAEd,0DAA0D;IAC1D,YAAY,EAAE,MAAM,CAAA;IAEpB,2EAA2E;IAC3E,SAAS,EAAE,MAAM,CAAA;CAClB;AAGD;;;GAGG;AACH,MAAM,WAAW,oBAAoB,CAAC,UAAU;IAC9C,MAAM,EAAE,UAAU,CAAA;IAClB,QAAQ,EAAE,cAAc,CAAC,UAAU,CAAC,EAAE,CAAA;IACtC,MAAM,EAAE,WAAW,CAAA;IACnB,KAAK,EAAE,mBAAmB,CAAA;CAC3B;AAGD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,wBAAgB,mBAAmB,CAAC,UAAU,EAC1C,MAAM,EAAE,UAAU,EAClB,MAAM,EAAE,UAAU,CAAC,UAAU,CAAC,EAC9B,IAAI,EAAE,MAAM,EACZ,eAAe,CAAC,EACd,CAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,GAAG,SAAS,KAAM,IAAI,GAChF,oBAAoB,CAAC,UAAU,CAAC,CA4GnC"}