@bldrs-ai/conway 1.389.1221 → 1.398.1224

package/compiled/src/step/parsing/step_parser.d.ts

@@ -116,6 +116,20 @@ export default class StepParser<TypeIDType> extends StepHeaderParser {
      * @return {BlockParseResult} The parsing result, including the index and result enum.
      */
     parseDataBlock(input: ParsingBuffer, onProgress?: ParseProgressCallback): BlockParseResult<TypeIDType>;
+    /**
+     * Streaming driver over parseDataBlockIncremental: identical parse (same
+     * generator body) but invokes `onRecordBoundary` at every top-level record
+     * boundary so a caller feeding the parser from a moving window can slide
+     * that window forward while the rewind stack is empty. See
+     * streaming_index_builder.ts for the coordinator that owns the window.
+     *
+     * @param input The input parsing buffer, positioned at the data section.
+     * @param onRecordBoundary Called at each top-level record boundary with the
+     * buffer; the callback may rebase the buffer's window in place.
+     * @param onProgress Optional byte-cursor progress callback.
+     * @return {BlockParseResult} The parsing result, including the index and result enum.
+     */
+    parseDataBlockStreamed(input: ParsingBuffer, onRecordBoundary: (input: ParsingBuffer) => void, onProgress?: ParseProgressCallback): BlockParseResult<TypeIDType>;
     /**
      * Cooperative variant of parseDataBlock: identical parse (same generator
      * body), but periodically awaits a macrotask so the event loop can run —

package/compiled/src/step/parsing/step_parser.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"step_parser.d.ts","sourceRoot":"","sources":["../../../../src/step/parsing/step_parser.ts"],"names":[],"mappings":"AACA,OAAO,SAAS,MAAM,2BAA2B,CAAA;AAEjD,OAAO,aAAa,MAAM,8BAA8B,CAAA;AAcxD,OAAO,iBAAiB,EAAE,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAA;AAEpE,MAAM,WAAW,kBAAkB,CAAC,UAAU;IAE5C,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,UAAU,CAAA;IACnB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,YAAY,CAAC,EAAE,kBAAkB,CAAC,UAAU,CAAC,EAAE,CAAA;IAE/C,cAAc,CAAC,EAAE,oBAAoB,CAAC,UAAU,CAAC,EAAE,CAAA;CACpD;AAED,MAAM,WAAW,oBAAoB,CAAC,UAAU,CAAE,SAAQ,kBAAkB,CAAC,UAAU,CAAC;IAEtF,SAAS,CAAC,EAAE,SAAS,CAAA;CACtB;AAED,MAAM,WAAW,cAAc,CAAC,UAAU,CAAE,SAAQ,kBAAkB,CAAC,UAAU,CAAC;IAEhF,SAAS,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,SAAS,CAAC,UAAU;IAEnC,QAAQ,EAAE,cAAc,CAAC,UAAU,CAAC,EAAE,CAAA;CAEvC;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,GAAG,CAAA;CACX;AAID;;GAEG;AAGH,oBAAY,WAAW;IAErB,QAAQ,IAAI;IACZ,UAAU,IAAI;IACd,YAAY,IAAI;IAChB,YAAY,IAAI;IAChB,YAAY,IAAI;CACjB;AAyBD,MAAM,MAAM,gBAAgB,CAAC,UAAU,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,WAAW,CAAC,CAAA;AAC/E,MAAM,MAAM,uBAAuB,CAAC,UAAU,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,WAAW,CAAC,CAAA;AAEtF;;;;;GAKG;AACH,MAAM,MAAM,qBAAqB,GAAG,CAAE,WAAW,EAAE,MAAM,KAAM,IAAI,CAAA;AAWnE,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAC7B;AAED,MAAM,MAAM,iBAAiB,GAAG,CAAC,UAAU,EAAE,WAAW,CAAC,CAAA;AAGzD;;GAEG;AACH,qBAAa,gBAAgB;IAE3B,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAkB;IAE3C;;;;OAIG;IACH,WAAkB,QAAQ,IAAI,gBAAgB,CAQ7C;IAED;;;;;OAKG;IACI,WAAW,CAAC,KAAK,EAAE,aAAa,GAAG,iBAAiB;CA0L5D;AACD;;GAEG;AACH,MAAM,CAAC,OAAO,OAAO,UAAU,CAAC,UAAU,CAAE,SAAQ,gBAAgB;IAQtD,OAAO,CAAC,QAAQ,CAAC,MAAM;IANnC;;;;;OAKG;gBAC0B,MAAM,EAAE,QAAQ,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IAKpE;;;;;;;;;;;;;OAaG;IACI,gBAAgB,CACnB,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,iBAAiB,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS;IA+D9E;;;;;;;;;;OAUG;IACI,cAAc,CACjB,KAAK,EAAE,aAAa,EACpB,UAAU,CAAC,EAAE,qBAAqB,GAAI,gBAAgB,CAAC,UAAU,CAAC;IAgBtE;;;;;;;;;;;OAWG;IACU,mBAAmB,CAC5B,KAAK,EAAE,aAAa,EACpB,UAAU,CAAC,EAAE,qBAAqB,EAClC,eAAe,GAAE,MAAwC,GACzD,OAAO,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;IAuBzC;;;;;;;;;OASG;IACH,OAAO,CAAE,yBAAyB;IAyZlC;;;;;;;OAOG;IACI,gBAAgB,CAAC,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,GAAG,CAAC,GAAG,EAAE,WAAW,CAAC;CAiarF"}
+{"version":3,"file":"step_parser.d.ts","sourceRoot":"","sources":["../../../../src/step/parsing/step_parser.ts"],"names":[],"mappings":"AACA,OAAO,SAAS,MAAM,2BAA2B,CAAA;AAEjD,OAAO,aAAa,MAAM,8BAA8B,CAAA;AAcxD,OAAO,iBAAiB,EAAE,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAA;AAEpE,MAAM,WAAW,kBAAkB,CAAC,UAAU;IAE5C,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,UAAU,CAAA;IACnB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,YAAY,CAAC,EAAE,kBAAkB,CAAC,UAAU,CAAC,EAAE,CAAA;IAE/C,cAAc,CAAC,EAAE,oBAAoB,CAAC,UAAU,CAAC,EAAE,CAAA;CACpD;AAED,MAAM,WAAW,oBAAoB,CAAC,UAAU,CAAE,SAAQ,kBAAkB,CAAC,UAAU,CAAC;IAEtF,SAAS,CAAC,EAAE,SAAS,CAAA;CACtB;AAED,MAAM,WAAW,cAAc,CAAC,UAAU,CAAE,SAAQ,kBAAkB,CAAC,UAAU,CAAC;IAEhF,SAAS,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,SAAS,CAAC,UAAU;IAEnC,QAAQ,EAAE,cAAc,CAAC,UAAU,CAAC,EAAE,CAAA;CAEvC;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,GAAG,CAAA;CACX;AAID;;GAEG;AAGH,oBAAY,WAAW;IAErB,QAAQ,IAAI;IACZ,UAAU,IAAI;IACd,YAAY,IAAI;IAChB,YAAY,IAAI;IAChB,YAAY,IAAI;CACjB;AAyBD,MAAM,MAAM,gBAAgB,CAAC,UAAU,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,WAAW,CAAC,CAAA;AAC/E,MAAM,MAAM,uBAAuB,CAAC,UAAU,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,WAAW,CAAC,CAAA;AAEtF;;;;;GAKG;AACH,MAAM,MAAM,qBAAqB,GAAG,CAAE,WAAW,EAAE,MAAM,KAAM,IAAI,CAAA;AAWnE,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAC7B;AAED,MAAM,MAAM,iBAAiB,GAAG,CAAC,UAAU,EAAE,WAAW,CAAC,CAAA;AAGzD;;GAEG;AACH,qBAAa,gBAAgB;IAE3B,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAkB;IAE3C;;;;OAIG;IACH,WAAkB,QAAQ,IAAI,gBAAgB,CAQ7C;IAED;;;;;OAKG;IACI,WAAW,CAAC,KAAK,EAAE,aAAa,GAAG,iBAAiB;CA0L5D;AACD;;GAEG;AACH,MAAM,CAAC,OAAO,OAAO,UAAU,CAAC,UAAU,CAAE,SAAQ,gBAAgB;IAQtD,OAAO,CAAC,QAAQ,CAAC,MAAM;IANnC;;;;;OAKG;gBAC0B,MAAM,EAAE,QAAQ,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IAKpE;;;;;;;;;;;;;OAaG;IACI,gBAAgB,CACnB,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,iBAAiB,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS;IA+D9E;;;;;;;;;;OAUG;IACI,cAAc,CACjB,KAAK,EAAE,aAAa,EACpB,UAAU,CAAC,EAAE,qBAAqB,GAAI,gBAAgB,CAAC,UAAU,CAAC;IAgBtE;;;;;;;;;;;;OAYG;IACI,sBAAsB,CACzB,KAAK,EAAE,aAAa,EACpB,gBAAgB,EAAE,CAAE,KAAK,EAAE,aAAa,KAAM,IAAI,EAClD,UAAU,CAAC,EAAE,qBAAqB,GAAI,gBAAgB,CAAC,UAAU,CAAC;IAgBtE;;;;;;;;;;;OAWG;IACU,mBAAmB,CAC5B,KAAK,EAAE,aAAa,EACpB,UAAU,CAAC,EAAE,qBAAqB,EAClC,eAAe,GAAE,MAAwC,GACzD,OAAO,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;IAuBzC;;;;;;;;;OASG;IACH,OAAO,CAAE,yBAAyB;IA0alC;;;;;;;OAOG;IACI,gBAAgB,CAAC,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,GAAG,CAAC,GAAG,EAAE,WAAW,CAAC;CAiarF"}

package/compiled/src/step/parsing/step_parser.js

@@ -310,6 +310,29 @@ export default class StepParser extends StepHeaderParser {
             onProgress?.(next.value);
         }
     }
+    /**
+     * Streaming driver over parseDataBlockIncremental: identical parse (same
+     * generator body) but invokes `onRecordBoundary` at every top-level record
+     * boundary so a caller feeding the parser from a moving window can slide
+     * that window forward while the rewind stack is empty. See
+     * streaming_index_builder.ts for the coordinator that owns the window.
+     *
+     * @param input The input parsing buffer, positioned at the data section.
+     * @param onRecordBoundary Called at each top-level record boundary with the
+     * buffer; the callback may rebase the buffer's window in place.
+     * @param onProgress Optional byte-cursor progress callback.
+     * @return {BlockParseResult} The parsing result, including the index and result enum.
+     */
+    parseDataBlockStreamed(input, onRecordBoundary, onProgress) {
+        const parser = this.parseDataBlockIncremental(input, onRecordBoundary);
+        while (true) {
+            const next = parser.next();
+            if (next.done === true) {
+                return next.value;
+            }
+            onProgress?.(next.value);
+        }
+    }
     /**
      * Cooperative variant of parseDataBlock: identical parse (same generator
      * body), but periodically awaits a macrotask so the event loop can run —
@@ -347,7 +370,7 @@ export default class StepParser extends StepHeaderParser {
      * @yields {number} The current byte cursor within the input buffer.
      * @return {BlockParseResult} The parsing result, including the index and result enum.
      */
-    *parseDataBlockIncremental(input) {
+    *parseDataBlockIncremental(input, onRecordBoundary) {
         const indexResult = { elements: [] };
         const match = input.match;
         const comment = () => match(commentParser);
@@ -394,7 +417,10 @@ export default class StepParser extends StepHeaderParser {
                 return syntaxError();
             }
             whitespace();
-            const startElement = input.cursor;
+            // File-absolute (== input.cursor when initialOffset is 0, i.e. every
+            // resident parse; file-absolute under a streaming moving window so the
+            // recorded address/length stay stable as the window slides).
+            const startElement = input.address;
             let stackDepth = 1;
             const savedInlineElements = inlineElements;
             inlineElements = void 0;
@@ -492,6 +518,11 @@ export default class StepParser extends StepHeaderParser {
             if ((++parsedElementCount & PARSE_PROGRESS_ELEMENT_MASK) === 0) {
                 yield input.cursor;
             }
+            // Top-level record boundary: the rewind stack is empty and the cursor
+            // sits at the next record's leading whitespace/`#`. A streaming driver
+            // uses this to slide its moving window forward (see
+            // streaming_index_builder.ts). No-op on the resident path.
+            onRecordBoundary?.(input);
             if (!charws(HASH)) {
                 if (tokenws(END_SECTION)) {
                     return parseResult(ParseResult.COMPLETE);
@@ -516,7 +547,10 @@ export default class StepParser extends StepHeaderParser {
                 }
                 whitespace();
                 inlineElements = void 0;
-                const startElement = input.cursor;
+                // File-absolute (== input.cursor when initialOffset is 0, i.e. every
+                // resident parse; file-absolute under a streaming moving window so the
+                // recorded address/length stay stable as the window slides).
+                const startElement = input.address;
                 // todo, loop and read inline then add them to a special "External Mapping" node.
                 while (!charws(CLOSE_PAREN)) {
                     input.begin();
@@ -547,7 +581,10 @@ export default class StepParser extends StepHeaderParser {
                 return syntaxError();
             }
             whitespace();
-            const startElement = input.cursor;
+            // File-absolute (== input.cursor when initialOffset is 0, i.e. every
+            // resident parse; file-absolute under a streaming moving window so the
+            // recorded address/length stay stable as the window slides).
+            const startElement = input.address;
             let stackDepth = 1;
             inlineElements = void 0;
             while (stackDepth > 0) {

package/compiled/src/step/parsing/streaming_index_builder.d.ts

@@ -0,0 +1,60 @@
+import { ByteSource } from "./byte_source.js";
+import StepParser, { ParseResult, StepHeader, StepIndexEntry } from "./step_parser.js";
+/**
+ * Diagnostics from a streaming index build — how hard the moving window had
+ * to work, so a pool-size sweep can compare cost against a resident parse.
+ */
+export interface StreamingIndexStats {
+    /** Target window size in bytes (the pool). */
+    pool: number;
+    /** Physical window buffer size actually used (grows past `pool` only if a
+     * single record exceeds the pool). */
+    windowBytes: number;
+    /** Number of times the window slid forward. */
+    slides: number;
+    /** Largest single top-level record observed, in bytes. */
+    maxRecordLen: number;
+    /** Total bytes read from the source (≈ file size + re-reads on growth). */
+    bytesRead: number;
+}
+/**
+ * The output of a streaming index build: the same element index a resident
+ * `parseDataBlock` produces, plus the header and window diagnostics.
+ */
+export interface StreamingIndexResult<TypeIDType> {
+    header: StepHeader;
+    elements: StepIndexEntry<TypeIDType>[];
+    result: ParseResult;
+    stats: StreamingIndexStats;
+}
+/**
+ * Build the entity index by streaming a `ByteSource` through a fixed-size
+ * moving window, instead of parsing one resident buffer. The parse loop is
+ * byte-for-byte the resident one (`StepParser.parseDataBlockStreamed` drives
+ * the same generator); this coordinator only owns the window: it fills it,
+ * parses the header from the first fill, then slides the window forward at
+ * top-level record boundaries as the parser advances.
+ *
+ * Because the parser records file-absolute addresses and its rewind stack is
+ * empty at every top-level boundary, sliding there is transparent: the
+ * unconsumed tail is copied to the front, fresh bytes are appended, and the
+ * buffer is rebased so `address` keeps its file-absolute value. Peak JS heap
+ * for the index build is therefore `window + index columns`, independent of
+ * file size.
+ *
+ * The window slides only once the cursor passes `pool / 2`, bounding the
+ * memmove frequency; a record up to `pool / 2` bytes always fits after a
+ * slide. If a single record exceeds that (never on the current corpus, whose
+ * largest STEP record is ~25 KB), the whole parse restarts from the
+ * beginning with the window doubled, and repeats until every record fits —
+ * correctness over the pathological case, at the cost of a re-scan. (M1's
+ * production path will instead grow in place / restart from the last
+ * boundary; from-scratch keeps the spike simple.)
+ *
+ * @param source The byte source.
+ * @param parser The STEP parser (typed to the schema).
+ * @param pool Target window size in bytes.
+ * @return {StreamingIndexResult} The index, header, result and diagnostics.
+ */
+export declare function buildIndexStreaming<TypeIDType>(source: ByteSource, parser: StepParser<TypeIDType>, pool: number): StreamingIndexResult<TypeIDType>;
+//# sourceMappingURL=streaming_index_builder.d.ts.map

package/compiled/src/step/parsing/streaming_index_builder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"streaming_index_builder.d.ts","sourceRoot":"","sources":["../../../../src/step/parsing/streaming_index_builder.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC1C,OAAO,UAAU,EAAE,EACjB,WAAW,EACX,UAAU,EACV,cAAc,EACf,MAAM,eAAe,CAAA;AAGtB;;;GAGG;AACH,MAAM,WAAW,mBAAmB;IAElC,8CAA8C;IAC9C,IAAI,EAAE,MAAM,CAAA;IAEZ;0CACsC;IACtC,WAAW,EAAE,MAAM,CAAA;IAEnB,+CAA+C;IAC/C,MAAM,EAAE,MAAM,CAAA;IAEd,0DAA0D;IAC1D,YAAY,EAAE,MAAM,CAAA;IAEpB,2EAA2E;IAC3E,SAAS,EAAE,MAAM,CAAA;CAClB;AAGD;;;GAGG;AACH,MAAM,WAAW,oBAAoB,CAAC,UAAU;IAC9C,MAAM,EAAE,UAAU,CAAA;IAClB,QAAQ,EAAE,cAAc,CAAC,UAAU,CAAC,EAAE,CAAA;IACtC,MAAM,EAAE,WAAW,CAAA;IACnB,KAAK,EAAE,mBAAmB,CAAA;CAC3B;AAGD;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,wBAAgB,mBAAmB,CAAC,UAAU,EAC1C,MAAM,EAAE,UAAU,EAClB,MAAM,EAAE,UAAU,CAAC,UAAU,CAAC,EAC9B,IAAI,EAAE,MAAM,GAAI,oBAAoB,CAAC,UAAU,CAAC,CA2GnD"}

package/compiled/src/step/parsing/streaming_index_builder.js

@@ -0,0 +1,115 @@
+import ParsingBuffer from "../../parsing/parsing_buffer.js";
+import { ParseResult, } from "./step_parser.js";
+/**
+ * Build the entity index by streaming a `ByteSource` through a fixed-size
+ * moving window, instead of parsing one resident buffer. The parse loop is
+ * byte-for-byte the resident one (`StepParser.parseDataBlockStreamed` drives
+ * the same generator); this coordinator only owns the window: it fills it,
+ * parses the header from the first fill, then slides the window forward at
+ * top-level record boundaries as the parser advances.
+ *
+ * Because the parser records file-absolute addresses and its rewind stack is
+ * empty at every top-level boundary, sliding there is transparent: the
+ * unconsumed tail is copied to the front, fresh bytes are appended, and the
+ * buffer is rebased so `address` keeps its file-absolute value. Peak JS heap
+ * for the index build is therefore `window + index columns`, independent of
+ * file size.
+ *
+ * The window slides only once the cursor passes `pool / 2`, bounding the
+ * memmove frequency; a record up to `pool / 2` bytes always fits after a
+ * slide. If a single record exceeds that (never on the current corpus, whose
+ * largest STEP record is ~25 KB), the whole parse restarts from the
+ * beginning with the window doubled, and repeats until every record fits —
+ * correctness over the pathological case, at the cost of a re-scan. (M1's
+ * production path will instead grow in place / restart from the last
+ * boundary; from-scratch keeps the spike simple.)
+ *
+ * @param source The byte source.
+ * @param parser The STEP parser (typed to the schema).
+ * @param pool Target window size in bytes.
+ * @return {StreamingIndexResult} The index, header, result and diagnostics.
+ */
+export function buildIndexStreaming(source, parser, pool) {
+    const fileSize = source.byteLength;
+    // Grow-and-restart loop: the body runs a full streamed parse at the current
+    // window size. It only re-runs if a single record couldn't fit the window
+    // (false end-of-file before true EOF), doubling the window each time.
+    let windowBytes = Math.max(pool, MIN_WINDOW);
+    for (;;) {
+        const window = new Uint8Array(windowBytes);
+        let windowStartFile = 0;
+        let windowLen = source.read(0, windowBytes, window, 0);
+        let bytesRead = windowLen;
+        const input = new ParsingBuffer(window, 0, windowLen);
+        const [header, headerResult] = parser.parseHeader(input);
+        if (headerResult !== ParseResult.COMPLETE) {
+            // Header didn't complete in the first window: either malformed, or a
+            // header larger than the pool. Report it as-is — the resident parse
+            // would surface the same header result.
+            return {
+                header,
+                elements: [],
+                result: headerResult,
+                stats: { pool, windowBytes, slides: 0, maxRecordLen: 0, bytesRead },
+            };
+        }
+        // Slide once the cursor is past half the window; a record up to this much
+        // headroom always fits without a false EOF.
+        const slideThreshold = windowBytes >> 1;
+        let slides = 0;
+        let maxRecordLen = 0;
+        let prevBoundaryFile = windowStartFile + input.cursor;
+        const onRecordBoundary = (buffer) => {
+            const cursor = buffer.cursor;
+            const recordFileStart = windowStartFile + cursor;
+            const recordLen = recordFileStart - prevBoundaryFile;
+            if (recordLen > maxRecordLen) {
+                maxRecordLen = recordLen;
+            }
+            prevBoundaryFile = recordFileStart;
+            // Window already spans to EOF — nothing left to slide in.
+            if (windowStartFile + windowLen >= fileSize) {
+                return;
+            }
+            // Enough headroom remains; defer the slide to bound memmove frequency.
+            if (cursor < slideThreshold) {
+                return;
+            }
+            // Slide: move the unconsumed tail to the front, refill behind it.
+            const tail = windowLen - cursor;
+            window.copyWithin(0, cursor, windowLen);
+            const want = windowBytes - tail;
+            const got = source.read(windowStartFile + windowLen, want, window, tail);
+            bytesRead += got;
+            windowLen = tail + got;
+            windowStartFile = recordFileStart;
+            buffer.rebaseWindow(window, 0, windowLen, windowStartFile);
+            ++slides;
+        };
+        const [index, result] = parser.parseDataBlockStreamed(input, onRecordBoundary);
+        // A parse that stopped short of true EOF, with the window not spanning to
+        // EOF, means a single record overflowed the window: grow and retry.
+        const stoppedShort = result !== ParseResult.COMPLETE;
+        const notAtEof = windowStartFile + windowLen < fileSize;
+        if (stoppedShort && notAtEof) {
+            windowBytes *= 2;
+            continue;
+        }
+        // Capture the final record's length (no boundary fires after the last).
+        const lastRecordLen = (windowStartFile + input.cursor) - prevBoundaryFile;
+        if (lastRecordLen > maxRecordLen) {
+            maxRecordLen = lastRecordLen;
+        }
+        return {
+            header,
+            elements: index.elements,
+            result,
+            stats: { pool, windowBytes, slides, maxRecordLen, bytesRead },
+        };
+    }
+}
+// Floor on the window so a pathologically tiny pool still holds the header
+// and a record with slide headroom. Tests drive pools down to this to force
+// many slides on a small fixture; the pool sweep uses ≥ 128 KB anyway.
+// eslint-disable-next-line no-magic-numbers
+const MIN_WINDOW = 4 * 1024;

package/compiled/src/step/parsing/streaming_index_builder.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=streaming_index_builder.test.d.ts.map

package/compiled/src/step/parsing/streaming_index_builder.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"streaming_index_builder.test.d.ts","sourceRoot":"","sources":["../../../../src/step/parsing/streaming_index_builder.test.ts"],"names":[],"mappings":""}

package/compiled/src/step/parsing/streaming_index_builder.test.js

@@ -0,0 +1,107 @@
+/* eslint-disable no-magic-numbers */
+// M0 spike: streaming index build must be byte-identical to the resident
+// parse. The parser reads file-absolute addresses and only rewinds within a
+// single top-level record, so feeding it from a moving window that slides at
+// record boundaries must produce the exact same index columns — at every
+// pool size, including one small enough to force many slides + straddles.
+import * as fs from "fs";
+import { beforeAll, describe, expect, test } from "@jest/globals";
+import ParsingBuffer from "../../parsing/parsing_buffer.js";
+import IfcStepParser from "../../ifc/ifc_step_parser.js";
+import { BufferByteSource } from "./byte_source.js";
+import { buildIndexStreaming } from "./streaming_index_builder.js";
+import { ParseResult } from "./step_parser.js";
+/**
+ * Flatten the index (top-level records + their inline / multi-mapping
+ * children) into a stable, comparable column dump: every field the streaming
+ * parse must reproduce identically.
+ *
+ * @param elements The index elements.
+ * @return {object[]} A flat list of column tuples in traversal order.
+ */
+function columns(elements) {
+    const out = [];
+    const visit = (e) => {
+        out.push({
+            address: e.address,
+            length: e.length,
+            typeID: e.typeID,
+            expressID: e.expressID ?? null,
+        });
+        for (const child of e.inlineEntities ?? []) {
+            visit(child);
+        }
+        for (const child of e.multiMapping ?? []) {
+            visit(child);
+        }
+    };
+    for (const e of elements) {
+        visit(e);
+    }
+    return out;
+}
+let bytes;
+let residentColumns;
+let residentCount;
+beforeAll(() => {
+    bytes = new Uint8Array(fs.readFileSync("data/index.ifc"));
+    const input = new ParsingBuffer(bytes);
+    IfcStepParser.Instance.parseHeader(input);
+    const [index, result] = IfcStepParser.Instance.parseDataBlock(input);
+    expect(result).toBe(ParseResult.COMPLETE);
+    residentColumns = columns(index.elements);
+    residentCount = index.elements.length;
+});
+describe("buildIndexStreaming", () => {
+    // A tiny pool relative to the 18 KB fixture forces repeated slides and at
+    // least one record straddling a window boundary; a large one takes the
+    // whole file in a single window (no slide). Both must match the resident
+    // parse exactly.
+    const POOLS = [4 * 1024, 8 * 1024, 16 * 1024, 128 * 1024];
+    test.each(POOLS)("produces the resident index at pool=%d bytes", (pool) => {
+        const source = new BufferByteSource(bytes);
+        const result = buildIndexStreaming(source, IfcStepParser.Instance, pool);
+        expect(result.result).toBe(ParseResult.COMPLETE);
+        expect(result.elements.length).toBe(residentCount);
+        expect(columns(result.elements)).toEqual(residentColumns);
+    });
+    test("the small pool actually slides the window (straddle exercised)", () => {
+        const source = new BufferByteSource(bytes);
+        const result = buildIndexStreaming(source, IfcStepParser.Instance, 4 * 1024);
+        // 18 KB fixture through a 4 KB window must slide several times.
+        expect(result.stats.slides).toBeGreaterThan(1);
+        // Physical window never grew past the pool (no record exceeds it).
+        expect(result.stats.windowBytes).toBe(4 * 1024);
+        // Every byte read once (no growth re-reads) — bounded to file size.
+        expect(result.stats.bytesRead).toBe(bytes.length);
+    });
+    test("the large pool takes the whole file in one window (no slide)", () => {
+        const source = new BufferByteSource(bytes);
+        const result = buildIndexStreaming(source, IfcStepParser.Instance, 128 * 1024);
+        expect(result.stats.slides).toBe(0);
+        expect(result.stats.maxRecordLen).toBeGreaterThan(0);
+    });
+    test("grows the window and restarts when a single record exceeds the pool", () => {
+        // Synthetic file with one record far larger than a tiny pool — the
+        // corpus never hits this (largest STEP record ~25 KB), so exercise the
+        // grow-and-restart safety valve directly. A long text literal makes one
+        // record ~12 KB; a 4 KB pool must grow past it and still index it.
+        const bigValue = "X".repeat(12000);
+        const text = "ISO-10303-21;\nHEADER;\nENDSEC;\nDATA;\n" +
+            "#1=IFCWALL('a');\n" +
+            `#2=IFCPROPERTYSINGLEVALUE('${bigValue}');\n` +
+            "#3=IFCWALL('c');\nENDSEC;\n";
+        const synthetic = new TextEncoder().encode(text);
+        const resident = new ParsingBuffer(synthetic);
+        IfcStepParser.Instance.parseHeader(resident);
+        const [residentIndex, residentResult] = IfcStepParser.Instance.parseDataBlock(resident);
+        expect(residentResult).toBe(ParseResult.COMPLETE);
+        const streamed = buildIndexStreaming(new BufferByteSource(synthetic), IfcStepParser.Instance, 4 * 1024);
+        expect(streamed.result).toBe(ParseResult.COMPLETE);
+        // Window grew past the 4 KB pool to fit the ~12 KB record.
+        expect(streamed.stats.windowBytes).toBeGreaterThan(4 * 1024);
+        expect(streamed.stats.maxRecordLen).toBeGreaterThan(12000);
+        // ...and the index is still byte-identical to the resident parse.
+        expect(columns(streamed.elements)).toEqual(columns(residentIndex.elements));
+    });
+});

package/compiled/src/version/version.js

@@ -5,5 +5,5 @@
 // only the first segment (major) is meaningful and is the one CI carries forward.
 // Must stay in `vN.N.N` shape: the CI stamp regex, scripts/updateVersion.mjs, and
 // statistics.ts all match `v\d+\.\d+\.\d+`.
-const versionString = 'Conway v1.389.1221';
+const versionString = 'Conway v1.398.1224';
 export { versionString };