@bldrs-ai/conway 1.389.1221 → 1.394.1504

package/compiled/src/step/parsing/record_field_cursor.js

@@ -0,0 +1,189 @@
+import StepVtableBuilder from "./step_vtable_builder.js";
+import { stepExtractString } from "./step_deserialization_functions.js";
+/** ASCII '#', which introduces an express reference. */
+const HASH = 35;
+/** ASCII '0' and '9', bounding the digits of a reference. */
+const DIGIT_0 = 48;
+const DIGIT_9 = 57;
+/** Everything at or below this code point is STEP whitespace. */
+const LAST_WHITESPACE = 32;
+/** Base of the decimal express IDs. */
+const DECIMAL = 10;
+/**
+ * Reads individual attributes out of a record's raw bytes, without a model and
+ * without materialising an entity (M2, issue #393).
+ *
+ * This is what makes a parse-path consumer viable at all: the streaming
+ * parse's record event hands over the record's attribute list as a window
+ * range, and a consumer that wants two strings out of it should not have to
+ * wait for the model, page the record back in, and construct an entity to get
+ * them. Field positions come from the production tokenizer
+ * ({@link StepParser.extractDataEntry}), so a record this reads and a record
+ * the model reads agree by construction — including on the awkward cases
+ * (quoted commas, nested parens, escaped quotes) a hand-rolled splitter gets
+ * wrong.
+ *
+ * One instance is reused across records: {@link open} clears and refills a
+ * single vtable buffer, so a consumer running over millions of records
+ * allocates nothing per record. Attribute indices are the same ones the
+ * generated accessors use (`extractString( 2, … )` for `IfcRoot.Name`, and so
+ * on), read against the *record's own* attribute list — inherited attributes
+ * included, since STEP serialises the flattened list.
+ */
+export class RecordFieldCursor {
+    /**
+     * @param parser_ The schema's parser — used only for its record tokenizer.
+     */
+    constructor(parser_) {
+        this.parser_ = parser_;
+        this.vtable_ = new StepVtableBuilder();
+        this.buffer_ = void 0;
+        this.indexMark_ = 0;
+        this.fieldCount_ = 0;
+        this.endCursor_ = 0;
+    }
+    /**
+     * Point this at one record's attribute list.
+     *
+     * @param buffer The buffer holding the record (the live parse window, for an
+     * event consumer — so everything read here must be read before returning).
+     * @param byteOffset Offset of the record's first attribute byte.
+     * @param byteLength Length of the attribute list in bytes.
+     * @return {number} The number of top-level attributes found; 0 if the record
+     * could not be tokenized (a truncated or malformed record — the caller
+     * should skip it rather than guess).
+     */
+    open(buffer, byteOffset, byteLength) {
+        this.vtable_.clear();
+        const end = byteOffset + byteLength;
+        const extracted = this.parser_.extractDataEntry(buffer, byteOffset, end, this.vtable_);
+        if (extracted === void 0) {
+            this.buffer_ = void 0;
+            this.fieldCount_ = 0;
+            return 0;
+        }
+        this.buffer_ = buffer;
+        this.indexMark_ = extracted[0];
+        this.fieldCount_ = extracted[1];
+        this.endCursor_ = extracted[2];
+        return this.fieldCount_;
+    }
+    /**
+     * The byte range of one attribute, as `[start, end)`.
+     *
+     * @param index The attribute index.
+     * @return {[number, number] | undefined} The range, or undefined when the
+     * record has no such attribute.
+     */
+    range(index) {
+        if (this.buffer_ === void 0 || index < 0 || index >= this.fieldCount_) {
+            return void 0;
+        }
+        const table = this.vtable_.buffer;
+        const start = table[this.indexMark_ + index];
+        // Each entry marks an attribute's first byte, so an attribute runs to the
+        // next entry's separator; the last runs to the record's end cursor.
+        const end = index + 1 < this.fieldCount_ ?
+            table[this.indexMark_ + index + 1] - 1 :
+            this.endCursor_;
+        return [start, end];
+    }
+    /**
+     * Read a string attribute.
+     *
+     * @param index The attribute index.
+     * @return {string | undefined} The decoded string, or undefined when the
+     * attribute is absent, `$`, or not a string.
+     */
+    string(index) {
+        const range = this.range(index);
+        if (range === void 0) {
+            return void 0;
+        }
+        return stepExtractString(this.buffer_, range[0], range[1]);
+    }
+    /**
+     * Read a single express reference attribute (`#123`).
+     *
+     * @param index The attribute index.
+     * @return {number | undefined} The express ID, or undefined when the
+     * attribute is absent or is not a reference.
+     */
+    reference(index) {
+        const range = this.range(index);
+        if (range === void 0) {
+            return void 0;
+        }
+        const buffer = this.buffer_;
+        let cursor = range[0];
+        while (cursor < range[1] && buffer[cursor] !== HASH) {
+            // Only whitespace may precede the '#'; anything else means this
+            // attribute is not a reference (`$`, an inline value, a typed value).
+            if (buffer[cursor] > LAST_WHITESPACE) {
+                return void 0;
+            }
+            ++cursor;
+        }
+        return this.readReferenceAt(cursor, range[1]);
+    }
+    /**
+     * Visit every express reference in an attribute, which is how a reference
+     * list (`(#1,#2,#3)`) is read — the enclosing parens and separators need no
+     * interpretation, since a reference is the only thing a `#` can start.
+     *
+     * @param index The attribute index.
+     * @param visit Called with each express ID, in order.
+     */
+    forEachReference(index, visit) {
+        const range = this.range(index);
+        if (range === void 0) {
+            return;
+        }
+        const buffer = this.buffer_;
+        const end = range[1];
+        for (let cursor = range[0]; cursor < end; ++cursor) {
+            if (buffer[cursor] !== HASH) {
+                continue;
+            }
+            const expressID = this.readReferenceAt(cursor, end);
+            if (expressID === void 0) {
+                continue;
+            }
+            visit(expressID);
+            // Skip the digits just consumed; the loop's ++ steps off the last one.
+            while (cursor + 1 < end && this.isDigit(buffer[cursor + 1])) {
+                ++cursor;
+            }
+        }
+    }
+    /**
+     * Read the digits of a reference whose `#` sits at `cursor`.
+     *
+     * @param cursor Offset of the `#`.
+     * @param end Exclusive end of the attribute.
+     * @return {number | undefined} The express ID, or undefined if `cursor` is
+     * not a `#` followed by at least one digit.
+     */
+    readReferenceAt(cursor, end) {
+        const buffer = this.buffer_;
+        if (cursor >= end || buffer[cursor] !== HASH) {
+            return void 0;
+        }
+        let at = cursor + 1;
+        let value = 0;
+        let digits = 0;
+        while (at < end && this.isDigit(buffer[at])) {
+            value = value * DECIMAL + (buffer[at] - DIGIT_0);
+            ++at;
+            ++digits;
+        }
+        return digits === 0 ? void 0 : value;
+    }
+    /**
+     * @param code A byte.
+     * @return {boolean} Whether it is an ASCII digit.
+     */
+    isDigit(code) {
+        return code >= DIGIT_0 && code <= DIGIT_9;
+    }
+}

package/compiled/src/step/parsing/step_deserialization_functions.d.ts

@@ -118,6 +118,35 @@ export declare function stepExtractArrayToken(buffer: Uint8Array, cursor: number
  * @return {IterableIterator} Lazy iterator to extract value.
  */
 export declare function stepExtractArray(buffer: Uint8Array, cursor: number, endCursor: number): IterableIterator<number>;
+/**
+ * Step past a `TYPENAME(` wrapper to the value inside it, when the value at
+ * the cursor is written in its typed form and the type name matches.
+ *
+ * A select member may be written either bare or typed — `.NULL.` or
+ * `NULL_STYLE(.NULL.)` — and inside a SELECT the typed form is how a reader
+ * tells which member it is, so exporters legitimately use it. Entity-valued
+ * members already handle both, because the parser indexes an inline
+ * `TYPENAME(...)` entity and `extractBufferReference` resolves it by address.
+ * A member whose type is an enum has no entity to index, so its deserializer
+ * is a bare parse starting at the cursor, and the wrapper makes it fail.
+ * That cost 274 of the public regression baseline's error rows — every
+ * `PRESENTATION_STYLE_ASSIGNMENT((NULL_STYLE(.NULL.)))` in the corpus threw
+ * and lost its styled item. See bldrs-ai/conway#489.
+ *
+ * The type name is verified rather than skipped past, so this accepts only
+ * the value's own typed form: a mismatched wrapper stays a type error
+ * instead of being silently read as the member it isn't.
+ *
+ * @param buffer The buffer to read from.
+ * @param cursor The position the value starts at.
+ * @param endCursor The last position accessible for this read in the buffer.
+ * @param typeName The expected STEP type name of the wrapper, upper case —
+ * StepEntityIdentifierParser matches `[A-Z][A-Z0-9_]*`, so a lower-case
+ * spelling is not an identifier here and never reaches the comparison.
+ * @return {number} The position just inside the wrapper's parenthesis, or
+ * `cursor` unchanged when the value is not this type's typed form.
+ */
+export declare function stepEnterTypedValue(buffer: Uint8Array, cursor: number, endCursor: number, typeName: string): number;
 /**
  * Extracts an inline element and returns its cursor, or none if it can't be found.
  *

package/compiled/src/step/parsing/step_deserialization_functions.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"step_deserialization_functions.d.ts","sourceRoot":"","sources":["../../../../src/step/parsing/step_deserialization_functions.ts"],"names":[],"mappings":"AAkCA;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,2BAA2B,IAAI,IAAI,CAElD;AAGD;;;;;;GAMG;AACH,wBAAgB,GAAG,CAAE,CAAC,EAAI,SAAS,EAAE,CAAC,GAAG,IAAI,GAAG,SAAS,EAAE,YAAY,EAAE,CAAC,GAAI,CAAC,CAE9E;AAGD;;;;;GAKG;AACH,wBAAgB,OAAO,CAAE,CAAC,EAAI,SAAS,EAAE,CAAC,EAAE,GAAG,IAAI,GAAG,SAAS,GAAI,MAAM,CAExE;AAED;;;;;GAKG;AACH,wBAAgB,MAAM,CAAE,CAAC,EAAI,SAAS,EAAE,CAAC,EAAE,GAAG,IAAI,GAAG,SAAS,GAAI,MAAM,CAEvE;AAGD;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAC/B,MAAM,EAAE,UAAU,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAUvC;AAED;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAC7B,MAAM,EAAE,UAAU,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAAI,MAAM,GAAG,SAAS,CAM1C;AAGD;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAC9B,MAAM,EAAE,UAAU,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAAI,OAAO,GAAG,IAAI,GAAG,SAAS,CAwBlD;AAGD;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAC9B,MAAM,EAAE,UAAU,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAAI,OAAO,GAAG,SAAS,CAgB3C;AAED;;;;;;;;;GASG;AACH,wBAAgB,SAAS,CACrB,MAAM,EAAE,UAAU,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAAK,MAAM,CA2E/B;AAED;;;;;;;;;GASG;AACH,wBAAgB,qBAAqB,CACjC,MAAM,EAAE,UAAU,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAAI,MAAM,CA0B9B;AAED;;;;;;;;;GASG;AACH,wBAAgB,qBAAqB,CACjC,MAAM,EAAE,UAAU,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAAI,MAAM,CA0C9B;AAED;;;;;;;;;GASG;AACH,wBAAiB,gBAAgB,CAC7B,MAAM,EAAE,UAAU,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAAI,gBAAgB,CAAE,MAAM,CAAE,CAkGlD;AAGD;;;;;;;;GAQG;AACH,wBAAgB,0BAA0B,CACtC,MAAM,EAAE,UAAU,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAAI,MAAM,GAAG,SAAS,CA4C1C;AAID;;;;;;;;;GASG;AACH,wBAAgB,oBAAoB,CAChC,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAAK,MAAM,GAAG,SAAS,CAsC3C;AAED,QAAA,MAAM,eAAe,8JAAkC,CAAA;AAEvD,OAAO,EAAE,eAAe,EAAE,CAAA;AAE1B,QAAA,MAAM,iBAAiB,iGAAoC,CAAA;AAE3D,OAAO,EAAE,iBAAiB,EAAE,CAAA;AAE5B,QAAA,MAAM,iBAAiB,4FAAiC,CAAA;AAExD,OAAO,EAAE,iBAAiB,EAAE,CAAA"}
+{"version":3,"file":"step_deserialization_functions.d.ts","sourceRoot":"","sources":["../../../../src/step/parsing/step_deserialization_functions.ts"],"names":[],"mappings":"AAkCA;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,2BAA2B,IAAI,IAAI,CAElD;AAGD;;;;;;GAMG;AACH,wBAAgB,GAAG,CAAE,CAAC,EAAI,SAAS,EAAE,CAAC,GAAG,IAAI,GAAG,SAAS,EAAE,YAAY,EAAE,CAAC,GAAI,CAAC,CAE9E;AAGD;;;;;GAKG;AACH,wBAAgB,OAAO,CAAE,CAAC,EAAI,SAAS,EAAE,CAAC,EAAE,GAAG,IAAI,GAAG,SAAS,GAAI,MAAM,CAExE;AAED;;;;;GAKG;AACH,wBAAgB,MAAM,CAAE,CAAC,EAAI,SAAS,EAAE,CAAC,EAAE,GAAG,IAAI,GAAG,SAAS,GAAI,MAAM,CAEvE;AAGD;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAC/B,MAAM,EAAE,UAAU,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAUvC;AAED;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAC7B,MAAM,EAAE,UAAU,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAAI,MAAM,GAAG,SAAS,CAM1C;AAGD;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAC9B,MAAM,EAAE,UAAU,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAAI,OAAO,GAAG,IAAI,GAAG,SAAS,CAwBlD;AAGD;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAC9B,MAAM,EAAE,UAAU,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAAI,OAAO,GAAG,SAAS,CAgB3C;AAED;;;;;;;;;GASG;AACH,wBAAgB,SAAS,CACrB,MAAM,EAAE,UAAU,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAAK,MAAM,CA2E/B;AAED;;;;;;;;;GASG;AACH,wBAAgB,qBAAqB,CACjC,MAAM,EAAE,UAAU,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAAI,MAAM,CA0B9B;AAED;;;;;;;;;GASG;AACH,wBAAgB,qBAAqB,CACjC,MAAM,EAAE,UAAU,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAAI,MAAM,CA0C9B;AAED;;;;;;;;;GASG;AACH,wBAAiB,gBAAgB,CAC7B,MAAM,EAAE,UAAU,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAAI,gBAAgB,CAAE,MAAM,CAAE,CAkGlD;AAGD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,mBAAmB,CAC/B,MAAM,EAAE,UAAU,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,GAAI,MAAM,CAqB7B;AAGD;;;;;;;;GAQG;AACH,wBAAgB,0BAA0B,CACtC,MAAM,EAAE,UAAU,EAClB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAAI,MAAM,GAAG,SAAS,CA4C1C;AAID;;;;;;;;;GASG;AACH,wBAAgB,oBAAoB,CAChC,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAAK,MAAM,GAAG,SAAS,CAsC3C;AAED,QAAA,MAAM,eAAe,8JAAkC,CAAA;AAEvD,OAAO,EAAE,eAAe,EAAE,CAAA;AAE1B,QAAA,MAAM,iBAAiB,iGAAoC,CAAA;AAE3D,OAAO,EAAE,iBAAiB,EAAE,CAAA;AAE5B,QAAA,MAAM,iBAAiB,4FAAiC,CAAA;AAExD,OAAO,EAAE,iBAAiB,EAAE,CAAA"}

package/compiled/src/step/parsing/step_deserialization_functions.js

@@ -367,6 +367,51 @@ export function* stepExtractArray(buffer, cursor, endCursor) {
         throw new Error("Unterminated array");
     }
 }
+/**
+ * Step past a `TYPENAME(` wrapper to the value inside it, when the value at
+ * the cursor is written in its typed form and the type name matches.
+ *
+ * A select member may be written either bare or typed — `.NULL.` or
+ * `NULL_STYLE(.NULL.)` — and inside a SELECT the typed form is how a reader
+ * tells which member it is, so exporters legitimately use it. Entity-valued
+ * members already handle both, because the parser indexes an inline
+ * `TYPENAME(...)` entity and `extractBufferReference` resolves it by address.
+ * A member whose type is an enum has no entity to index, so its deserializer
+ * is a bare parse starting at the cursor, and the wrapper makes it fail.
+ * That cost 274 of the public regression baseline's error rows — every
+ * `PRESENTATION_STYLE_ASSIGNMENT((NULL_STYLE(.NULL.)))` in the corpus threw
+ * and lost its styled item. See bldrs-ai/conway#489.
+ *
+ * The type name is verified rather than skipped past, so this accepts only
+ * the value's own typed form: a mismatched wrapper stays a type error
+ * instead of being silently read as the member it isn't.
+ *
+ * @param buffer The buffer to read from.
+ * @param cursor The position the value starts at.
+ * @param endCursor The last position accessible for this read in the buffer.
+ * @param typeName The expected STEP type name of the wrapper, upper case —
+ * StepEntityIdentifierParser matches `[A-Z][A-Z0-9_]*`, so a lower-case
+ * spelling is not an identifier here and never reaches the comparison.
+ * @return {number} The position just inside the wrapper's parenthesis, or
+ * `cursor` unchanged when the value is not this type's typed form.
+ */
+export function stepEnterTypedValue(buffer, cursor, endCursor, typeName) {
+    const identifierEnd = identifierParser.match(buffer, cursor, endCursor);
+    if (identifierEnd === void 0 || (identifierEnd - cursor) !== typeName.length) {
+        return cursor;
+    }
+    for (let where = 0; where < typeName.length; ++where) {
+        if (buffer[cursor + where] !== typeName.charCodeAt(where)) {
+            return cursor;
+        }
+    }
+    // Delegate the rest to the inline-element reader rather than repeating its
+    // identifier/whitespace/comment/paren walk. Sharing it is not just less
+    // code: entity-valued select members reach their value through that exact
+    // function, so a divergence here would mean the two kinds of member in the
+    // SAME select disagreed about where a typed value begins.
+    return stepExtractInlineElemement(buffer, cursor, endCursor) ?? cursor;
+}
 /**
  * Extracts an inline element and returns its cursor, or none if it can't be found.
  *

package/compiled/src/step/parsing/step_deserialization_functions.test.js

@@ -7,7 +7,7 @@
 // unreachable after spill + release, where before it was retained via
 // Context[parsingBufferReusable] → ParsingBuffer.buffer).
 import { describe, expect, test } from "@jest/globals";
-import { releaseScratchParsingBuffer, stepExtractNumber, } from "./step_deserialization_functions.js";
+import { releaseScratchParsingBuffer, stepEnterTypedValue, stepExtractNumber, } from "./step_deserialization_functions.js";
 const REAL_VALUE = 42.5;
 const INT_VALUE = 7;
 describe("releaseScratchParsingBuffer", () => {
@@ -25,3 +25,61 @@ describe("releaseScratchParsingBuffer", () => {
         expect(stepExtractNumber(bytes, 0, bytes.length)).toBe(REAL_VALUE);
     });
 });
+describe("stepEnterTypedValue", () => {
+    /**
+     * Encode and enter, returning both the cursor and what it points at, so a
+     * failed expectation says which byte the function landed on.
+     *
+     * @param text The STEP text to read from position 0.
+     * @param typeName The expected wrapper type name.
+     * @return {[number, string]} The returned cursor and the character there.
+     */
+    const enter = (text, typeName) => {
+        const bytes = new TextEncoder().encode(text);
+        const cursor = stepEnterTypedValue(bytes, 0, bytes.length, typeName);
+        return [cursor, text[cursor] ?? ""];
+    };
+    test("enters a matching wrapper, landing on the value", () => {
+        // The case from bldrs-ai/conway#489: 274 occurrences in the public
+        // regression baseline, every one of them throwing and losing its
+        // styled item because the enum parse started on the 'N'.
+        // Positions derived rather than written out, so the expectation says
+        // "just past the wrapper" rather than asserting an opaque index.
+        expect(enter("NULL_STYLE(.NULL.)", "NULL_STYLE"))
+            .toEqual(["NULL_STYLE(".length, "."]);
+        expect(enter("IFCNULLSTYLE(.NULL.)", "IFCNULLSTYLE"))
+            .toEqual(["IFCNULLSTYLE(".length, "."]);
+    });
+    test("leaves a bare value alone", () => {
+        // Both forms are legal for a select member, so entering must be a no-op
+        // on the bare one rather than a precondition.
+        expect(enter(".NULL.", "NULL_STYLE")).toEqual([0, "."]);
+    });
+    test("does not treat a lower-case spelling as the type name", () => {
+        // Not a policy choice here: StepEntityIdentifierParser matches
+        // `[A-Z][A-Z0-9_]*`, so a lower-case wrapper is not an identifier at all
+        // and the value is left to be read bare (where it then fails as the
+        // malformed input it is).
+        expect(enter("null_style(.NULL.)", "NULL_STYLE")).toEqual([0, "n"]);
+    });
+    test("skips whitespace and comments before and after the paren", () => {
+        const spaced = "NULL_STYLE /* c */ ( /* d */ .NULL.)";
+        expect(enter(spaced, "NULL_STYLE")).toEqual([spaced.indexOf(".NULL."), "."]);
+    });
+    test("refuses a wrapper that is not this type", () => {
+        // The point of verifying the name rather than skipping any identifier:
+        // a mismatched wrapper stays a type error instead of being silently
+        // read as the member it is not.
+        expect(enter("SOME_OTHER_TYPE(.NULL.)", "NULL_STYLE")).toEqual([0, "S"]);
+        // A prefix must not match either - length is checked before the bytes.
+        expect(enter("NULL_STYLE_EXTRA(.NULL.)", "NULL_STYLE")).toEqual([0, "N"]);
+    });
+    test("refuses an identifier with no parenthesis", () => {
+        expect(enter("NULL_STYLE", "NULL_STYLE")).toEqual([0, "N"]);
+        expect(enter("NULL_STYLE.NULL.", "NULL_STYLE")).toEqual([0, "N"]);
+    });
+    test("refuses a reference or a bare number", () => {
+        expect(enter("#4085", "NULL_STYLE")).toEqual([0, "#"]);
+        expect(enter("0.7", "NULL_STYLE")).toEqual([0, "0"]);
+    });
+});

package/compiled/src/step/parsing/step_parser.d.ts

@@ -1,5 +1,6 @@
 import TypeIndex from "../../indexing/type_index.js";
 import ParsingBuffer from "../../parsing/parsing_buffer.js";
+import { RecordEventHandler } from "./record_event.js";
 import StepVtableBuilder, { IndexMark } from "./step_vtable_builder.js";
 export interface StepIndexEntryBase<TypeIDType> {
     address: number;
@@ -18,6 +19,27 @@ export interface StepIndexEntry<TypeIDType> extends StepIndexEntryBase<TypeIDTyp
 export interface StepIndex<TypeIDType> {
     elements: StepIndexEntry<TypeIDType>[];
 }
+/**
+ * A consumer of completed top-level index entries, used to divert the parse
+ * away from accumulating the object-form `StepIndex` — the columnar sink
+ * (columnar_index.ts) encodes each entry into typed-array columns and drops
+ * the object unless it carries children. When a sink is supplied, the parse's
+ * returned `StepIndex.elements` stays empty; the sink holds the index.
+ */
+export interface StepIndexSink<TypeIDType> {
+    /**
+     * Consume one completed top-level entry (its inlineEntities / multiMapping,
+     * when present, are attached and complete).
+     *
+     * @param entry The completed entry. Ownership transfers to the sink.
+     */
+    pushTopLevel(entry: StepIndexEntry<TypeIDType>): void;
+    /**
+     * Rewind to empty — the streaming builder's grow-and-restart re-parses from
+     * the start and re-pushes every record.
+     */
+    reset(): void;
+}
 export interface Argument {
     type: number;
     value: any;
@@ -46,7 +68,7 @@ export type LineArgumentParseResult<TypeIDType> = [
  * parser stays decoupled from the higher-level progress event contract in
  * core/progress.ts (ProgressTracker maps this into a ProgressEvent).
  */
-export type ParseProgressCallback = (cursorBytes: number) => void;
+export type ParseProgressCallback = (cursorBytes: number) => unknown;
 export interface StepHeader {
     headers: Map<string, string>;
 }
@@ -116,6 +138,49 @@ export default class StepParser<TypeIDType> extends StepHeaderParser {
      * @return {BlockParseResult} The parsing result, including the index and result enum.
      */
     parseDataBlock(input: ParsingBuffer, onProgress?: ParseProgressCallback): BlockParseResult<TypeIDType>;
+    /**
+     * Streaming driver over parseDataBlockIncremental: identical parse (same
+     * generator body) but invokes `onRecordBoundary` at every top-level record
+     * boundary so a caller feeding the parser from a moving window can slide
+     * that window forward while the rewind stack is empty. See
+     * streaming_index_builder.ts for the coordinator that owns the window.
+     *
+     * @param input The input parsing buffer, positioned at the data section.
+     * @param onRecordBoundary Called at each top-level record boundary with the
+     * buffer; the callback may rebase the buffer's window in place. Must be
+     * synchronous on this driver — a returned Promise is a caller bug.
+     * @param onRecordIndexed Called as each top-level record is indexed, with
+     * its localID, expressID, typeID (0 for external-mapping records) and the
+     * record's bytes as an un-materialised `(buffer, byteOffset, byteLength)`
+     * window view — the seam for incremental semantic consumers (type index,
+     * roots registry, names skeleton). See {@link RecordEventHandler} for the
+     * window-lifetime rule. Must be synchronous and cheap; expensive work
+     * belongs on a demand queue, not the parse path.
+     * @param onProgress Optional byte-cursor progress callback.
+     * @return {BlockParseResult} The parsing result, including the index and result enum.
+     */
+    parseDataBlockStreamed(input: ParsingBuffer, onRecordBoundary: (input: ParsingBuffer) => void | Promise<void>, onRecordIndexed?: RecordEventHandler<TypeIDType>, onProgress?: ParseProgressCallback, sink?: StepIndexSink<TypeIDType>): BlockParseResult<TypeIDType>;
+    /**
+     * Cooperative variant of parseDataBlockStreamed: identical parse and
+     * window-slide behaviour (same generator body, same boundary callback),
+     * but periodically awaits a macrotask so the event loop can run —
+     * browsers repaint progress UI during a large streamed parse instead of
+     * flagging the tab as stalled. Issue #301 §2 for the streamed path.
+     *
+     * @param input The input parsing buffer, positioned at the data section.
+     * @param onRecordBoundary Called at each top-level record boundary with the
+     * buffer; the callback may rebase the buffer's window in place. May return
+     * a Promise when the slide itself is I/O (an async ByteSource); this
+     * driver awaits it before the next record is lexed.
+     * @param onRecordIndexed Called as each top-level record is indexed — see
+     * parseDataBlockStreamed.
+     * @param onProgress Optional byte-cursor progress callback.
+     * @param sink Optional index sink (columnar builds).
+     * @param yieldIntervalMs Minimum ms between event-loop yields.
+     * @return {Promise<BlockParseResult>} The parsing result, including the
+     * index and result enum.
+     */
+    parseDataBlockStreamedAsync(input: ParsingBuffer, onRecordBoundary: (input: ParsingBuffer) => void | Promise<void>, onRecordIndexed?: RecordEventHandler<TypeIDType>, onProgress?: ParseProgressCallback, sink?: StepIndexSink<TypeIDType>, yieldIntervalMs?: number): Promise<BlockParseResult<TypeIDType>>;
     /**
      * Cooperative variant of parseDataBlock: identical parse (same generator
      * body), but periodically awaits a macrotask so the event loop can run —

package/compiled/src/step/parsing/step_parser.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"step_parser.d.ts","sourceRoot":"","sources":["../../../../src/step/parsing/step_parser.ts"],"names":[],"mappings":"AACA,OAAO,SAAS,MAAM,2BAA2B,CAAA;AAEjD,OAAO,aAAa,MAAM,8BAA8B,CAAA;AAcxD,OAAO,iBAAiB,EAAE,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAA;AAEpE,MAAM,WAAW,kBAAkB,CAAC,UAAU;IAE5C,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,UAAU,CAAA;IACnB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,YAAY,CAAC,EAAE,kBAAkB,CAAC,UAAU,CAAC,EAAE,CAAA;IAE/C,cAAc,CAAC,EAAE,oBAAoB,CAAC,UAAU,CAAC,EAAE,CAAA;CACpD;AAED,MAAM,WAAW,oBAAoB,CAAC,UAAU,CAAE,SAAQ,kBAAkB,CAAC,UAAU,CAAC;IAEtF,SAAS,CAAC,EAAE,SAAS,CAAA;CACtB;AAED,MAAM,WAAW,cAAc,CAAC,UAAU,CAAE,SAAQ,kBAAkB,CAAC,UAAU,CAAC;IAEhF,SAAS,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,SAAS,CAAC,UAAU;IAEnC,QAAQ,EAAE,cAAc,CAAC,UAAU,CAAC,EAAE,CAAA;CAEvC;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,GAAG,CAAA;CACX;AAID;;GAEG;AAGH,oBAAY,WAAW;IAErB,QAAQ,IAAI;IACZ,UAAU,IAAI;IACd,YAAY,IAAI;IAChB,YAAY,IAAI;IAChB,YAAY,IAAI;CACjB;AAyBD,MAAM,MAAM,gBAAgB,CAAC,UAAU,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,WAAW,CAAC,CAAA;AAC/E,MAAM,MAAM,uBAAuB,CAAC,UAAU,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,WAAW,CAAC,CAAA;AAEtF;;;;;GAKG;AACH,MAAM,MAAM,qBAAqB,GAAG,CAAE,WAAW,EAAE,MAAM,KAAM,IAAI,CAAA;AAWnE,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAC7B;AAED,MAAM,MAAM,iBAAiB,GAAG,CAAC,UAAU,EAAE,WAAW,CAAC,CAAA;AAGzD;;GAEG;AACH,qBAAa,gBAAgB;IAE3B,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAkB;IAE3C;;;;OAIG;IACH,WAAkB,QAAQ,IAAI,gBAAgB,CAQ7C;IAED;;;;;OAKG;IACI,WAAW,CAAC,KAAK,EAAE,aAAa,GAAG,iBAAiB;CA0L5D;AACD;;GAEG;AACH,MAAM,CAAC,OAAO,OAAO,UAAU,CAAC,UAAU,CAAE,SAAQ,gBAAgB;IAQtD,OAAO,CAAC,QAAQ,CAAC,MAAM;IANnC;;;;;OAKG;gBAC0B,MAAM,EAAE,QAAQ,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IAKpE;;;;;;;;;;;;;OAaG;IACI,gBAAgB,CACnB,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,iBAAiB,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS;IA+D9E;;;;;;;;;;OAUG;IACI,cAAc,CACjB,KAAK,EAAE,aAAa,EACpB,UAAU,CAAC,EAAE,qBAAqB,GAAI,gBAAgB,CAAC,UAAU,CAAC;IAgBtE;;;;;;;;;;;OAWG;IACU,mBAAmB,CAC5B,KAAK,EAAE,aAAa,EACpB,UAAU,CAAC,EAAE,qBAAqB,EAClC,eAAe,GAAE,MAAwC,GACzD,OAAO,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;IAuBzC;;;;;;;;;OASG;IACH,OAAO,CAAE,yBAAyB;IAyZlC;;;;;;;OAOG;IACI,gBAAgB,CAAC,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,GAAG,CAAC,GAAG,EAAE,WAAW,CAAC;CAiarF"}
+{"version":3,"file":"step_parser.d.ts","sourceRoot":"","sources":["../../../../src/step/parsing/step_parser.ts"],"names":[],"mappings":"AACA,OAAO,SAAS,MAAM,2BAA2B,CAAA;AAEjD,OAAO,aAAa,MAAM,8BAA8B,CAAA;AAOxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAA;AAQnD,OAAO,iBAAiB,EAAE,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAA;AAEpE,MAAM,WAAW,kBAAkB,CAAC,UAAU;IAE5C,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,UAAU,CAAA;IACnB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,YAAY,CAAC,EAAE,kBAAkB,CAAC,UAAU,CAAC,EAAE,CAAA;IAE/C,cAAc,CAAC,EAAE,oBAAoB,CAAC,UAAU,CAAC,EAAE,CAAA;CACpD;AAED,MAAM,WAAW,oBAAoB,CAAC,UAAU,CAAE,SAAQ,kBAAkB,CAAC,UAAU,CAAC;IAEtF,SAAS,CAAC,EAAE,SAAS,CAAA;CACtB;AAED,MAAM,WAAW,cAAc,CAAC,UAAU,CAAE,SAAQ,kBAAkB,CAAC,UAAU,CAAC;IAEhF,SAAS,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,SAAS,CAAC,UAAU;IAEnC,QAAQ,EAAE,cAAc,CAAC,UAAU,CAAC,EAAE,CAAA;CAEvC;AAED;;;;;;GAMG;AACH,MAAM,WAAW,aAAa,CAAC,UAAU;IAEvC;;;;;OAKG;IACH,YAAY,CAAE,KAAK,EAAE,cAAc,CAAC,UAAU,CAAC,GAAI,IAAI,CAAA;IAEvD;;;OAGG;IACH,KAAK,IAAI,IAAI,CAAA;CACd;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,GAAG,CAAA;CACX;AAID;;GAEG;AAGH,oBAAY,WAAW;IAErB,QAAQ,IAAI;IACZ,UAAU,IAAI;IACd,YAAY,IAAI;IAChB,YAAY,IAAI;IAChB,YAAY,IAAI;CACjB;AAyBD,MAAM,MAAM,gBAAgB,CAAC,UAAU,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,WAAW,CAAC,CAAA;AAC/E,MAAM,MAAM,uBAAuB,CAAC,UAAU,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,WAAW,CAAC,CAAA;AAEtF;;;;;GAKG;AACH,MAAM,MAAM,qBAAqB,GAC/B,CAAE,WAAW,EAAE,MAAM,KAAM,OAAO,CAAA;AAWpC,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAC7B;AAED,MAAM,MAAM,iBAAiB,GAAG,CAAC,UAAU,EAAE,WAAW,CAAC,CAAA;AAGzD;;GAEG;AACH,qBAAa,gBAAgB;IAE3B,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAkB;IAE3C;;;;OAIG;IACH,WAAkB,QAAQ,IAAI,gBAAgB,CAQ7C;IAED;;;;;OAKG;IACI,WAAW,CAAC,KAAK,EAAE,aAAa,GAAG,iBAAiB;CA0L5D;AACD;;GAEG;AACH,MAAM,CAAC,OAAO,OAAO,UAAU,CAAC,UAAU,CAAE,SAAQ,gBAAgB;IAQtD,OAAO,CAAC,QAAQ,CAAC,MAAM;IANnC;;;;;OAKG;gBAC0B,MAAM,EAAE,QAAQ,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;IAKpE;;;;;;;;;;;;;OAaG;IACI,gBAAgB,CACnB,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,EACjB,aAAa,EAAE,iBAAiB,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS;IA+D9E;;;;;;;;;;OAUG;IACI,cAAc,CACjB,KAAK,EAAE,aAAa,EACpB,UAAU,CAAC,EAAE,qBAAqB,GAAI,gBAAgB,CAAC,UAAU,CAAC;IAkBtE;;;;;;;;;;;;;;;;;;;;OAoBG;IACI,sBAAsB,CACzB,KAAK,EAAE,aAAa,EACpB,gBAAgB,EAAE,CAAE,KAAK,EAAE,aAAa,KAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAClE,eAAe,CAAC,EAAE,kBAAkB,CAAC,UAAU,CAAC,EAChD,UAAU,CAAC,EAAE,qBAAqB,EAClC,IAAI,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,GAAI,gBAAgB,CAAC,UAAU,CAAC;IAsBpE;;;;;;;;;;;;;;;;;;;OAmBG;IACU,2BAA2B,CACpC,KAAK,EAAE,aAAa,EACpB,gBAAgB,EAAE,CAAE,KAAK,EAAE,aAAa,KAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAClE,eAAe,CAAC,EAAE,kBAAkB,CAAC,UAAU,CAAC,EAChD,UAAU,CAAC,EAAE,qBAAqB,EAClC,IAAI,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,EAChC,eAAe,GAAE,MAAwC,GACzD,OAAO,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;IAkCzC;;;;;;;;;;;OAWG;IACU,mBAAmB,CAC5B,KAAK,EAAE,aAAa,EACpB,UAAU,CAAC,EAAE,qBAAqB,EAClC,eAAe,GAAE,MAAwC,GACzD,OAAO,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;IAyBzC;;;;;;;;;OASG;IACH,OAAO,CAAE,yBAAyB;IAsflC;;;;;;;OAOG;IACI,gBAAgB,CAAC,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,GAAG,CAAC,GAAG,EAAE,WAAW,CAAC;CAiarF"}