@ifc-lite/parser 4.3.2 → 5.0.0

package/dist/step-lexing.js

@@ -19,6 +19,8 @@ const SLASH = 0x2f; // '/'
 const STAR = 0x2a; // '*'
 const NEWLINE = 0x0a; // '\n'
 const QUOTE = 0x27; // '\''
+const LPAREN = 0x28; // '('
+const RPAREN = 0x29; // ')'
 // Whether a comment opens at `pos`.
 export function opensComment(buf, pos, len) {
     return buf[pos] === SLASH && pos + 1 < len && buf[pos + 1] === STAR;
@@ -41,21 +43,6 @@ export function opensComment(buf, pos, len) {
 // string literal. Callers skip literals with skipStringLiteral first, which is
 // what keeps a /* inside a HEADER description from opening a comment.
 //
-// Known, shared with the Rust EntityScanner, and deliberately NOT fixed here:
-// ISO 10303-21 allows a comment anywhere whitespace is allowed, and none of the
-// scanners treat one as trivia *within* a record. Two shapes, both pre-existing
-// and both unchanged by this file:
-//
-//   #1 /* note */ = IFCWALL();        header: fails the '=' check, no record
-//   #1=IFCWALL('a', /* n; */ $);      body: ends early at the ';' in the comment
-//
-// That is true of scanEntities, of scanEntitiesFast, of the worker scanner, and
-// of the Rust side, whose next_entity wants '=' straight after the digits and
-// whose find_entity_end jumps to the next quote or semicolon with memchr2.
-// Correcting it on the TypeScript side alone would put the JS fallback and the
-// wasm scan at odds on well-formed files, so it wants one change across all
-// four loops rather than a partial one here. Separate defect.
-//
 // The literal skip above is a smaller version of the same tension and is worth
 // naming rather than hiding. Rust has no literal skip outside a record, so on a
 // malformed file carrying an unpaired quote in DATA the two now disagree: this
@@ -71,8 +58,10 @@ export function skipComment(buf, pos, len) {
     }
     return -1;
 }
-// Index just past the closing quote of the literal opening at `pos`, or `len`
-// when it never closes. A doubled '' is an escaped quote and stays inside.
+// Index just past the closing quote of the literal opening at `pos`, or -1
+// when it never closes -- the same signal `skipComment` gives, not the old
+// `len` return that read as "closed at the last byte". A doubled '' is an
+// escaped quote and stays inside.
 //
 // Scanners consume a literal whole so nothing inside it can be mistaken for
 // syntax. Without this, a HEADER description reading `'rev /* pending'` opens a
@@ -92,7 +81,7 @@ export function skipStringLiteral(buf, pos, len) {
         }
         p++;
     }
-    return len;
+    return -1;
 }
 // Skip a string literal or a comment starting at `pos`. Callers test
 // `opensLiteralOrComment` first, so this is only reached on a byte that starts
@@ -100,6 +89,9 @@ export function skipStringLiteral(buf, pos, len) {
 export function skipLexical(buf, pos, len) {
     if (buf[pos] === QUOTE) {
         const next = skipStringLiteral(buf, pos, len);
+        if (next < 0) {
+            return { next: len, lines: countNewlines(buf, pos, len), stop: true };
+        }
         return { next, lines: countNewlines(buf, pos, next), stop: false };
     }
     const next = skipComment(buf, pos, len);
@@ -112,6 +104,59 @@ export function skipLexical(buf, pos, len) {
 export function opensLiteralOrComment(buf, pos, len) {
     return buf[pos] === QUOTE || opensComment(buf, pos, len);
 }
+// ASCII whitespace per ISO 10303-21. Not /\s/: that also matches U+00A0 and the
+// other Unicode space separators, which the byte scanners and the Rust half do
+// not treat as whitespace.
+//
+// The full six-byte set (space, tab, LF, CR, form feed 0x0c, vertical tab
+// 0x0b) matches `isAsciiSpace` below, which this file's header-string scanner
+// already used for the same six bytes -- and matches `is_step_space` in
+// `rust/export/src/source_header.rs`, which spells the set out for the same
+// reason this one now does: a stdlib "ASCII whitespace" helper is not
+// trustworthy here. Rust's `u8::is_ascii_whitespace` follows the WhatWG
+// definition and excludes vertical tab, so reaching for it on either side
+// reintroduces the exact FF/VT divergence issue #3733 reported (dropped
+// entity on a leading form feed) rather than closing it.
+function isSpaceByte(b) {
+    return (b === 0x20 || b === 0x09 || b === 0x0d || b === NEWLINE || b === 0x0c || b === 0x0b);
+}
+// Skip STEP trivia from `pos`: whitespace, comments, and any run of the two.
+//
+// ISO 10303-21 allows a comment ANYWHERE whitespace is allowed, which includes
+// inside a record -- between an instance name and its '=', between the '=' and
+// the type name, and between the type name and its '('. A scanner that skips
+// only whitespace at those points reads `#1 /* was #7 */ = IFCWALL(...)` as no
+// record at all. This is the one home for that rule on the byte side; the
+// matching Rust half is `skip_step_trivia` in rust/core/src/parser/lexical.rs
+// and the two are changed together.
+//
+// Callers hold to the composition order the rest of this file uses: a string
+// literal is consumed whole first, so a '/*' inside one is text; a comment is
+// consumed whole here, so a quote inside one is text and cannot open a literal.
+//
+// `stop` is true when a comment opens and never closes. Everything from there
+// to end of input is inside it, so there is nothing left to find -- the same
+// answer `skipLexical` gives, and the same answer Rust's `skip_step_comment`
+// gives by refusing.
+export function skipTrivia(buf, pos, len) {
+    let p = pos;
+    let lines = 0;
+    for (;;) {
+        while (p < len && isSpaceByte(buf[p])) {
+            if (buf[p] === NEWLINE)
+                lines++;
+            p++;
+        }
+        if (!opensComment(buf, p, len))
+            return { next: p, lines, stop: false };
+        const end = skipComment(buf, p, len);
+        if (end < 0) {
+            return { next: len, lines: lines + countNewlines(buf, p, len), stop: true };
+        }
+        lines += countNewlines(buf, p, end);
+        p = end;
+    }
+}
 // Newlines in [from, to), so a skipped region does not desync line numbers.
 export function countNewlines(buf, from, to) {
     let n = 0;
@@ -121,6 +166,59 @@ export function countNewlines(buf, from, to) {
     }
     return n;
 }
+// Byte length of the record that starts at `startOffset` and whose argument
+// list opens at `pos`: the span up to and including the ')' balancing that '(',
+// or 0 when the input runs out first.
+//
+// A string literal is jumped over by skipStringLiteral above rather than
+// counted, so the '(' in 'Storey (Level 1)' is text and not depth, and STEP's
+// doubled-quote escape ('') stays inside the literal instead of closing it.
+// Sharing that helper is what keeps the escape rule in one place: an
+// open-coded `inString` flag here would be a second copy of it, free to drift
+// from the one every other scanner in this file uses.
+//
+// A comment is jumped over for the same reason, in the same order: the literal
+// test comes first, so a '/*' inside a value is text; the comment is then taken
+// whole, so a '(' or a quote inside it is text. Without that, the comment in
+// `#1=IFCWALL('a', /* see IFCWALL( */ $);` opened a paren depth that never
+// closed and the record came back with length 0.
+export function findEntityLength(buf, pos, startOffset) {
+    const len = buf.length;
+    let depth = 0;
+    while (pos < len) {
+        const char = buf[pos];
+        if (char === QUOTE) {
+            // Returns `-1` on an unterminated literal (see skipStringLiteral's own
+            // doc): no balancing ')' can follow, same 0 as running off the end.
+            const next = skipStringLiteral(buf, pos, len);
+            if (next < 0)
+                return 0;
+            pos = next;
+        }
+        else if (opensComment(buf, pos, len)) {
+            const end = skipComment(buf, pos, len);
+            // Unterminated: the rest of the input is inside the comment, so no
+            // balancing ')' can follow. Same 0 as running off the end.
+            if (end < 0)
+                return 0;
+            pos = end;
+        }
+        else if (char === LPAREN) {
+            depth++;
+            pos++;
+        }
+        else if (char === RPAREN) {
+            depth--;
+            pos++;
+            if (depth === 0)
+                return pos - startOffset;
+        }
+        else {
+            pos++;
+        }
+    }
+    return 0; // no matching ')'
+}
 // ---------------------------------------------------------------------------
 // The same rule over a decoded string.
 //

package/dist/step-lexing.js.map

@@ -1 +1 @@
-{"version":3,"file":"step-lexing.js","sourceRoot":"","sources":["../src/step-lexing.ts"],"names":[],"mappings":"AAAA;;+DAE+D;AAE/D,6EAA6E;AAC7E,2EAA2E;AAC3E,+EAA+E;AAC/E,gDAAgD;AAChD,EAAE;AACF,uEAAuE;AACvE,gFAAgF;AAChF,gFAAgF;AAChF,+EAA+E;AAC/E,qDAAqD;AACrD,EAAE;AACF,8EAA8E;AAC9E,gFAAgF;AAChF,oEAAoE;AAEpE,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,MAAM;AAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,MAAM;AACzB,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,OAAO;AAC7B,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,OAAO;AAE3B,oCAAoC;AACpC,MAAM,UAAU,YAAY,CAAC,GAAe,EAAE,GAAW,EAAE,GAAW;IACpE,OAAO,GAAG,CAAC,GAAG,CAAC,KAAK,KAAK,IAAI,GAAG,GAAG,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC;AACtE,CAAC;AAED,gFAAgF;AAChF,mBAAmB;AACnB,EAAE;AACF,gFAAgF;AAChF,6EAA6E;AAC7E,gFAAgF;AAChF,gFAAgF;AAChF,4EAA4E;AAC5E,gEAAgE;AAChE,EAAE;AACF,gFAAgF;AAChF,8EAA8E;AAC9E,uCAAuC;AACvC,EAAE;AACF,yEAAyE;AACzE,+EAA+E;AAC/E,sEAAsE;AACtE,EAAE;AACF,8EAA8E;AAC9E,gFAAgF;AAChF,gFAAgF;AAChF,mCAAmC;AACnC,EAAE;AACF,6EAA6E;AAC7E,iFAAiF;AACjF,EAAE;AACF,gFAAgF;AAChF,8EAA8E;AAC9E,2EAA2E;AAC3E,+EAA+E;AAC/E,4EAA4E;AAC5E,8DAA8D;AAC9D,EAAE;AACF,+EAA+E;AAC/E,gFAAgF;AAChF,+EAA+E;AAC/E,8EAA8E;AAC9E,8EAA8E;AAC9E,0EAA0E;AAC1E,MAAM,UAAU,WAAW,CAAC,GAAe,EAAE,GAAW,EAAE,GAAW;IACnE,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,EAAE,CAAC;QACnB,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,KAAK;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC;QAC1D,CAAC,EAAE,CAAC;IACN,CAAC;IACD,OAAO,CAAC,CAAC,CAAC;AACZ,CAAC;AAED,8EAA8E;AAC9E,2EAA2E;AAC3E,EAAE;AACF,4EAA4E;AAC5E,gFAAgF;AAChF,8EAA8E;AAC9E,gFAAgF;AAChF,gFAAgF;AAChF,uDAAuD;AACvD,MAAM,UAAU,iBAAiB,CAAC,GAAe,EAAE,GAAW,EAAE,GAAW;IACzE,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,GAAG,EAAE,CAAC;QACf,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,KAAK,EAAE,CAAC;YACrB,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,KAAK,EAAE,CAAC;gBACxC,CAAC,IAAI,CAAC,CAAC;gBACP,SAAS;YACX,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,CAAC;QACf,CAAC;QACD,CAAC,EAAE,CAAC;IACN,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAYD,qEAAqE;AACrE,+EAA+E;AAC/E,+EAA+E;AAC/E,MAAM,UAAU,WAAW,CAAC,GAAe,EAAE,GAAW,EAAE,GAAW;IACnE,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,KAAK,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,iBAAiB,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;QAC9C,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,aAAa,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IACrE,CAAC;IACD,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;IACxC,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;QACb,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,aAAa,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACxE,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,aAAa,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;AACrE,CAAC;AAED,yDAAyD;AACzD,MAAM,UAAU,qBAAqB,CAAC,GAAe,EAAE,GAAW,EAAE,GAAW;IAC7E,OAAO,GAAG,CAAC,GAAG,CAAC,KAAK,KAAK,IAAI,YAAY,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;AAC3D,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,aAAa,CAAC,GAAe,EAAE,IAAY,EAAE,EAAU;IACrE,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,KAAK,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,OAAO;YAAE,CAAC,EAAE,CAAC;IAC9B,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,8EAA8E;AAC9E,uCAAuC;AACvC,EAAE;AACF,gFAAgF;AAChF,+EAA+E;AAC/E,8EAA8E;AAC9E,8EAA8E;AAC9E,gDAAgD;AAChD,EAAE;AACF,8EAA8E;AAC9E,2EAA2E;AAC3E,+EAA+E;AAC/E,+EAA+E;AAC/E,+EAA+E;AAC/E,+BAA+B;AAC/B,8EAA8E;AAE9E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,MAAM,OAAO,YAAY;IASM;IARrB,WAAW,GAAG,CAAC,CAAC;IAChB,QAAQ,GAAG,KAAK,CAAC;IAEzB,iFAAiF;IACjF,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED,YAA6B,IAAY;QAAZ,SAAI,GAAJ,IAAI,CAAQ;IAAG,CAAC;IAE7C;;;;;;;;OAQG;IACK,aAAa,CAAC,CAAS;QAC7B,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;QACtB,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG;YAAE,OAAO,CAAC,CAAC,CAAC;QACtD,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO,CAAC,CAAC,CAAC;QAC7B,IAAI,CAAC,WAAW,EAAE,CAAC;QACnB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACxC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;YACrB,OAAO,CAAC,CAAC,CAAC;QACZ,CAAC;QACD,OAAO,KAAK,GAAG,CAAC,CAAC;IACnB,CAAC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACH,aAAa,CAAC,CAAS;QACrB,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;QACtB,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YACpB,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACzC,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG;oBAAE,SAAS;gBAC9B,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;oBAAC,CAAC,EAAE,CAAC;oBAAC,SAAS;gBAAC,CAAC,CAAC,wCAAwC;gBACpF,OAAO,CAAC,GAAG,CAAC,CAAC;YACf,CAAC;YACD,OAAO,IAAI,CAAC,MAAM,CAAC;QACrB,CAAC;QACD,OAAO,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC;IAED;;;;;;;;OAQG;IACH,UAAU,CAAC,CAAS;QAClB,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;QACtB,SAAS,CAAC;YACR,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAAE,CAAC,EAAE,CAAC;YACrD,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;YACnC,IAAI,IAAI,GAAG,CAAC;gBAAE,OAAO,CAAC,CAAC;YACvB,CAAC,GAAG,IAAI,CAAC;QACX,CAAC;IACH,CAAC;CACF;AAED,wEAAwE;AACxE,SAAS,YAAY,CAAC,EAAsB;IAC1C,OAAO,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,CAAC;AAC/F,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAAE,CAAS,EAAE,OAAe;IACvE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,IAAI,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,yCAAyC;QACzE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,GAAG;YAAE,CAAC,IAAI,EAAE,CAAC;QACjC,IAAI,CAAC,KAAK,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;IAChD,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}
+{"version":3,"file":"step-lexing.js","sourceRoot":"","sources":["../src/step-lexing.ts"],"names":[],"mappings":"AAAA;;+DAE+D;AAE/D,6EAA6E;AAC7E,2EAA2E;AAC3E,+EAA+E;AAC/E,gDAAgD;AAChD,EAAE;AACF,uEAAuE;AACvE,gFAAgF;AAChF,gFAAgF;AAChF,+EAA+E;AAC/E,qDAAqD;AACrD,EAAE;AACF,8EAA8E;AAC9E,gFAAgF;AAChF,oEAAoE;AAEpE,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,MAAM;AAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,MAAM;AACzB,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,OAAO;AAC7B,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,OAAO;AAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,MAAM;AAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,CAAC,MAAM;AAE3B,oCAAoC;AACpC,MAAM,UAAU,YAAY,CAAC,GAAe,EAAE,GAAW,EAAE,GAAW;IACpE,OAAO,GAAG,CAAC,GAAG,CAAC,KAAK,KAAK,IAAI,GAAG,GAAG,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC;AACtE,CAAC;AAED,gFAAgF;AAChF,mBAAmB;AACnB,EAAE;AACF,gFAAgF;AAChF,6EAA6E;AAC7E,gFAAgF;AAChF,gFAAgF;AAChF,4EAA4E;AAC5E,gEAAgE;AAChE,EAAE;AACF,gFAAgF;AAChF,8EAA8E;AAC9E,uCAAuC;AACvC,EAAE;AACF,yEAAyE;AACzE,+EAA+E;AAC/E,sEAAsE;AACtE,EAAE;AACF,+EAA+E;AAC/E,gFAAgF;AAChF,+EAA+E;AAC/E,8EAA8E;AAC9E,8EAA8E;AAC9E,0EAA0E;AAC1E,MAAM,UAAU,WAAW,CAAC,GAAe,EAAE,GAAW,EAAE,GAAW;IACnE,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,EAAE,CAAC;QACnB,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,KAAK;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC;QAC1D,CAAC,EAAE,CAAC;IACN,CAAC;IACD,OAAO,CAAC,CAAC,CAAC;AACZ,CAAC;AAED,2EAA2E;AAC3E,2EAA2E;AAC3E,0EAA0E;AAC1E,kCAAkC;AAClC,EAAE;AACF,4EAA4E;AAC5E,gFAAgF;AAChF,8EAA8E;AAC9E,gFAAgF;AAChF,gFAAgF;AAChF,uDAAuD;AACvD,MAAM,UAAU,iBAAiB,CAAC,GAAe,EAAE,GAAW,EAAE,GAAW;IACzE,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,GAAG,EAAE,CAAC;QACf,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,KAAK,EAAE,CAAC;YACrB,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,KAAK,EAAE,CAAC;gBACxC,CAAC,IAAI,CAAC,CAAC;gBACP,SAAS;YACX,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,CAAC;QACf,CAAC;QACD,CAAC,EAAE,CAAC;IACN,CAAC;IACD,OAAO,CAAC,CAAC,CAAC;AACZ,CAAC;AAcD,qEAAqE;AACrE,+EAA+E;AAC/E,+EAA+E;AAC/E,MAAM,UAAU,WAAW,CAAC,GAAe,EAAE,GAAW,EAAE,GAAW;IACnE,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,KAAK,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,iBAAiB,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;QAC9C,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;YACb,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,aAAa,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACxE,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,aAAa,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IACrE,CAAC;IACD,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;IACxC,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;QACb,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,aAAa,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACxE,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,aAAa,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;AACrE,CAAC;AAED,yDAAyD;AACzD,MAAM,UAAU,qBAAqB,CAAC,GAAe,EAAE,GAAW,EAAE,GAAW;IAC7E,OAAO,GAAG,CAAC,GAAG,CAAC,KAAK,KAAK,IAAI,YAAY,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;AAC3D,CAAC;AAED,gFAAgF;AAChF,+EAA+E;AAC/E,2BAA2B;AAC3B,EAAE;AACF,0EAA0E;AAC1E,8EAA8E;AAC9E,wEAAwE;AACxE,4EAA4E;AAC5E,sEAAsE;AACtE,wEAAwE;AACxE,0EAA0E;AAC1E,wEAAwE;AACxE,yDAAyD;AACzD,SAAS,WAAW,CAAC,CAAS;IAC5B,OAAO,CACL,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,OAAO,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,CACpF,CAAC;AACJ,CAAC;AAED,6EAA6E;AAC7E,EAAE;AACF,+EAA+E;AAC/E,+EAA+E;AAC/E,6EAA6E;AAC7E,+EAA+E;AAC/E,0EAA0E;AAC1E,8EAA8E;AAC9E,oCAAoC;AACpC,EAAE;AACF,6EAA6E;AAC7E,8EAA8E;AAC9E,gFAAgF;AAChF,EAAE;AACF,8EAA8E;AAC9E,6EAA6E;AAC7E,6EAA6E;AAC7E,qBAAqB;AACrB,MAAM,UAAU,UAAU,CAAC,GAAe,EAAE,GAAW,EAAE,GAAW;IAClE,IAAI,CAAC,GAAG,GAAG,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,SAAS,CAAC;QACR,OAAO,CAAC,GAAG,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACtC,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,OAAO;gBAAE,KAAK,EAAE,CAAC;YAChC,CAAC,EAAE,CAAC;QACN,CAAC;QACD,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,CAAC,EAAE,GAAG,CAAC;YAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;QAEvE,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;QACrC,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;YACZ,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,GAAG,aAAa,CAAC,GAAG,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAC9E,CAAC;QACD,KAAK,IAAI,aAAa,CAAC,GAAG,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;QACpC,CAAC,GAAG,GAAG,CAAC;IACV,CAAC;AACH,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,aAAa,CAAC,GAAe,EAAE,IAAY,EAAE,EAAU;IACrE,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,KAAK,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,OAAO;YAAE,CAAC,EAAE,CAAC;IAC9B,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,4EAA4E;AAC5E,gFAAgF;AAChF,sCAAsC;AACtC,EAAE;AACF,yEAAyE;AACzE,8EAA8E;AAC9E,4EAA4E;AAC5E,qEAAqE;AACrE,8EAA8E;AAC9E,sDAAsD;AACtD,EAAE;AACF,+EAA+E;AAC/E,gFAAgF;AAChF,6EAA6E;AAC7E,2EAA2E;AAC3E,iDAAiD;AACjD,MAAM,UAAU,gBAAgB,CAAC,GAAe,EAAE,GAAW,EAAE,WAAmB;IAChF,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC;IACvB,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,OAAO,GAAG,GAAG,GAAG,EAAE,CAAC;QACjB,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;QAEtB,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;YACnB,uEAAuE;YACvE,oEAAoE;YACpE,MAAM,IAAI,GAAG,iBAAiB,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;YAC9C,IAAI,IAAI,GAAG,CAAC;gBAAE,OAAO,CAAC,CAAC;YACvB,GAAG,GAAG,IAAI,CAAC;QACb,CAAC;aAAM,IAAI,YAAY,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;YACvC,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;YACvC,mEAAmE;YACnE,2DAA2D;YAC3D,IAAI,GAAG,GAAG,CAAC;gBAAE,OAAO,CAAC,CAAC;YACtB,GAAG,GAAG,GAAG,CAAC;QACZ,CAAC;aAAM,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;YAC3B,KAAK,EAAE,CAAC;YACR,GAAG,EAAE,CAAC;QACR,CAAC;aAAM,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;YAC3B,KAAK,EAAE,CAAC;YACR,GAAG,EAAE,CAAC;YACN,IAAI,KAAK,KAAK,CAAC;gBAAE,OAAO,GAAG,GAAG,WAAW,CAAC;QAC5C,CAAC;aAAM,CAAC;YACN,GAAG,EAAE,CAAC;QACR,CAAC;IACH,CAAC;IAED,OAAO,CAAC,CAAC,CAAC,kBAAkB;AAC9B,CAAC;AAED,8EAA8E;AAC9E,uCAAuC;AACvC,EAAE;AACF,gFAAgF;AAChF,+EAA+E;AAC/E,8EAA8E;AAC9E,8EAA8E;AAC9E,gDAAgD;AAChD,EAAE;AACF,8EAA8E;AAC9E,2EAA2E;AAC3E,+EAA+E;AAC/E,+EAA+E;AAC/E,+EAA+E;AAC/E,+BAA+B;AAC/B,8EAA8E;AAE9E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,MAAM,OAAO,YAAY;IASM;IARrB,WAAW,GAAG,CAAC,CAAC;IAChB,QAAQ,GAAG,KAAK,CAAC;IAEzB,iFAAiF;IACjF,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED,YAA6B,IAAY;QAAZ,SAAI,GAAJ,IAAI,CAAQ;IAAG,CAAC;IAE7C;;;;;;;;OAQG;IACK,aAAa,CAAC,CAAS;QAC7B,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;QACtB,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG;YAAE,OAAO,CAAC,CAAC,CAAC;QACtD,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO,CAAC,CAAC,CAAC;QAC7B,IAAI,CAAC,WAAW,EAAE,CAAC;QACnB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACxC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;YACrB,OAAO,CAAC,CAAC,CAAC;QACZ,CAAC;QACD,OAAO,KAAK,GAAG,CAAC,CAAC;IACnB,CAAC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACH,aAAa,CAAC,CAAS;QACrB,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;QACtB,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YACpB,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACzC,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,GAAG;oBAAE,SAAS;gBAC9B,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;oBAAC,CAAC,EAAE,CAAC;oBAAC,SAAS;gBAAC,CAAC,CAAC,wCAAwC;gBACpF,OAAO,CAAC,GAAG,CAAC,CAAC;YACf,CAAC;YACD,OAAO,IAAI,CAAC,MAAM,CAAC;QACrB,CAAC;QACD,OAAO,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC;IAED;;;;;;;;OAQG;IACH,UAAU,CAAC,CAAS;QAClB,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;QACtB,SAAS,CAAC;YACR,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAAE,CAAC,EAAE,CAAC;YACrD,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;YACnC,IAAI,IAAI,GAAG,CAAC;gBAAE,OAAO,CAAC,CAAC;YACvB,CAAC,GAAG,IAAI,CAAC;QACX,CAAC;IACH,CAAC;CACF;AAED,wEAAwE;AACxE,SAAS,YAAY,CAAC,EAAsB;IAC1C,OAAO,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,CAAC;AAC/F,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY,EAAE,CAAS,EAAE,OAAe;IACvE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,IAAI,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,yCAAyC;QACzE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,GAAG;YAAE,CAAC,IAAI,EAAE,CAAC;QACjC,IAAI,CAAC,KAAK,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;IAChD,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/step-trivia.d.ts

@@ -0,0 +1,60 @@
+/**
+ * Regex source for ISO 10303-21 "trivia": a run of ASCII whitespace and
+ * non-nesting `/* ... *​/` comments, in any order, ANYWHERE whitespace is
+ * legal in a record — including between a type name and its `(`.
+ *
+ * String-side counterpart to `skipTrivia` in `step-lexing.ts`, for callers
+ * that already hold a decoded record as a JS string and build a regex around
+ * it (entity/typed-value extraction and rewriting), rather than scanning raw
+ * bytes. Both mirror `skip_step_trivia` in `rust/core/src/parser/lexical.rs`.
+ *
+ * Deliberately ASCII-only (`[ \t\n\r\x0b\x0c]`, matching `is_step_space` /
+ * `isSpaceByte` in `step-lexing.ts`) rather than `\s`, which also matches
+ * U+00A0 and other Unicode space separators Rust's byte scanner does not
+ * treat as whitespace — the exact TS/Rust divergence class issue #3733
+ * fixed for the byte scanners; a new trivia matcher should not reintroduce
+ * it under a different name.
+ *
+ * ## Why each alternative is shaped the way it is
+ *
+ * The outer `(?:A|B)*` must have exactly ONE way to partition any input into
+ * iterations. Where it has more than one, a failing suffix makes the engine
+ * enumerate them, which is exponential in the length of the trivia run. Both
+ * alternatives below are written to keep that count at one, and BOTH shapes
+ * have been measured — the two axes fail independently, so neither is
+ * theoretical:
+ *
+ * - The comment body is `(?:[^*]|\*(?!/))*`, NOT a lazy `[\s\S]*?`. A lazy
+ *   body looks unambiguous locally, but when the overall pattern fails past
+ *   a comment the engine retries it against every later `*​/`, so one comment
+ *   can absorb the ones after it and the two alternatives start overlapping
+ *   on the same span. Requiring every `*` in the body to not be followed by
+ *   `/` gives the body exactly one maximal extent, so a comment can never
+ *   swallow the next one.
+ * - The whitespace alternative is a SINGLE-CHARACTER class, NOT `[...]+`.
+ *   This is the counter-intuitive half: `+` looks like it collapses a run
+ *   into one iteration, but the outer `*` can still split an n-character run
+ *   into any composition of `+` matches (2^(n-1) of them) and walks all of
+ *   them on failure — the textbook `(?:A+|B)*` blowup. A single-character
+ *   class has exactly one partition: n iterations of one character each.
+ *
+ * Both hazards are pinned by `packages/parser/test/step-trivia-redos.test.ts`,
+ * one case per axis, and the test comment there carries the measurements.
+ * Change either alternative only with that test in front of you.
+ *
+ * ## Relation to the byte scanners
+ *
+ * The comment body also brings this pattern into line with
+ * `skip_step_trivia`, which stops a comment at its FIRST `*​/`. A lazy body
+ * did not: on backtracking it would accept `/* a *​/ *​/` as one comment,
+ * where Rust reads a comment followed by junk and refuses the record. The
+ * shared vector "reject: comments do not nest, so the trailing `*​/` is junk"
+ * in `rust/core/tests/fixtures/type_paren_trivia_vectors.json` pins that
+ * agreement, so this is a deliberate narrowing, not an accident.
+ *
+ * An unpaired, unterminated `/*` still fails to match rather than hanging:
+ * the body runs out of input, backtracking is O(1) per position, and the
+ * required `\*​/` after it never appears.
+ */
+export declare const STEP_TRIVIA = "(?:[ \\t\\n\\r\\x0b\\x0c]|/\\*(?:[^*]|\\*(?!/))*\\*/)*";
+//# sourceMappingURL=step-trivia.d.ts.map

package/dist/step-trivia.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"step-trivia.d.ts","sourceRoot":"","sources":["../src/step-trivia.ts"],"names":[],"mappings":"AAIA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyDG;AACH,eAAO,MAAM,WAAW,2DAA2D,CAAC"}

package/dist/step-trivia.js

@@ -0,0 +1,63 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+/**
+ * Regex source for ISO 10303-21 "trivia": a run of ASCII whitespace and
+ * non-nesting `/* ... *​/` comments, in any order, ANYWHERE whitespace is
+ * legal in a record — including between a type name and its `(`.
+ *
+ * String-side counterpart to `skipTrivia` in `step-lexing.ts`, for callers
+ * that already hold a decoded record as a JS string and build a regex around
+ * it (entity/typed-value extraction and rewriting), rather than scanning raw
+ * bytes. Both mirror `skip_step_trivia` in `rust/core/src/parser/lexical.rs`.
+ *
+ * Deliberately ASCII-only (`[ \t\n\r\x0b\x0c]`, matching `is_step_space` /
+ * `isSpaceByte` in `step-lexing.ts`) rather than `\s`, which also matches
+ * U+00A0 and other Unicode space separators Rust's byte scanner does not
+ * treat as whitespace — the exact TS/Rust divergence class issue #3733
+ * fixed for the byte scanners; a new trivia matcher should not reintroduce
+ * it under a different name.
+ *
+ * ## Why each alternative is shaped the way it is
+ *
+ * The outer `(?:A|B)*` must have exactly ONE way to partition any input into
+ * iterations. Where it has more than one, a failing suffix makes the engine
+ * enumerate them, which is exponential in the length of the trivia run. Both
+ * alternatives below are written to keep that count at one, and BOTH shapes
+ * have been measured — the two axes fail independently, so neither is
+ * theoretical:
+ *
+ * - The comment body is `(?:[^*]|\*(?!/))*`, NOT a lazy `[\s\S]*?`. A lazy
+ *   body looks unambiguous locally, but when the overall pattern fails past
+ *   a comment the engine retries it against every later `*​/`, so one comment
+ *   can absorb the ones after it and the two alternatives start overlapping
+ *   on the same span. Requiring every `*` in the body to not be followed by
+ *   `/` gives the body exactly one maximal extent, so a comment can never
+ *   swallow the next one.
+ * - The whitespace alternative is a SINGLE-CHARACTER class, NOT `[...]+`.
+ *   This is the counter-intuitive half: `+` looks like it collapses a run
+ *   into one iteration, but the outer `*` can still split an n-character run
+ *   into any composition of `+` matches (2^(n-1) of them) and walks all of
+ *   them on failure — the textbook `(?:A+|B)*` blowup. A single-character
+ *   class has exactly one partition: n iterations of one character each.
+ *
+ * Both hazards are pinned by `packages/parser/test/step-trivia-redos.test.ts`,
+ * one case per axis, and the test comment there carries the measurements.
+ * Change either alternative only with that test in front of you.
+ *
+ * ## Relation to the byte scanners
+ *
+ * The comment body also brings this pattern into line with
+ * `skip_step_trivia`, which stops a comment at its FIRST `*​/`. A lazy body
+ * did not: on backtracking it would accept `/* a *​/ *​/` as one comment,
+ * where Rust reads a comment followed by junk and refuses the record. The
+ * shared vector "reject: comments do not nest, so the trailing `*​/` is junk"
+ * in `rust/core/tests/fixtures/type_paren_trivia_vectors.json` pins that
+ * agreement, so this is a deliberate narrowing, not an accident.
+ *
+ * An unpaired, unterminated `/*` still fails to match rather than hanging:
+ * the body runs out of input, backtracking is O(1) per position, and the
+ * required `\*​/` after it never appears.
+ */
+export const STEP_TRIVIA = '(?:[ \\t\\n\\r\\x0b\\x0c]|/\\*(?:[^*]|\\*(?!/))*\\*/)*';
+//# sourceMappingURL=step-trivia.js.map

package/dist/step-trivia.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"step-trivia.js","sourceRoot":"","sources":["../src/step-trivia.ts"],"names":[],"mappings":"AAAA;;+DAE+D;AAE/D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyDG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,wDAAwD,CAAC"}

package/dist/tokenizer.d.ts

@@ -1,33 +1,32 @@
+import { type ScannedEntityRef } from './scan-entities-balanced.js';
 export declare class StepTokenizer {
     private buffer;
-    private position;
-    private lineNumber;
+    private oversizedIds;
+    private malformedRecords;
     constructor(buffer: Uint8Array);
+    /** Records the last scan refused for an out-of-contract express id
+     *  (express-id.ts, #3395). Reset per scan; the caller reports it. */
+    get oversizedIdCount(): number;
+    /** 0 or 1: whether the last `scanEntitiesFast`/`scanEntities` run stopped
+     *  early on an unclosed `'` string, an unclosed block comment, or a
+     *  declaration cut off before its own '(' -- never a count of how many,
+     *  since the scan has no reliable way to resume past the first one it
+     *  hits. Reset at the start of every scan; the caller reports it. */
+    get malformedRecordCount(): number;
     /**
      * Scan for all entity declarations (#EXPRESS_ID = TYPE(...))
-     * Returns entity references without parsing full content
+     * Returns entity references without parsing full content.
+     *
+     * Closes each record on the ')' balancing its argument list. The scan itself
+     * lives in `scan-entities-balanced.ts`; only the refusal count comes back
+     * here, and it comes back in a `finally` so an abandoned generator still
+     * reports what it refused.
      */
-    scanEntities(): Generator<{
-        expressId: number;
-        type: string;
-        offset: number;
-        length: number;
-        line: number;
-    }>;
+    scanEntities(): Generator<ScannedEntityRef>;
     /**
      * FAST scan - skips to semicolon instead of matching parentheses
      * ~5-10x faster for large files, yields length=0 (calculate on-demand)
      */
-    scanEntitiesFast(): Generator<{
-        expressId: number;
-        type: string;
-        offset: number;
-        length: number;
-        line: number;
-    }>;
-    private readExpressId;
-    private readTypeName;
-    private skipWhitespace;
-    private findEntityLength;
+    scanEntitiesFast(): Generator<ScannedEntityRef>;
 }
 //# sourceMappingURL=tokenizer.d.ts.map

package/dist/tokenizer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAaA,qBAAa,aAAa;IACxB,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,QAAQ,CAAa;IAC7B,OAAO,CAAC,UAAU,CAAa;gBAEnB,MAAM,EAAE,UAAU;IAI9B;;;OAGG;IACF,YAAY,IAAI,SAAS,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IA0F7G;;;OAGG;IACF,gBAAgB,IAAI,SAAS,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAuKjH,OAAO,CAAC,aAAa;IAqBrB,OAAO,CAAC,YAAY;IA+BpB,OAAO,CAAC,cAAc;IAYtB,OAAO,CAAC,gBAAgB;CA6CzB"}
+{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAUA,OAAO,EAAsB,KAAK,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAUxF,qBAAa,aAAa;IACxB,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,YAAY,CAAa;IACjC,OAAO,CAAC,gBAAgB,CAAa;gBAEzB,MAAM,EAAE,UAAU;IAI9B;yEACqE;IACrE,IAAI,gBAAgB,IAAI,MAAM,CAA8B;IAE5D;;;;yEAIqE;IACrE,IAAI,oBAAoB,IAAI,MAAM,CAAkC;IAEpE;;;;;;;;OAQG;IACF,YAAY,IAAI,SAAS,CAAC,gBAAgB,CAAC;IAY5C;;;OAGG;IACF,gBAAgB,IAAI,SAAS,CAAC,gBAAgB,CAAC;CAqQjD"}