@ifc-lite/parser 4.3.1 → 5.0.0

package/dist/step-trivia.d.ts

@@ -0,0 +1,60 @@
+/**
+ * Regex source for ISO 10303-21 "trivia": a run of ASCII whitespace and
+ * non-nesting `/* ... *​/` comments, in any order, ANYWHERE whitespace is
+ * legal in a record — including between a type name and its `(`.
+ *
+ * String-side counterpart to `skipTrivia` in `step-lexing.ts`, for callers
+ * that already hold a decoded record as a JS string and build a regex around
+ * it (entity/typed-value extraction and rewriting), rather than scanning raw
+ * bytes. Both mirror `skip_step_trivia` in `rust/core/src/parser/lexical.rs`.
+ *
+ * Deliberately ASCII-only (`[ \t\n\r\x0b\x0c]`, matching `is_step_space` /
+ * `isSpaceByte` in `step-lexing.ts`) rather than `\s`, which also matches
+ * U+00A0 and other Unicode space separators Rust's byte scanner does not
+ * treat as whitespace — the exact TS/Rust divergence class issue #3733
+ * fixed for the byte scanners; a new trivia matcher should not reintroduce
+ * it under a different name.
+ *
+ * ## Why each alternative is shaped the way it is
+ *
+ * The outer `(?:A|B)*` must have exactly ONE way to partition any input into
+ * iterations. Where it has more than one, a failing suffix makes the engine
+ * enumerate them, which is exponential in the length of the trivia run. Both
+ * alternatives below are written to keep that count at one, and BOTH shapes
+ * have been measured — the two axes fail independently, so neither is
+ * theoretical:
+ *
+ * - The comment body is `(?:[^*]|\*(?!/))*`, NOT a lazy `[\s\S]*?`. A lazy
+ *   body looks unambiguous locally, but when the overall pattern fails past
+ *   a comment the engine retries it against every later `*​/`, so one comment
+ *   can absorb the ones after it and the two alternatives start overlapping
+ *   on the same span. Requiring every `*` in the body to not be followed by
+ *   `/` gives the body exactly one maximal extent, so a comment can never
+ *   swallow the next one.
+ * - The whitespace alternative is a SINGLE-CHARACTER class, NOT `[...]+`.
+ *   This is the counter-intuitive half: `+` looks like it collapses a run
+ *   into one iteration, but the outer `*` can still split an n-character run
+ *   into any composition of `+` matches (2^(n-1) of them) and walks all of
+ *   them on failure — the textbook `(?:A+|B)*` blowup. A single-character
+ *   class has exactly one partition: n iterations of one character each.
+ *
+ * Both hazards are pinned by `packages/parser/test/step-trivia-redos.test.ts`,
+ * one case per axis, and the test comment there carries the measurements.
+ * Change either alternative only with that test in front of you.
+ *
+ * ## Relation to the byte scanners
+ *
+ * The comment body also brings this pattern into line with
+ * `skip_step_trivia`, which stops a comment at its FIRST `*​/`. A lazy body
+ * did not: on backtracking it would accept `/* a *​/ *​/` as one comment,
+ * where Rust reads a comment followed by junk and refuses the record. The
+ * shared vector "reject: comments do not nest, so the trailing `*​/` is junk"
+ * in `rust/core/tests/fixtures/type_paren_trivia_vectors.json` pins that
+ * agreement, so this is a deliberate narrowing, not an accident.
+ *
+ * An unpaired, unterminated `/*` still fails to match rather than hanging:
+ * the body runs out of input, backtracking is O(1) per position, and the
+ * required `\*​/` after it never appears.
+ */
+export declare const STEP_TRIVIA = "(?:[ \\t\\n\\r\\x0b\\x0c]|/\\*(?:[^*]|\\*(?!/))*\\*/)*";
+//# sourceMappingURL=step-trivia.d.ts.map

package/dist/step-trivia.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"step-trivia.d.ts","sourceRoot":"","sources":["../src/step-trivia.ts"],"names":[],"mappings":"AAIA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyDG;AACH,eAAO,MAAM,WAAW,2DAA2D,CAAC"}

package/dist/step-trivia.js

@@ -0,0 +1,63 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+/**
+ * Regex source for ISO 10303-21 "trivia": a run of ASCII whitespace and
+ * non-nesting `/* ... *​/` comments, in any order, ANYWHERE whitespace is
+ * legal in a record — including between a type name and its `(`.
+ *
+ * String-side counterpart to `skipTrivia` in `step-lexing.ts`, for callers
+ * that already hold a decoded record as a JS string and build a regex around
+ * it (entity/typed-value extraction and rewriting), rather than scanning raw
+ * bytes. Both mirror `skip_step_trivia` in `rust/core/src/parser/lexical.rs`.
+ *
+ * Deliberately ASCII-only (`[ \t\n\r\x0b\x0c]`, matching `is_step_space` /
+ * `isSpaceByte` in `step-lexing.ts`) rather than `\s`, which also matches
+ * U+00A0 and other Unicode space separators Rust's byte scanner does not
+ * treat as whitespace — the exact TS/Rust divergence class issue #3733
+ * fixed for the byte scanners; a new trivia matcher should not reintroduce
+ * it under a different name.
+ *
+ * ## Why each alternative is shaped the way it is
+ *
+ * The outer `(?:A|B)*` must have exactly ONE way to partition any input into
+ * iterations. Where it has more than one, a failing suffix makes the engine
+ * enumerate them, which is exponential in the length of the trivia run. Both
+ * alternatives below are written to keep that count at one, and BOTH shapes
+ * have been measured — the two axes fail independently, so neither is
+ * theoretical:
+ *
+ * - The comment body is `(?:[^*]|\*(?!/))*`, NOT a lazy `[\s\S]*?`. A lazy
+ *   body looks unambiguous locally, but when the overall pattern fails past
+ *   a comment the engine retries it against every later `*​/`, so one comment
+ *   can absorb the ones after it and the two alternatives start overlapping
+ *   on the same span. Requiring every `*` in the body to not be followed by
+ *   `/` gives the body exactly one maximal extent, so a comment can never
+ *   swallow the next one.
+ * - The whitespace alternative is a SINGLE-CHARACTER class, NOT `[...]+`.
+ *   This is the counter-intuitive half: `+` looks like it collapses a run
+ *   into one iteration, but the outer `*` can still split an n-character run
+ *   into any composition of `+` matches (2^(n-1) of them) and walks all of
+ *   them on failure — the textbook `(?:A+|B)*` blowup. A single-character
+ *   class has exactly one partition: n iterations of one character each.
+ *
+ * Both hazards are pinned by `packages/parser/test/step-trivia-redos.test.ts`,
+ * one case per axis, and the test comment there carries the measurements.
+ * Change either alternative only with that test in front of you.
+ *
+ * ## Relation to the byte scanners
+ *
+ * The comment body also brings this pattern into line with
+ * `skip_step_trivia`, which stops a comment at its FIRST `*​/`. A lazy body
+ * did not: on backtracking it would accept `/* a *​/ *​/` as one comment,
+ * where Rust reads a comment followed by junk and refuses the record. The
+ * shared vector "reject: comments do not nest, so the trailing `*​/` is junk"
+ * in `rust/core/tests/fixtures/type_paren_trivia_vectors.json` pins that
+ * agreement, so this is a deliberate narrowing, not an accident.
+ *
+ * An unpaired, unterminated `/*` still fails to match rather than hanging:
+ * the body runs out of input, backtracking is O(1) per position, and the
+ * required `\*​/` after it never appears.
+ */
+export const STEP_TRIVIA = '(?:[ \\t\\n\\r\\x0b\\x0c]|/\\*(?:[^*]|\\*(?!/))*\\*/)*';
+//# sourceMappingURL=step-trivia.js.map

package/dist/step-trivia.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"step-trivia.js","sourceRoot":"","sources":["../src/step-trivia.ts"],"names":[],"mappings":"AAAA;;+DAE+D;AAE/D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyDG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,wDAAwD,CAAC"}

package/dist/tokenizer.d.ts

@@ -1,33 +1,32 @@
+import { type ScannedEntityRef } from './scan-entities-balanced.js';
 export declare class StepTokenizer {
     private buffer;
-    private position;
-    private lineNumber;
+    private oversizedIds;
+    private malformedRecords;
     constructor(buffer: Uint8Array);
+    /** Records the last scan refused for an out-of-contract express id
+     *  (express-id.ts, #3395). Reset per scan; the caller reports it. */
+    get oversizedIdCount(): number;
+    /** 0 or 1: whether the last `scanEntitiesFast`/`scanEntities` run stopped
+     *  early on an unclosed `'` string, an unclosed block comment, or a
+     *  declaration cut off before its own '(' -- never a count of how many,
+     *  since the scan has no reliable way to resume past the first one it
+     *  hits. Reset at the start of every scan; the caller reports it. */
+    get malformedRecordCount(): number;
     /**
      * Scan for all entity declarations (#EXPRESS_ID = TYPE(...))
-     * Returns entity references without parsing full content
+     * Returns entity references without parsing full content.
+     *
+     * Closes each record on the ')' balancing its argument list. The scan itself
+     * lives in `scan-entities-balanced.ts`; only the refusal count comes back
+     * here, and it comes back in a `finally` so an abandoned generator still
+     * reports what it refused.
      */
-    scanEntities(): Generator<{
-        expressId: number;
-        type: string;
-        offset: number;
-        length: number;
-        line: number;
-    }>;
+    scanEntities(): Generator<ScannedEntityRef>;
     /**
      * FAST scan - skips to semicolon instead of matching parentheses
      * ~5-10x faster for large files, yields length=0 (calculate on-demand)
      */
-    scanEntitiesFast(): Generator<{
-        expressId: number;
-        type: string;
-        offset: number;
-        length: number;
-        line: number;
-    }>;
-    private readExpressId;
-    private readTypeName;
-    private skipWhitespace;
-    private findEntityLength;
+    scanEntitiesFast(): Generator<ScannedEntityRef>;
 }
 //# sourceMappingURL=tokenizer.d.ts.map

package/dist/tokenizer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAaA,qBAAa,aAAa;IACxB,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,QAAQ,CAAa;IAC7B,OAAO,CAAC,UAAU,CAAa;gBAEnB,MAAM,EAAE,UAAU;IAI9B;;;OAGG;IACF,YAAY,IAAI,SAAS,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IA0F7G;;;OAGG;IACF,gBAAgB,IAAI,SAAS,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAuKjH,OAAO,CAAC,aAAa;IAqBrB,OAAO,CAAC,YAAY;IA+BpB,OAAO,CAAC,cAAc;IAYtB,OAAO,CAAC,gBAAgB;CA6CzB"}
+{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAUA,OAAO,EAAsB,KAAK,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAUxF,qBAAa,aAAa;IACxB,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,YAAY,CAAa;IACjC,OAAO,CAAC,gBAAgB,CAAa;gBAEzB,MAAM,EAAE,UAAU;IAI9B;yEACqE;IACrE,IAAI,gBAAgB,IAAI,MAAM,CAA8B;IAE5D;;;;yEAIqE;IACrE,IAAI,oBAAoB,IAAI,MAAM,CAAkC;IAEpE;;;;;;;;OAQG;IACF,YAAY,IAAI,SAAS,CAAC,gBAAgB,CAAC;IAY5C;;;OAGG;IACF,gBAAgB,IAAI,SAAS,CAAC,gBAAgB,CAAC;CAqQjD"}

package/dist/tokenizer.js

@@ -5,97 +5,44 @@
  * STEP tokenizer - fast byte-level scanning for entity markers
  * Leverages Spike 1 approach: ~1,259 MB/s throughput
  */
-import { safeUtf8Decode } from '@ifc-lite/data';
-import { countNewlines, opensLiteralOrComment, skipLexical } from './step-lexing.js';
+import { isIndexableExpressId } from './express-id.js';
+import { BalancedEntityScan } from './scan-entities-balanced.js';
+import { countNewlines, opensComment, opensLiteralOrComment, skipComment, skipLexical, skipTrivia, } from './step-lexing.js';
 export class StepTokenizer {
     buffer;
-    position = 0;
-    lineNumber = 1;
+    oversizedIds = 0;
+    malformedRecords = 0;
     constructor(buffer) {
         this.buffer = buffer;
     }
+    /** Records the last scan refused for an out-of-contract express id
+     *  (express-id.ts, #3395). Reset per scan; the caller reports it. */
+    get oversizedIdCount() { return this.oversizedIds; }
+    /** 0 or 1: whether the last `scanEntitiesFast`/`scanEntities` run stopped
+     *  early on an unclosed `'` string, an unclosed block comment, or a
+     *  declaration cut off before its own '(' -- never a count of how many,
+     *  since the scan has no reliable way to resume past the first one it
+     *  hits. Reset at the start of every scan; the caller reports it. */
+    get malformedRecordCount() { return this.malformedRecords; }
     /**
      * Scan for all entity declarations (#EXPRESS_ID = TYPE(...))
-     * Returns entity references without parsing full content
+     * Returns entity references without parsing full content.
+     *
+     * Closes each record on the ')' balancing its argument list. The scan itself
+     * lives in `scan-entities-balanced.ts`; only the refusal count comes back
+     * here, and it comes back in a `finally` so an abandoned generator still
+     * reports what it refused.
      */
     *scanEntities() {
-        this.position = 0;
-        this.lineNumber = 1;
-        while (this.position < this.buffer.length) {
-            // Look for '#' character (entity ID marker)
-            if (this.buffer[this.position] === 0x23) { // '#'
-                const startOffset = this.position;
-                const startLine = this.lineNumber;
-                // Read express ID
-                const expressId = this.readExpressId();
-                if (expressId === null) {
-                    this.position++;
-                    continue;
-                }
-                // Skip whitespace
-                this.skipWhitespace();
-                // Check for '=' (assignment)
-                if (this.position >= this.buffer.length || this.buffer[this.position] !== 0x3D) {
-                    this.position++;
-                    continue;
-                }
-                this.position++; // Skip '='
-                // Skip whitespace
-                this.skipWhitespace();
-                // Read type name
-                const type = this.readTypeName();
-                if (!type) {
-                    this.position++;
-                    continue;
-                }
-                // Skip whitespace
-                this.skipWhitespace();
-                // Check for '(' (start of parameters)
-                if (this.position >= this.buffer.length || this.buffer[this.position] !== 0x28) {
-                    this.position++;
-                    continue;
-                }
-                // Find matching closing parenthesis to get full entity length
-                const entityLength = this.findEntityLength(startOffset);
-                if (entityLength > 0) {
-                    // Step past the whole record, as Rust's next_entity does. Leaving
-                    // `position` at the '(' made this loop re-walk the body, which was
-                    // harmless only while it ignored quotes and comments.
-                    //
-                    // Count from `position`, not from `startOffset`: `position` is on the
-                    // '(' here, and every newline before it was already counted by the
-                    // three skipWhitespace calls above. Counting the whole record instead
-                    // double-counts a newline written between `#1=` and its type name,
-                    // which is ordinary whitespace and legal.
-                    this.lineNumber += countNewlines(this.buffer, this.position, startOffset + entityLength);
-                    this.position = startOffset + entityLength;
-                    yield {
-                        expressId,
-                        type,
-                        offset: startOffset,
-                        length: entityLength,
-                        line: startLine,
-                    };
-                }
-            }
-            else if (this.buffer[this.position] === 0x0A) {
-                // Newline
-                this.lineNumber++;
-                this.position++;
-            }
-            else if (opensLiteralOrComment(this.buffer, this.position, this.buffer.length)) {
-                // A commented-out record satisfies every check above, so a comment has
-                // to be skipped as a region; a literal has to be skipped so its
-                // contents cannot look like one. See step-lexing.
-                const skip = skipLexical(this.buffer, this.position, this.buffer.length);
-                this.lineNumber += skip.lines;
-                this.position = skip.next;
-                if (skip.stop)
-                    return;
-            }
-            else {
-                this.position++;
-            }
+        const scan = new BalancedEntityScan(this.buffer);
+        this.oversizedIds = 0;
+        this.malformedRecords = 0;
+        try {
+            yield* scan.run();
+        }
+        finally {
+            this.oversizedIds = scan.oversizedIdCount;
+            this.malformedRecords = scan.malformedRecordCount;
         }
     }
     /**
@@ -103,8 +50,8 @@ export class StepTokenizer {
      * ~5-10x faster for large files, yields length=0 (calculate on-demand)
      */
     *scanEntitiesFast() {
-        this.position = 0;
-        this.lineNumber = 1;
+        this.oversizedIds = 0;
+        this.malformedRecords = 0;
         // Pre-compute common byte codes
         const HASH = 0x23; // '#'
         const EQUALS = 0x3D; // '='
@@ -112,6 +59,7 @@ export class StepTokenizer {
         const SEMICOLON = 0x3B; // ';'
         const QUOTE = 0x27; // '\''
         const NEWLINE = 0x0A; // '\n'
+        const SLASH = 0x2F; // '/'
         const buf = this.buffer;
         const len = buf.length;
         let pos = 0;
@@ -119,6 +67,22 @@ export class StepTokenizer {
         // Cache type name strings: IFC files have ~776 unique types repeated
         // across 8M+ entities. Caching avoids millions of String.fromCharCode allocations.
         const typeCache = new Map();
+        // Set on the way to the single post-loop check at the bottom of this
+        // function, not counted at each site: `stopped` for an unclosed string or
+        // comment that ran the scan to end of buffer with nothing left to find,
+        // `declOpen` while a `#id=TYPE(` header is incomplete. `declOpen` stays
+        // armed ONLY when the reason for abandoning is running out of buffer
+        // (`pos >= len`); a mismatch with buffer still left (bad byte, oversized
+        // id) clears it, because the scan resumes byte-by-byte from wherever it
+        // gave up, and a `#ref` token inside the abandoned record's own argument
+        // list reads as a fresh, equally incomplete attempt -- one that must not
+        // report "cut off" just because nothing later happens to clear it.
+        // Per-site increments used to miss whole shapes -- a leading unterminated
+        // comment before '=', or a declaration cut off before its own '(' --
+        // because each site only knew about its own exit, never the scan's final
+        // state.
+        let stopped = false;
+        let declOpen = false;
         while (pos < len) {
             const char = buf[pos];
             if (char === HASH) {
@@ -141,10 +105,15 @@ export class StepTokenizer {
                 }
                 if (!hasDigits)
                     continue;
-                // Skip whitespace (inline)
+                declOpen = true;
+                // Skip whitespace (inline). Kept byte-for-byte in sync with
+                // `isSpaceByte` in step-lexing.ts (space, tab, CR, LF, form feed,
+                // vertical tab) -- this loop, its two twins below in this method, and
+                // the worker's copy in scan-worker-source.ts are the same rule
+                // hand-duplicated for speed, not four independent decisions.
                 while (pos < len) {
                     const c = buf[pos];
-                    if (c === 0x20 || c === 0x09 || c === 0x0D) {
+                    if (c === 0x20 || c === 0x09 || c === 0x0D || c === 0x0C || c === 0x0B) {
                         pos++;
                     }
                     else if (c === NEWLINE) {
@@ -154,14 +123,50 @@ export class StepTokenizer {
                     else
                         break;
                 }
-                // Check for '='
-                if (pos >= len || buf[pos] !== EQUALS)
+                // 10303-21 allows a comment wherever whitespace is allowed, so
+                // `#1 /* was #7 */ =` is a declaration. The inline loop above stays
+                // for the common case; this runs only once a comment actually opens,
+                // and skipTrivia (step-lexing) then takes the whole run of both.
+                if (opensComment(buf, pos, len)) {
+                    const t = skipTrivia(buf, pos, len);
+                    line += t.lines;
+                    pos = t.next;
+                    if (t.stop) {
+                        stopped = true;
+                        break;
+                    }
+                }
+                // Check for '='. A byte that is not '=' with buffer left to scan is
+                // not a truncation -- clear declOpen so a reference token inside a
+                // LATER abandoned record's argument list (see the oversized-id note
+                // below) cannot leave it stuck armed with nothing left to clear it.
+                if (pos >= len)
+                    continue;
+                if (buf[pos] !== EQUALS) {
+                    declOpen = false;
                     continue;
+                }
                 pos++;
+                // Storage contract, not just overflow: see express-id.ts (#3395).
+                // Tested only now that `#<digits>[ws]*=` has matched, which is the
+                // DECLARATION shape Rust's `EntityScanner` validates before it
+                // refuses. Refusing above the '=' check counted references too: the
+                // `continue` resumes inside the refused record's argument list
+                // (unlike the accepted path, which skips to the terminating ';'), so
+                // `#4294967297=IFCWALL(#4294967298,#4294967299,...)` reported three
+                // skipped records for the one record actually dropped. A count that
+                // overstates is the same class of defect as one that undercounts.
+                // Refused for being out of range, not for running out of buffer, so
+                // it does not belong to `declOpen`'s "cut off by EOF" story either.
+                if (!isIndexableExpressId(expressId)) {
+                    this.oversizedIds++;
+                    declOpen = false;
+                    continue;
+                }
                 // Skip whitespace
                 while (pos < len) {
                     const c = buf[pos];
-                    if (c === 0x20 || c === 0x09 || c === 0x0D) {
+                    if (c === 0x20 || c === 0x09 || c === 0x0D || c === 0x0C || c === 0x0B) {
                         pos++;
                     }
                     else if (c === NEWLINE) {
@@ -171,10 +176,24 @@ export class StepTokenizer {
                     else
                         break;
                 }
-                // Read type name (inline)
+                if (opensComment(buf, pos, len)) {
+                    const t = skipTrivia(buf, pos, len);
+                    line += t.lines;
+                    pos = t.next;
+                    if (t.stop) {
+                        stopped = true;
+                        break;
+                    }
+                }
+                // Read type name (inline). Must start A-Z; a bad start byte with
+                // buffer left clears declOpen for the same reason as the '=' check.
                 const typeStart = pos;
-                if (pos >= len || buf[pos] < 0x41 || buf[pos] > 0x5A)
-                    continue; // Must start A-Z
+                if (pos >= len)
+                    continue;
+                if (buf[pos] < 0x41 || buf[pos] > 0x5A) {
+                    declOpen = false;
+                    continue;
+                }
                 while (pos < len) {
                     const c = buf[pos];
                     if ((c >= 0x41 && c <= 0x5A) || (c >= 0x61 && c <= 0x7A) ||
@@ -218,7 +237,7 @@ export class StepTokenizer {
                 // Skip whitespace
                 while (pos < len) {
                     const c = buf[pos];
-                    if (c === 0x20 || c === 0x09 || c === 0x0D) {
+                    if (c === 0x20 || c === 0x09 || c === 0x0D || c === 0x0C || c === 0x0B) {
                         pos++;
                     }
                     else if (c === NEWLINE) {
@@ -228,11 +247,26 @@ export class StepTokenizer {
                     else
                         break;
                 }
-                // Check for '('
-                if (pos >= len || buf[pos] !== LPAREN)
+                if (opensComment(buf, pos, len)) {
+                    const t = skipTrivia(buf, pos, len);
+                    line += t.lines;
+                    pos = t.next;
+                    if (t.stop) {
+                        stopped = true;
+                        break;
+                    }
+                }
+                // Check for '('. Same EOF-vs-mismatch split as '=' and the type name.
+                if (pos >= len)
                     continue;
+                if (buf[pos] !== LPAREN) {
+                    declOpen = false;
+                    continue;
+                }
+                declOpen = false; // Header complete: '(' found.
                 // FAST: Skip to semicolon (handling strings)
                 let inString = false;
+                let foundTerminator = false;
                 while (pos < len) {
                     const c = buf[pos];
                     if (c === QUOTE) {
@@ -242,11 +276,29 @@ export class StepTokenizer {
                         }
                         inString = !inString;
                     }
+                    else if (c === SLASH && !inString && opensComment(buf, pos, len)) {
+                        // The ';' that ends a record can be preceded by a comment holding
+                        // its own ';'. Take the comment whole -- which also makes the
+                        // quotes and parens inside it text, the other half of the rule the
+                        // literal skip above provides in the opposite direction.
+                        const end = skipComment(buf, pos, len);
+                        if (end < 0) {
+                            // Unterminated: this record has no terminator, and neither has
+                            // anything after it. Drop it and stop, which is the None Rust's
+                            // find_entity_end returns on the same input.
+                            pos = len;
+                            break;
+                        }
+                        line += countNewlines(buf, pos, end);
+                        pos = end;
+                        continue;
+                    }
                     else if (c === SEMICOLON && !inString) {
                         // Found end of entity
                         const entityLength = pos - startOffset + 1; // Include semicolon
                         yield { expressId, type, offset: startOffset, length: entityLength, line: startLine };
                         pos++;
+                        foundTerminator = true;
                         break;
                     }
                     else if (c === NEWLINE) {
@@ -254,6 +306,14 @@ export class StepTokenizer {
                     }
                     pos++;
                 }
+                // Ran off the end without an unquoted ';' — usually an unescaped `'`
+                // left open, or the unterminated-comment break above; `pos` is
+                // already `len`, ending the scan here. Not resynced: with no known
+                // terminator, guessing a resume point risks fabricating entities from
+                // misaligned bytes. Recorded in `stopped`, not incremented here --
+                // see the post-loop check below.
+                if (!foundTerminator)
+                    stopped = true;
             }
             else if (char === NEWLINE) {
                 line++;
@@ -268,120 +328,21 @@ export class StepTokenizer {
                 line += skip.lines;
                 pos = skip.next;
                 if (skip.stop) {
-                    this.position = len;
-                    this.lineNumber = line;
-                    return;
-                }
-            }
-            else {
-                pos++;
-            }
-        }
-        this.position = pos;
-        this.lineNumber = line;
-    }
-    readExpressId() {
-        let id = 0;
-        let digits = 0;
-        let pos = this.position + 1; // Skip '#'
-        while (pos < this.buffer.length) {
-            const char = this.buffer[pos];
-            if (char >= 0x30 && char <= 0x39) { // '0'-'9'
-                id = id * 10 + (char - 0x30);
-                digits++;
-                pos++;
-            }
-            else {
-                break;
-            }
-        }
-        if (digits === 0)
-            return null;
-        this.position = pos;
-        return id;
-    }
-    readTypeName() {
-        let start = this.position;
-        let end = start;
-        // Type names start with uppercase letter
-        if (this.position >= this.buffer.length || this.buffer[this.position] < 0x41 || this.buffer[this.position] > 0x5A) {
-            return null;
-        }
-        while (end < this.buffer.length) {
-            const char = this.buffer[end];
-            // Allow letters, numbers, and underscore
-            if ((char >= 0x41 && char <= 0x5A) || // A-Z
-                (char >= 0x61 && char <= 0x7A) || // a-z
-                (char >= 0x30 && char <= 0x39) || // 0-9
-                char === 0x5F // _
-            ) {
-                end++;
-            }
-            else {
-                break;
-            }
-        }
-        if (end === start)
-            return null;
-        const typeName = safeUtf8Decode(this.buffer, start, end);
-        this.position = end;
-        return typeName;
-    }
-    skipWhitespace() {
-        while (this.position < this.buffer.length) {
-            const char = this.buffer[this.position];
-            if (char === 0x20 || char === 0x09 || char === 0x0D || char === 0x0A) { // space, tab, CR, LF
-                if (char === 0x0A)
-                    this.lineNumber++;
-                this.position++;
-            }
-            else {
-                break;
-            }
-        }
-    }
-    findEntityLength(startOffset) {
-        let pos = this.position;
-        let depth = 0;
-        let inString = false;
-        while (pos < this.buffer.length) {
-            const char = this.buffer[pos];
-            if (char === 0x27) { // Single quote (string delimiter)
-                if (inString) {
-                    // Check for escaped quote ('') - STEP uses doubled quotes
-                    if (pos + 1 < this.buffer.length && this.buffer[pos + 1] === 0x27) {
-                        pos += 2; // Skip escaped quote
-                        continue;
-                    }
-                    inString = false;
-                }
-                else {
-                    inString = true;
-                }
-                pos++;
-                continue;
-            }
-            if (inString) {
-                pos++;
-                continue;
-            }
-            if (char === 0x28) { // '('
-                depth++;
-                pos++;
-            }
-            else if (char === 0x29) { // ')'
-                depth--;
-                pos++;
-                if (depth === 0) {
-                    // Found matching closing parenthesis
-                    return pos - startOffset;
+                    stopped = true;
+                    break;
                 }
             }
             else {
                 pos++;
             }
         }
-        return 0; // No matching closing parenthesis found
+        // ONE post-loop check, not an increment at every exit site above: the
+        // scan stopped early if it hit an explicit "no terminator" boundary
+        // (`stopped`), or the last `#id=TYPE(` header was cut short before its
+        // '(' was found (`declOpen`). Always 0 or 1 -- the scan stops at the
+        // first one, so there is nothing further to accumulate.
+        if (stopped || declOpen)
+            this.malformedRecords = 1;
     }
 }
 //# sourceMappingURL=tokenizer.js.map