@ifc-lite/parser 4.3.2 → 5.1.0

package/dist/tokenizer.js

@@ -5,97 +5,44 @@
  * STEP tokenizer - fast byte-level scanning for entity markers
  * Leverages Spike 1 approach: ~1,259 MB/s throughput
  */
-import { safeUtf8Decode } from '@ifc-lite/data';
-import { countNewlines, opensLiteralOrComment, skipLexical } from './step-lexing.js';
+import { isIndexableExpressId } from './express-id.js';
+import { BalancedEntityScan } from './scan-entities-balanced.js';
+import { countNewlines, opensComment, opensLiteralOrComment, skipComment, skipLexical, skipTrivia, } from './step-lexing.js';
 export class StepTokenizer {
     buffer;
-    position = 0;
-    lineNumber = 1;
+    oversizedIds = 0;
+    malformedRecords = 0;
     constructor(buffer) {
         this.buffer = buffer;
     }
+    /** Records the last scan refused for an out-of-contract express id
+     *  (express-id.ts, #3395). Reset per scan; the caller reports it. */
+    get oversizedIdCount() { return this.oversizedIds; }
+    /** 0 or 1: whether the last `scanEntitiesFast`/`scanEntities` run stopped
+     *  early on an unclosed `'` string, an unclosed block comment, or a
+     *  declaration cut off before its own '(' -- never a count of how many,
+     *  since the scan has no reliable way to resume past the first one it
+     *  hits. Reset at the start of every scan; the caller reports it. */
+    get malformedRecordCount() { return this.malformedRecords; }
     /**
      * Scan for all entity declarations (#EXPRESS_ID = TYPE(...))
-     * Returns entity references without parsing full content
+     * Returns entity references without parsing full content.
+     *
+     * Closes each record on the ')' balancing its argument list. The scan itself
+     * lives in `scan-entities-balanced.ts`; only the refusal count comes back
+     * here, and it comes back in a `finally` so an abandoned generator still
+     * reports what it refused.
      */
     *scanEntities() {
-        this.position = 0;
-        this.lineNumber = 1;
-        while (this.position < this.buffer.length) {
-            // Look for '#' character (entity ID marker)
-            if (this.buffer[this.position] === 0x23) { // '#'
-                const startOffset = this.position;
-                const startLine = this.lineNumber;
-                // Read express ID
-                const expressId = this.readExpressId();
-                if (expressId === null) {
-                    this.position++;
-                    continue;
-                }
-                // Skip whitespace
-                this.skipWhitespace();
-                // Check for '=' (assignment)
-                if (this.position >= this.buffer.length || this.buffer[this.position] !== 0x3D) {
-                    this.position++;
-                    continue;
-                }
-                this.position++; // Skip '='
-                // Skip whitespace
-                this.skipWhitespace();
-                // Read type name
-                const type = this.readTypeName();
-                if (!type) {
-                    this.position++;
-                    continue;
-                }
-                // Skip whitespace
-                this.skipWhitespace();
-                // Check for '(' (start of parameters)
-                if (this.position >= this.buffer.length || this.buffer[this.position] !== 0x28) {
-                    this.position++;
-                    continue;
-                }
-                // Find matching closing parenthesis to get full entity length
-                const entityLength = this.findEntityLength(startOffset);
-                if (entityLength > 0) {
-                    // Step past the whole record, as Rust's next_entity does. Leaving
-                    // `position` at the '(' made this loop re-walk the body, which was
-                    // harmless only while it ignored quotes and comments.
-                    //
-                    // Count from `position`, not from `startOffset`: `position` is on the
-                    // '(' here, and every newline before it was already counted by the
-                    // three skipWhitespace calls above. Counting the whole record instead
-                    // double-counts a newline written between `#1=` and its type name,
-                    // which is ordinary whitespace and legal.
-                    this.lineNumber += countNewlines(this.buffer, this.position, startOffset + entityLength);
-                    this.position = startOffset + entityLength;
-                    yield {
-                        expressId,
-                        type,
-                        offset: startOffset,
-                        length: entityLength,
-                        line: startLine,
-                    };
-                }
-            }
-            else if (this.buffer[this.position] === 0x0A) {
-                // Newline
-                this.lineNumber++;
-                this.position++;
-            }
-            else if (opensLiteralOrComment(this.buffer, this.position, this.buffer.length)) {
-                // A commented-out record satisfies every check above, so a comment has
-                // to be skipped as a region; a literal has to be skipped so its
-                // contents cannot look like one. See step-lexing.
-                const skip = skipLexical(this.buffer, this.position, this.buffer.length);
-                this.lineNumber += skip.lines;
-                this.position = skip.next;
-                if (skip.stop)
-                    return;
-            }
-            else {
-                this.position++;
-            }
+        const scan = new BalancedEntityScan(this.buffer);
+        this.oversizedIds = 0;
+        this.malformedRecords = 0;
+        try {
+            yield* scan.run();
+        }
+        finally {
+            this.oversizedIds = scan.oversizedIdCount;
+            this.malformedRecords = scan.malformedRecordCount;
         }
     }
     /**
@@ -103,8 +50,8 @@ export class StepTokenizer {
      * ~5-10x faster for large files, yields length=0 (calculate on-demand)
      */
     *scanEntitiesFast() {
-        this.position = 0;
-        this.lineNumber = 1;
+        this.oversizedIds = 0;
+        this.malformedRecords = 0;
         // Pre-compute common byte codes
         const HASH = 0x23; // '#'
         const EQUALS = 0x3D; // '='
@@ -112,6 +59,7 @@ export class StepTokenizer {
         const SEMICOLON = 0x3B; // ';'
         const QUOTE = 0x27; // '\''
         const NEWLINE = 0x0A; // '\n'
+        const SLASH = 0x2F; // '/'
         const buf = this.buffer;
         const len = buf.length;
         let pos = 0;
@@ -119,6 +67,22 @@ export class StepTokenizer {
         // Cache type name strings: IFC files have ~776 unique types repeated
         // across 8M+ entities. Caching avoids millions of String.fromCharCode allocations.
         const typeCache = new Map();
+        // Set on the way to the single post-loop check at the bottom of this
+        // function, not counted at each site: `stopped` for an unclosed string or
+        // comment that ran the scan to end of buffer with nothing left to find,
+        // `declOpen` while a `#id=TYPE(` header is incomplete. `declOpen` stays
+        // armed ONLY when the reason for abandoning is running out of buffer
+        // (`pos >= len`); a mismatch with buffer still left (bad byte, oversized
+        // id) clears it, because the scan resumes byte-by-byte from wherever it
+        // gave up, and a `#ref` token inside the abandoned record's own argument
+        // list reads as a fresh, equally incomplete attempt -- one that must not
+        // report "cut off" just because nothing later happens to clear it.
+        // Per-site increments used to miss whole shapes -- a leading unterminated
+        // comment before '=', or a declaration cut off before its own '(' --
+        // because each site only knew about its own exit, never the scan's final
+        // state.
+        let stopped = false;
+        let declOpen = false;
         while (pos < len) {
             const char = buf[pos];
             if (char === HASH) {
@@ -141,10 +105,15 @@ export class StepTokenizer {
                 }
                 if (!hasDigits)
                     continue;
-                // Skip whitespace (inline)
+                declOpen = true;
+                // Skip whitespace (inline). Kept byte-for-byte in sync with
+                // `isSpaceByte` in step-lexing.ts (space, tab, CR, LF, form feed,
+                // vertical tab) -- this loop, its two twins below in this method, and
+                // the worker's copy in scan-worker-source.ts are the same rule
+                // hand-duplicated for speed, not four independent decisions.
                 while (pos < len) {
                     const c = buf[pos];
-                    if (c === 0x20 || c === 0x09 || c === 0x0D) {
+                    if (c === 0x20 || c === 0x09 || c === 0x0D || c === 0x0C || c === 0x0B) {
                         pos++;
                     }
                     else if (c === NEWLINE) {
@@ -154,14 +123,50 @@ export class StepTokenizer {
                     else
                         break;
                 }
-                // Check for '='
-                if (pos >= len || buf[pos] !== EQUALS)
+                // 10303-21 allows a comment wherever whitespace is allowed, so
+                // `#1 /* was #7 */ =` is a declaration. The inline loop above stays
+                // for the common case; this runs only once a comment actually opens,
+                // and skipTrivia (step-lexing) then takes the whole run of both.
+                if (opensComment(buf, pos, len)) {
+                    const t = skipTrivia(buf, pos, len);
+                    line += t.lines;
+                    pos = t.next;
+                    if (t.stop) {
+                        stopped = true;
+                        break;
+                    }
+                }
+                // Check for '='. A byte that is not '=' with buffer left to scan is
+                // not a truncation -- clear declOpen so a reference token inside a
+                // LATER abandoned record's argument list (see the oversized-id note
+                // below) cannot leave it stuck armed with nothing left to clear it.
+                if (pos >= len)
+                    continue;
+                if (buf[pos] !== EQUALS) {
+                    declOpen = false;
                     continue;
+                }
                 pos++;
+                // Storage contract, not just overflow: see express-id.ts (#3395).
+                // Tested only now that `#<digits>[ws]*=` has matched, which is the
+                // DECLARATION shape Rust's `EntityScanner` validates before it
+                // refuses. Refusing above the '=' check counted references too: the
+                // `continue` resumes inside the refused record's argument list
+                // (unlike the accepted path, which skips to the terminating ';'), so
+                // `#4294967297=IFCWALL(#4294967298,#4294967299,...)` reported three
+                // skipped records for the one record actually dropped. A count that
+                // overstates is the same class of defect as one that undercounts.
+                // Refused for being out of range, not for running out of buffer, so
+                // it does not belong to `declOpen`'s "cut off by EOF" story either.
+                if (!isIndexableExpressId(expressId)) {
+                    this.oversizedIds++;
+                    declOpen = false;
+                    continue;
+                }
                 // Skip whitespace
                 while (pos < len) {
                     const c = buf[pos];
-                    if (c === 0x20 || c === 0x09 || c === 0x0D) {
+                    if (c === 0x20 || c === 0x09 || c === 0x0D || c === 0x0C || c === 0x0B) {
                         pos++;
                     }
                     else if (c === NEWLINE) {
@@ -171,10 +176,24 @@ export class StepTokenizer {
                     else
                         break;
                 }
-                // Read type name (inline)
+                if (opensComment(buf, pos, len)) {
+                    const t = skipTrivia(buf, pos, len);
+                    line += t.lines;
+                    pos = t.next;
+                    if (t.stop) {
+                        stopped = true;
+                        break;
+                    }
+                }
+                // Read type name (inline). Must start A-Z; a bad start byte with
+                // buffer left clears declOpen for the same reason as the '=' check.
                 const typeStart = pos;
-                if (pos >= len || buf[pos] < 0x41 || buf[pos] > 0x5A)
-                    continue; // Must start A-Z
+                if (pos >= len)
+                    continue;
+                if (buf[pos] < 0x41 || buf[pos] > 0x5A) {
+                    declOpen = false;
+                    continue;
+                }
                 while (pos < len) {
                     const c = buf[pos];
                     if ((c >= 0x41 && c <= 0x5A) || (c >= 0x61 && c <= 0x7A) ||
@@ -218,7 +237,7 @@ export class StepTokenizer {
                 // Skip whitespace
                 while (pos < len) {
                     const c = buf[pos];
-                    if (c === 0x20 || c === 0x09 || c === 0x0D) {
+                    if (c === 0x20 || c === 0x09 || c === 0x0D || c === 0x0C || c === 0x0B) {
                         pos++;
                     }
                     else if (c === NEWLINE) {
@@ -228,11 +247,26 @@ export class StepTokenizer {
                     else
                         break;
                 }
-                // Check for '('
-                if (pos >= len || buf[pos] !== LPAREN)
+                if (opensComment(buf, pos, len)) {
+                    const t = skipTrivia(buf, pos, len);
+                    line += t.lines;
+                    pos = t.next;
+                    if (t.stop) {
+                        stopped = true;
+                        break;
+                    }
+                }
+                // Check for '('. Same EOF-vs-mismatch split as '=' and the type name.
+                if (pos >= len)
                     continue;
+                if (buf[pos] !== LPAREN) {
+                    declOpen = false;
+                    continue;
+                }
+                declOpen = false; // Header complete: '(' found.
                 // FAST: Skip to semicolon (handling strings)
                 let inString = false;
+                let foundTerminator = false;
                 while (pos < len) {
                     const c = buf[pos];
                     if (c === QUOTE) {
@@ -242,11 +276,29 @@ export class StepTokenizer {
                         }
                         inString = !inString;
                     }
+                    else if (c === SLASH && !inString && opensComment(buf, pos, len)) {
+                        // The ';' that ends a record can be preceded by a comment holding
+                        // its own ';'. Take the comment whole -- which also makes the
+                        // quotes and parens inside it text, the other half of the rule the
+                        // literal skip above provides in the opposite direction.
+                        const end = skipComment(buf, pos, len);
+                        if (end < 0) {
+                            // Unterminated: this record has no terminator, and neither has
+                            // anything after it. Drop it and stop, which is the None Rust's
+                            // find_entity_end returns on the same input.
+                            pos = len;
+                            break;
+                        }
+                        line += countNewlines(buf, pos, end);
+                        pos = end;
+                        continue;
+                    }
                     else if (c === SEMICOLON && !inString) {
                         // Found end of entity
                         const entityLength = pos - startOffset + 1; // Include semicolon
                         yield { expressId, type, offset: startOffset, length: entityLength, line: startLine };
                         pos++;
+                        foundTerminator = true;
                         break;
                     }
                     else if (c === NEWLINE) {
@@ -254,6 +306,14 @@ export class StepTokenizer {
                     }
                     pos++;
                 }
+                // Ran off the end without an unquoted ';' — usually an unescaped `'`
+                // left open, or the unterminated-comment break above; `pos` is
+                // already `len`, ending the scan here. Not resynced: with no known
+                // terminator, guessing a resume point risks fabricating entities from
+                // misaligned bytes. Recorded in `stopped`, not incremented here --
+                // see the post-loop check below.
+                if (!foundTerminator)
+                    stopped = true;
             }
             else if (char === NEWLINE) {
                 line++;
@@ -268,120 +328,21 @@ export class StepTokenizer {
                 line += skip.lines;
                 pos = skip.next;
                 if (skip.stop) {
-                    this.position = len;
-                    this.lineNumber = line;
-                    return;
-                }
-            }
-            else {
-                pos++;
-            }
-        }
-        this.position = pos;
-        this.lineNumber = line;
-    }
-    readExpressId() {
-        let id = 0;
-        let digits = 0;
-        let pos = this.position + 1; // Skip '#'
-        while (pos < this.buffer.length) {
-            const char = this.buffer[pos];
-            if (char >= 0x30 && char <= 0x39) { // '0'-'9'
-                id = id * 10 + (char - 0x30);
-                digits++;
-                pos++;
-            }
-            else {
-                break;
-            }
-        }
-        if (digits === 0)
-            return null;
-        this.position = pos;
-        return id;
-    }
-    readTypeName() {
-        let start = this.position;
-        let end = start;
-        // Type names start with uppercase letter
-        if (this.position >= this.buffer.length || this.buffer[this.position] < 0x41 || this.buffer[this.position] > 0x5A) {
-            return null;
-        }
-        while (end < this.buffer.length) {
-            const char = this.buffer[end];
-            // Allow letters, numbers, and underscore
-            if ((char >= 0x41 && char <= 0x5A) || // A-Z
-                (char >= 0x61 && char <= 0x7A) || // a-z
-                (char >= 0x30 && char <= 0x39) || // 0-9
-                char === 0x5F // _
-            ) {
-                end++;
-            }
-            else {
-                break;
-            }
-        }
-        if (end === start)
-            return null;
-        const typeName = safeUtf8Decode(this.buffer, start, end);
-        this.position = end;
-        return typeName;
-    }
-    skipWhitespace() {
-        while (this.position < this.buffer.length) {
-            const char = this.buffer[this.position];
-            if (char === 0x20 || char === 0x09 || char === 0x0D || char === 0x0A) { // space, tab, CR, LF
-                if (char === 0x0A)
-                    this.lineNumber++;
-                this.position++;
-            }
-            else {
-                break;
-            }
-        }
-    }
-    findEntityLength(startOffset) {
-        let pos = this.position;
-        let depth = 0;
-        let inString = false;
-        while (pos < this.buffer.length) {
-            const char = this.buffer[pos];
-            if (char === 0x27) { // Single quote (string delimiter)
-                if (inString) {
-                    // Check for escaped quote ('') - STEP uses doubled quotes
-                    if (pos + 1 < this.buffer.length && this.buffer[pos + 1] === 0x27) {
-                        pos += 2; // Skip escaped quote
-                        continue;
-                    }
-                    inString = false;
-                }
-                else {
-                    inString = true;
-                }
-                pos++;
-                continue;
-            }
-            if (inString) {
-                pos++;
-                continue;
-            }
-            if (char === 0x28) { // '('
-                depth++;
-                pos++;
-            }
-            else if (char === 0x29) { // ')'
-                depth--;
-                pos++;
-                if (depth === 0) {
-                    // Found matching closing parenthesis
-                    return pos - startOffset;
+                    stopped = true;
+                    break;
                 }
             }
             else {
                 pos++;
             }
         }
-        return 0; // No matching closing parenthesis found
+        // ONE post-loop check, not an increment at every exit site above: the
+        // scan stopped early if it hit an explicit "no terminator" boundary
+        // (`stopped`), or the last `#id=TYPE(` header was cut short before its
+        // '(' was found (`declOpen`). Always 0 or 1 -- the scan stops at the
+        // first one, so there is nothing further to accumulate.
+        if (stopped || declOpen)
+            this.malformedRecords = 1;
     }
 }
 //# sourceMappingURL=tokenizer.js.map

package/dist/tokenizer.js.map

@@ -1 +1 @@
-{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAAA;;+DAE+D;AAE/D;;;GAGG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAEhD,OAAO,EAAE,aAAa,EAAE,qBAAqB,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAErF,MAAM,OAAO,aAAa;IAChB,MAAM,CAAa;IACnB,QAAQ,GAAW,CAAC,CAAC;IACrB,UAAU,GAAW,CAAC,CAAC;IAE/B,YAAY,MAAkB;QAC5B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED;;;OAGG;IACH,CAAC,YAAY;QACX,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;QAClB,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC;QAEpB,OAAO,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YAC1C,4CAA4C;YAC5C,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM;gBAC/C,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC;gBAClC,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC;gBAElC,kBAAkB;gBAClB,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;gBACvC,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;oBACvB,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAChB,SAAS;gBACX,CAAC;gBAED,kBAAkB;gBAClB,IAAI,CAAC,cAAc,EAAE,CAAC;gBAEtB,6BAA6B;gBAC7B,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC;oBAC/E,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAChB,SAAS;gBACX,CAAC;gBACD,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,WAAW;gBAE5B,kBAAkB;gBAClB,IAAI,CAAC,cAAc,EAAE,CAAC;gBAEtB,iBAAiB;gBACjB,MAAM,IAAI,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;gBACjC,IAAI,CAAC,IAAI,EAAE,CAAC;oBACV,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAChB,SAAS;gBACX,CAAC;gBAED,kBAAkB;gBAClB,IAAI,CAAC,cAAc,EAAE,CAAC;gBAEtB,sCAAsC;gBACtC,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC;oBAC/E,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAChB,SAAS;gBACX,CAAC;gBAED,8DAA8D;gBAC9D,MAAM,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;gBACxD,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;oBACrB,kEAAkE;oBAClE,mEAAmE;oBACnE,sDAAsD;oBACtD,EAAE;oBACF,sEAAsE;oBACtE,mEAAmE;oBACnE,sEAAsE;oBACtE,mEAAmE;oBACnE,0CAA0C;oBAC1C,IAAI,CAAC,UAAU,IAAI,aAAa,CAC9B,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,QAAQ,EACb,WAAW,GAAG,YAAY,CAC3B,CAAC;oBACF,IAAI,CAAC,QAAQ,GAAG,WAAW,GAAG,YAAY,CAAC;oBAC3C,MAAM;wBACJ,SAAS;wBACT,IAAI;wBACJ,MAAM,EAAE,WAAW;wBACnB,MAAM,EAAE,YAAY;wBACpB,IAAI,EAAE,SAAS;qBAChB,CAAC;gBACJ,CAAC;YACH,CAAC;iBAAM,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC;gBAC/C,UAAU;gBACV,IAAI,CAAC,UAAU,EAAE,CAAC;gBAClB,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,CAAC;iBAAM,IAAI,qBAAqB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;gBACjF,uEAAuE;gBACvE,gEAAgE;gBAChE,kDAAkD;gBAClD,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBACzE,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,KAAK,CAAC;gBAC9B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC;gBAC1B,IAAI,IAAI,CAAC,IAAI;oBAAE,OAAO;YACxB,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,CAAC,gBAAgB;QACf,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;QAClB,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC;QAEpB,gCAAgC;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAM,MAAM;QAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,CAAI,MAAM;QAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,CAAI,MAAM;QAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,MAAM;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,CAAK,OAAO;QAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,CAAG,OAAO;QAE/B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC;QACxB,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC;QACvB,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,IAAI,IAAI,GAAG,CAAC,CAAC;QAEb,qEAAqE;QACrE,mFAAmF;QACnF,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;QAE5C,OAAO,GAAG,GAAG,GAAG,EAAE,CAAC;YACjB,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;YAEtB,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;gBAClB,MAAM,WAAW,GAAG,GAAG,CAAC;gBACxB,MAAM,SAAS,GAAG,IAAI,CAAC;gBACvB,GAAG,EAAE,CAAC,CAAC,WAAW;gBAElB,qCAAqC;gBACrC,IAAI,SAAS,GAAG,CAAC,CAAC;gBAClB,IAAI,SAAS,GAAG,KAAK,CAAC;gBACtB,OAAO,GAAG,GAAG,GAAG,EAAE,CAAC;oBACjB,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;oBACnB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,UAAU;wBACtC,SAAS,GAAG,SAAS,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;wBACxC,SAAS,GAAG,IAAI,CAAC;wBACjB,GAAG,EAAE,CAAC;oBACR,CAAC;yBAAM,CAAC;wBACN,MAAM;oBACR,CAAC;gBACH,CAAC;gBAED,IAAI,CAAC,SAAS;oBAAE,SAAS;gBAEzB,2BAA2B;gBAC3B,OAAO,GAAG,GAAG,GAAG,EAAE,CAAC;oBACjB,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;oBACnB,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;wBAAC,GAAG,EAAE,CAAC;oBAAC,CAAC;yBACjD,IAAI,CAAC,KAAK,OAAO,EAAE,CAAC;wBAAC,IAAI,EAAE,CAAC;wBAAC,GAAG,EAAE,CAAC;oBAAC,CAAC;;wBACrC,MAAM;gBACb,CAAC;gBAED,gBAAgB;gBAChB,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,MAAM;oBAAE,SAAS;gBAChD,GAAG,EAAE,CAAC;gBAEN,kBAAkB;gBAClB,OAAO,GAAG,GAAG,GAAG,EAAE,CAAC;oBACjB,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;oBACnB,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;wBAAC,GAAG,EAAE,CAAC;oBAAC,CAAC;yBACjD,IAAI,CAAC,KAAK,OAAO,EAAE,CAAC;wBAAC,IAAI,EAAE,CAAC;wBAAC,GAAG,EAAE,CAAC;oBAAC,CAAC;;wBACrC,MAAM;gBACb,CAAC;gBAED,0BAA0B;gBAC1B,MAAM,SAAS,GAAG,GAAG,CAAC;gBACtB,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI;oBAAE,SAAS,CAAC,iBAAiB;gBAEjF,OAAO,GAAG,GAAG,GAAG,EAAE,CAAC;oBACjB,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;oBACnB,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC;wBACpD,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;wBAC3C,GAAG,EAAE,CAAC;oBACR,CAAC;yBAAM,CAAC;wBACN,MAAM;oBACR,CAAC;gBACH,CAAC;gBAED,IAAI,GAAG,KAAK,SAAS;oBAAE,SAAS;gBAEhC,mFAAmF;gBACnF,mFAAmF;gBACnF,iFAAiF;gBACjF,mFAAmF;gBACnF,gFAAgF;gBAChF,MAAM,OAAO,GAAG,GAAG,GAAG,SAAS,CAAC;gBAChC,IAAI,QAAQ,GAAG,OAAO,CAAC;gBACvB,KAAK,IAAI,CAAC,GAAG,SAAS,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;oBACrC,QAAQ,GAAG,CAAC,QAAQ,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBAC1C,CAAC;gBACD,MAAM,QAAQ,GAAG,GAAG,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC1C,IAAI,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACnC,IAAI,eAAe,GAAG,KAAK,CAAC;gBAC5B,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;oBAClD,eAAe,GAAG,IAAI,CAAC;oBACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;wBACjC,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,SAAS,GAAG,CAAC,CAAC,EAAE,CAAC;4BAC9C,eAAe,GAAG,KAAK,CAAC;4BACxB,MAAM;wBACR,CAAC;oBACH,CAAC;gBACH,CAAC;gBACD,qEAAqE;gBACrE,mEAAmE;gBACnE,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,eAAe,EAAE,CAAC;oBAC3C,IAAI,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC;oBAC5D,SAAS,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;gBAChC,CAAC;gBAED,kBAAkB;gBAClB,OAAO,GAAG,GAAG,GAAG,EAAE,CAAC;oBACjB,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;oBACnB,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;wBAAC,GAAG,EAAE,CAAC;oBAAC,CAAC;yBACjD,IAAI,CAAC,KAAK,OAAO,EAAE,CAAC;wBAAC,IAAI,EAAE,CAAC;wBAAC,GAAG,EAAE,CAAC;oBAAC,CAAC;;wBACrC,MAAM;gBACb,CAAC;gBAED,gBAAgB;gBAChB,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,MAAM;oBAAE,SAAS;gBAEhD,6CAA6C;gBAC7C,IAAI,QAAQ,GAAG,KAAK,CAAC;gBACrB,OAAO,GAAG,GAAG,GAAG,EAAE,CAAC;oBACjB,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;oBACnB,IAAI,CAAC,KAAK,KAAK,EAAE,CAAC;wBAChB,IAAI,QAAQ,IAAI,GAAG,GAAG,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,KAAK,EAAE,CAAC;4BACxD,GAAG,IAAI,CAAC,CAAC,CAAC,qBAAqB;4BAC/B,SAAS;wBACX,CAAC;wBACD,QAAQ,GAAG,CAAC,QAAQ,CAAC;oBACvB,CAAC;yBAAM,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,QAAQ,EAAE,CAAC;wBACxC,sBAAsB;wBACtB,MAAM,YAAY,GAAG,GAAG,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,oBAAoB;wBAChE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;wBACtF,GAAG,EAAE,CAAC;wBACN,MAAM;oBACR,CAAC;yBAAM,IAAI,CAAC,KAAK,OAAO,EAAE,CAAC;wBACzB,IAAI,EAAE,CAAC;oBACT,CAAC;oBACD,GAAG,EAAE,CAAC;gBACR,CAAC;YACH,CAAC;iBAAM,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;gBAC5B,IAAI,EAAE,CAAC;gBACP,GAAG,EAAE,CAAC;YACR,CAAC;iBAAM,IAAI,qBAAqB,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;gBAChD,mEAAmE;gBACnE,mEAAmE;gBACnE,sDAAsD;gBACtD,kEAAkE;gBAClE,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;gBACxC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC;gBACnB,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC;gBAChB,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;oBACd,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC;oBACpB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;oBACvB,OAAO;gBACT,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,GAAG,EAAE,CAAC;YACR,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC;QACpB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;IACzB,CAAC;IAEO,aAAa;QACnB,IAAI,EAAE,GAAG,CAAC,CAAC;QACX,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,GAAG,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,WAAW;QAExC,OAAO,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC9B,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC,UAAU;gBAC5C,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;gBAC7B,MAAM,EAAE,CAAC;gBACT,GAAG,EAAE,CAAC;YACR,CAAC;iBAAM,CAAC;gBACN,MAAM;YACR,CAAC;QACH,CAAC;QAED,IAAI,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAC9B,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC;QACpB,OAAO,EAAE,CAAC;IACZ,CAAC;IAEO,YAAY;QAClB,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC1B,IAAI,GAAG,GAAG,KAAK,CAAC;QAEhB,yCAAyC;QACzC,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,IAAI,EAAE,CAAC;YAClH,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC9B,yCAAyC;YACzC,IACE,CAAC,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,MAAM;gBACxC,CAAC,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,MAAM;gBACxC,CAAC,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,MAAM;gBACxC,IAAI,KAAK,IAAI,CAAC,IAAI;cAClB,CAAC;gBACD,GAAG,EAAE,CAAC;YACR,CAAC;iBAAM,CAAC;gBACN,MAAM;YACR,CAAC;QACH,CAAC;QAED,IAAI,GAAG,KAAK,KAAK;YAAE,OAAO,IAAI,CAAC;QAE/B,MAAM,QAAQ,GAAG,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;QACzD,IAAI,CAAC,QAAQ,GAAG,GAAG,CAAC;QACpB,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,cAAc;QACpB,OAAO,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YAC1C,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACxC,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC,CAAC,qBAAqB;gBAC3F,IAAI,IAAI,KAAK,IAAI;oBAAE,IAAI,CAAC,UAAU,EAAE,CAAC;gBACrC,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,CAAC;iBAAM,CAAC;gBACN,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAEO,gBAAgB,CAAC,WAAmB;QAC1C,IAAI,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC;QACxB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,OAAO,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAE9B,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC,CAAC,kCAAkC;gBACrD,IAAI,QAAQ,EAAE,CAAC;oBACb,0DAA0D;oBAC1D,IAAI,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;wBAClE,GAAG,IAAI,CAAC,CAAC,CAAC,qBAAqB;wBAC/B,SAAS;oBACX,CAAC;oBACD,QAAQ,GAAG,KAAK,CAAC;gBACnB,CAAC;qBAAM,CAAC;oBACN,QAAQ,GAAG,IAAI,CAAC;gBAClB,CAAC;gBACD,GAAG,EAAE,CAAC;gBACN,SAAS;YACX,CAAC;YAED,IAAI,QAAQ,EAAE,CAAC;gBACb,GAAG,EAAE,CAAC;gBACN,SAAS;YACX,CAAC;YAED,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM;gBACzB,KAAK,EAAE,CAAC;gBACR,GAAG,EAAE,CAAC;YACR,CAAC;iBAAM,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM;gBAChC,KAAK,EAAE,CAAC;gBACR,GAAG,EAAE,CAAC;gBACN,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;oBAChB,qCAAqC;oBACrC,OAAO,GAAG,GAAG,WAAW,CAAC;gBAC3B,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,GAAG,EAAE,CAAC;YACR,CAAC;QACH,CAAC;QAED,OAAO,CAAC,CAAC,CAAC,wCAAwC;IACpD,CAAC;CACF"}
+{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAAA;;+DAE+D;AAE/D;;;GAGG;AAEH,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,kBAAkB,EAAyB,MAAM,6BAA6B,CAAC;AACxF,OAAO,EACL,aAAa,EACb,YAAY,EACZ,qBAAqB,EACrB,WAAW,EACX,WAAW,EACX,UAAU,GACX,MAAM,kBAAkB,CAAC;AAE1B,MAAM,OAAO,aAAa;IAChB,MAAM,CAAa;IACnB,YAAY,GAAW,CAAC,CAAC;IACzB,gBAAgB,GAAW,CAAC,CAAC;IAErC,YAAY,MAAkB;QAC5B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED;yEACqE;IACrE,IAAI,gBAAgB,KAAa,OAAO,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;IAE5D;;;;yEAIqE;IACrE,IAAI,oBAAoB,KAAa,OAAO,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC;IAEpE;;;;;;;;OAQG;IACH,CAAC,YAAY;QACX,MAAM,IAAI,GAAG,IAAI,kBAAkB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACjD,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;QACtB,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC;QAC1B,IAAI,CAAC;YACH,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;QACpB,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC;YAC1C,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,oBAAoB,CAAC;QACpD,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,CAAC,gBAAgB;QACf,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;QACtB,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC;QAE1B,gCAAgC;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAM,MAAM;QAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,CAAI,MAAM;QAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,CAAI,MAAM;QAC9B,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,MAAM;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,CAAK,OAAO;QAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,CAAG,OAAO;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,CAAK,MAAM;QAE9B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC;QACxB,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC;QACvB,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,IAAI,IAAI,GAAG,CAAC,CAAC;QAEb,qEAAqE;QACrE,mFAAmF;QACnF,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;QAE5C,qEAAqE;QACrE,0EAA0E;QAC1E,wEAAwE;QACxE,wEAAwE;QACxE,qEAAqE;QACrE,yEAAyE;QACzE,wEAAwE;QACxE,yEAAyE;QACzE,yEAAyE;QACzE,mEAAmE;QACnE,0EAA0E;QAC1E,qEAAqE;QACrE,yEAAyE;QACzE,SAAS;QACT,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,IAAI,QAAQ,GAAG,KAAK,CAAC;QAErB,OAAO,GAAG,GAAG,GAAG,EAAE,CAAC;YACjB,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;YAEtB,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;gBAClB,MAAM,WAAW,GAAG,GAAG,CAAC;gBACxB,MAAM,SAAS,GAAG,IAAI,CAAC;gBACvB,GAAG,EAAE,CAAC,CAAC,WAAW;gBAElB,qCAAqC;gBACrC,IAAI,SAAS,GAAG,CAAC,CAAC;gBAClB,IAAI,SAAS,GAAG,KAAK,CAAC;gBACtB,OAAO,GAAG,GAAG,GAAG,EAAE,CAAC;oBACjB,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;oBACnB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,UAAU;wBACtC,SAAS,GAAG,SAAS,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;wBACxC,SAAS,GAAG,IAAI,CAAC;wBACjB,GAAG,EAAE,CAAC;oBACR,CAAC;yBAAM,CAAC;wBACN,MAAM;oBACR,CAAC;gBACH,CAAC;gBAED,IAAI,CAAC,SAAS;oBAAE,SAAS;gBACzB,QAAQ,GAAG,IAAI,CAAC;gBAEhB,4DAA4D;gBAC5D,kEAAkE;gBAClE,sEAAsE;gBACtE,+DAA+D;gBAC/D,6DAA6D;gBAC7D,OAAO,GAAG,GAAG,GAAG,EAAE,CAAC;oBACjB,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;oBACnB,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;wBAAC,GAAG,EAAE,CAAC;oBAAC,CAAC;yBAC7E,IAAI,CAAC,KAAK,OAAO,EAAE,CAAC;wBAAC,IAAI,EAAE,CAAC;wBAAC,GAAG,EAAE,CAAC;oBAAC,CAAC;;wBACrC,MAAM;gBACb,CAAC;gBAED,+DAA+D;gBAC/D,oEAAoE;gBACpE,qEAAqE;gBACrE,iEAAiE;gBACjE,IAAI,YAAY,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;oBAChC,MAAM,CAAC,GAAG,UAAU,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;oBACpC,IAAI,IAAI,CAAC,CAAC,KAAK,CAAC;oBAChB,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC;oBACb,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;wBAAC,OAAO,GAAG,IAAI,CAAC;wBAAC,MAAM;oBAAC,CAAC;gBACxC,CAAC;gBAED,oEAAoE;gBACpE,mEAAmE;gBACnE,oEAAoE;gBACpE,oEAAoE;gBACpE,IAAI,GAAG,IAAI,GAAG;oBAAE,SAAS;gBACzB,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,MAAM,EAAE,CAAC;oBAAC,QAAQ,GAAG,KAAK,CAAC;oBAAC,SAAS;gBAAC,CAAC;gBACxD,GAAG,EAAE,CAAC;gBAEN,kEAAkE;gBAClE,mEAAmE;gBACnE,+DAA+D;gBAC/D,oEAAoE;gBACpE,+DAA+D;gBAC/D,qEAAqE;gBACrE,oEAAoE;gBACpE,oEAAoE;gBACpE,kEAAkE;gBAClE,oEAAoE;gBACpE,oEAAoE;gBACpE,IAAI,CAAC,oBAAoB,CAAC,SAAS,CAAC,EAAE,CAAC;oBAAC,IAAI,CAAC,YAAY,EAAE,CAAC;oBAAC,QAAQ,GAAG,KAAK,CAAC;oBAAC,SAAS;gBAAC,CAAC;gBAE1F,kBAAkB;gBAClB,OAAO,GAAG,GAAG,GAAG,EAAE,CAAC;oBACjB,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;oBACnB,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;wBAAC,GAAG,EAAE,CAAC;oBAAC,CAAC;yBAC7E,IAAI,CAAC,KAAK,OAAO,EAAE,CAAC;wBAAC,IAAI,EAAE,CAAC;wBAAC,GAAG,EAAE,CAAC;oBAAC,CAAC;;wBACrC,MAAM;gBACb,CAAC;gBAED,IAAI,YAAY,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;oBAChC,MAAM,CAAC,GAAG,UAAU,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;oBACpC,IAAI,IAAI,CAAC,CAAC,KAAK,CAAC;oBAChB,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC;oBACb,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;wBAAC,OAAO,GAAG,IAAI,CAAC;wBAAC,MAAM;oBAAC,CAAC;gBACxC,CAAC;gBAED,iEAAiE;gBACjE,oEAAoE;gBACpE,MAAM,SAAS,GAAG,GAAG,CAAC;gBACtB,IAAI,GAAG,IAAI,GAAG;oBAAE,SAAS;gBACzB,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,CAAC;oBAAC,QAAQ,GAAG,KAAK,CAAC;oBAAC,SAAS;gBAAC,CAAC;gBAEvE,OAAO,GAAG,GAAG,GAAG,EAAE,CAAC;oBACjB,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;oBACnB,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC;wBACpD,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;wBAC3C,GAAG,EAAE,CAAC;oBACR,CAAC;yBAAM,CAAC;wBACN,MAAM;oBACR,CAAC;gBACH,CAAC;gBAED,IAAI,GAAG,KAAK,SAAS;oBAAE,SAAS;gBAEhC,mFAAmF;gBACnF,mFAAmF;gBACnF,iFAAiF;gBACjF,mFAAmF;gBACnF,gFAAgF;gBAChF,MAAM,OAAO,GAAG,GAAG,GAAG,SAAS,CAAC;gBAChC,IAAI,QAAQ,GAAG,OAAO,CAAC;gBACvB,KAAK,IAAI,CAAC,GAAG,SAAS,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;oBACrC,QAAQ,GAAG,CAAC,QAAQ,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBAC1C,CAAC;gBACD,MAAM,QAAQ,GAAG,GAAG,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC1C,IAAI,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBACnC,IAAI,eAAe,GAAG,KAAK,CAAC;gBAC5B,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;oBAClD,eAAe,GAAG,IAAI,CAAC;oBACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;wBACjC,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,SAAS,GAAG,CAAC,CAAC,EAAE,CAAC;4BAC9C,eAAe,GAAG,KAAK,CAAC;4BACxB,MAAM;wBACR,CAAC;oBACH,CAAC;gBACH,CAAC;gBACD,qEAAqE;gBACrE,mEAAmE;gBACnE,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,eAAe,EAAE,CAAC;oBAC3C,IAAI,GAAG,MAAM,CAAC,YAAY,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC;oBAC5D,SAAS,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;gBAChC,CAAC;gBAED,kBAAkB;gBAClB,OAAO,GAAG,GAAG,GAAG,EAAE,CAAC;oBACjB,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;oBACnB,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;wBAAC,GAAG,EAAE,CAAC;oBAAC,CAAC;yBAC7E,IAAI,CAAC,KAAK,OAAO,EAAE,CAAC;wBAAC,IAAI,EAAE,CAAC;wBAAC,GAAG,EAAE,CAAC;oBAAC,CAAC;;wBACrC,MAAM;gBACb,CAAC;gBAED,IAAI,YAAY,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;oBAChC,MAAM,CAAC,GAAG,UAAU,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;oBACpC,IAAI,IAAI,CAAC,CAAC,KAAK,CAAC;oBAChB,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC;oBACb,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;wBAAC,OAAO,GAAG,IAAI,CAAC;wBAAC,MAAM;oBAAC,CAAC;gBACxC,CAAC;gBAED,sEAAsE;gBACtE,IAAI,GAAG,IAAI,GAAG;oBAAE,SAAS;gBACzB,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,MAAM,EAAE,CAAC;oBAAC,QAAQ,GAAG,KAAK,CAAC;oBAAC,SAAS;gBAAC,CAAC;gBACxD,QAAQ,GAAG,KAAK,CAAC,CAAC,8BAA8B;gBAEhD,6CAA6C;gBAC7C,IAAI,QAAQ,GAAG,KAAK,CAAC;gBACrB,IAAI,eAAe,GAAG,KAAK,CAAC;gBAC5B,OAAO,GAAG,GAAG,GAAG,EAAE,CAAC;oBACjB,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;oBACnB,IAAI,CAAC,KAAK,KAAK,EAAE,CAAC;wBAChB,IAAI,QAAQ,IAAI,GAAG,GAAG,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,KAAK,EAAE,CAAC;4BACxD,GAAG,IAAI,CAAC,CAAC,CAAC,qBAAqB;4BAC/B,SAAS;wBACX,CAAC;wBACD,QAAQ,GAAG,CAAC,QAAQ,CAAC;oBACvB,CAAC;yBAAM,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;wBACnE,kEAAkE;wBAClE,8DAA8D;wBAC9D,mEAAmE;wBACnE,yDAAyD;wBACzD,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;wBACvC,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;4BACZ,+DAA+D;4BAC/D,gEAAgE;4BAChE,6CAA6C;4BAC7C,GAAG,GAAG,GAAG,CAAC;4BACV,MAAM;wBACR,CAAC;wBACD,IAAI,IAAI,aAAa,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;wBACrC,GAAG,GAAG,GAAG,CAAC;wBACV,SAAS;oBACX,CAAC;yBAAM,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,QAAQ,EAAE,CAAC;wBACxC,sBAAsB;wBACtB,MAAM,YAAY,GAAG,GAAG,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,oBAAoB;wBAChE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;wBACtF,GAAG,EAAE,CAAC;wBACN,eAAe,GAAG,IAAI,CAAC;wBACvB,MAAM;oBACR,CAAC;yBAAM,IAAI,CAAC,KAAK,OAAO,EAAE,CAAC;wBACzB,IAAI,EAAE,CAAC;oBACT,CAAC;oBACD,GAAG,EAAE,CAAC;gBACR,CAAC;gBAED,qEAAqE;gBACrE,+DAA+D;gBAC/D,mEAAmE;gBACnE,sEAAsE;gBACtE,mEAAmE;gBACnE,iCAAiC;gBACjC,IAAI,CAAC,eAAe;oBAAE,OAAO,GAAG,IAAI,CAAC;YACvC,CAAC;iBAAM,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;gBAC5B,IAAI,EAAE,CAAC;gBACP,GAAG,EAAE,CAAC;YACR,CAAC;iBAAM,IAAI,qBAAqB,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;gBAChD,mEAAmE;gBACnE,mEAAmE;gBACnE,sDAAsD;gBACtD,kEAAkE;gBAClE,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;gBACxC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC;gBACnB,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC;gBAChB,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;oBAAC,OAAO,GAAG,IAAI,CAAC;oBAAC,MAAM;gBAAC,CAAC;YAC3C,CAAC;iBAAM,CAAC;gBACN,GAAG,EAAE,CAAC;YACR,CAAC;QACH,CAAC;QAED,sEAAsE;QACtE,oEAAoE;QACpE,uEAAuE;QACvE,qEAAqE;QACrE,wDAAwD;QACxD,IAAI,OAAO,IAAI,QAAQ;YAAE,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC;IACrD,CAAC;CACF"}

package/dist/unit-extractor.d.ts

@@ -6,6 +6,8 @@
  */
 import type { EntityRef } from './types.js';
 import type { IfcSourceBytes } from './source-bytes.js';
+import { type RelatedLookup } from './owning-project.js';
+export { resolveOwningIfcProjectId, type RelatedLookup } from './owning-project.js';
 /**
  * SI Prefix multipliers, keyed by the members of the `IfcSIPrefix` EXPRESS
  * enumeration — that enumeration is the authority for which prefixes a unit
@@ -43,6 +45,12 @@ export declare const CONVERSION_BASED_UNIT_FACTORS: Record<string, number>;
  *
  * @param source - The IFC file bytes, either raw or behind {@link IfcSourceBytes}
  * @param entityIndex - Entity index with byId and byType maps
+ * @param projectId - Optional express id of the specific `IFCPROJECT` to read
+ *   units from. Most files declare exactly one, so the default (the file's
+ *   first `IFCPROJECT`) is correct for the overwhelmingly common case; pass
+ *   this explicitly when resolving units for an entity that may belong to a
+ *   later `IFCPROJECT` in a multi-project (federated-merge, see #1332) file
+ *   — see {@link resolveOwningIfcProjectId}.
  * @returns Scale factor to apply to length values (e.g., 0.001 for millimeters)
  */
 export declare function extractLengthUnitScale(source: Uint8Array | IfcSourceBytes, entityIndex: {
@@ -50,5 +58,39 @@ export declare function extractLengthUnitScale(source: Uint8Array | IfcSourceByt
         get(expressId: number): EntityRef | undefined;
     };
     byType: Map<string, number[]>;
-}): number;
+}, projectId?: number): number;
+/**
+ * Resolve the length unit scale that applies to ONE entity, correctly for a
+ * multi-`IfcProject` file (a {@link MergedExporter} `unitReconciliation: 'auto'`
+ * federated export: a model whose length unit differs from the first model's
+ * keeps its own `IfcProject`/`IfcUnitAssignment` rather than being rescaled —
+ * see that module's docs).
+ *
+ * {@link extractLengthUnitScale} (and the `dataStore.lengthUnitScale` it feeds)
+ * answers for the FIRST `IfcProject` only. That is exactly right for the
+ * overwhelmingly common single-project file, but silently wrong for a
+ * federated entity that belongs to a LATER project: e.g. a material layer's
+ * `LayerThickness` is a raw literal in ITS OWN project's unit, and scaling it
+ * by the first project's factor corrupts the value by whatever ratio
+ * separates the two units (a millimetre federated model read back with the
+ * metres factor turns a 300 mm layer into a fabricated "300 m" one).
+ *
+ * Resolves the entity's owning project via {@link resolveOwningIfcProjectId} and
+ * reads that project's units, falling back to the first project's scale when
+ * the walk can't place the entity, OR when the owning project's own
+ * `UnitsInContext` declares no `LENGTHUNIT` at all (`UnitsInContext` is
+ * OPTIONAL on `IfcContext`, so a federated model can legitimately arrive
+ * with none). {@link extractLengthUnitScale} answers an unconfirmed `1.0`
+ * for that case too - absence reading as success - which would silently
+ * rescale a millimetre-authored value owned by such a project by 1000x
+ * instead of taking the file-wide answer, the same safe-miss direction
+ * {@link extractLengthUnitScale} already documents for every other ambiguous
+ * case.
+ */
+export declare function resolveEntityLengthUnitScale(source: Uint8Array | IfcSourceBytes, entityIndex: {
+    byId: {
+        get(expressId: number): EntityRef | undefined;
+    };
+    byType: Map<string, number[]>;
+}, relationships: RelatedLookup, expressId: number): number;
 //# sourceMappingURL=unit-extractor.d.ts.map

package/dist/unit-extractor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"unit-extractor.d.ts","sourceRoot":"","sources":["../src/unit-extractor.ts"],"names":[],"mappings":"AAIA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAE5C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAExD;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,qBAAqB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAiBxD,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,6BAA6B,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAgBhE,CAAC;AAyBF;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,sBAAsB,CACpC,MAAM,EAAE,UAAU,GAAG,cAAc,EACnC,WAAW,EAAE;IAAE,IAAI,EAAE;QAAE,GAAG,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,SAAS,CAAA;KAAE,CAAC;IAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;CAAE,GACtG,MAAM,CA8LR"}
+{"version":3,"file":"unit-extractor.d.ts","sourceRoot":"","sources":["../src/unit-extractor.ts"],"names":[],"mappings":"AAIA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAE5C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACxD,OAAO,EAA6B,KAAK,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAGpF,OAAO,EAAE,yBAAyB,EAAE,KAAK,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEpF;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,qBAAqB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAiBxD,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,6BAA6B,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAgBhE,CAAC;AAyBF;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,sBAAsB,CACpC,MAAM,EAAE,UAAU,GAAG,cAAc,EACnC,WAAW,EAAE;IAAE,IAAI,EAAE;QAAE,GAAG,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,SAAS,CAAA;KAAE,CAAC;IAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;CAAE,EACvG,SAAS,CAAC,EAAE,MAAM,GACjB,MAAM,CAUR;AAuMD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,UAAU,GAAG,cAAc,EACnC,WAAW,EAAE;IAAE,IAAI,EAAE;QAAE,GAAG,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,SAAS,CAAA;KAAE,CAAC;IAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAA;CAAE,EACvG,aAAa,EAAE,aAAa,EAC5B,SAAS,EAAE,MAAM,GAChB,MAAM,CASR"}