@zzzen/pyright-internal 1.2.0-dev.20231119 → 1.2.0-dev.20231126

package/dist/parser/tokenizer.js

@@ -20,90 +20,90 @@ const characterStream_1 = require("./characterStream");
 const tokenizerTypes_1 = require("./tokenizerTypes");
 // This must be a Map, as operations like {}["constructor"] succeed.
 const PythonKeywords = new Map([
-    ['and', 0 /* And */],
-    ['as', 1 /* As */],
-    ['assert', 2 /* Assert */],
-    ['async', 3 /* Async */],
-    ['await', 4 /* Await */],
-    ['break', 5 /* Break */],
-    ['case', 6 /* Case */],
-    ['class', 7 /* Class */],
-    ['continue', 8 /* Continue */],
-    ['__debug__', 9 /* Debug */],
-    ['def', 10 /* Def */],
-    ['del', 11 /* Del */],
-    ['elif', 12 /* Elif */],
-    ['else', 13 /* Else */],
-    ['except', 14 /* Except */],
-    ['finally', 16 /* Finally */],
-    ['for', 17 /* For */],
-    ['from', 18 /* From */],
-    ['global', 19 /* Global */],
-    ['if', 20 /* If */],
-    ['import', 21 /* Import */],
-    ['in', 22 /* In */],
-    ['is', 23 /* Is */],
-    ['lambda', 24 /* Lambda */],
-    ['match', 25 /* Match */],
-    ['nonlocal', 27 /* Nonlocal */],
-    ['not', 28 /* Not */],
-    ['or', 29 /* Or */],
-    ['pass', 30 /* Pass */],
-    ['raise', 31 /* Raise */],
-    ['return', 32 /* Return */],
-    ['try', 34 /* Try */],
-    ['type', 35 /* Type */],
-    ['while', 36 /* While */],
-    ['with', 37 /* With */],
-    ['yield', 38 /* Yield */],
-    ['False', 15 /* False */],
-    ['None', 26 /* None */],
-    ['True', 33 /* True */],
+    ['and', 0 /* KeywordType.And */],
+    ['as', 1 /* KeywordType.As */],
+    ['assert', 2 /* KeywordType.Assert */],
+    ['async', 3 /* KeywordType.Async */],
+    ['await', 4 /* KeywordType.Await */],
+    ['break', 5 /* KeywordType.Break */],
+    ['case', 6 /* KeywordType.Case */],
+    ['class', 7 /* KeywordType.Class */],
+    ['continue', 8 /* KeywordType.Continue */],
+    ['__debug__', 9 /* KeywordType.Debug */],
+    ['def', 10 /* KeywordType.Def */],
+    ['del', 11 /* KeywordType.Del */],
+    ['elif', 12 /* KeywordType.Elif */],
+    ['else', 13 /* KeywordType.Else */],
+    ['except', 14 /* KeywordType.Except */],
+    ['finally', 16 /* KeywordType.Finally */],
+    ['for', 17 /* KeywordType.For */],
+    ['from', 18 /* KeywordType.From */],
+    ['global', 19 /* KeywordType.Global */],
+    ['if', 20 /* KeywordType.If */],
+    ['import', 21 /* KeywordType.Import */],
+    ['in', 22 /* KeywordType.In */],
+    ['is', 23 /* KeywordType.Is */],
+    ['lambda', 24 /* KeywordType.Lambda */],
+    ['match', 25 /* KeywordType.Match */],
+    ['nonlocal', 27 /* KeywordType.Nonlocal */],
+    ['not', 28 /* KeywordType.Not */],
+    ['or', 29 /* KeywordType.Or */],
+    ['pass', 30 /* KeywordType.Pass */],
+    ['raise', 31 /* KeywordType.Raise */],
+    ['return', 32 /* KeywordType.Return */],
+    ['try', 34 /* KeywordType.Try */],
+    ['type', 35 /* KeywordType.Type */],
+    ['while', 36 /* KeywordType.While */],
+    ['with', 37 /* KeywordType.With */],
+    ['yield', 38 /* KeywordType.Yield */],
+    ['False', 15 /* KeywordType.False */],
+    ['None', 26 /* KeywordType.None */],
+    ['True', 33 /* KeywordType.True */],
 ]);
 const _softKeywords = new Set(['match', 'case', 'type']);
 const _operatorInfo = {
-    [0 /* Add */]: 1 /* Unary */ | 2 /* Binary */,
-    [1 /* AddEqual */]: 4 /* Assignment */,
-    [2 /* Assign */]: 4 /* Assignment */,
-    [3 /* BitwiseAnd */]: 2 /* Binary */,
-    [4 /* BitwiseAndEqual */]: 4 /* Assignment */,
-    [5 /* BitwiseInvert */]: 1 /* Unary */,
-    [6 /* BitwiseOr */]: 2 /* Binary */,
-    [7 /* BitwiseOrEqual */]: 4 /* Assignment */,
-    [8 /* BitwiseXor */]: 2 /* Binary */,
-    [9 /* BitwiseXorEqual */]: 4 /* Assignment */,
-    [10 /* Divide */]: 2 /* Binary */,
-    [11 /* DivideEqual */]: 4 /* Assignment */,
-    [12 /* Equals */]: 2 /* Binary */ | 8 /* Comparison */,
-    [13 /* FloorDivide */]: 2 /* Binary */,
-    [14 /* FloorDivideEqual */]: 4 /* Assignment */,
-    [15 /* GreaterThan */]: 2 /* Binary */ | 8 /* Comparison */,
-    [16 /* GreaterThanOrEqual */]: 2 /* Binary */ | 8 /* Comparison */,
-    [17 /* LeftShift */]: 2 /* Binary */,
-    [18 /* LeftShiftEqual */]: 4 /* Assignment */,
-    [19 /* LessOrGreaterThan */]: 2 /* Binary */ | 8 /* Comparison */ | 16 /* Deprecated */,
-    [20 /* LessThan */]: 2 /* Binary */ | 8 /* Comparison */,
-    [21 /* LessThanOrEqual */]: 2 /* Binary */ | 8 /* Comparison */,
-    [22 /* MatrixMultiply */]: 2 /* Binary */,
-    [23 /* MatrixMultiplyEqual */]: 4 /* Assignment */,
-    [24 /* Mod */]: 2 /* Binary */,
-    [25 /* ModEqual */]: 4 /* Assignment */,
-    [26 /* Multiply */]: 2 /* Binary */,
-    [27 /* MultiplyEqual */]: 4 /* Assignment */,
-    [28 /* NotEquals */]: 2 /* Binary */ | 8 /* Comparison */,
-    [29 /* Power */]: 2 /* Binary */,
-    [30 /* PowerEqual */]: 4 /* Assignment */,
-    [31 /* RightShift */]: 2 /* Binary */,
-    [32 /* RightShiftEqual */]: 4 /* Assignment */,
-    [33 /* Subtract */]: 2 /* Binary */,
-    [34 /* SubtractEqual */]: 4 /* Assignment */,
-    [36 /* And */]: 2 /* Binary */,
-    [37 /* Or */]: 2 /* Binary */,
-    [38 /* Not */]: 1 /* Unary */,
-    [39 /* Is */]: 2 /* Binary */,
-    [40 /* IsNot */]: 2 /* Binary */,
-    [41 /* In */]: 2 /* Binary */,
-    [42 /* NotIn */]: 2 /* Binary */,
+    [0 /* OperatorType.Add */]: 1 /* OperatorFlags.Unary */ | 2 /* OperatorFlags.Binary */,
+    [1 /* OperatorType.AddEqual */]: 4 /* OperatorFlags.Assignment */,
+    [2 /* OperatorType.Assign */]: 4 /* OperatorFlags.Assignment */,
+    [3 /* OperatorType.BitwiseAnd */]: 2 /* OperatorFlags.Binary */,
+    [4 /* OperatorType.BitwiseAndEqual */]: 4 /* OperatorFlags.Assignment */,
+    [5 /* OperatorType.BitwiseInvert */]: 1 /* OperatorFlags.Unary */,
+    [6 /* OperatorType.BitwiseOr */]: 2 /* OperatorFlags.Binary */,
+    [7 /* OperatorType.BitwiseOrEqual */]: 4 /* OperatorFlags.Assignment */,
+    [8 /* OperatorType.BitwiseXor */]: 2 /* OperatorFlags.Binary */,
+    [9 /* OperatorType.BitwiseXorEqual */]: 4 /* OperatorFlags.Assignment */,
+    [10 /* OperatorType.Divide */]: 2 /* OperatorFlags.Binary */,
+    [11 /* OperatorType.DivideEqual */]: 4 /* OperatorFlags.Assignment */,
+    [12 /* OperatorType.Equals */]: 2 /* OperatorFlags.Binary */ | 8 /* OperatorFlags.Comparison */,
+    [13 /* OperatorType.FloorDivide */]: 2 /* OperatorFlags.Binary */,
+    [14 /* OperatorType.FloorDivideEqual */]: 4 /* OperatorFlags.Assignment */,
+    [15 /* OperatorType.GreaterThan */]: 2 /* OperatorFlags.Binary */ | 8 /* OperatorFlags.Comparison */,
+    [16 /* OperatorType.GreaterThanOrEqual */]: 2 /* OperatorFlags.Binary */ | 8 /* OperatorFlags.Comparison */,
+    [17 /* OperatorType.LeftShift */]: 2 /* OperatorFlags.Binary */,
+    [18 /* OperatorType.LeftShiftEqual */]: 4 /* OperatorFlags.Assignment */,
+    [19 /* OperatorType.LessOrGreaterThan */]: 2 /* OperatorFlags.Binary */ | 8 /* OperatorFlags.Comparison */ | 16 /* OperatorFlags.Deprecated */,
+    [20 /* OperatorType.LessThan */]: 2 /* OperatorFlags.Binary */ | 8 /* OperatorFlags.Comparison */,
+    [21 /* OperatorType.LessThanOrEqual */]: 2 /* OperatorFlags.Binary */ | 8 /* OperatorFlags.Comparison */,
+    [22 /* OperatorType.MatrixMultiply */]: 2 /* OperatorFlags.Binary */,
+    [23 /* OperatorType.MatrixMultiplyEqual */]: 4 /* OperatorFlags.Assignment */,
+    [24 /* OperatorType.Mod */]: 2 /* OperatorFlags.Binary */,
+    [25 /* OperatorType.ModEqual */]: 4 /* OperatorFlags.Assignment */,
+    [26 /* OperatorType.Multiply */]: 2 /* OperatorFlags.Binary */,
+    [27 /* OperatorType.MultiplyEqual */]: 4 /* OperatorFlags.Assignment */,
+    [28 /* OperatorType.NotEquals */]: 2 /* OperatorFlags.Binary */ | 8 /* OperatorFlags.Comparison */,
+    [29 /* OperatorType.Power */]: 2 /* OperatorFlags.Binary */,
+    [30 /* OperatorType.PowerEqual */]: 4 /* OperatorFlags.Assignment */,
+    [31 /* OperatorType.RightShift */]: 2 /* OperatorFlags.Binary */,
+    [32 /* OperatorType.RightShiftEqual */]: 4 /* OperatorFlags.Assignment */,
+    [33 /* OperatorType.Subtract */]: 2 /* OperatorFlags.Binary */,
+    [34 /* OperatorType.SubtractEqual */]: 4 /* OperatorFlags.Assignment */,
+    [36 /* OperatorType.And */]: 2 /* OperatorFlags.Binary */,
+    [37 /* OperatorType.Or */]: 2 /* OperatorFlags.Binary */,
+    [38 /* OperatorType.Not */]: 1 /* OperatorFlags.Unary */,
+    [39 /* OperatorType.Is */]: 2 /* OperatorFlags.Binary */,
+    [40 /* OperatorType.IsNot */]: 2 /* OperatorFlags.Binary */,
+    [41 /* OperatorType.In */]: 2 /* OperatorFlags.Binary */,
+    [42 /* OperatorType.NotIn */]: 2 /* OperatorFlags.Binary */,
 };
 const _byteOrderMarker = 0xfeff;
 const defaultTabSize = 8;
@@ -176,24 +176,24 @@ class Tokenizer {
         }
         // Insert any implied FStringEnd tokens.
         while (this._activeFString) {
-            this._tokens.push(tokenizerTypes_1.FStringEndToken.create(this._cs.position, 0, this._activeFString.startToken.flags | 65536 /* Unterminated */));
+            this._tokens.push(tokenizerTypes_1.FStringEndToken.create(this._cs.position, 0, this._activeFString.startToken.flags | 65536 /* StringTokenFlags.Unterminated */));
             this._activeFString = this._fStringStack.pop();
         }
         // Insert an implied new line to make parsing easier.
-        if (this._tokens.length === 0 || this._tokens[this._tokens.length - 1].type !== 2 /* NewLine */) {
-            this._tokens.push(tokenizerTypes_1.NewLineToken.create(this._cs.position, 0, 3 /* Implied */, this._getComments()));
+        if (this._tokens.length === 0 || this._tokens[this._tokens.length - 1].type !== 2 /* TokenType.NewLine */) {
+            this._tokens.push(tokenizerTypes_1.NewLineToken.create(this._cs.position, 0, 3 /* NewLineType.Implied */, this._getComments()));
         }
         // Insert any implied dedent tokens.
         this._setIndent(this._cs.position, 0, 0, /* isSpacePresent */ false, /* isTabPresent */ false);
         // Add a final end-of-stream token to make parsing easier.
-        this._tokens.push(tokenizerTypes_1.Token.create(1 /* EndOfStream */, this._cs.position, 0, this._getComments()));
+        this._tokens.push(tokenizerTypes_1.Token.create(1 /* TokenType.EndOfStream */, this._cs.position, 0, this._getComments()));
         // Add the final line range.
         this._addLineRange();
         // If the last line ended in a line-end character, add an empty line.
         if (this._lineRanges.length > 0) {
             const lastLine = this._lineRanges[this._lineRanges.length - 1];
             const lastCharOfLastLine = text.charCodeAt(lastLine.start + lastLine.length - 1);
-            if (lastCharOfLastLine === 13 /* CarriageReturn */ || lastCharOfLastLine === 10 /* LineFeed */) {
+            if (lastCharOfLastLine === 13 /* Char.CarriageReturn */ || lastCharOfLastLine === 10 /* Char.LineFeed */) {
                 this._lineRanges.push({ start: this._cs.position, length: 0 });
             }
         }
@@ -257,13 +257,13 @@ class Tokenizer {
         if (operatorType === undefined || _operatorInfo[operatorType] === undefined) {
             return false;
         }
-        return (_operatorInfo[operatorType] & 4 /* Assignment */) !== 0;
+        return (_operatorInfo[operatorType] & 4 /* OperatorFlags.Assignment */) !== 0;
     }
     static isOperatorComparison(operatorType) {
         if (operatorType === undefined || _operatorInfo[operatorType] === undefined) {
             return false;
         }
-        return (_operatorInfo[operatorType] & 8 /* Comparison */) !== 0;
+        return (_operatorInfo[operatorType] & 8 /* OperatorFlags.Comparison */) !== 0;
     }
     _addNextToken() {
         // Are we in the middle of an f-string but not in a replacement field?
@@ -297,25 +297,25 @@ class Tokenizer {
                 this._cs.advance(stringPrefixLength);
             }
             const quoteTypeFlags = this._getQuoteTypeFlags(stringPrefix);
-            if (quoteTypeFlags !== 0 /* None */) {
+            if (quoteTypeFlags !== 0 /* StringTokenFlags.None */) {
                 this._handleString(quoteTypeFlags, stringPrefixLength);
                 return true;
             }
         }
-        if (this._cs.currentChar === 35 /* Hash */) {
+        if (this._cs.currentChar === 35 /* Char.Hash */) {
             this._handleComment();
             return true;
         }
         if (this._ipythonMode) {
             const kind = this._getIPythonMagicsKind();
             if (kind === 'line') {
-                this._handleIPythonMagics(this._cs.currentChar === 37 /* Percent */ ? 1 /* IPythonMagic */ : 2 /* IPythonShellEscape */);
+                this._handleIPythonMagics(this._cs.currentChar === 37 /* Char.Percent */ ? 1 /* CommentType.IPythonMagic */ : 2 /* CommentType.IPythonShellEscape */);
                 return true;
             }
             if (kind === 'cell') {
-                this._handleIPythonMagics(this._cs.currentChar === 37 /* Percent */
-                    ? 3 /* IPythonCellMagic */
-                    : 4 /* IPythonCellShellEscape */);
+                this._handleIPythonMagics(this._cs.currentChar === 37 /* Char.Percent */
+                    ? 3 /* CommentType.IPythonCellMagic */
+                    : 4 /* CommentType.IPythonCellShellEscape */);
                 return true;
             }
         }
@@ -327,19 +327,19 @@ class Tokenizer {
                 }
                 return this._handleInvalid();
             }
-            case 13 /* CarriageReturn */: {
-                const length = this._cs.nextChar === 10 /* LineFeed */ ? 2 : 1;
-                const newLineType = length === 2 ? 2 /* CarriageReturnLineFeed */ : 0 /* CarriageReturn */;
+            case 13 /* Char.CarriageReturn */: {
+                const length = this._cs.nextChar === 10 /* Char.LineFeed */ ? 2 : 1;
+                const newLineType = length === 2 ? 2 /* NewLineType.CarriageReturnLineFeed */ : 0 /* NewLineType.CarriageReturn */;
                 this._handleNewLine(length, newLineType);
                 return true;
             }
-            case 10 /* LineFeed */: {
-                this._handleNewLine(1, 1 /* LineFeed */);
+            case 10 /* Char.LineFeed */: {
+                this._handleNewLine(1, 1 /* NewLineType.LineFeed */);
                 return true;
             }
-            case 92 /* Backslash */: {
-                if (this._cs.nextChar === 13 /* CarriageReturn */) {
-                    if (this._cs.lookAhead(2) === 10 /* LineFeed */) {
+            case 92 /* Char.Backslash */: {
+                if (this._cs.nextChar === 13 /* Char.CarriageReturn */) {
+                    if (this._cs.lookAhead(2) === 10 /* Char.LineFeed */) {
                         this._cs.advance(3);
                     }
                     else {
@@ -348,40 +348,40 @@ class Tokenizer {
                     this._addLineRange();
                     return true;
                 }
-                else if (this._cs.nextChar === 10 /* LineFeed */) {
+                else if (this._cs.nextChar === 10 /* Char.LineFeed */) {
                     this._cs.advance(2);
                     this._addLineRange();
                     return true;
                 }
                 return this._handleInvalid();
             }
-            case 40 /* OpenParenthesis */: {
+            case 40 /* Char.OpenParenthesis */: {
                 this._parenDepth++;
-                this._tokens.push(tokenizerTypes_1.Token.create(13 /* OpenParenthesis */, this._cs.position, 1, this._getComments()));
+                this._tokens.push(tokenizerTypes_1.Token.create(13 /* TokenType.OpenParenthesis */, this._cs.position, 1, this._getComments()));
                 break;
             }
-            case 41 /* CloseParenthesis */: {
+            case 41 /* Char.CloseParenthesis */: {
                 if (this._parenDepth > 0) {
                     this._parenDepth--;
                 }
-                this._tokens.push(tokenizerTypes_1.Token.create(14 /* CloseParenthesis */, this._cs.position, 1, this._getComments()));
+                this._tokens.push(tokenizerTypes_1.Token.create(14 /* TokenType.CloseParenthesis */, this._cs.position, 1, this._getComments()));
                 break;
             }
-            case 91 /* OpenBracket */: {
+            case 91 /* Char.OpenBracket */: {
                 this._parenDepth++;
-                this._tokens.push(tokenizerTypes_1.Token.create(15 /* OpenBracket */, this._cs.position, 1, this._getComments()));
+                this._tokens.push(tokenizerTypes_1.Token.create(15 /* TokenType.OpenBracket */, this._cs.position, 1, this._getComments()));
                 break;
             }
-            case 93 /* CloseBracket */: {
+            case 93 /* Char.CloseBracket */: {
                 if (this._parenDepth > 0) {
                     this._parenDepth--;
                 }
-                this._tokens.push(tokenizerTypes_1.Token.create(16 /* CloseBracket */, this._cs.position, 1, this._getComments()));
+                this._tokens.push(tokenizerTypes_1.Token.create(16 /* TokenType.CloseBracket */, this._cs.position, 1, this._getComments()));
                 break;
             }
-            case 123 /* OpenBrace */: {
+            case 123 /* Char.OpenBrace */: {
                 this._parenDepth++;
-                this._tokens.push(tokenizerTypes_1.Token.create(17 /* OpenCurlyBrace */, this._cs.position, 1, this._getComments()));
+                this._tokens.push(tokenizerTypes_1.Token.create(17 /* TokenType.OpenCurlyBrace */, this._cs.position, 1, this._getComments()));
                 if (this._activeFString) {
                     // Are we starting a new replacement field?
                     if (!this._activeFString.activeReplacementField ||
@@ -400,7 +400,7 @@ class Tokenizer {
                 }
                 break;
             }
-            case 125 /* CloseBrace */: {
+            case 125 /* Char.CloseBrace */: {
                 if (this._activeFString &&
                     ((_a = this._activeFString.activeReplacementField) === null || _a === void 0 ? void 0 : _a.parenDepth) === this._parenDepth) {
                     this._activeFString.activeReplacementField = this._activeFString.replacementFieldStack.pop();
@@ -408,32 +408,32 @@ class Tokenizer {
                 if (this._parenDepth > 0) {
                     this._parenDepth--;
                 }
-                this._tokens.push(tokenizerTypes_1.Token.create(18 /* CloseCurlyBrace */, this._cs.position, 1, this._getComments()));
+                this._tokens.push(tokenizerTypes_1.Token.create(18 /* TokenType.CloseCurlyBrace */, this._cs.position, 1, this._getComments()));
                 break;
             }
-            case 44 /* Comma */: {
-                this._tokens.push(tokenizerTypes_1.Token.create(12 /* Comma */, this._cs.position, 1, this._getComments()));
+            case 44 /* Char.Comma */: {
+                this._tokens.push(tokenizerTypes_1.Token.create(12 /* TokenType.Comma */, this._cs.position, 1, this._getComments()));
                 break;
             }
-            case 96 /* Backtick */: {
-                this._tokens.push(tokenizerTypes_1.Token.create(22 /* Backtick */, this._cs.position, 1, this._getComments()));
+            case 96 /* Char.Backtick */: {
+                this._tokens.push(tokenizerTypes_1.Token.create(22 /* TokenType.Backtick */, this._cs.position, 1, this._getComments()));
                 break;
             }
-            case 59 /* Semicolon */: {
-                this._tokens.push(tokenizerTypes_1.Token.create(11 /* Semicolon */, this._cs.position, 1, this._getComments()));
+            case 59 /* Char.Semicolon */: {
+                this._tokens.push(tokenizerTypes_1.Token.create(11 /* TokenType.Semicolon */, this._cs.position, 1, this._getComments()));
                 break;
             }
-            case 58 /* Colon */: {
-                if (this._cs.nextChar === 61 /* Equal */) {
+            case 58 /* Char.Colon */: {
+                if (this._cs.nextChar === 61 /* Char.Equal */) {
                     if (!this._activeFString ||
                         !this._activeFString.activeReplacementField ||
                         this._activeFString.activeReplacementField.parenDepth !== this._parenDepth) {
-                        this._tokens.push(tokenizerTypes_1.OperatorToken.create(this._cs.position, 2, 35 /* Walrus */, this._getComments()));
+                        this._tokens.push(tokenizerTypes_1.OperatorToken.create(this._cs.position, 2, 35 /* OperatorType.Walrus */, this._getComments()));
                         this._cs.advance(1);
                         break;
                     }
                 }
-                this._tokens.push(tokenizerTypes_1.Token.create(10 /* Colon */, this._cs.position, 1, this._getComments()));
+                this._tokens.push(tokenizerTypes_1.Token.create(10 /* TokenType.Colon */, this._cs.position, 1, this._getComments()));
                 if (((_b = this._activeFString) === null || _b === void 0 ? void 0 : _b.activeReplacementField) &&
                     this._parenDepth === this._activeFString.activeReplacementField.parenDepth) {
                     this._activeFString.activeReplacementField.inFormatSpecifier = true;
@@ -446,13 +446,13 @@ class Tokenizer {
                         return true;
                     }
                 }
-                if (this._cs.currentChar === 46 /* Period */) {
-                    if (this._cs.nextChar === 46 /* Period */ && this._cs.lookAhead(2) === 46 /* Period */) {
-                        this._tokens.push(tokenizerTypes_1.Token.create(19 /* Ellipsis */, this._cs.position, 3, this._getComments()));
+                if (this._cs.currentChar === 46 /* Char.Period */) {
+                    if (this._cs.nextChar === 46 /* Char.Period */ && this._cs.lookAhead(2) === 46 /* Char.Period */) {
+                        this._tokens.push(tokenizerTypes_1.Token.create(19 /* TokenType.Ellipsis */, this._cs.position, 3, this._getComments()));
                         this._cs.advance(3);
                         return true;
                     }
-                    this._tokens.push(tokenizerTypes_1.Token.create(20 /* Dot */, this._cs.position, 1, this._getComments()));
+                    this._tokens.push(tokenizerTypes_1.Token.create(20 /* TokenType.Dot */, this._cs.position, 1, this._getComments()));
                     break;
                 }
                 if (!this._tryIdentifier()) {
@@ -473,17 +473,17 @@ class Tokenizer {
         this._prevLineStart = this._cs.position;
     }
     _handleNewLine(length, newLineType) {
-        if (this._parenDepth === 0 && newLineType !== 3 /* Implied */) {
+        if (this._parenDepth === 0 && newLineType !== 3 /* NewLineType.Implied */) {
             // New lines are ignored within parentheses.
             // We'll also avoid adding multiple newlines in a row to simplify parsing.
-            if (this._tokens.length === 0 || this._tokens[this._tokens.length - 1].type !== 2 /* NewLine */) {
+            if (this._tokens.length === 0 || this._tokens[this._tokens.length - 1].type !== 2 /* TokenType.NewLine */) {
                 this._tokens.push(tokenizerTypes_1.NewLineToken.create(this._cs.position, length, newLineType, this._getComments()));
             }
         }
-        if (newLineType === 0 /* CarriageReturn */) {
+        if (newLineType === 0 /* NewLineType.CarriageReturn */) {
             this._crCount++;
         }
-        else if (newLineType === 2 /* CarriageReturnLineFeed */) {
+        else if (newLineType === 2 /* NewLineType.CarriageReturnLineFeed */) {
             this._crLfCount++;
         }
         else {
@@ -501,13 +501,13 @@ class Tokenizer {
         const startOffset = this._cs.position;
         while (!this._cs.isEndOfStream()) {
             switch (this._cs.currentChar) {
-                case 32 /* Space */:
+                case 32 /* Char.Space */:
                     tab1Spaces++;
                     tab8Spaces++;
                     isSpacePresent = true;
                     this._cs.moveNext();
                     break;
-                case 9 /* Tab */:
+                case 9 /* Char.Tab */:
                     // Translate tabs into spaces assuming both 1-space
                     // and 8-space tab stops.
                     tab1Spaces++;
@@ -515,7 +515,7 @@ class Tokenizer {
                     isTabPresent = true;
                     this._cs.moveNext();
                     break;
-                case 12 /* FormFeed */:
+                case 12 /* Char.FormFeed */:
                     tab1Spaces = 0;
                     tab8Spaces = 0;
                     isTabPresent = false;
@@ -526,9 +526,9 @@ class Tokenizer {
                     // Non-blank line. Set the current indent level.
                     this._setIndent(startOffset, tab1Spaces, tab8Spaces, isSpacePresent, isTabPresent);
                     return;
-                case 35 /* Hash */:
-                case 10 /* LineFeed */:
-                case 13 /* CarriageReturn */:
+                case 35 /* Char.Hash */:
+                case 10 /* Char.LineFeed */:
+                case 13 /* Char.CarriageReturn */:
                     // Blank line -- no need to adjust indentation.
                     return;
             }
@@ -658,18 +658,18 @@ class Tokenizer {
         if ((0, characters_1.isDecimal)(this._cs.currentChar)) {
             return true;
         }
-        if (this._cs.currentChar === 46 /* Period */ && (0, characters_1.isDecimal)(this._cs.nextChar)) {
+        if (this._cs.currentChar === 46 /* Char.Period */ && (0, characters_1.isDecimal)(this._cs.nextChar)) {
             return true;
         }
         return false;
     }
     _tryNumber() {
         const start = this._cs.position;
-        if (this._cs.currentChar === 48 /* _0 */) {
+        if (this._cs.currentChar === 48 /* Char._0 */) {
             let radix = 0;
             let leadingChars = 0;
             // Try hex => hexinteger: "0" ("x" | "X") (["_"] hexdigit)+
-            if ((this._cs.nextChar === 120 /* x */ || this._cs.nextChar === 88 /* X */) && (0, characters_1.isHex)(this._cs.lookAhead(2))) {
+            if ((this._cs.nextChar === 120 /* Char.x */ || this._cs.nextChar === 88 /* Char.X */) && (0, characters_1.isHex)(this._cs.lookAhead(2))) {
                 this._cs.advance(2);
                 leadingChars = 2;
                 while ((0, characters_1.isHex)(this._cs.currentChar)) {
@@ -678,7 +678,7 @@ class Tokenizer {
                 radix = 16;
             }
             // Try binary => bininteger: "0" ("b" | "B") (["_"] bindigit)+
-            else if ((this._cs.nextChar === 98 /* b */ || this._cs.nextChar === 66 /* B */) &&
+            else if ((this._cs.nextChar === 98 /* Char.b */ || this._cs.nextChar === 66 /* Char.B */) &&
                 (0, characters_1.isBinary)(this._cs.lookAhead(2))) {
                 this._cs.advance(2);
                 leadingChars = 2;
@@ -688,7 +688,7 @@ class Tokenizer {
                 radix = 2;
             }
             // Try octal => octinteger: "0" ("o" | "O") (["_"] octdigit)+
-            else if ((this._cs.nextChar === 111 /* o */ || this._cs.nextChar === 79 /* O */) && (0, characters_1.isOctal)(this._cs.lookAhead(2))) {
+            else if ((this._cs.nextChar === 111 /* Char.o */ || this._cs.nextChar === 79 /* Char.O */) && (0, characters_1.isOctal)(this._cs.lookAhead(2))) {
                 this._cs.advance(2);
                 leadingChars = 2;
                 while ((0, characters_1.isOctal)(this._cs.currentChar)) {
@@ -718,27 +718,27 @@ class Tokenizer {
         //    decinteger: nonzerodigit (["_"] digit)* | "0" (["_"] "0")*
         //    nonzerodigit: "1"..."9"
         //    digit: "0"..."9"
-        if (this._cs.currentChar >= 49 /* _1 */ && this._cs.currentChar <= 57 /* _9 */) {
+        if (this._cs.currentChar >= 49 /* Char._1 */ && this._cs.currentChar <= 57 /* Char._9 */) {
             while ((0, characters_1.isDecimal)(this._cs.currentChar)) {
                 mightBeFloatingPoint = true;
                 this._cs.moveNext();
             }
             isDecimalInteger =
-                this._cs.currentChar !== 46 /* Period */ &&
-                    this._cs.currentChar !== 101 /* e */ &&
-                    this._cs.currentChar !== 69 /* E */;
+                this._cs.currentChar !== 46 /* Char.Period */ &&
+                    this._cs.currentChar !== 101 /* Char.e */ &&
+                    this._cs.currentChar !== 69 /* Char.E */;
         }
         // "0" (["_"] "0")*
-        if (this._cs.currentChar === 48 /* _0 */) {
+        if (this._cs.currentChar === 48 /* Char._0 */) {
             mightBeFloatingPoint = true;
-            while (this._cs.currentChar === 48 /* _0 */ || this._cs.currentChar === 95 /* Underscore */) {
+            while (this._cs.currentChar === 48 /* Char._0 */ || this._cs.currentChar === 95 /* Char.Underscore */) {
                 this._cs.moveNext();
             }
             isDecimalInteger =
-                this._cs.currentChar !== 46 /* Period */ &&
-                    this._cs.currentChar !== 101 /* e */ &&
-                    this._cs.currentChar !== 69 /* E */ &&
-                    (this._cs.currentChar < 49 /* _1 */ || this._cs.currentChar > 57 /* _9 */);
+                this._cs.currentChar !== 46 /* Char.Period */ &&
+                    this._cs.currentChar !== 101 /* Char.e */ &&
+                    this._cs.currentChar !== 69 /* Char.E */ &&
+                    (this._cs.currentChar < 49 /* Char._1 */ || this._cs.currentChar > 57 /* Char._9 */);
         }
         if (isDecimalInteger) {
             let text = this._cs.getText().substring(start, this._cs.position);
@@ -752,7 +752,7 @@ class Tokenizer {
                     bigIntValue > Number.MAX_SAFE_INTEGER) {
                     intValue = bigIntValue;
                 }
-                if (this._cs.currentChar === 106 /* j */ || this._cs.currentChar === 74 /* J */) {
+                if (this._cs.currentChar === 106 /* Char.j */ || this._cs.currentChar === 74 /* Char.J */) {
                     isImaginary = true;
                     text += String.fromCharCode(this._cs.currentChar);
                     this._cs.moveNext();
@@ -764,13 +764,13 @@ class Tokenizer {
         // Floating point. Sign and leading digits were already skipped over.
         this._cs.position = start;
         if (mightBeFloatingPoint ||
-            (this._cs.currentChar === 46 /* Period */ && this._cs.nextChar >= 48 /* _0 */ && this._cs.nextChar <= 57 /* _9 */)) {
+            (this._cs.currentChar === 46 /* Char.Period */ && this._cs.nextChar >= 48 /* Char._0 */ && this._cs.nextChar <= 57 /* Char._9 */)) {
             if (this._skipFloatingPointCandidate()) {
                 let text = this._cs.getText().substring(start, this._cs.position);
                 const value = parseFloat(text);
                 if (!isNaN(value)) {
                     let isImaginary = false;
-                    if (this._cs.currentChar === 106 /* j */ || this._cs.currentChar === 74 /* J */) {
+                    if (this._cs.currentChar === 106 /* Char.j */ || this._cs.currentChar === 74 /* Char.J */) {
                         isImaginary = true;
                         text += String.fromCharCode(this._cs.currentChar);
                         this._cs.moveNext();
@@ -789,111 +789,111 @@ class Tokenizer {
         const nextChar = this._cs.nextChar;
         let operatorType;
         switch (this._cs.currentChar) {
-            case 43 /* Plus */:
-                length = nextChar === 61 /* Equal */ ? 2 : 1;
-                operatorType = length === 2 ? 1 /* AddEqual */ : 0 /* Add */;
+            case 43 /* Char.Plus */:
+                length = nextChar === 61 /* Char.Equal */ ? 2 : 1;
+                operatorType = length === 2 ? 1 /* OperatorType.AddEqual */ : 0 /* OperatorType.Add */;
                 break;
-            case 38 /* Ampersand */:
-                length = nextChar === 61 /* Equal */ ? 2 : 1;
-                operatorType = length === 2 ? 4 /* BitwiseAndEqual */ : 3 /* BitwiseAnd */;
+            case 38 /* Char.Ampersand */:
+                length = nextChar === 61 /* Char.Equal */ ? 2 : 1;
+                operatorType = length === 2 ? 4 /* OperatorType.BitwiseAndEqual */ : 3 /* OperatorType.BitwiseAnd */;
                 break;
-            case 124 /* Bar */:
-                length = nextChar === 61 /* Equal */ ? 2 : 1;
-                operatorType = length === 2 ? 7 /* BitwiseOrEqual */ : 6 /* BitwiseOr */;
+            case 124 /* Char.Bar */:
+                length = nextChar === 61 /* Char.Equal */ ? 2 : 1;
+                operatorType = length === 2 ? 7 /* OperatorType.BitwiseOrEqual */ : 6 /* OperatorType.BitwiseOr */;
                 break;
-            case 94 /* Caret */:
-                length = nextChar === 61 /* Equal */ ? 2 : 1;
-                operatorType = length === 2 ? 9 /* BitwiseXorEqual */ : 8 /* BitwiseXor */;
+            case 94 /* Char.Caret */:
+                length = nextChar === 61 /* Char.Equal */ ? 2 : 1;
+                operatorType = length === 2 ? 9 /* OperatorType.BitwiseXorEqual */ : 8 /* OperatorType.BitwiseXor */;
                 break;
-            case 61 /* Equal */:
+            case 61 /* Char.Equal */:
                 if (((_a = this._activeFString) === null || _a === void 0 ? void 0 : _a.activeReplacementField) &&
                     ((_b = this._activeFString) === null || _b === void 0 ? void 0 : _b.activeReplacementField.parenDepth) === this._parenDepth &&
                     !this._activeFString.activeReplacementField.inFormatSpecifier &&
-                    nextChar !== 61 /* Equal */) {
+                    nextChar !== 61 /* Char.Equal */) {
                     length = 1;
-                    operatorType = 2 /* Assign */;
+                    operatorType = 2 /* OperatorType.Assign */;
                     break;
                 }
-                length = nextChar === 61 /* Equal */ ? 2 : 1;
-                operatorType = length === 2 ? 12 /* Equals */ : 2 /* Assign */;
+                length = nextChar === 61 /* Char.Equal */ ? 2 : 1;
+                operatorType = length === 2 ? 12 /* OperatorType.Equals */ : 2 /* OperatorType.Assign */;
                 break;
-            case 33 /* ExclamationMark */:
-                if (nextChar !== 61 /* Equal */) {
+            case 33 /* Char.ExclamationMark */:
+                if (nextChar !== 61 /* Char.Equal */) {
                     if (this._activeFString) {
                         // Handle the conversion separator (!) within an f-string.
-                        this._tokens.push(tokenizerTypes_1.Token.create(23 /* ExclamationMark */, this._cs.position, 1, this._getComments()));
+                        this._tokens.push(tokenizerTypes_1.Token.create(23 /* TokenType.ExclamationMark */, this._cs.position, 1, this._getComments()));
                         this._cs.advance(1);
                         return true;
                     }
                     return false;
                 }
                 length = 2;
-                operatorType = 28 /* NotEquals */;
+                operatorType = 28 /* OperatorType.NotEquals */;
                 break;
-            case 37 /* Percent */:
-                length = nextChar === 61 /* Equal */ ? 2 : 1;
-                operatorType = length === 2 ? 25 /* ModEqual */ : 24 /* Mod */;
+            case 37 /* Char.Percent */:
+                length = nextChar === 61 /* Char.Equal */ ? 2 : 1;
+                operatorType = length === 2 ? 25 /* OperatorType.ModEqual */ : 24 /* OperatorType.Mod */;
                 break;
-            case 126 /* Tilde */:
+            case 126 /* Char.Tilde */:
                 length = 1;
-                operatorType = 5 /* BitwiseInvert */;
+                operatorType = 5 /* OperatorType.BitwiseInvert */;
                 break;
-            case 45 /* Hyphen */:
-                if (nextChar === 62 /* Greater */) {
-                    this._tokens.push(tokenizerTypes_1.Token.create(21 /* Arrow */, this._cs.position, 2, this._getComments()));
+            case 45 /* Char.Hyphen */:
+                if (nextChar === 62 /* Char.Greater */) {
+                    this._tokens.push(tokenizerTypes_1.Token.create(21 /* TokenType.Arrow */, this._cs.position, 2, this._getComments()));
                     this._cs.advance(2);
                     return true;
                 }
-                length = nextChar === 61 /* Equal */ ? 2 : 1;
-                operatorType = length === 2 ? 34 /* SubtractEqual */ : 33 /* Subtract */;
+                length = nextChar === 61 /* Char.Equal */ ? 2 : 1;
+                operatorType = length === 2 ? 34 /* OperatorType.SubtractEqual */ : 33 /* OperatorType.Subtract */;
                 break;
-            case 42 /* Asterisk */:
-                if (nextChar === 42 /* Asterisk */) {
-                    length = this._cs.lookAhead(2) === 61 /* Equal */ ? 3 : 2;
-                    operatorType = length === 3 ? 30 /* PowerEqual */ : 29 /* Power */;
+            case 42 /* Char.Asterisk */:
+                if (nextChar === 42 /* Char.Asterisk */) {
+                    length = this._cs.lookAhead(2) === 61 /* Char.Equal */ ? 3 : 2;
+                    operatorType = length === 3 ? 30 /* OperatorType.PowerEqual */ : 29 /* OperatorType.Power */;
                 }
                 else {
-                    length = nextChar === 61 /* Equal */ ? 2 : 1;
-                    operatorType = length === 2 ? 27 /* MultiplyEqual */ : 26 /* Multiply */;
+                    length = nextChar === 61 /* Char.Equal */ ? 2 : 1;
+                    operatorType = length === 2 ? 27 /* OperatorType.MultiplyEqual */ : 26 /* OperatorType.Multiply */;
                 }
                 break;
-            case 47 /* Slash */:
-                if (nextChar === 47 /* Slash */) {
-                    length = this._cs.lookAhead(2) === 61 /* Equal */ ? 3 : 2;
-                    operatorType = length === 3 ? 14 /* FloorDivideEqual */ : 13 /* FloorDivide */;
+            case 47 /* Char.Slash */:
+                if (nextChar === 47 /* Char.Slash */) {
+                    length = this._cs.lookAhead(2) === 61 /* Char.Equal */ ? 3 : 2;
+                    operatorType = length === 3 ? 14 /* OperatorType.FloorDivideEqual */ : 13 /* OperatorType.FloorDivide */;
                 }
                 else {
-                    length = nextChar === 61 /* Equal */ ? 2 : 1;
-                    operatorType = length === 2 ? 11 /* DivideEqual */ : 10 /* Divide */;
+                    length = nextChar === 61 /* Char.Equal */ ? 2 : 1;
+                    operatorType = length === 2 ? 11 /* OperatorType.DivideEqual */ : 10 /* OperatorType.Divide */;
                 }
                 break;
-            case 60 /* Less */:
-                if (nextChar === 60 /* Less */) {
-                    length = this._cs.lookAhead(2) === 61 /* Equal */ ? 3 : 2;
-                    operatorType = length === 3 ? 18 /* LeftShiftEqual */ : 17 /* LeftShift */;
+            case 60 /* Char.Less */:
+                if (nextChar === 60 /* Char.Less */) {
+                    length = this._cs.lookAhead(2) === 61 /* Char.Equal */ ? 3 : 2;
+                    operatorType = length === 3 ? 18 /* OperatorType.LeftShiftEqual */ : 17 /* OperatorType.LeftShift */;
                 }
-                else if (nextChar === 62 /* Greater */) {
+                else if (nextChar === 62 /* Char.Greater */) {
                     length = 2;
-                    operatorType = 19 /* LessOrGreaterThan */;
+                    operatorType = 19 /* OperatorType.LessOrGreaterThan */;
                 }
                 else {
-                    length = nextChar === 61 /* Equal */ ? 2 : 1;
-                    operatorType = length === 2 ? 21 /* LessThanOrEqual */ : 20 /* LessThan */;
+                    length = nextChar === 61 /* Char.Equal */ ? 2 : 1;
+                    operatorType = length === 2 ? 21 /* OperatorType.LessThanOrEqual */ : 20 /* OperatorType.LessThan */;
                 }
                 break;
-            case 62 /* Greater */:
-                if (nextChar === 62 /* Greater */) {
-                    length = this._cs.lookAhead(2) === 61 /* Equal */ ? 3 : 2;
-                    operatorType = length === 3 ? 32 /* RightShiftEqual */ : 31 /* RightShift */;
+            case 62 /* Char.Greater */:
+                if (nextChar === 62 /* Char.Greater */) {
+                    length = this._cs.lookAhead(2) === 61 /* Char.Equal */ ? 3 : 2;
+                    operatorType = length === 3 ? 32 /* OperatorType.RightShiftEqual */ : 31 /* OperatorType.RightShift */;
                 }
                 else {
-                    length = nextChar === 61 /* Equal */ ? 2 : 1;
-                    operatorType = length === 2 ? 16 /* GreaterThanOrEqual */ : 15 /* GreaterThan */;
+                    length = nextChar === 61 /* Char.Equal */ ? 2 : 1;
+                    operatorType = length === 2 ? 16 /* OperatorType.GreaterThanOrEqual */ : 15 /* OperatorType.GreaterThan */;
                 }
                 break;
-            case 64 /* At */:
-                length = nextChar === 61 /* Equal */ ? 2 : 1;
-                operatorType = length === 2 ? 23 /* MatrixMultiplyEqual */ : 22 /* MatrixMultiply */;
+            case 64 /* Char.At */:
+                length = nextChar === 61 /* Char.Equal */ ? 2 : 1;
+                operatorType = length === 2 ? 23 /* OperatorType.MatrixMultiplyEqual */ : 22 /* OperatorType.MatrixMultiply */;
                 break;
             default:
                 return false;
@@ -905,8 +905,8 @@ class Tokenizer {
     _handleInvalid() {
         const start = this._cs.position;
         while (true) {
-            if (this._cs.currentChar === 10 /* LineFeed */ ||
-                this._cs.currentChar === 13 /* CarriageReturn */ ||
+            if (this._cs.currentChar === 10 /* Char.LineFeed */ ||
+                this._cs.currentChar === 13 /* Char.CarriageReturn */ ||
                 this._cs.isAtWhiteSpace() ||
                 this._cs.isEndOfStream()) {
                 break;
@@ -921,7 +921,7 @@ class Tokenizer {
         }
         const length = this._cs.position - start;
         if (length > 0) {
-            this._tokens.push(tokenizerTypes_1.Token.create(0 /* Invalid */, start, length, this._getComments()));
+            this._tokens.push(tokenizerTypes_1.Token.create(0 /* TokenType.Invalid */, start, length, this._getComments()));
             return true;
         }
         return false;
@@ -946,7 +946,7 @@ class Tokenizer {
         }
         return 'line';
         function isMagicChar(ch) {
-            return ch === 37 /* Percent */ || ch === 33 /* ExclamationMark */;
+            return ch === 37 /* Char.Percent */ || ch === 33 /* Char.ExclamationMark */;
         }
     }
     _handleIPythonMagics(type) {
@@ -954,7 +954,7 @@ class Tokenizer {
         let begin = start;
         do {
             this._cs.skipToEol();
-            if (type === 1 /* IPythonMagic */ || type === 2 /* IPythonShellEscape */) {
+            if (type === 1 /* CommentType.IPythonMagic */ || type === 2 /* CommentType.IPythonShellEscape */) {
                 const length = this._cs.position - begin;
                 const value = this._cs.getText().substring(begin, begin + length);
                 // is it multiline magics?
@@ -990,7 +990,7 @@ class Tokenizer {
                 range: textRange,
                 rulesList: this._getIgnoreCommentRulesList(commentStart, typeIgnoreRegexMatch),
             };
-            if (this._tokens.findIndex((t) => t.type !== 2 /* NewLine */ && t && t.type !== 3 /* Indent */) < 0) {
+            if (this._tokens.findIndex((t) => t.type !== 2 /* TokenType.NewLine */ && t && t.type !== 3 /* TokenType.Indent */) < 0) {
                 this._typeIgnoreAll = ignoreComment;
             }
             else {
@@ -1043,27 +1043,27 @@ class Tokenizer {
         }
     }
     _getStringPrefixLength() {
-        if (this._cs.currentChar === 39 /* SingleQuote */ || this._cs.currentChar === 34 /* DoubleQuote */) {
+        if (this._cs.currentChar === 39 /* Char.SingleQuote */ || this._cs.currentChar === 34 /* Char.DoubleQuote */) {
             // Simple string, no prefix
             return 0;
         }
-        if (this._cs.nextChar === 39 /* SingleQuote */ || this._cs.nextChar === 34 /* DoubleQuote */) {
+        if (this._cs.nextChar === 39 /* Char.SingleQuote */ || this._cs.nextChar === 34 /* Char.DoubleQuote */) {
             switch (this._cs.currentChar) {
-                case 102 /* f */:
-                case 70 /* F */:
-                case 114 /* r */:
-                case 82 /* R */:
-                case 98 /* b */:
-                case 66 /* B */:
-                case 117 /* u */:
-                case 85 /* U */:
+                case 102 /* Char.f */:
+                case 70 /* Char.F */:
+                case 114 /* Char.r */:
+                case 82 /* Char.R */:
+                case 98 /* Char.b */:
+                case 66 /* Char.B */:
+                case 117 /* Char.u */:
+                case 85 /* Char.U */:
                     // Single-char prefix like u"" or r""
                     return 1;
                 default:
                     break;
             }
         }
-        if (this._cs.lookAhead(2) === 39 /* SingleQuote */ || this._cs.lookAhead(2) === 34 /* DoubleQuote */) {
+        if (this._cs.lookAhead(2) === 39 /* Char.SingleQuote */ || this._cs.lookAhead(2) === 34 /* Char.DoubleQuote */) {
             const prefix = this._cs
                 .getText()
                 .substring(this._cs.position, this._cs.position + 2)
@@ -1083,34 +1083,34 @@ class Tokenizer {
         return -1;
     }
     _getQuoteTypeFlags(prefix) {
-        let flags = 0 /* None */;
+        let flags = 0 /* StringTokenFlags.None */;
         prefix = prefix.toLowerCase();
         for (let i = 0; i < prefix.length; i++) {
             switch (prefix[i]) {
                 case 'u':
-                    flags |= 16 /* Unicode */;
+                    flags |= 16 /* StringTokenFlags.Unicode */;
                     break;
                 case 'b':
-                    flags |= 32 /* Bytes */;
+                    flags |= 32 /* StringTokenFlags.Bytes */;
                     break;
                 case 'r':
-                    flags |= 8 /* Raw */;
+                    flags |= 8 /* StringTokenFlags.Raw */;
                     break;
                 case 'f':
-                    flags |= 64 /* Format */;
+                    flags |= 64 /* StringTokenFlags.Format */;
                     break;
             }
         }
-        if (this._cs.currentChar === 39 /* SingleQuote */) {
-            flags |= 1 /* SingleQuote */;
-            if (this._cs.nextChar === 39 /* SingleQuote */ && this._cs.lookAhead(2) === 39 /* SingleQuote */) {
-                flags |= 4 /* Triplicate */;
+        if (this._cs.currentChar === 39 /* Char.SingleQuote */) {
+            flags |= 1 /* StringTokenFlags.SingleQuote */;
+            if (this._cs.nextChar === 39 /* Char.SingleQuote */ && this._cs.lookAhead(2) === 39 /* Char.SingleQuote */) {
+                flags |= 4 /* StringTokenFlags.Triplicate */;
             }
         }
-        else if (this._cs.currentChar === 34 /* DoubleQuote */) {
-            flags |= 2 /* DoubleQuote */;
-            if (this._cs.nextChar === 34 /* DoubleQuote */ && this._cs.lookAhead(2) === 34 /* DoubleQuote */) {
-                flags |= 4 /* Triplicate */;
+        else if (this._cs.currentChar === 34 /* Char.DoubleQuote */) {
+            flags |= 2 /* StringTokenFlags.DoubleQuote */;
+            if (this._cs.nextChar === 34 /* Char.DoubleQuote */ && this._cs.lookAhead(2) === 34 /* Char.DoubleQuote */) {
+                flags |= 4 /* StringTokenFlags.Triplicate */;
             }
         }
         return flags;
@@ -1118,8 +1118,8 @@ class Tokenizer {
     _handleString(flags, stringPrefixLength) {
         var _a;
         const start = this._cs.position - stringPrefixLength;
-        if (flags & 64 /* Format */) {
-            if (flags & 4 /* Triplicate */) {
+        if (flags & 64 /* StringTokenFlags.Format */) {
+            if (flags & 4 /* StringTokenFlags.Triplicate */) {
                 this._cs.advance(3);
             }
             else {
@@ -1139,12 +1139,12 @@ class Tokenizer {
             this._tokens.push(fStringStartToken);
         }
         else {
-            if (flags & 4 /* Triplicate */) {
+            if (flags & 4 /* StringTokenFlags.Triplicate */) {
                 this._cs.advance(3);
             }
             else {
                 this._cs.moveNext();
-                if (flags & 1 /* SingleQuote */) {
+                if (flags & 1 /* StringTokenFlags.SingleQuote */) {
                     this._singleQuoteCount++;
                 }
                 else {
@@ -1157,15 +1157,15 @@ class Tokenizer {
             // of an active f-string. If so, we'll treat it as an f-string end
             // token rather than an unterminated regular string. This helps with
             // parse error recovery if a closing bracket is missing in an f-string.
-            if ((stringLiteralInfo.flags & 65536 /* Unterminated */) !== 0 &&
+            if ((stringLiteralInfo.flags & 65536 /* StringTokenFlags.Unterminated */) !== 0 &&
                 ((_a = this._activeFString) === null || _a === void 0 ? void 0 : _a.activeReplacementField)) {
                 if ((flags &
-                    (32 /* Bytes */ |
-                        16 /* Unicode */ |
-                        8 /* Raw */ |
-                        64 /* Format */)) ===
+                    (32 /* StringTokenFlags.Bytes */ |
+                        16 /* StringTokenFlags.Unicode */ |
+                        8 /* StringTokenFlags.Raw */ |
+                        64 /* StringTokenFlags.Format */)) ===
                     0) {
-                    const quoteTypeMask = 4 /* Triplicate */ | 2 /* DoubleQuote */ | 1 /* SingleQuote */;
+                    const quoteTypeMask = 4 /* StringTokenFlags.Triplicate */ | 2 /* StringTokenFlags.DoubleQuote */ | 1 /* StringTokenFlags.SingleQuote */;
                     if ((this._activeFString.startToken.flags & quoteTypeMask) === (flags & quoteTypeMask)) {
                         // Unwind to the start of this string token and terminate any replacement fields
                         // that are active. This will cause the tokenizer to re-process the quote as an
@@ -1193,9 +1193,9 @@ class Tokenizer {
         const flags = activeFString.startToken.flags;
         const stringLiteralInfo = this._skipToEndOfStringLiteral(flags, inFormatSpecifier);
         const end = this._cs.position;
-        const isUnterminated = (stringLiteralInfo.flags & 65536 /* Unterminated */) !== 0;
-        const sawReplacementFieldStart = (stringLiteralInfo.flags & 128 /* ReplacementFieldStart */) !== 0;
-        const sawReplacementFieldEnd = (stringLiteralInfo.flags & 256 /* ReplacementFieldEnd */) !== 0;
+        const isUnterminated = (stringLiteralInfo.flags & 65536 /* StringTokenFlags.Unterminated */) !== 0;
+        const sawReplacementFieldStart = (stringLiteralInfo.flags & 128 /* StringTokenFlags.ReplacementFieldStart */) !== 0;
+        const sawReplacementFieldEnd = (stringLiteralInfo.flags & 256 /* StringTokenFlags.ReplacementFieldEnd */) !== 0;
         const sawEndQuote = !isUnterminated && !sawReplacementFieldStart && !sawReplacementFieldEnd;
         let middleTokenLength = end - start;
         if (sawEndQuote) {
@@ -1213,39 +1213,39 @@ class Tokenizer {
         }
     }
     _skipToEndOfStringLiteral(flags, inFormatSpecifier = false) {
-        const quoteChar = flags & 1 /* SingleQuote */ ? 39 /* SingleQuote */ : 34 /* DoubleQuote */;
-        const isTriplicate = (flags & 4 /* Triplicate */) !== 0;
-        const isFString = (flags & 64 /* Format */) !== 0;
+        const quoteChar = flags & 1 /* StringTokenFlags.SingleQuote */ ? 39 /* Char.SingleQuote */ : 34 /* Char.DoubleQuote */;
+        const isTriplicate = (flags & 4 /* StringTokenFlags.Triplicate */) !== 0;
+        const isFString = (flags & 64 /* StringTokenFlags.Format */) !== 0;
         let isInNamedUnicodeEscape = false;
         let escapedValueParts = [];
         while (true) {
             if (this._cs.isEndOfStream()) {
                 // Hit the end of file without a termination.
-                flags |= 65536 /* Unterminated */;
+                flags |= 65536 /* StringTokenFlags.Unterminated */;
                 return { escapedValue: String.fromCharCode.apply(undefined, escapedValueParts), flags };
             }
-            if (this._cs.currentChar === 92 /* Backslash */) {
+            if (this._cs.currentChar === 92 /* Char.Backslash */) {
                 escapedValueParts.push(this._cs.currentChar);
                 // Move past the escape (backslash) character.
                 this._cs.moveNext();
                 // Handle the special escape sequence /N{name} for unicode characters.
                 if (!isInNamedUnicodeEscape &&
-                    this._cs.getCurrentChar() === 78 /* N */ &&
-                    this._cs.nextChar === 123 /* OpenBrace */) {
+                    this._cs.getCurrentChar() === 78 /* Char.N */ &&
+                    this._cs.nextChar === 123 /* Char.OpenBrace */) {
                     isInNamedUnicodeEscape = true;
                 }
                 else {
                     // If this is an f-string, the only escapes that are allowed is for
                     // a single or double quote symbol or a newline/carriage return.
-                    const isEscapedQuote = this._cs.getCurrentChar() === 39 /* SingleQuote */ ||
-                        this._cs.getCurrentChar() === 34 /* DoubleQuote */;
-                    const isEscapedNewLine = this._cs.getCurrentChar() === 13 /* CarriageReturn */ ||
-                        this._cs.getCurrentChar() === 10 /* LineFeed */;
-                    const isEscapedBackslash = this._cs.getCurrentChar() === 92 /* Backslash */;
+                    const isEscapedQuote = this._cs.getCurrentChar() === 39 /* Char.SingleQuote */ ||
+                        this._cs.getCurrentChar() === 34 /* Char.DoubleQuote */;
+                    const isEscapedNewLine = this._cs.getCurrentChar() === 13 /* Char.CarriageReturn */ ||
+                        this._cs.getCurrentChar() === 10 /* Char.LineFeed */;
+                    const isEscapedBackslash = this._cs.getCurrentChar() === 92 /* Char.Backslash */;
                     if (!isFString || isEscapedBackslash || isEscapedQuote || isEscapedNewLine) {
                         if (isEscapedNewLine) {
-                            if (this._cs.getCurrentChar() === 13 /* CarriageReturn */ &&
-                                this._cs.nextChar === 10 /* LineFeed */) {
+                            if (this._cs.getCurrentChar() === 13 /* Char.CarriageReturn */ &&
+                                this._cs.nextChar === 10 /* Char.LineFeed */) {
                                 escapedValueParts.push(this._cs.currentChar);
                                 this._cs.moveNext();
                             }
@@ -1260,14 +1260,14 @@ class Tokenizer {
                     }
                 }
             }
-            else if (this._cs.currentChar === 10 /* LineFeed */ || this._cs.currentChar === 13 /* CarriageReturn */) {
+            else if (this._cs.currentChar === 10 /* Char.LineFeed */ || this._cs.currentChar === 13 /* Char.CarriageReturn */) {
                 if (!isTriplicate && !isFString) {
                     // Unterminated single-line string
-                    flags |= 65536 /* Unterminated */;
+                    flags |= 65536 /* StringTokenFlags.Unterminated */;
                     return { escapedValue: String.fromCharCode.apply(undefined, escapedValueParts), flags };
                 }
                 // Skip over the new line (either one or two characters).
-                if (this._cs.currentChar === 13 /* CarriageReturn */ && this._cs.nextChar === 10 /* LineFeed */) {
+                if (this._cs.currentChar === 13 /* Char.CarriageReturn */ && this._cs.nextChar === 10 /* Char.LineFeed */) {
                     escapedValueParts.push(this._cs.currentChar);
                     this._cs.moveNext();
                 }
@@ -1286,9 +1286,9 @@ class Tokenizer {
                 this._cs.advance(3);
                 break;
             }
-            else if (!isInNamedUnicodeEscape && isFString && this._cs.currentChar === 123 /* OpenBrace */) {
-                if (inFormatSpecifier || this._cs.nextChar !== 123 /* OpenBrace */) {
-                    flags |= 128 /* ReplacementFieldStart */;
+            else if (!isInNamedUnicodeEscape && isFString && this._cs.currentChar === 123 /* Char.OpenBrace */) {
+                if (inFormatSpecifier || this._cs.nextChar !== 123 /* Char.OpenBrace */) {
+                    flags |= 128 /* StringTokenFlags.ReplacementFieldStart */;
                     break;
                 }
                 else {
@@ -1298,14 +1298,14 @@ class Tokenizer {
                     this._cs.moveNext();
                 }
             }
-            else if (isInNamedUnicodeEscape && this._cs.currentChar === 125 /* CloseBrace */) {
+            else if (isInNamedUnicodeEscape && this._cs.currentChar === 125 /* Char.CloseBrace */) {
                 isInNamedUnicodeEscape = false;
                 escapedValueParts.push(this._cs.currentChar);
                 this._cs.moveNext();
             }
-            else if (isFString && this._cs.currentChar === 125 /* CloseBrace */) {
-                if (inFormatSpecifier || this._cs.nextChar !== 125 /* CloseBrace */) {
-                    flags |= 256 /* ReplacementFieldEnd */;
+            else if (isFString && this._cs.currentChar === 125 /* Char.CloseBrace */) {
+                if (inFormatSpecifier || this._cs.nextChar !== 125 /* Char.CloseBrace */) {
+                    flags |= 256 /* StringTokenFlags.ReplacementFieldEnd */;
                     break;
                 }
                 else {
@@ -1326,7 +1326,7 @@ class Tokenizer {
         // a string literal or a docstring, so this should be fine.
         if (escapedValueParts.length > exports.maxStringTokenLength) {
             escapedValueParts = escapedValueParts.slice(0, exports.maxStringTokenLength);
-            flags |= 131072 /* ExceedsMaxSize */;
+            flags |= 131072 /* StringTokenFlags.ExceedsMaxSize */;
         }
         return { escapedValue: String.fromCharCode.apply(undefined, escapedValueParts), flags };
     }
@@ -1336,7 +1336,7 @@ class Tokenizer {
         this._skipFractionalNumber();
         if (this._cs.position > start) {
             // Optional exponent sign
-            if (this._cs.currentChar === 101 /* e */ || this._cs.currentChar === 69 /* E */) {
+            if (this._cs.currentChar === 101 /* Char.e */ || this._cs.currentChar === 69 /* Char.E */) {
                 this._cs.moveNext();
                 // Skip exponent value
                 this._skipDecimalNumber(/* allowSign */ true);
@@ -1346,14 +1346,14 @@ class Tokenizer {
     }
     _skipFractionalNumber() {
         this._skipDecimalNumber(false);
-        if (this._cs.currentChar === 46 /* Period */) {
+        if (this._cs.currentChar === 46 /* Char.Period */) {
             // Optional period
             this._cs.moveNext();
         }
         this._skipDecimalNumber(false);
     }
     _skipDecimalNumber(allowSign) {
-        if (allowSign && (this._cs.currentChar === 45 /* Hyphen */ || this._cs.currentChar === 43 /* Plus */)) {
+        if (allowSign && (this._cs.currentChar === 45 /* Char.Hyphen */ || this._cs.currentChar === 43 /* Char.Plus */)) {
             // Optional sign
             this._cs.moveNext();
         }