@zzzen/pyright-internal 1.1.254

package/dist/parser/tokenizer.js

@@ -0,0 +1,1111 @@
+"use strict";
+/*
+ * tokenizer.ts
+ * Copyright (c) Microsoft Corporation.
+ * Licensed under the MIT license.
+ * Author: Eric Traut
+ *
+ * Based on code from vscode-python repository:
+ *  https://github.com/Microsoft/vscode-python
+ *
+ * Converts a Python program text stream into a stream of tokens.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Tokenizer = exports.defaultTabSize = void 0;
+const textRangeCollection_1 = require("../common/textRangeCollection");
+const characters_1 = require("./characters");
+const characterStream_1 = require("./characterStream");
+const tokenizerTypes_1 = require("./tokenizerTypes");
+// This must be a Map, as operations like {}["constructor"] succeed.
+const _keywords = new Map([
+    ['and', 0 /* And */],
+    ['as', 1 /* As */],
+    ['assert', 2 /* Assert */],
+    ['async', 3 /* Async */],
+    ['await', 4 /* Await */],
+    ['break', 5 /* Break */],
+    ['case', 6 /* Case */],
+    ['class', 7 /* Class */],
+    ['continue', 8 /* Continue */],
+    ['__debug__', 9 /* Debug */],
+    ['def', 10 /* Def */],
+    ['del', 11 /* Del */],
+    ['elif', 12 /* Elif */],
+    ['else', 13 /* Else */],
+    ['except', 14 /* Except */],
+    ['finally', 16 /* Finally */],
+    ['for', 17 /* For */],
+    ['from', 18 /* From */],
+    ['global', 19 /* Global */],
+    ['if', 20 /* If */],
+    ['import', 21 /* Import */],
+    ['in', 22 /* In */],
+    ['is', 23 /* Is */],
+    ['lambda', 24 /* Lambda */],
+    ['match', 25 /* Match */],
+    ['nonlocal', 27 /* Nonlocal */],
+    ['not', 28 /* Not */],
+    ['or', 29 /* Or */],
+    ['pass', 30 /* Pass */],
+    ['raise', 31 /* Raise */],
+    ['return', 32 /* Return */],
+    ['try', 34 /* Try */],
+    ['while', 35 /* While */],
+    ['with', 36 /* With */],
+    ['yield', 37 /* Yield */],
+    ['False', 15 /* False */],
+    ['None', 26 /* None */],
+    ['True', 33 /* True */],
+]);
+const _operatorInfo = {
+    [0 /* Add */]: 1 /* Unary */ | 2 /* Binary */,
+    [1 /* AddEqual */]: 4 /* Assignment */,
+    [2 /* Assign */]: 4 /* Assignment */,
+    [3 /* BitwiseAnd */]: 2 /* Binary */,
+    [4 /* BitwiseAndEqual */]: 4 /* Assignment */,
+    [5 /* BitwiseInvert */]: 1 /* Unary */,
+    [6 /* BitwiseOr */]: 2 /* Binary */,
+    [7 /* BitwiseOrEqual */]: 4 /* Assignment */,
+    [8 /* BitwiseXor */]: 2 /* Binary */,
+    [9 /* BitwiseXorEqual */]: 4 /* Assignment */,
+    [10 /* Divide */]: 2 /* Binary */,
+    [11 /* DivideEqual */]: 4 /* Assignment */,
+    [12 /* Equals */]: 2 /* Binary */ | 8 /* Comparison */,
+    [13 /* FloorDivide */]: 2 /* Binary */,
+    [14 /* FloorDivideEqual */]: 4 /* Assignment */,
+    [15 /* GreaterThan */]: 2 /* Binary */ | 8 /* Comparison */,
+    [16 /* GreaterThanOrEqual */]: 2 /* Binary */ | 8 /* Comparison */,
+    [17 /* LeftShift */]: 2 /* Binary */,
+    [18 /* LeftShiftEqual */]: 4 /* Assignment */,
+    [19 /* LessOrGreaterThan */]: 2 /* Binary */ | 8 /* Comparison */ | 16 /* Deprecated */,
+    [20 /* LessThan */]: 2 /* Binary */ | 8 /* Comparison */,
+    [21 /* LessThanOrEqual */]: 2 /* Binary */ | 8 /* Comparison */,
+    [22 /* MatrixMultiply */]: 2 /* Binary */,
+    [23 /* MatrixMultiplyEqual */]: 4 /* Assignment */,
+    [24 /* Mod */]: 2 /* Binary */,
+    [25 /* ModEqual */]: 4 /* Assignment */,
+    [26 /* Multiply */]: 2 /* Binary */,
+    [27 /* MultiplyEqual */]: 4 /* Assignment */,
+    [28 /* NotEquals */]: 2 /* Binary */ | 8 /* Comparison */,
+    [29 /* Power */]: 2 /* Binary */,
+    [30 /* PowerEqual */]: 4 /* Assignment */,
+    [31 /* RightShift */]: 2 /* Binary */,
+    [32 /* RightShiftEqual */]: 4 /* Assignment */,
+    [33 /* Subtract */]: 2 /* Binary */,
+    [34 /* SubtractEqual */]: 4 /* Assignment */,
+    [36 /* And */]: 2 /* Binary */,
+    [37 /* Or */]: 2 /* Binary */,
+    [38 /* Not */]: 1 /* Unary */,
+    [39 /* Is */]: 2 /* Binary */,
+    [40 /* IsNot */]: 2 /* Binary */,
+    [41 /* In */]: 2 /* Binary */,
+    [42 /* NotIn */]: 2 /* Binary */,
+};
+const _byteOrderMarker = 0xfeff;
+const _maxStringTokenLength = 32 * 1024;
+exports.defaultTabSize = 8;
+class Tokenizer {
+    constructor() {
+        this._cs = new characterStream_1.CharacterStream('');
+        this._tokens = [];
+        this._prevLineStart = 0;
+        this._parenDepth = 0;
+        this._lineRanges = [];
+        this._indentAmounts = [];
+        this._typeIgnoreLines = new Map();
+        this._pyrightIgnoreLines = new Map();
+        // Total times CR, CR/LF, and LF are used to terminate
+        // lines. Used to determine the predominant line ending.
+        this._crCount = 0;
+        this._crLfCount = 0;
+        this._lfCount = 0;
+        // Number of times an indent token is emitted.
+        this._indentCount = 0;
+        // Number of times an indent token is emitted and a tab character
+        // is present (used to determine predominant tab sequence).
+        this._indentTabCount = 0;
+        // Number of spaces that are added for an indent token
+        // (used to determine predominant tab sequence).
+        this._indentSpacesTotal = 0;
+        // Number of single or double quote string literals found
+        // in the code.
+        this._singleQuoteCount = 0;
+        this._doubleQuoteCount = 0;
+        // ipython mode
+        this._ipythonMode = false;
+    }
+    tokenize(text, start, length, initialParenDepth = 0, ipythonMode = false) {
+        if (start === undefined) {
+            start = 0;
+        }
+        else if (start < 0 || start > text.length) {
+            throw new Error(`Invalid range start (start=${start}, text.length=${text.length})`);
+        }
+        if (length === undefined) {
+            length = text.length;
+        }
+        else if (length < 0 || start + length > text.length) {
+            throw new Error(`Invalid range length (start=${start}, length=${length}, text.length=${text.length})`);
+        }
+        else if (start + length < text.length) {
+            text = text.substring(0, start + length);
+        }
+        this._cs = new characterStream_1.CharacterStream(text);
+        this._cs.position = start;
+        this._tokens = [];
+        this._prevLineStart = 0;
+        this._parenDepth = initialParenDepth;
+        this._lineRanges = [];
+        this._indentAmounts = [];
+        this._ipythonMode = ipythonMode;
+        const end = start + length;
+        if (start === 0) {
+            this._readIndentationAfterNewLine();
+        }
+        while (!this._cs.isEndOfStream()) {
+            this._addNextToken();
+            if (this._cs.position >= end) {
+                break;
+            }
+        }
+        // Insert an implied new line to make parsing easier.
+        if (this._tokens.length === 0 || this._tokens[this._tokens.length - 1].type !== 2 /* NewLine */) {
+            this._tokens.push(tokenizerTypes_1.NewLineToken.create(this._cs.position, 0, 3 /* Implied */, this._getComments()));
+        }
+        // Insert any implied dedent tokens.
+        this._setIndent(0, 0, /* isSpacePresent */ false, /* isTabPresent */ false);
+        // Add a final end-of-stream token to make parsing easier.
+        this._tokens.push(tokenizerTypes_1.Token.create(1 /* EndOfStream */, this._cs.position, 0, this._getComments()));
+        // Add the final line range.
+        this._addLineRange();
+        let predominantEndOfLineSequence = '\n';
+        if (this._crCount > this._crLfCount && this._crCount > this._lfCount) {
+            predominantEndOfLineSequence = '\r';
+        }
+        else if (this._crLfCount > this._crCount && this._crLfCount > this._lfCount) {
+            predominantEndOfLineSequence = '\r\n';
+        }
+        let predominantTabSequence = '    ';
+        // If more than half of the indents use tab sequences,
+        // assume we're using tabs rather than spaces.
+        if (this._indentTabCount > this._indentCount / 2) {
+            predominantTabSequence = '\t';
+        }
+        else if (this._indentCount > 0) {
+            // Compute the average number of spaces per indent
+            // to estimate the predominant tab value.
+            let averageSpacePerIndent = Math.round(this._indentSpacesTotal / this._indentCount);
+            if (averageSpacePerIndent < 1) {
+                averageSpacePerIndent = 1;
+            }
+            else if (averageSpacePerIndent > exports.defaultTabSize) {
+                averageSpacePerIndent = exports.defaultTabSize;
+            }
+            predominantTabSequence = '';
+            for (let i = 0; i < averageSpacePerIndent; i++) {
+                predominantTabSequence += ' ';
+            }
+        }
+        return {
+            tokens: new textRangeCollection_1.TextRangeCollection(this._tokens),
+            lines: new textRangeCollection_1.TextRangeCollection(this._lineRanges),
+            typeIgnoreLines: this._typeIgnoreLines,
+            typeIgnoreAll: this._typeIgnoreAll,
+            pyrightIgnoreLines: this._pyrightIgnoreLines,
+            predominantEndOfLineSequence,
+            predominantTabSequence,
+            predominantSingleQuoteCharacter: this._singleQuoteCount >= this._doubleQuoteCount ? "'" : '"',
+        };
+    }
+    static getOperatorInfo(operatorType) {
+        return _operatorInfo[operatorType];
+    }
+    static isOperatorAssignment(operatorType) {
+        if (operatorType === undefined || _operatorInfo[operatorType] === undefined) {
+            return false;
+        }
+        return (_operatorInfo[operatorType] & 4 /* Assignment */) !== 0;
+    }
+    static isOperatorComparison(operatorType) {
+        if (operatorType === undefined || _operatorInfo[operatorType] === undefined) {
+            return false;
+        }
+        return (_operatorInfo[operatorType] & 8 /* Comparison */) !== 0;
+    }
+    _addNextToken() {
+        this._cs.skipWhitespace();
+        if (this._cs.isEndOfStream()) {
+            return;
+        }
+        if (!this._handleCharacter()) {
+            this._cs.moveNext();
+        }
+    }
+    _handleCharacter() {
+        // f-strings, b-strings, etc
+        const stringPrefixLength = this._getStringPrefixLength();
+        if (stringPrefixLength >= 0) {
+            let stringPrefix = '';
+            if (stringPrefixLength > 0) {
+                stringPrefix = this._cs.getText().substring(this._cs.position, this._cs.position + stringPrefixLength);
+                // Indeed a string
+                this._cs.advance(stringPrefixLength);
+            }
+            const quoteTypeFlags = this._getQuoteTypeFlags(stringPrefix);
+            if (quoteTypeFlags !== 0 /* None */) {
+                this._handleString(quoteTypeFlags, stringPrefixLength);
+                return true;
+            }
+        }
+        if (this._cs.currentChar === 35 /* Hash */) {
+            this._handleComment();
+            return true;
+        }
+        if (this._ipythonMode && this._isIPythonMagics()) {
+            this._handleIPythonMagics(this._cs.currentChar === 37 /* Percent */ ? 1 /* IPythonMagic */ : 2 /* IPythonShellEscape */);
+            return true;
+        }
+        switch (this._cs.currentChar) {
+            case _byteOrderMarker: {
+                // Skip the BOM if it's at the start of the file.
+                if (this._cs.position === 0) {
+                    return false;
+                }
+                return this._handleInvalid();
+            }
+            case 13 /* CarriageReturn */: {
+                const length = this._cs.nextChar === 10 /* LineFeed */ ? 2 : 1;
+                const newLineType = length === 2 ? 2 /* CarriageReturnLineFeed */ : 0 /* CarriageReturn */;
+                this._handleNewLine(length, newLineType);
+                return true;
+            }
+            case 10 /* LineFeed */: {
+                this._handleNewLine(1, 1 /* LineFeed */);
+                return true;
+            }
+            case 92 /* Backslash */: {
+                if (this._cs.nextChar === 13 /* CarriageReturn */) {
+                    if (this._cs.lookAhead(2) === 10 /* LineFeed */) {
+                        this._cs.advance(3);
+                    }
+                    else {
+                        this._cs.advance(2);
+                    }
+                    this._addLineRange();
+                    return true;
+                }
+                else if (this._cs.nextChar === 10 /* LineFeed */) {
+                    this._cs.advance(2);
+                    this._addLineRange();
+                    return true;
+                }
+                return this._handleInvalid();
+            }
+            case 40 /* OpenParenthesis */: {
+                this._parenDepth++;
+                this._tokens.push(tokenizerTypes_1.Token.create(13 /* OpenParenthesis */, this._cs.position, 1, this._getComments()));
+                break;
+            }
+            case 41 /* CloseParenthesis */: {
+                if (this._parenDepth > 0) {
+                    this._parenDepth--;
+                }
+                this._tokens.push(tokenizerTypes_1.Token.create(14 /* CloseParenthesis */, this._cs.position, 1, this._getComments()));
+                break;
+            }
+            case 91 /* OpenBracket */: {
+                this._parenDepth++;
+                this._tokens.push(tokenizerTypes_1.Token.create(15 /* OpenBracket */, this._cs.position, 1, this._getComments()));
+                break;
+            }
+            case 93 /* CloseBracket */: {
+                if (this._parenDepth > 0) {
+                    this._parenDepth--;
+                }
+                this._tokens.push(tokenizerTypes_1.Token.create(16 /* CloseBracket */, this._cs.position, 1, this._getComments()));
+                break;
+            }
+            case 123 /* OpenBrace */: {
+                this._parenDepth++;
+                this._tokens.push(tokenizerTypes_1.Token.create(17 /* OpenCurlyBrace */, this._cs.position, 1, this._getComments()));
+                break;
+            }
+            case 125 /* CloseBrace */: {
+                if (this._parenDepth > 0) {
+                    this._parenDepth--;
+                }
+                this._tokens.push(tokenizerTypes_1.Token.create(18 /* CloseCurlyBrace */, this._cs.position, 1, this._getComments()));
+                break;
+            }
+            case 44 /* Comma */: {
+                this._tokens.push(tokenizerTypes_1.Token.create(12 /* Comma */, this._cs.position, 1, this._getComments()));
+                break;
+            }
+            case 96 /* Backtick */: {
+                this._tokens.push(tokenizerTypes_1.Token.create(22 /* Backtick */, this._cs.position, 1, this._getComments()));
+                break;
+            }
+            case 59 /* Semicolon */: {
+                this._tokens.push(tokenizerTypes_1.Token.create(11 /* Semicolon */, this._cs.position, 1, this._getComments()));
+                break;
+            }
+            case 58 /* Colon */: {
+                if (this._cs.nextChar === 61 /* Equal */) {
+                    this._tokens.push(tokenizerTypes_1.OperatorToken.create(this._cs.position, 2, 35 /* Walrus */, this._getComments()));
+                    this._cs.advance(1);
+                    break;
+                }
+                this._tokens.push(tokenizerTypes_1.Token.create(10 /* Colon */, this._cs.position, 1, this._getComments()));
+                break;
+            }
+            default: {
+                if (this._isPossibleNumber()) {
+                    if (this._tryNumber()) {
+                        return true;
+                    }
+                }
+                if (this._cs.currentChar === 46 /* Period */) {
+                    if (this._cs.nextChar === 46 /* Period */ && this._cs.lookAhead(2) === 46 /* Period */) {
+                        this._tokens.push(tokenizerTypes_1.Token.create(19 /* Ellipsis */, this._cs.position, 3, this._getComments()));
+                        this._cs.advance(3);
+                        return true;
+                    }
+                    this._tokens.push(tokenizerTypes_1.Token.create(20 /* Dot */, this._cs.position, 1, this._getComments()));
+                    break;
+                }
+                if (!this._tryIdentifier()) {
+                    if (!this._tryOperator()) {
+                        return this._handleInvalid();
+                    }
+                }
+                return true;
+            }
+        }
+        return false;
+    }
+    _addLineRange() {
+        const lineLength = this._cs.position - this._prevLineStart;
+        if (lineLength > 0) {
+            this._lineRanges.push({ start: this._prevLineStart, length: lineLength });
+        }
+        this._prevLineStart = this._cs.position;
+    }
+    _handleNewLine(length, newLineType) {
+        if (this._parenDepth === 0 && newLineType !== 3 /* Implied */) {
+            // New lines are ignored within parentheses.
+            // We'll also avoid adding multiple newlines in a row to simplify parsing.
+            if (this._tokens.length === 0 || this._tokens[this._tokens.length - 1].type !== 2 /* NewLine */) {
+                this._tokens.push(tokenizerTypes_1.NewLineToken.create(this._cs.position, length, newLineType, this._getComments()));
+            }
+        }
+        if (newLineType === 0 /* CarriageReturn */) {
+            this._crCount++;
+        }
+        else if (newLineType === 2 /* CarriageReturnLineFeed */) {
+            this._crLfCount++;
+        }
+        else {
+            this._lfCount++;
+        }
+        this._cs.advance(length);
+        this._addLineRange();
+        this._readIndentationAfterNewLine();
+    }
+    _readIndentationAfterNewLine() {
+        let tab1Spaces = 0;
+        let tab8Spaces = 0;
+        let isTabPresent = false;
+        let isSpacePresent = false;
+        while (!this._cs.isEndOfStream()) {
+            switch (this._cs.currentChar) {
+                case 32 /* Space */:
+                    tab1Spaces++;
+                    tab8Spaces++;
+                    isSpacePresent = true;
+                    this._cs.moveNext();
+                    break;
+                case 9 /* Tab */:
+                    // Translate tabs into spaces assuming both 1-space
+                    // and 8-space tab stops.
+                    tab1Spaces++;
+                    tab8Spaces += exports.defaultTabSize - (tab8Spaces % exports.defaultTabSize);
+                    isTabPresent = true;
+                    this._cs.moveNext();
+                    break;
+                case 12 /* FormFeed */:
+                    tab1Spaces = 0;
+                    tab8Spaces = 0;
+                    isTabPresent = false;
+                    isSpacePresent = false;
+                    this._cs.moveNext();
+                    break;
+                default:
+                    // Non-blank line. Set the current indent level.
+                    this._setIndent(tab1Spaces, tab8Spaces, isSpacePresent, isTabPresent);
+                    return;
+                case 35 /* Hash */:
+                case 10 /* LineFeed */:
+                case 13 /* CarriageReturn */:
+                    // Blank line -- no need to adjust indentation.
+                    return;
+            }
+        }
+    }
+    // The caller must specify two space count values. The first assumes
+    // that tabs are translated into one-space tab stops. The second assumes
+    // that tabs are translated into eight-space tab stops.
+    _setIndent(tab1Spaces, tab8Spaces, isSpacePresent, isTabPresent) {
+        // Indentations are ignored within a parenthesized clause.
+        if (this._parenDepth > 0) {
+            return;
+        }
+        // Insert indent or dedent tokens as necessary.
+        if (this._indentAmounts.length === 0) {
+            if (tab8Spaces > 0) {
+                this._indentCount++;
+                if (isTabPresent) {
+                    this._indentTabCount++;
+                }
+                this._indentSpacesTotal += tab8Spaces;
+                this._indentAmounts.push({
+                    tab1Spaces,
+                    tab8Spaces,
+                    isSpacePresent,
+                    isTabPresent,
+                });
+                this._tokens.push(tokenizerTypes_1.IndentToken.create(this._cs.position, 0, tab8Spaces, false, this._getComments()));
+            }
+        }
+        else {
+            const prevTabInfo = this._indentAmounts[this._indentAmounts.length - 1];
+            if (prevTabInfo.tab8Spaces < tab8Spaces) {
+                // The Python spec says that if there is ambiguity about how tabs should
+                // be translated into spaces because the user has intermixed tabs and
+                // spaces, it should be an error. We'll record this condition in the token
+                // so the parser can later report it.
+                const isIndentAmbiguous = ((prevTabInfo.isSpacePresent && isTabPresent) || (prevTabInfo.isTabPresent && isSpacePresent)) &&
+                    prevTabInfo.tab1Spaces >= tab1Spaces;
+                this._indentCount++;
+                if (isTabPresent) {
+                    this._indentTabCount++;
+                }
+                this._indentSpacesTotal += tab8Spaces - this._indentAmounts[this._indentAmounts.length - 1].tab8Spaces;
+                this._indentAmounts.push({
+                    tab1Spaces,
+                    tab8Spaces,
+                    isSpacePresent,
+                    isTabPresent,
+                });
+                this._tokens.push(tokenizerTypes_1.IndentToken.create(this._cs.position, 0, tab8Spaces, isIndentAmbiguous, this._getComments()));
+            }
+            else if (prevTabInfo.tab8Spaces === tab8Spaces) {
+                // The Python spec says that if there is ambiguity about how tabs should
+                // be translated into spaces because the user has intermixed tabs and
+                // spaces, it should be an error. We'll record this condition in the token
+                // so the parser can later report it.
+                if ((prevTabInfo.isSpacePresent && isTabPresent) || (prevTabInfo.isTabPresent && isSpacePresent)) {
+                    this._tokens.push(tokenizerTypes_1.IndentToken.create(this._cs.position, 0, tab8Spaces, true, this._getComments()));
+                }
+            }
+            else {
+                // The Python spec says that if there is ambiguity about how tabs should
+                // be translated into spaces because the user has intermixed tabs and
+                // spaces, it should be an error. We'll record this condition in the token
+                // so the parser can later report it.
+                let isDedentAmbiguous = (prevTabInfo.isSpacePresent && isTabPresent) || (prevTabInfo.isTabPresent && isSpacePresent);
+                // The Python spec says that dedent amounts need to match the indent
+                // amount exactly. An error is generated at runtime if it doesn't.
+                // We'll record that error condition within the token, allowing the
+                // parser to report it later.
+                const dedentPoints = [];
+                while (this._indentAmounts.length > 0 &&
+                    this._indentAmounts[this._indentAmounts.length - 1].tab8Spaces > tab8Spaces) {
+                    dedentPoints.push(this._indentAmounts.length > 1
+                        ? this._indentAmounts[this._indentAmounts.length - 2].tab8Spaces
+                        : 0);
+                    this._indentAmounts.pop();
+                }
+                dedentPoints.forEach((dedentAmount, index) => {
+                    const matchesIndent = index < dedentPoints.length - 1 || dedentAmount === tab8Spaces;
+                    const actualDedentAmount = index < dedentPoints.length - 1 ? dedentAmount : tab8Spaces;
+                    this._tokens.push(tokenizerTypes_1.DedentToken.create(this._cs.position, 0, actualDedentAmount, matchesIndent, isDedentAmbiguous, this._getComments()));
+                    isDedentAmbiguous = false;
+                });
+            }
+        }
+    }
+    _tryIdentifier() {
+        const swallowRemainingChars = () => {
+            while (true) {
+                if ((0, characters_1.isIdentifierChar)(this._cs.currentChar)) {
+                    this._cs.moveNext();
+                }
+                else if ((0, characters_1.isIdentifierChar)(this._cs.currentChar, this._cs.nextChar)) {
+                    this._cs.moveNext();
+                    this._cs.moveNext();
+                }
+                else {
+                    break;
+                }
+            }
+        };
+        const start = this._cs.position;
+        if ((0, characters_1.isIdentifierStartChar)(this._cs.currentChar)) {
+            this._cs.moveNext();
+            swallowRemainingChars();
+        }
+        else if ((0, characters_1.isIdentifierStartChar)(this._cs.currentChar, this._cs.nextChar)) {
+            this._cs.moveNext();
+            this._cs.moveNext();
+            swallowRemainingChars();
+        }
+        if (this._cs.position > start) {
+            const value = this._cs.getText().substring(start, this._cs.position);
+            if (_keywords.has(value)) {
+                this._tokens.push(tokenizerTypes_1.KeywordToken.create(start, this._cs.position - start, _keywords.get(value), this._getComments()));
+            }
+            else {
+                this._tokens.push(tokenizerTypes_1.IdentifierToken.create(start, this._cs.position - start, value, this._getComments()));
+            }
+            return true;
+        }
+        return false;
+    }
+    _isPossibleNumber() {
+        if ((0, characters_1.isDecimal)(this._cs.currentChar)) {
+            return true;
+        }
+        if (this._cs.currentChar === 46 /* Period */ && (0, characters_1.isDecimal)(this._cs.nextChar)) {
+            return true;
+        }
+        return false;
+    }
+    _tryNumber() {
+        const start = this._cs.position;
+        if (this._cs.currentChar === 48 /* _0 */) {
+            let radix = 0;
+            let leadingChars = 0;
+            // Try hex => hexinteger: "0" ("x" | "X") (["_"] hexdigit)+
+            if ((this._cs.nextChar === 120 /* x */ || this._cs.nextChar === 88 /* X */) && (0, characters_1.isHex)(this._cs.lookAhead(2))) {
+                this._cs.advance(2);
+                leadingChars = 2;
+                while ((0, characters_1.isHex)(this._cs.currentChar)) {
+                    this._cs.moveNext();
+                }
+                radix = 16;
+            }
+            // Try binary => bininteger: "0" ("b" | "B") (["_"] bindigit)+
+            else if ((this._cs.nextChar === 98 /* b */ || this._cs.nextChar === 66 /* B */) &&
+                (0, characters_1.isBinary)(this._cs.lookAhead(2))) {
+                this._cs.advance(2);
+                leadingChars = 2;
+                while ((0, characters_1.isBinary)(this._cs.currentChar)) {
+                    this._cs.moveNext();
+                }
+                radix = 2;
+            }
+            // Try octal => octinteger: "0" ("o" | "O") (["_"] octdigit)+
+            else if ((this._cs.nextChar === 111 /* o */ || this._cs.nextChar === 79 /* O */) && (0, characters_1.isOctal)(this._cs.lookAhead(2))) {
+                this._cs.advance(2);
+                leadingChars = 2;
+                while ((0, characters_1.isOctal)(this._cs.currentChar)) {
+                    this._cs.moveNext();
+                }
+                radix = 8;
+            }
+            if (radix > 0) {
+                const text = this._cs.getText().substring(start, this._cs.position);
+                const simpleIntText = text.replace(/_/g, '');
+                let intValue = parseInt(simpleIntText.substring(leadingChars), radix);
+                if (!isNaN(intValue)) {
+                    const bigIntValue = BigInt(simpleIntText);
+                    if (!isFinite(intValue) || BigInt(intValue) !== bigIntValue) {
+                        intValue = bigIntValue;
+                    }
+                    this._tokens.push(tokenizerTypes_1.NumberToken.create(start, text.length, intValue, true, false, this._getComments()));
+                    return true;
+                }
+            }
+        }
+        let isDecimalInteger = false;
+        let mightBeFloatingPoint = false;
+        // Try decimal int =>
+        //    decinteger: nonzerodigit (["_"] digit)* | "0" (["_"] "0")*
+        //    nonzerodigit: "1"..."9"
+        //    digit: "0"..."9"
+        if (this._cs.currentChar >= 49 /* _1 */ && this._cs.currentChar <= 57 /* _9 */) {
+            while ((0, characters_1.isDecimal)(this._cs.currentChar)) {
+                mightBeFloatingPoint = true;
+                this._cs.moveNext();
+            }
+            isDecimalInteger =
+                this._cs.currentChar !== 46 /* Period */ &&
+                    this._cs.currentChar !== 101 /* e */ &&
+                    this._cs.currentChar !== 69 /* E */;
+        }
+        // "0" (["_"] "0")*
+        if (this._cs.currentChar === 48 /* _0 */) {
+            mightBeFloatingPoint = true;
+            while (this._cs.currentChar === 48 /* _0 */ || this._cs.currentChar === 95 /* Underscore */) {
+                this._cs.moveNext();
+            }
+            isDecimalInteger =
+                this._cs.currentChar !== 46 /* Period */ &&
+                    this._cs.currentChar !== 101 /* e */ &&
+                    this._cs.currentChar !== 69 /* E */;
+        }
+        if (isDecimalInteger) {
+            let text = this._cs.getText().substring(start, this._cs.position);
+            const simpleIntText = text.replace(/_/g, '');
+            let intValue = parseInt(simpleIntText, 10);
+            if (!isNaN(intValue)) {
+                let isImaginary = false;
+                const bigIntValue = BigInt(simpleIntText);
+                if (!isFinite(intValue) || BigInt(intValue) !== bigIntValue) {
+                    intValue = bigIntValue;
+                }
+                if (this._cs.currentChar === 106 /* j */ || this._cs.currentChar === 74 /* J */) {
+                    isImaginary = true;
+                    text += String.fromCharCode(this._cs.currentChar);
+                    this._cs.moveNext();
+                }
+                this._tokens.push(tokenizerTypes_1.NumberToken.create(start, text.length, intValue, true, isImaginary, this._getComments()));
+                return true;
+            }
+        }
+        // Floating point. Sign and leading digits were already skipped over.
+        this._cs.position = start;
+        if (mightBeFloatingPoint ||
+            (this._cs.currentChar === 46 /* Period */ && this._cs.nextChar >= 48 /* _0 */ && this._cs.nextChar <= 57 /* _9 */)) {
+            if (this._skipFloatingPointCandidate()) {
+                let text = this._cs.getText().substring(start, this._cs.position);
+                const value = parseFloat(text);
+                if (!isNaN(value)) {
+                    let isImaginary = false;
+                    if (this._cs.currentChar === 106 /* j */ || this._cs.currentChar === 74 /* J */) {
+                        isImaginary = true;
+                        text += String.fromCharCode(this._cs.currentChar);
+                        this._cs.moveNext();
+                    }
+                    this._tokens.push(tokenizerTypes_1.NumberToken.create(start, this._cs.position - start, value, false, isImaginary, this._getComments()));
+                    return true;
+                }
+            }
+        }
+        this._cs.position = start;
+        return false;
+    }
+    _tryOperator() {
+        let length = 0;
+        const nextChar = this._cs.nextChar;
+        let operatorType;
+        switch (this._cs.currentChar) {
+            case 43 /* Plus */:
+                length = nextChar === 61 /* Equal */ ? 2 : 1;
+                operatorType = length === 2 ? 1 /* AddEqual */ : 0 /* Add */;
+                break;
+            case 38 /* Ampersand */:
+                length = nextChar === 61 /* Equal */ ? 2 : 1;
+                operatorType = length === 2 ? 4 /* BitwiseAndEqual */ : 3 /* BitwiseAnd */;
+                break;
+            case 124 /* Bar */:
+                length = nextChar === 61 /* Equal */ ? 2 : 1;
+                operatorType = length === 2 ? 7 /* BitwiseOrEqual */ : 6 /* BitwiseOr */;
+                break;
+            case 94 /* Caret */:
+                length = nextChar === 61 /* Equal */ ? 2 : 1;
+                operatorType = length === 2 ? 9 /* BitwiseXorEqual */ : 8 /* BitwiseXor */;
+                break;
+            case 61 /* Equal */:
+                length = nextChar === 61 /* Equal */ ? 2 : 1;
+                operatorType = length === 2 ? 12 /* Equals */ : 2 /* Assign */;
+                break;
+            case 33 /* ExclamationMark */:
+                if (nextChar !== 61 /* Equal */) {
+                    return false;
+                }
+                length = 2;
+                operatorType = 28 /* NotEquals */;
+                break;
+            case 37 /* Percent */:
+                length = nextChar === 61 /* Equal */ ? 2 : 1;
+                operatorType = length === 2 ? 25 /* ModEqual */ : 24 /* Mod */;
+                break;
+            case 126 /* Tilde */:
+                length = 1;
+                operatorType = 5 /* BitwiseInvert */;
+                break;
+            case 45 /* Hyphen */:
+                if (nextChar === 62 /* Greater */) {
+                    this._tokens.push(tokenizerTypes_1.Token.create(21 /* Arrow */, this._cs.position, 2, this._getComments()));
+                    this._cs.advance(2);
+                    return true;
+                }
+                length = nextChar === 61 /* Equal */ ? 2 : 1;
+                operatorType = length === 2 ? 34 /* SubtractEqual */ : 33 /* Subtract */;
+                break;
+            case 42 /* Asterisk */:
+                if (nextChar === 42 /* Asterisk */) {
+                    length = this._cs.lookAhead(2) === 61 /* Equal */ ? 3 : 2;
+                    operatorType = length === 3 ? 30 /* PowerEqual */ : 29 /* Power */;
+                }
+                else {
+                    length = nextChar === 61 /* Equal */ ? 2 : 1;
+                    operatorType = length === 2 ? 27 /* MultiplyEqual */ : 26 /* Multiply */;
+                }
+                break;
+            case 47 /* Slash */:
+                if (nextChar === 47 /* Slash */) {
+                    length = this._cs.lookAhead(2) === 61 /* Equal */ ? 3 : 2;
+                    operatorType = length === 3 ? 14 /* FloorDivideEqual */ : 13 /* FloorDivide */;
+                }
+                else {
+                    length = nextChar === 61 /* Equal */ ? 2 : 1;
+                    operatorType = length === 2 ? 11 /* DivideEqual */ : 10 /* Divide */;
+                }
+                break;
+            case 60 /* Less */:
+                if (nextChar === 60 /* Less */) {
+                    length = this._cs.lookAhead(2) === 61 /* Equal */ ? 3 : 2;
+                    operatorType = length === 3 ? 18 /* LeftShiftEqual */ : 17 /* LeftShift */;
+                }
+                else if (nextChar === 62 /* Greater */) {
+                    length = 2;
+                    operatorType = 19 /* LessOrGreaterThan */;
+                }
+                else {
+                    length = nextChar === 61 /* Equal */ ? 2 : 1;
+                    operatorType = length === 2 ? 21 /* LessThanOrEqual */ : 20 /* LessThan */;
+                }
+                break;
+            case 62 /* Greater */:
+                if (nextChar === 62 /* Greater */) {
+                    length = this._cs.lookAhead(2) === 61 /* Equal */ ? 3 : 2;
+                    operatorType = length === 3 ? 32 /* RightShiftEqual */ : 31 /* RightShift */;
+                }
+                else {
+                    length = nextChar === 61 /* Equal */ ? 2 : 1;
+                    operatorType = length === 2 ? 16 /* GreaterThanOrEqual */ : 15 /* GreaterThan */;
+                }
+                break;
+            case 64 /* At */:
+                length = nextChar === 61 /* Equal */ ? 2 : 1;
+                operatorType = length === 2 ? 23 /* MatrixMultiplyEqual */ : 22 /* MatrixMultiply */;
+                break;
+            default:
+                return false;
+        }
+        this._tokens.push(tokenizerTypes_1.OperatorToken.create(this._cs.position, length, operatorType, this._getComments()));
+        this._cs.advance(length);
+        return length > 0;
+    }
+    _handleInvalid() {
+        const start = this._cs.position;
+        while (true) {
+            if (this._cs.currentChar === 10 /* LineFeed */ ||
+                this._cs.currentChar === 13 /* CarriageReturn */ ||
+                this._cs.isAtWhiteSpace() ||
+                this._cs.isEndOfStream()) {
+                break;
+            }
+            if ((0, characters_1.isSurrogateChar)(this._cs.currentChar)) {
+                this._cs.moveNext();
+                this._cs.moveNext();
+            }
+            else {
+                this._cs.moveNext();
+            }
+        }
+        const length = this._cs.position - start;
+        if (length > 0) {
+            this._tokens.push(tokenizerTypes_1.Token.create(0 /* Invalid */, start, length, this._getComments()));
+            return true;
+        }
+        return false;
+    }
+    _getComments() {
+        const prevComments = this._comments;
+        this._comments = undefined;
+        return prevComments;
+    }
+    _isIPythonMagics() {
+        const prevToken = this._tokens.length > 0 ? this._tokens[this._tokens.length - 1] : undefined;
+        return ((prevToken === undefined || prevToken.type === 2 /* NewLine */ || prevToken.type === 3 /* Indent */) &&
+            (this._cs.currentChar === 37 /* Percent */ || this._cs.currentChar === 33 /* ExclamationMark */));
+    }
+    _handleIPythonMagics(type) {
+        const start = this._cs.position + 1;
+        let begin = start;
+        do {
+            this._cs.skipToEol();
+            const length = this._cs.position - begin;
+            const value = this._cs.getText().substring(begin, begin + length);
+            // is it multiline magics?
+            // %magic command \
+            //        next arguments
+            if (!value.match(/\\\s*$/)) {
+                break;
+            }
+            begin = this._cs.position + 1;
+        } while (!this._cs.isEndOfStream());
+        const length = this._cs.position - start;
+        const value = this._cs.getText().substring(start, start + length);
+        const comment = tokenizerTypes_1.Comment.create(start, length, value, type);
+        this._addComments(comment);
+    }
+    _handleComment() {
+        const start = this._cs.position + 1;
+        this._cs.skipToEol();
+        const length = this._cs.position - start;
+        const value = this._cs.getText().substring(start, start + length);
+        const comment = tokenizerTypes_1.Comment.create(start, length, value);
+        const typeIgnoreRegexMatch = value.match(/^\s*type:\s*ignore(\s*\[([\s*\w-,]*)\]|\s|$)/);
+        if (typeIgnoreRegexMatch) {
+            const textRange = { start, length: typeIgnoreRegexMatch[0].length };
+            const ignoreComment = {
+                range: textRange,
+                rulesList: this._getIgnoreCommentRulesList(start, typeIgnoreRegexMatch),
+            };
+            if (this._tokens.findIndex((t) => t.type !== 2 /* NewLine */ && t && t.type !== 3 /* Indent */) < 0) {
+                this._typeIgnoreAll = ignoreComment;
+            }
+            else {
+                this._typeIgnoreLines.set(this._lineRanges.length, ignoreComment);
+            }
+        }
+        const pyrightIgnoreRegexMatch = value.match(/^\s*pyright:\s*ignore(\s*\[([\s*\w-,]*)\]|\s|$)/);
+        if (pyrightIgnoreRegexMatch) {
+            const textRange = { start, length: pyrightIgnoreRegexMatch[0].length };
+            const ignoreComment = {
+                range: textRange,
+                rulesList: this._getIgnoreCommentRulesList(start, pyrightIgnoreRegexMatch),
+            };
+            this._pyrightIgnoreLines.set(this._lineRanges.length, ignoreComment);
+        }
+        this._addComments(comment);
+    }
+    // Extracts the individual rules within a "type: ignore [x, y, z]" comment.
+    _getIgnoreCommentRulesList(start, match) {
+        if (match.length < 3 || match[2] === undefined) {
+            return undefined;
+        }
+        const splitElements = match[2].split(',');
+        const commentRules = [];
+        let currentOffset = start + match[0].indexOf('[') + 1;
+        for (const element of splitElements) {
+            const frontTrimmed = element.trimStart();
+            currentOffset += element.length - frontTrimmed.length;
+            const endTrimmed = frontTrimmed.trimEnd();
+            if (endTrimmed.length > 0) {
+                commentRules.push({
+                    range: { start: currentOffset, length: endTrimmed.length },
+                    text: endTrimmed,
+                });
+            }
+            currentOffset += frontTrimmed.length + 1;
+        }
+        return commentRules;
+    }
+    _addComments(comment) {
+        if (this._comments) {
+            this._comments.push(comment);
+        }
+        else {
+            this._comments = [comment];
+        }
+    }
+    _getStringPrefixLength() {
+        if (this._cs.currentChar === 39 /* SingleQuote */ || this._cs.currentChar === 34 /* DoubleQuote */) {
+            // Simple string, no prefix
+            return 0;
+        }
+        if (this._cs.nextChar === 39 /* SingleQuote */ || this._cs.nextChar === 34 /* DoubleQuote */) {
+            switch (this._cs.currentChar) {
+                case 102 /* f */:
+                case 70 /* F */:
+                case 114 /* r */:
+                case 82 /* R */:
+                case 98 /* b */:
+                case 66 /* B */:
+                case 117 /* u */:
+                case 85 /* U */:
+                    // Single-char prefix like u"" or r""
+                    return 1;
+                default:
+                    break;
+            }
+        }
+        if (this._cs.lookAhead(2) === 39 /* SingleQuote */ || this._cs.lookAhead(2) === 34 /* DoubleQuote */) {
+            const prefix = this._cs
+                .getText()
+                .substring(this._cs.position, this._cs.position + 2)
+                .toLowerCase();
+            switch (prefix) {
+                case 'rf':
+                case 'fr':
+                case 'ur':
+                case 'ru':
+                case 'br':
+                case 'rb':
+                    return 2;
+                default:
+                    break;
+            }
+        }
+        return -1;
+    }
+    _getQuoteTypeFlags(prefix) {
+        let flags = 0 /* None */;
+        prefix = prefix.toLowerCase();
+        for (let i = 0; i < prefix.length; i++) {
+            switch (prefix[i]) {
+                case 'u':
+                    flags |= 16 /* Unicode */;
+                    break;
+                case 'b':
+                    flags |= 32 /* Bytes */;
+                    break;
+                case 'r':
+                    flags |= 8 /* Raw */;
+                    break;
+                case 'f':
+                    flags |= 64 /* Format */;
+                    break;
+            }
+        }
+        if (this._cs.currentChar === 39 /* SingleQuote */) {
+            flags |= 1 /* SingleQuote */;
+            if (this._cs.nextChar === 39 /* SingleQuote */ && this._cs.lookAhead(2) === 39 /* SingleQuote */) {
+                flags |= 4 /* Triplicate */;
+            }
+        }
+        else if (this._cs.currentChar === 34 /* DoubleQuote */) {
+            flags |= 2 /* DoubleQuote */;
+            if (this._cs.nextChar === 34 /* DoubleQuote */ && this._cs.lookAhead(2) === 34 /* DoubleQuote */) {
+                flags |= 4 /* Triplicate */;
+            }
+        }
+        return flags;
+    }
+    _handleString(flags, stringPrefixLength) {
+        const start = this._cs.position - stringPrefixLength;
+        if (flags & 4 /* Triplicate */) {
+            this._cs.advance(3);
+        }
+        else {
+            this._cs.moveNext();
+            if (flags & 1 /* SingleQuote */) {
+                this._singleQuoteCount++;
+            }
+            else {
+                this._doubleQuoteCount++;
+            }
+        }
+        const stringLiteralInfo = this._skipToEndOfStringLiteral(flags);
+        const end = this._cs.position;
+        this._tokens.push(tokenizerTypes_1.StringToken.create(start, end - start, stringLiteralInfo.flags, stringLiteralInfo.escapedValue, stringPrefixLength, this._getComments()));
+    }
+    _skipToEndOfStringLiteral(flags) {
+        const quoteChar = flags & 1 /* SingleQuote */ ? 39 /* SingleQuote */ : 34 /* DoubleQuote */;
+        const isTriplicate = (flags & 4 /* Triplicate */) !== 0;
+        let escapedValueParts = [];
+        while (true) {
+            if (this._cs.isEndOfStream()) {
+                // Hit the end of file without a termination.
+                flags |= 65536 /* Unterminated */;
+                return { escapedValue: String.fromCharCode.apply(undefined, escapedValueParts), flags };
+            }
+            if (this._cs.currentChar === 92 /* Backslash */) {
+                escapedValueParts.push(this._cs.currentChar);
+                // Move past the escape (backslash) character.
+                this._cs.moveNext();
+                if (this._cs.getCurrentChar() === 13 /* CarriageReturn */ || this._cs.getCurrentChar() === 10 /* LineFeed */) {
+                    if (this._cs.getCurrentChar() === 13 /* CarriageReturn */ && this._cs.nextChar === 10 /* LineFeed */) {
+                        escapedValueParts.push(this._cs.currentChar);
+                        this._cs.moveNext();
+                    }
+                    escapedValueParts.push(this._cs.currentChar);
+                    this._cs.moveNext();
+                    this._addLineRange();
+                }
+                else {
+                    escapedValueParts.push(this._cs.currentChar);
+                    this._cs.moveNext();
+                }
+            }
+            else if (this._cs.currentChar === 10 /* LineFeed */ || this._cs.currentChar === 13 /* CarriageReturn */) {
+                if (!isTriplicate) {
+                    // Unterminated single-line string
+                    flags |= 65536 /* Unterminated */;
+                    return { escapedValue: String.fromCharCode.apply(undefined, escapedValueParts), flags };
+                }
+                // Skip over the new line (either one or two characters).
+                if (this._cs.currentChar === 13 /* CarriageReturn */ && this._cs.nextChar === 10 /* LineFeed */) {
+                    escapedValueParts.push(this._cs.currentChar);
+                    this._cs.moveNext();
+                }
+                escapedValueParts.push(this._cs.currentChar);
+                this._cs.moveNext();
+                this._addLineRange();
+            }
+            else if (!isTriplicate && this._cs.currentChar === quoteChar) {
+                this._cs.moveNext();
+                break;
+            }
+            else if (isTriplicate &&
+                this._cs.currentChar === quoteChar &&
+                this._cs.nextChar === quoteChar &&
+                this._cs.lookAhead(2) === quoteChar) {
+                this._cs.advance(3);
+                break;
+            }
+            else {
+                escapedValueParts.push(this._cs.currentChar);
+                this._cs.moveNext();
+            }
+        }
+        // String.fromCharCode.apply crashes (stack overflow) if passed an array
+        // that is too long. Cut off the extra characters in this case to avoid
+        // the crash. It's unlikely that the full string value will be used as
+        // a string literal, an f-string, or a docstring, so this should be fine.
+        if (escapedValueParts.length > _maxStringTokenLength) {
+            escapedValueParts = escapedValueParts.slice(0, _maxStringTokenLength);
+            flags |= 131072 /* ExceedsMaxSize */;
+        }
+        return { escapedValue: String.fromCharCode.apply(undefined, escapedValueParts), flags };
+    }
+    _skipFloatingPointCandidate() {
+        // Determine end of the potential floating point number
+        const start = this._cs.position;
+        this._skipFractionalNumber();
+        if (this._cs.position > start) {
+            // Optional exponent sign
+            if (this._cs.currentChar === 101 /* e */ || this._cs.currentChar === 69 /* E */) {
+                this._cs.moveNext();
+                // Skip exponent value
+                this._skipDecimalNumber(/* allowSign */ true);
+            }
+        }
+        return this._cs.position > start;
+    }
+    _skipFractionalNumber() {
+        this._skipDecimalNumber(false);
+        if (this._cs.currentChar === 46 /* Period */) {
+            // Optional period
+            this._cs.moveNext();
+        }
+        this._skipDecimalNumber(false);
+    }
+    _skipDecimalNumber(allowSign) {
+        if (allowSign && (this._cs.currentChar === 45 /* Hyphen */ || this._cs.currentChar === 43 /* Plus */)) {
+            // Optional sign
+            this._cs.moveNext();
+        }
+        while ((0, characters_1.isDecimal)(this._cs.currentChar)) {
+            // Skip integer part
+            this._cs.moveNext();
+        }
+    }
+}
+exports.Tokenizer = Tokenizer;
+//# sourceMappingURL=tokenizer.js.map