@zzzen/pyright-internal 1.1.254 → 1.1.255

package/src/parser/tokenizer.ts

@@ -0,0 +1,1379 @@
+/*
+ * tokenizer.ts
+ * Copyright (c) Microsoft Corporation.
+ * Licensed under the MIT license.
+ * Author: Eric Traut
+ *
+ * Based on code from vscode-python repository:
+ *  https://github.com/Microsoft/vscode-python
+ *
+ * Converts a Python program text stream into a stream of tokens.
+ */
+
+import Char from 'typescript-char';
+
+import { TextRange } from '../common/textRange';
+import { TextRangeCollection } from '../common/textRangeCollection';
+import {
+    isBinary,
+    isDecimal,
+    isHex,
+    isIdentifierChar,
+    isIdentifierStartChar,
+    isOctal,
+    isSurrogateChar,
+} from './characters';
+import { CharacterStream } from './characterStream';
+import {
+    Comment,
+    CommentType,
+    DedentToken,
+    IdentifierToken,
+    IndentToken,
+    KeywordToken,
+    KeywordType,
+    NewLineToken,
+    NewLineType,
+    NumberToken,
+    OperatorFlags,
+    OperatorToken,
+    OperatorType,
+    StringToken,
+    StringTokenFlags,
+    Token,
+    TokenType,
+} from './tokenizerTypes';
+
+// This must be a Map, as operations like {}["constructor"] succeed.
+const _keywords: Map<string, KeywordType> = new Map([
+    ['and', KeywordType.And],
+    ['as', KeywordType.As],
+    ['assert', KeywordType.Assert],
+    ['async', KeywordType.Async],
+    ['await', KeywordType.Await],
+    ['break', KeywordType.Break],
+    ['case', KeywordType.Case],
+    ['class', KeywordType.Class],
+    ['continue', KeywordType.Continue],
+    ['__debug__', KeywordType.Debug],
+    ['def', KeywordType.Def],
+    ['del', KeywordType.Del],
+    ['elif', KeywordType.Elif],
+    ['else', KeywordType.Else],
+    ['except', KeywordType.Except],
+    ['finally', KeywordType.Finally],
+    ['for', KeywordType.For],
+    ['from', KeywordType.From],
+    ['global', KeywordType.Global],
+    ['if', KeywordType.If],
+    ['import', KeywordType.Import],
+    ['in', KeywordType.In],
+    ['is', KeywordType.Is],
+    ['lambda', KeywordType.Lambda],
+    ['match', KeywordType.Match],
+    ['nonlocal', KeywordType.Nonlocal],
+    ['not', KeywordType.Not],
+    ['or', KeywordType.Or],
+    ['pass', KeywordType.Pass],
+    ['raise', KeywordType.Raise],
+    ['return', KeywordType.Return],
+    ['try', KeywordType.Try],
+    ['while', KeywordType.While],
+    ['with', KeywordType.With],
+    ['yield', KeywordType.Yield],
+    ['False', KeywordType.False],
+    ['None', KeywordType.None],
+    ['True', KeywordType.True],
+]);
+
+const _operatorInfo: { [key: number]: OperatorFlags } = {
+    [OperatorType.Add]: OperatorFlags.Unary | OperatorFlags.Binary,
+    [OperatorType.AddEqual]: OperatorFlags.Assignment,
+    [OperatorType.Assign]: OperatorFlags.Assignment,
+    [OperatorType.BitwiseAnd]: OperatorFlags.Binary,
+    [OperatorType.BitwiseAndEqual]: OperatorFlags.Assignment,
+    [OperatorType.BitwiseInvert]: OperatorFlags.Unary,
+    [OperatorType.BitwiseOr]: OperatorFlags.Binary,
+    [OperatorType.BitwiseOrEqual]: OperatorFlags.Assignment,
+    [OperatorType.BitwiseXor]: OperatorFlags.Binary,
+    [OperatorType.BitwiseXorEqual]: OperatorFlags.Assignment,
+    [OperatorType.Divide]: OperatorFlags.Binary,
+    [OperatorType.DivideEqual]: OperatorFlags.Assignment,
+    [OperatorType.Equals]: OperatorFlags.Binary | OperatorFlags.Comparison,
+    [OperatorType.FloorDivide]: OperatorFlags.Binary,
+    [OperatorType.FloorDivideEqual]: OperatorFlags.Assignment,
+    [OperatorType.GreaterThan]: OperatorFlags.Binary | OperatorFlags.Comparison,
+    [OperatorType.GreaterThanOrEqual]: OperatorFlags.Binary | OperatorFlags.Comparison,
+    [OperatorType.LeftShift]: OperatorFlags.Binary,
+    [OperatorType.LeftShiftEqual]: OperatorFlags.Assignment,
+    [OperatorType.LessOrGreaterThan]: OperatorFlags.Binary | OperatorFlags.Comparison | OperatorFlags.Deprecated,
+    [OperatorType.LessThan]: OperatorFlags.Binary | OperatorFlags.Comparison,
+    [OperatorType.LessThanOrEqual]: OperatorFlags.Binary | OperatorFlags.Comparison,
+    [OperatorType.MatrixMultiply]: OperatorFlags.Binary,
+    [OperatorType.MatrixMultiplyEqual]: OperatorFlags.Assignment,
+    [OperatorType.Mod]: OperatorFlags.Binary,
+    [OperatorType.ModEqual]: OperatorFlags.Assignment,
+    [OperatorType.Multiply]: OperatorFlags.Binary,
+    [OperatorType.MultiplyEqual]: OperatorFlags.Assignment,
+    [OperatorType.NotEquals]: OperatorFlags.Binary | OperatorFlags.Comparison,
+    [OperatorType.Power]: OperatorFlags.Binary,
+    [OperatorType.PowerEqual]: OperatorFlags.Assignment,
+    [OperatorType.RightShift]: OperatorFlags.Binary,
+    [OperatorType.RightShiftEqual]: OperatorFlags.Assignment,
+    [OperatorType.Subtract]: OperatorFlags.Binary,
+    [OperatorType.SubtractEqual]: OperatorFlags.Assignment,
+
+    [OperatorType.And]: OperatorFlags.Binary,
+    [OperatorType.Or]: OperatorFlags.Binary,
+    [OperatorType.Not]: OperatorFlags.Unary,
+    [OperatorType.Is]: OperatorFlags.Binary,
+    [OperatorType.IsNot]: OperatorFlags.Binary,
+    [OperatorType.In]: OperatorFlags.Binary,
+    [OperatorType.NotIn]: OperatorFlags.Binary,
+};
+
+const _byteOrderMarker = 0xfeff;
+
+const _maxStringTokenLength = 32 * 1024;
+
+export const defaultTabSize = 8;
+
+export interface TokenizerOutput {
+    // List of all tokens.
+    tokens: TextRangeCollection<Token>;
+
+    // List of ranges that comprise the lines.
+    lines: TextRangeCollection<TextRange>;
+
+    // Map of all line numbers that end in a "type: ignore" comment.
+    typeIgnoreLines: Map<number, IgnoreComment>;
+
+    // Map of all line numbers that end in a "pyright: ignore" comment.
+    pyrightIgnoreLines: Map<number, IgnoreComment>;
+
+    // Program starts with a "type: ignore" comment.
+    typeIgnoreAll: IgnoreComment | undefined;
+
+    // Line-end sequence ('/n', '/r', or '/r/n').
+    predominantEndOfLineSequence: string;
+
+    // Tab sequence ('/t or consecutive spaces).
+    predominantTabSequence: string;
+
+    // Does the code mostly use single or double quote
+    // characters for string literals?
+    predominantSingleQuoteCharacter: string;
+}
+
+interface StringScannerOutput {
+    escapedValue: string;
+    flags: StringTokenFlags;
+}
+
+interface IndentInfo {
+    tab1Spaces: number;
+    tab8Spaces: number;
+    isSpacePresent: boolean;
+    isTabPresent: boolean;
+}
+
+export interface IgnoreCommentRule {
+    text: string;
+    range: TextRange;
+}
+
+export interface IgnoreComment {
+    range: TextRange;
+    rulesList: IgnoreCommentRule[] | undefined;
+}
+
+export class Tokenizer {
+    private _cs = new CharacterStream('');
+    private _tokens: Token[] = [];
+    private _prevLineStart = 0;
+    private _parenDepth = 0;
+    private _lineRanges: TextRange[] = [];
+    private _indentAmounts: IndentInfo[] = [];
+    private _typeIgnoreAll: IgnoreComment | undefined;
+    private _typeIgnoreLines = new Map<number, IgnoreComment>();
+    private _pyrightIgnoreLines = new Map<number, IgnoreComment>();
+    private _comments: Comment[] | undefined;
+
+    // Total times CR, CR/LF, and LF are used to terminate
+    // lines. Used to determine the predominant line ending.
+    private _crCount = 0;
+    private _crLfCount = 0;
+    private _lfCount = 0;
+
+    // Number of times an indent token is emitted.
+    private _indentCount = 0;
+
+    // Number of times an indent token is emitted and a tab character
+    // is present (used to determine predominant tab sequence).
+    private _indentTabCount = 0;
+
+    // Number of spaces that are added for an indent token
+    // (used to determine predominant tab sequence).
+    private _indentSpacesTotal = 0;
+
+    // Number of single or double quote string literals found
+    // in the code.
+    private _singleQuoteCount = 0;
+    private _doubleQuoteCount = 0;
+
+    // ipython mode
+    private _ipythonMode = false;
+
+    tokenize(
+        text: string,
+        start?: number,
+        length?: number,
+        initialParenDepth = 0,
+        ipythonMode = false
+    ): TokenizerOutput {
+        if (start === undefined) {
+            start = 0;
+        } else if (start < 0 || start > text.length) {
+            throw new Error(`Invalid range start (start=${start}, text.length=${text.length})`);
+        }
+
+        if (length === undefined) {
+            length = text.length;
+        } else if (length < 0 || start + length > text.length) {
+            throw new Error(`Invalid range length (start=${start}, length=${length}, text.length=${text.length})`);
+        } else if (start + length < text.length) {
+            text = text.substring(0, start + length);
+        }
+
+        this._cs = new CharacterStream(text);
+        this._cs.position = start;
+        this._tokens = [];
+        this._prevLineStart = 0;
+        this._parenDepth = initialParenDepth;
+        this._lineRanges = [];
+        this._indentAmounts = [];
+        this._ipythonMode = ipythonMode;
+
+        const end = start + length;
+
+        if (start === 0) {
+            this._readIndentationAfterNewLine();
+        }
+
+        while (!this._cs.isEndOfStream()) {
+            this._addNextToken();
+
+            if (this._cs.position >= end) {
+                break;
+            }
+        }
+
+        // Insert an implied new line to make parsing easier.
+        if (this._tokens.length === 0 || this._tokens[this._tokens.length - 1].type !== TokenType.NewLine) {
+            this._tokens.push(NewLineToken.create(this._cs.position, 0, NewLineType.Implied, this._getComments()));
+        }
+
+        // Insert any implied dedent tokens.
+        this._setIndent(0, 0, /* isSpacePresent */ false, /* isTabPresent */ false);
+
+        // Add a final end-of-stream token to make parsing easier.
+        this._tokens.push(Token.create(TokenType.EndOfStream, this._cs.position, 0, this._getComments()));
+
+        // Add the final line range.
+        this._addLineRange();
+
+        let predominantEndOfLineSequence = '\n';
+        if (this._crCount > this._crLfCount && this._crCount > this._lfCount) {
+            predominantEndOfLineSequence = '\r';
+        } else if (this._crLfCount > this._crCount && this._crLfCount > this._lfCount) {
+            predominantEndOfLineSequence = '\r\n';
+        }
+
+        let predominantTabSequence = '    ';
+        // If more than half of the indents use tab sequences,
+        // assume we're using tabs rather than spaces.
+        if (this._indentTabCount > this._indentCount / 2) {
+            predominantTabSequence = '\t';
+        } else if (this._indentCount > 0) {
+            // Compute the average number of spaces per indent
+            // to estimate the predominant tab value.
+            let averageSpacePerIndent = Math.round(this._indentSpacesTotal / this._indentCount);
+            if (averageSpacePerIndent < 1) {
+                averageSpacePerIndent = 1;
+            } else if (averageSpacePerIndent > defaultTabSize) {
+                averageSpacePerIndent = defaultTabSize;
+            }
+            predominantTabSequence = '';
+            for (let i = 0; i < averageSpacePerIndent; i++) {
+                predominantTabSequence += ' ';
+            }
+        }
+
+        return {
+            tokens: new TextRangeCollection(this._tokens),
+            lines: new TextRangeCollection(this._lineRanges),
+            typeIgnoreLines: this._typeIgnoreLines,
+            typeIgnoreAll: this._typeIgnoreAll,
+            pyrightIgnoreLines: this._pyrightIgnoreLines,
+            predominantEndOfLineSequence,
+            predominantTabSequence,
+            predominantSingleQuoteCharacter: this._singleQuoteCount >= this._doubleQuoteCount ? "'" : '"',
+        };
+    }
+
+    static getOperatorInfo(operatorType: OperatorType): OperatorFlags {
+        return _operatorInfo[operatorType];
+    }
+
+    static isOperatorAssignment(operatorType?: OperatorType): boolean {
+        if (operatorType === undefined || _operatorInfo[operatorType] === undefined) {
+            return false;
+        }
+        return (_operatorInfo[operatorType] & OperatorFlags.Assignment) !== 0;
+    }
+
+    static isOperatorComparison(operatorType?: OperatorType): boolean {
+        if (operatorType === undefined || _operatorInfo[operatorType] === undefined) {
+            return false;
+        }
+        return (_operatorInfo[operatorType] & OperatorFlags.Comparison) !== 0;
+    }
+
+    private _addNextToken(): void {
+        this._cs.skipWhitespace();
+
+        if (this._cs.isEndOfStream()) {
+            return;
+        }
+
+        if (!this._handleCharacter()) {
+            this._cs.moveNext();
+        }
+    }
+
+    private _handleCharacter(): boolean {
+        // f-strings, b-strings, etc
+        const stringPrefixLength = this._getStringPrefixLength();
+
+        if (stringPrefixLength >= 0) {
+            let stringPrefix = '';
+            if (stringPrefixLength > 0) {
+                stringPrefix = this._cs.getText().substring(this._cs.position, this._cs.position + stringPrefixLength);
+                // Indeed a string
+                this._cs.advance(stringPrefixLength);
+            }
+
+            const quoteTypeFlags = this._getQuoteTypeFlags(stringPrefix);
+            if (quoteTypeFlags !== StringTokenFlags.None) {
+                this._handleString(quoteTypeFlags, stringPrefixLength);
+                return true;
+            }
+        }
+
+        if (this._cs.currentChar === Char.Hash) {
+            this._handleComment();
+            return true;
+        }
+
+        if (this._ipythonMode && this._isIPythonMagics()) {
+            this._handleIPythonMagics(
+                this._cs.currentChar === Char.Percent ? CommentType.IPythonMagic : CommentType.IPythonShellEscape
+            );
+            return true;
+        }
+
+        switch (this._cs.currentChar) {
+            case _byteOrderMarker: {
+                // Skip the BOM if it's at the start of the file.
+                if (this._cs.position === 0) {
+                    return false;
+                }
+                return this._handleInvalid();
+            }
+
+            case Char.CarriageReturn: {
+                const length = this._cs.nextChar === Char.LineFeed ? 2 : 1;
+                const newLineType = length === 2 ? NewLineType.CarriageReturnLineFeed : NewLineType.CarriageReturn;
+                this._handleNewLine(length, newLineType);
+                return true;
+            }
+
+            case Char.LineFeed: {
+                this._handleNewLine(1, NewLineType.LineFeed);
+                return true;
+            }
+
+            case Char.Backslash: {
+                if (this._cs.nextChar === Char.CarriageReturn) {
+                    if (this._cs.lookAhead(2) === Char.LineFeed) {
+                        this._cs.advance(3);
+                    } else {
+                        this._cs.advance(2);
+                    }
+                    this._addLineRange();
+                    return true;
+                } else if (this._cs.nextChar === Char.LineFeed) {
+                    this._cs.advance(2);
+                    this._addLineRange();
+                    return true;
+                }
+                return this._handleInvalid();
+            }
+
+            case Char.OpenParenthesis: {
+                this._parenDepth++;
+                this._tokens.push(Token.create(TokenType.OpenParenthesis, this._cs.position, 1, this._getComments()));
+                break;
+            }
+
+            case Char.CloseParenthesis: {
+                if (this._parenDepth > 0) {
+                    this._parenDepth--;
+                }
+                this._tokens.push(Token.create(TokenType.CloseParenthesis, this._cs.position, 1, this._getComments()));
+                break;
+            }
+
+            case Char.OpenBracket: {
+                this._parenDepth++;
+                this._tokens.push(Token.create(TokenType.OpenBracket, this._cs.position, 1, this._getComments()));
+                break;
+            }
+
+            case Char.CloseBracket: {
+                if (this._parenDepth > 0) {
+                    this._parenDepth--;
+                }
+                this._tokens.push(Token.create(TokenType.CloseBracket, this._cs.position, 1, this._getComments()));
+                break;
+            }
+
+            case Char.OpenBrace: {
+                this._parenDepth++;
+                this._tokens.push(Token.create(TokenType.OpenCurlyBrace, this._cs.position, 1, this._getComments()));
+                break;
+            }
+
+            case Char.CloseBrace: {
+                if (this._parenDepth > 0) {
+                    this._parenDepth--;
+                }
+                this._tokens.push(Token.create(TokenType.CloseCurlyBrace, this._cs.position, 1, this._getComments()));
+                break;
+            }
+
+            case Char.Comma: {
+                this._tokens.push(Token.create(TokenType.Comma, this._cs.position, 1, this._getComments()));
+                break;
+            }
+
+            case Char.Backtick: {
+                this._tokens.push(Token.create(TokenType.Backtick, this._cs.position, 1, this._getComments()));
+                break;
+            }
+
+            case Char.Semicolon: {
+                this._tokens.push(Token.create(TokenType.Semicolon, this._cs.position, 1, this._getComments()));
+                break;
+            }
+
+            case Char.Colon: {
+                if (this._cs.nextChar === Char.Equal) {
+                    this._tokens.push(
+                        OperatorToken.create(this._cs.position, 2, OperatorType.Walrus, this._getComments())
+                    );
+                    this._cs.advance(1);
+                    break;
+                }
+                this._tokens.push(Token.create(TokenType.Colon, this._cs.position, 1, this._getComments()));
+                break;
+            }
+
+            default: {
+                if (this._isPossibleNumber()) {
+                    if (this._tryNumber()) {
+                        return true;
+                    }
+                }
+
+                if (this._cs.currentChar === Char.Period) {
+                    if (this._cs.nextChar === Char.Period && this._cs.lookAhead(2) === Char.Period) {
+                        this._tokens.push(Token.create(TokenType.Ellipsis, this._cs.position, 3, this._getComments()));
+                        this._cs.advance(3);
+                        return true;
+                    }
+                    this._tokens.push(Token.create(TokenType.Dot, this._cs.position, 1, this._getComments()));
+                    break;
+                }
+
+                if (!this._tryIdentifier()) {
+                    if (!this._tryOperator()) {
+                        return this._handleInvalid();
+                    }
+                }
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private _addLineRange() {
+        const lineLength = this._cs.position - this._prevLineStart;
+        if (lineLength > 0) {
+            this._lineRanges.push({ start: this._prevLineStart, length: lineLength });
+        }
+
+        this._prevLineStart = this._cs.position;
+    }
+
+    private _handleNewLine(length: number, newLineType: NewLineType) {
+        if (this._parenDepth === 0 && newLineType !== NewLineType.Implied) {
+            // New lines are ignored within parentheses.
+            // We'll also avoid adding multiple newlines in a row to simplify parsing.
+            if (this._tokens.length === 0 || this._tokens[this._tokens.length - 1].type !== TokenType.NewLine) {
+                this._tokens.push(NewLineToken.create(this._cs.position, length, newLineType, this._getComments()));
+            }
+        }
+        if (newLineType === NewLineType.CarriageReturn) {
+            this._crCount++;
+        } else if (newLineType === NewLineType.CarriageReturnLineFeed) {
+            this._crLfCount++;
+        } else {
+            this._lfCount++;
+        }
+        this._cs.advance(length);
+        this._addLineRange();
+        this._readIndentationAfterNewLine();
+    }
+
+    private _readIndentationAfterNewLine() {
+        let tab1Spaces = 0;
+        let tab8Spaces = 0;
+        let isTabPresent = false;
+        let isSpacePresent = false;
+
+        while (!this._cs.isEndOfStream()) {
+            switch (this._cs.currentChar) {
+                case Char.Space:
+                    tab1Spaces++;
+                    tab8Spaces++;
+                    isSpacePresent = true;
+                    this._cs.moveNext();
+                    break;
+
+                case Char.Tab:
+                    // Translate tabs into spaces assuming both 1-space
+                    // and 8-space tab stops.
+                    tab1Spaces++;
+                    tab8Spaces += defaultTabSize - (tab8Spaces % defaultTabSize);
+                    isTabPresent = true;
+                    this._cs.moveNext();
+                    break;
+
+                case Char.FormFeed:
+                    tab1Spaces = 0;
+                    tab8Spaces = 0;
+                    isTabPresent = false;
+                    isSpacePresent = false;
+                    this._cs.moveNext();
+                    break;
+
+                default:
+                    // Non-blank line. Set the current indent level.
+                    this._setIndent(tab1Spaces, tab8Spaces, isSpacePresent, isTabPresent);
+                    return;
+
+                case Char.Hash:
+                case Char.LineFeed:
+                case Char.CarriageReturn:
+                    // Blank line -- no need to adjust indentation.
+                    return;
+            }
+        }
+    }
+
+    // The caller must specify two space count values. The first assumes
+    // that tabs are translated into one-space tab stops. The second assumes
+    // that tabs are translated into eight-space tab stops.
+    private _setIndent(tab1Spaces: number, tab8Spaces: number, isSpacePresent: boolean, isTabPresent: boolean) {
+        // Indentations are ignored within a parenthesized clause.
+        if (this._parenDepth > 0) {
+            return;
+        }
+
+        // Insert indent or dedent tokens as necessary.
+        if (this._indentAmounts.length === 0) {
+            if (tab8Spaces > 0) {
+                this._indentCount++;
+                if (isTabPresent) {
+                    this._indentTabCount++;
+                }
+                this._indentSpacesTotal += tab8Spaces;
+
+                this._indentAmounts.push({
+                    tab1Spaces,
+                    tab8Spaces,
+                    isSpacePresent,
+                    isTabPresent,
+                });
+                this._tokens.push(IndentToken.create(this._cs.position, 0, tab8Spaces, false, this._getComments()));
+            }
+        } else {
+            const prevTabInfo = this._indentAmounts[this._indentAmounts.length - 1];
+            if (prevTabInfo.tab8Spaces < tab8Spaces) {
+                // The Python spec says that if there is ambiguity about how tabs should
+                // be translated into spaces because the user has intermixed tabs and
+                // spaces, it should be an error. We'll record this condition in the token
+                // so the parser can later report it.
+                const isIndentAmbiguous =
+                    ((prevTabInfo.isSpacePresent && isTabPresent) || (prevTabInfo.isTabPresent && isSpacePresent)) &&
+                    prevTabInfo.tab1Spaces >= tab1Spaces;
+
+                this._indentCount++;
+                if (isTabPresent) {
+                    this._indentTabCount++;
+                }
+                this._indentSpacesTotal += tab8Spaces - this._indentAmounts[this._indentAmounts.length - 1].tab8Spaces;
+
+                this._indentAmounts.push({
+                    tab1Spaces,
+                    tab8Spaces,
+                    isSpacePresent,
+                    isTabPresent,
+                });
+
+                this._tokens.push(
+                    IndentToken.create(this._cs.position, 0, tab8Spaces, isIndentAmbiguous, this._getComments())
+                );
+            } else if (prevTabInfo.tab8Spaces === tab8Spaces) {
+                // The Python spec says that if there is ambiguity about how tabs should
+                // be translated into spaces because the user has intermixed tabs and
+                // spaces, it should be an error. We'll record this condition in the token
+                // so the parser can later report it.
+                if ((prevTabInfo.isSpacePresent && isTabPresent) || (prevTabInfo.isTabPresent && isSpacePresent)) {
+                    this._tokens.push(IndentToken.create(this._cs.position, 0, tab8Spaces, true, this._getComments()));
+                }
+            } else {
+                // The Python spec says that if there is ambiguity about how tabs should
+                // be translated into spaces because the user has intermixed tabs and
+                // spaces, it should be an error. We'll record this condition in the token
+                // so the parser can later report it.
+                let isDedentAmbiguous =
+                    (prevTabInfo.isSpacePresent && isTabPresent) || (prevTabInfo.isTabPresent && isSpacePresent);
+
+                // The Python spec says that dedent amounts need to match the indent
+                // amount exactly. An error is generated at runtime if it doesn't.
+                // We'll record that error condition within the token, allowing the
+                // parser to report it later.
+                const dedentPoints: number[] = [];
+                while (
+                    this._indentAmounts.length > 0 &&
+                    this._indentAmounts[this._indentAmounts.length - 1].tab8Spaces > tab8Spaces
+                ) {
+                    dedentPoints.push(
+                        this._indentAmounts.length > 1
+                            ? this._indentAmounts[this._indentAmounts.length - 2].tab8Spaces
+                            : 0
+                    );
+                    this._indentAmounts.pop();
+                }
+
+                dedentPoints.forEach((dedentAmount, index) => {
+                    const matchesIndent = index < dedentPoints.length - 1 || dedentAmount === tab8Spaces;
+                    const actualDedentAmount = index < dedentPoints.length - 1 ? dedentAmount : tab8Spaces;
+                    this._tokens.push(
+                        DedentToken.create(
+                            this._cs.position,
+                            0,
+                            actualDedentAmount,
+                            matchesIndent,
+                            isDedentAmbiguous,
+                            this._getComments()
+                        )
+                    );
+
+                    isDedentAmbiguous = false;
+                });
+            }
+        }
+    }
+
+    private _tryIdentifier(): boolean {
+        const swallowRemainingChars = () => {
+            while (true) {
+                if (isIdentifierChar(this._cs.currentChar)) {
+                    this._cs.moveNext();
+                } else if (isIdentifierChar(this._cs.currentChar, this._cs.nextChar)) {
+                    this._cs.moveNext();
+                    this._cs.moveNext();
+                } else {
+                    break;
+                }
+            }
+        };
+
+        const start = this._cs.position;
+        if (isIdentifierStartChar(this._cs.currentChar)) {
+            this._cs.moveNext();
+            swallowRemainingChars();
+        } else if (isIdentifierStartChar(this._cs.currentChar, this._cs.nextChar)) {
+            this._cs.moveNext();
+            this._cs.moveNext();
+            swallowRemainingChars();
+        }
+
+        if (this._cs.position > start) {
+            const value = this._cs.getText().substring(start, this._cs.position);
+            if (_keywords.has(value)) {
+                this._tokens.push(
+                    KeywordToken.create(start, this._cs.position - start, _keywords.get(value)!, this._getComments())
+                );
+            } else {
+                this._tokens.push(IdentifierToken.create(start, this._cs.position - start, value, this._getComments()));
+            }
+            return true;
+        }
+        return false;
+    }
+
+    private _isPossibleNumber(): boolean {
+        if (isDecimal(this._cs.currentChar)) {
+            return true;
+        }
+
+        if (this._cs.currentChar === Char.Period && isDecimal(this._cs.nextChar)) {
+            return true;
+        }
+
+        return false;
+    }
+
+    private _tryNumber(): boolean {
+        const start = this._cs.position;
+
+        if (this._cs.currentChar === Char._0) {
+            let radix = 0;
+            let leadingChars = 0;
+
+            // Try hex => hexinteger: "0" ("x" | "X") (["_"] hexdigit)+
+            if ((this._cs.nextChar === Char.x || this._cs.nextChar === Char.X) && isHex(this._cs.lookAhead(2))) {
+                this._cs.advance(2);
+                leadingChars = 2;
+                while (isHex(this._cs.currentChar)) {
+                    this._cs.moveNext();
+                }
+                radix = 16;
+            }
+
+            // Try binary => bininteger: "0" ("b" | "B") (["_"] bindigit)+
+            else if (
+                (this._cs.nextChar === Char.b || this._cs.nextChar === Char.B) &&
+                isBinary(this._cs.lookAhead(2))
+            ) {
+                this._cs.advance(2);
+                leadingChars = 2;
+                while (isBinary(this._cs.currentChar)) {
+                    this._cs.moveNext();
+                }
+                radix = 2;
+            }
+
+            // Try octal => octinteger: "0" ("o" | "O") (["_"] octdigit)+
+            else if ((this._cs.nextChar === Char.o || this._cs.nextChar === Char.O) && isOctal(this._cs.lookAhead(2))) {
+                this._cs.advance(2);
+                leadingChars = 2;
+                while (isOctal(this._cs.currentChar)) {
+                    this._cs.moveNext();
+                }
+                radix = 8;
+            }
+
+            if (radix > 0) {
+                const text = this._cs.getText().substring(start, this._cs.position);
+                const simpleIntText = text.replace(/_/g, '');
+                let intValue: number | bigint = parseInt(simpleIntText.substring(leadingChars), radix);
+
+                if (!isNaN(intValue)) {
+                    const bigIntValue = BigInt(simpleIntText);
+                    if (!isFinite(intValue) || BigInt(intValue) !== bigIntValue) {
+                        intValue = bigIntValue;
+                    }
+
+                    this._tokens.push(
+                        NumberToken.create(start, text.length, intValue, true, false, this._getComments())
+                    );
+                    return true;
+                }
+            }
+        }
+
+        let isDecimalInteger = false;
+        let mightBeFloatingPoint = false;
+        // Try decimal int =>
+        //    decinteger: nonzerodigit (["_"] digit)* | "0" (["_"] "0")*
+        //    nonzerodigit: "1"..."9"
+        //    digit: "0"..."9"
+        if (this._cs.currentChar >= Char._1 && this._cs.currentChar <= Char._9) {
+            while (isDecimal(this._cs.currentChar)) {
+                mightBeFloatingPoint = true;
+                this._cs.moveNext();
+            }
+            isDecimalInteger =
+                this._cs.currentChar !== Char.Period &&
+                this._cs.currentChar !== Char.e &&
+                this._cs.currentChar !== Char.E;
+        }
+
+        // "0" (["_"] "0")*
+        if (this._cs.currentChar === Char._0) {
+            mightBeFloatingPoint = true;
+            while (this._cs.currentChar === Char._0 || this._cs.currentChar === Char.Underscore) {
+                this._cs.moveNext();
+            }
+            isDecimalInteger =
+                this._cs.currentChar !== Char.Period &&
+                this._cs.currentChar !== Char.e &&
+                this._cs.currentChar !== Char.E;
+        }
+
+        if (isDecimalInteger) {
+            let text = this._cs.getText().substring(start, this._cs.position);
+            const simpleIntText = text.replace(/_/g, '');
+            let intValue: number | bigint = parseInt(simpleIntText, 10);
+
+            if (!isNaN(intValue)) {
+                let isImaginary = false;
+
+                const bigIntValue = BigInt(simpleIntText);
+                if (!isFinite(intValue) || BigInt(intValue) !== bigIntValue) {
+                    intValue = bigIntValue;
+                }
+
+                if (this._cs.currentChar === Char.j || this._cs.currentChar === Char.J) {
+                    isImaginary = true;
+                    text += String.fromCharCode(this._cs.currentChar);
+                    this._cs.moveNext();
+                }
+
+                this._tokens.push(
+                    NumberToken.create(start, text.length, intValue, true, isImaginary, this._getComments())
+                );
+                return true;
+            }
+        }
+
+        // Floating point. Sign and leading digits were already skipped over.
+        this._cs.position = start;
+        if (
+            mightBeFloatingPoint ||
+            (this._cs.currentChar === Char.Period && this._cs.nextChar >= Char._0 && this._cs.nextChar <= Char._9)
+        ) {
+            if (this._skipFloatingPointCandidate()) {
+                let text = this._cs.getText().substring(start, this._cs.position);
+                const value = parseFloat(text);
+                if (!isNaN(value)) {
+                    let isImaginary = false;
+                    if (this._cs.currentChar === Char.j || this._cs.currentChar === Char.J) {
+                        isImaginary = true;
+                        text += String.fromCharCode(this._cs.currentChar);
+                        this._cs.moveNext();
+                    }
+                    this._tokens.push(
+                        NumberToken.create(
+                            start,
+                            this._cs.position - start,
+                            value,
+                            false,
+                            isImaginary,
+                            this._getComments()
+                        )
+                    );
+                    return true;
+                }
+            }
+        }
+
+        this._cs.position = start;
+        return false;
+    }
+
+    private _tryOperator(): boolean {
+        let length = 0;
+        const nextChar = this._cs.nextChar;
+        let operatorType: OperatorType;
+
+        switch (this._cs.currentChar) {
+            case Char.Plus:
+                length = nextChar === Char.Equal ? 2 : 1;
+                operatorType = length === 2 ? OperatorType.AddEqual : OperatorType.Add;
+                break;
+
+            case Char.Ampersand:
+                length = nextChar === Char.Equal ? 2 : 1;
+                operatorType = length === 2 ? OperatorType.BitwiseAndEqual : OperatorType.BitwiseAnd;
+                break;
+
+            case Char.Bar:
+                length = nextChar === Char.Equal ? 2 : 1;
+                operatorType = length === 2 ? OperatorType.BitwiseOrEqual : OperatorType.BitwiseOr;
+                break;
+
+            case Char.Caret:
+                length = nextChar === Char.Equal ? 2 : 1;
+                operatorType = length === 2 ? OperatorType.BitwiseXorEqual : OperatorType.BitwiseXor;
+                break;
+
+            case Char.Equal:
+                length = nextChar === Char.Equal ? 2 : 1;
+                operatorType = length === 2 ? OperatorType.Equals : OperatorType.Assign;
+                break;
+
+            case Char.ExclamationMark:
+                if (nextChar !== Char.Equal) {
+                    return false;
+                }
+                length = 2;
+                operatorType = OperatorType.NotEquals;
+                break;
+
+            case Char.Percent:
+                length = nextChar === Char.Equal ? 2 : 1;
+                operatorType = length === 2 ? OperatorType.ModEqual : OperatorType.Mod;
+                break;
+
+            case Char.Tilde:
+                length = 1;
+                operatorType = OperatorType.BitwiseInvert;
+                break;
+
+            case Char.Hyphen:
+                if (nextChar === Char.Greater) {
+                    this._tokens.push(Token.create(TokenType.Arrow, this._cs.position, 2, this._getComments()));
+                    this._cs.advance(2);
+                    return true;
+                }
+
+                length = nextChar === Char.Equal ? 2 : 1;
+                operatorType = length === 2 ? OperatorType.SubtractEqual : OperatorType.Subtract;
+                break;
+
+            case Char.Asterisk:
+                if (nextChar === Char.Asterisk) {
+                    length = this._cs.lookAhead(2) === Char.Equal ? 3 : 2;
+                    operatorType = length === 3 ? OperatorType.PowerEqual : OperatorType.Power;
+                } else {
+                    length = nextChar === Char.Equal ? 2 : 1;
+                    operatorType = length === 2 ? OperatorType.MultiplyEqual : OperatorType.Multiply;
+                }
+                break;
+
+            case Char.Slash:
+                if (nextChar === Char.Slash) {
+                    length = this._cs.lookAhead(2) === Char.Equal ? 3 : 2;
+                    operatorType = length === 3 ? OperatorType.FloorDivideEqual : OperatorType.FloorDivide;
+                } else {
+                    length = nextChar === Char.Equal ? 2 : 1;
+                    operatorType = length === 2 ? OperatorType.DivideEqual : OperatorType.Divide;
+                }
+                break;
+
+            case Char.Less:
+                if (nextChar === Char.Less) {
+                    length = this._cs.lookAhead(2) === Char.Equal ? 3 : 2;
+                    operatorType = length === 3 ? OperatorType.LeftShiftEqual : OperatorType.LeftShift;
+                } else if (nextChar === Char.Greater) {
+                    length = 2;
+                    operatorType = OperatorType.LessOrGreaterThan;
+                } else {
+                    length = nextChar === Char.Equal ? 2 : 1;
+                    operatorType = length === 2 ? OperatorType.LessThanOrEqual : OperatorType.LessThan;
+                }
+                break;
+
+            case Char.Greater:
+                if (nextChar === Char.Greater) {
+                    length = this._cs.lookAhead(2) === Char.Equal ? 3 : 2;
+                    operatorType = length === 3 ? OperatorType.RightShiftEqual : OperatorType.RightShift;
+                } else {
+                    length = nextChar === Char.Equal ? 2 : 1;
+                    operatorType = length === 2 ? OperatorType.GreaterThanOrEqual : OperatorType.GreaterThan;
+                }
+                break;
+
+            case Char.At:
+                length = nextChar === Char.Equal ? 2 : 1;
+                operatorType = length === 2 ? OperatorType.MatrixMultiplyEqual : OperatorType.MatrixMultiply;
+                break;
+
+            default:
+                return false;
+        }
+        this._tokens.push(OperatorToken.create(this._cs.position, length, operatorType, this._getComments()));
+        this._cs.advance(length);
+        return length > 0;
+    }
+
+    private _handleInvalid(): boolean {
+        const start = this._cs.position;
+        while (true) {
+            if (
+                this._cs.currentChar === Char.LineFeed ||
+                this._cs.currentChar === Char.CarriageReturn ||
+                this._cs.isAtWhiteSpace() ||
+                this._cs.isEndOfStream()
+            ) {
+                break;
+            }
+
+            if (isSurrogateChar(this._cs.currentChar)) {
+                this._cs.moveNext();
+                this._cs.moveNext();
+            } else {
+                this._cs.moveNext();
+            }
+        }
+        const length = this._cs.position - start;
+        if (length > 0) {
+            this._tokens.push(Token.create(TokenType.Invalid, start, length, this._getComments()));
+            return true;
+        }
+        return false;
+    }
+
+    private _getComments(): Comment[] | undefined {
+        const prevComments = this._comments;
+        this._comments = undefined;
+        return prevComments;
+    }
+
+    private _isIPythonMagics() {
+        const prevToken = this._tokens.length > 0 ? this._tokens[this._tokens.length - 1] : undefined;
+        return (
+            (prevToken === undefined || prevToken.type === TokenType.NewLine || prevToken.type === TokenType.Indent) &&
+            (this._cs.currentChar === Char.Percent || this._cs.currentChar === Char.ExclamationMark)
+        );
+    }
+
+    private _handleIPythonMagics(type: CommentType): void {
+        const start = this._cs.position + 1;
+
+        let begin = start;
+        do {
+            this._cs.skipToEol();
+
+            const length = this._cs.position - begin;
+            const value = this._cs.getText().substring(begin, begin + length);
+
+            // is it multiline magics?
+            // %magic command \
+            //        next arguments
+            if (!value.match(/\\\s*$/)) {
+                break;
+            }
+
+            begin = this._cs.position + 1;
+        } while (!this._cs.isEndOfStream());
+
+        const length = this._cs.position - start;
+        const value = this._cs.getText().substring(start, start + length);
+
+        const comment = Comment.create(start, length, value, type);
+        this._addComments(comment);
+    }
+
+    private _handleComment(): void {
+        const start = this._cs.position + 1;
+        this._cs.skipToEol();
+
+        const length = this._cs.position - start;
+        const value = this._cs.getText().substring(start, start + length);
+        const comment = Comment.create(start, length, value);
+
+        const typeIgnoreRegexMatch = value.match(/^\s*type:\s*ignore(\s*\[([\s*\w-,]*)\]|\s|$)/);
+        if (typeIgnoreRegexMatch) {
+            const textRange: TextRange = { start, length: typeIgnoreRegexMatch[0].length };
+            const ignoreComment: IgnoreComment = {
+                range: textRange,
+                rulesList: this._getIgnoreCommentRulesList(start, typeIgnoreRegexMatch),
+            };
+
+            if (this._tokens.findIndex((t) => t.type !== TokenType.NewLine && t && t.type !== TokenType.Indent) < 0) {
+                this._typeIgnoreAll = ignoreComment;
+            } else {
+                this._typeIgnoreLines.set(this._lineRanges.length, ignoreComment);
+            }
+        }
+
+        const pyrightIgnoreRegexMatch = value.match(/^\s*pyright:\s*ignore(\s*\[([\s*\w-,]*)\]|\s|$)/);
+        if (pyrightIgnoreRegexMatch) {
+            const textRange: TextRange = { start, length: pyrightIgnoreRegexMatch[0].length };
+            const ignoreComment: IgnoreComment = {
+                range: textRange,
+                rulesList: this._getIgnoreCommentRulesList(start, pyrightIgnoreRegexMatch),
+            };
+            this._pyrightIgnoreLines.set(this._lineRanges.length, ignoreComment);
+        }
+
+        this._addComments(comment);
+    }
+
+    // Extracts the individual rules within a "type: ignore [x, y, z]" comment.
+    private _getIgnoreCommentRulesList(start: number, match: RegExpMatchArray): IgnoreCommentRule[] | undefined {
+        if (match.length < 3 || match[2] === undefined) {
+            return undefined;
+        }
+
+        const splitElements = match[2].split(',');
+        const commentRules: IgnoreCommentRule[] = [];
+        let currentOffset = start + match[0].indexOf('[') + 1;
+
+        for (const element of splitElements) {
+            const frontTrimmed = element.trimStart();
+            currentOffset += element.length - frontTrimmed.length;
+            const endTrimmed = frontTrimmed.trimEnd();
+
+            if (endTrimmed.length > 0) {
+                commentRules.push({
+                    range: { start: currentOffset, length: endTrimmed.length },
+                    text: endTrimmed,
+                });
+            }
+
+            currentOffset += frontTrimmed.length + 1;
+        }
+
+        return commentRules;
+    }
+
+    private _addComments(comment: Comment) {
+        if (this._comments) {
+            this._comments.push(comment);
+        } else {
+            this._comments = [comment];
+        }
+    }
+
+    private _getStringPrefixLength(): number {
+        if (this._cs.currentChar === Char.SingleQuote || this._cs.currentChar === Char.DoubleQuote) {
+            // Simple string, no prefix
+            return 0;
+        }
+
+        if (this._cs.nextChar === Char.SingleQuote || this._cs.nextChar === Char.DoubleQuote) {
+            switch (this._cs.currentChar) {
+                case Char.f:
+                case Char.F:
+                case Char.r:
+                case Char.R:
+                case Char.b:
+                case Char.B:
+                case Char.u:
+                case Char.U:
+                    // Single-char prefix like u"" or r""
+                    return 1;
+                default:
+                    break;
+            }
+        }
+
+        if (this._cs.lookAhead(2) === Char.SingleQuote || this._cs.lookAhead(2) === Char.DoubleQuote) {
+            const prefix = this._cs
+                .getText()
+                .substring(this._cs.position, this._cs.position + 2)
+                .toLowerCase();
+            switch (prefix) {
+                case 'rf':
+                case 'fr':
+                case 'ur':
+                case 'ru':
+                case 'br':
+                case 'rb':
+                    return 2;
+                default:
+                    break;
+            }
+        }
+        return -1;
+    }
+
+    private _getQuoteTypeFlags(prefix: string): StringTokenFlags {
+        let flags = StringTokenFlags.None;
+
+        prefix = prefix.toLowerCase();
+        for (let i = 0; i < prefix.length; i++) {
+            switch (prefix[i]) {
+                case 'u':
+                    flags |= StringTokenFlags.Unicode;
+                    break;
+
+                case 'b':
+                    flags |= StringTokenFlags.Bytes;
+                    break;
+
+                case 'r':
+                    flags |= StringTokenFlags.Raw;
+                    break;
+
+                case 'f':
+                    flags |= StringTokenFlags.Format;
+                    break;
+            }
+        }
+
+        if (this._cs.currentChar === Char.SingleQuote) {
+            flags |= StringTokenFlags.SingleQuote;
+            if (this._cs.nextChar === Char.SingleQuote && this._cs.lookAhead(2) === Char.SingleQuote) {
+                flags |= StringTokenFlags.Triplicate;
+            }
+        } else if (this._cs.currentChar === Char.DoubleQuote) {
+            flags |= StringTokenFlags.DoubleQuote;
+            if (this._cs.nextChar === Char.DoubleQuote && this._cs.lookAhead(2) === Char.DoubleQuote) {
+                flags |= StringTokenFlags.Triplicate;
+            }
+        }
+
+        return flags;
+    }
+
+    private _handleString(flags: StringTokenFlags, stringPrefixLength: number): void {
+        const start = this._cs.position - stringPrefixLength;
+
+        if (flags & StringTokenFlags.Triplicate) {
+            this._cs.advance(3);
+        } else {
+            this._cs.moveNext();
+
+            if (flags & StringTokenFlags.SingleQuote) {
+                this._singleQuoteCount++;
+            } else {
+                this._doubleQuoteCount++;
+            }
+        }
+
+        const stringLiteralInfo = this._skipToEndOfStringLiteral(flags);
+
+        const end = this._cs.position;
+
+        this._tokens.push(
+            StringToken.create(
+                start,
+                end - start,
+                stringLiteralInfo.flags,
+                stringLiteralInfo.escapedValue,
+                stringPrefixLength,
+                this._getComments()
+            )
+        );
+    }
+
+    private _skipToEndOfStringLiteral(flags: StringTokenFlags): StringScannerOutput {
+        const quoteChar = flags & StringTokenFlags.SingleQuote ? Char.SingleQuote : Char.DoubleQuote;
+        const isTriplicate = (flags & StringTokenFlags.Triplicate) !== 0;
+        let escapedValueParts: number[] = [];
+
+        while (true) {
+            if (this._cs.isEndOfStream()) {
+                // Hit the end of file without a termination.
+                flags |= StringTokenFlags.Unterminated;
+                return { escapedValue: String.fromCharCode.apply(undefined, escapedValueParts), flags };
+            }
+
+            if (this._cs.currentChar === Char.Backslash) {
+                escapedValueParts.push(this._cs.currentChar);
+
+                // Move past the escape (backslash) character.
+                this._cs.moveNext();
+
+                if (this._cs.getCurrentChar() === Char.CarriageReturn || this._cs.getCurrentChar() === Char.LineFeed) {
+                    if (this._cs.getCurrentChar() === Char.CarriageReturn && this._cs.nextChar === Char.LineFeed) {
+                        escapedValueParts.push(this._cs.currentChar);
+                        this._cs.moveNext();
+                    }
+                    escapedValueParts.push(this._cs.currentChar);
+                    this._cs.moveNext();
+                    this._addLineRange();
+                } else {
+                    escapedValueParts.push(this._cs.currentChar);
+                    this._cs.moveNext();
+                }
+            } else if (this._cs.currentChar === Char.LineFeed || this._cs.currentChar === Char.CarriageReturn) {
+                if (!isTriplicate) {
+                    // Unterminated single-line string
+                    flags |= StringTokenFlags.Unterminated;
+                    return { escapedValue: String.fromCharCode.apply(undefined, escapedValueParts), flags };
+                }
+
+                // Skip over the new line (either one or two characters).
+                if (this._cs.currentChar === Char.CarriageReturn && this._cs.nextChar === Char.LineFeed) {
+                    escapedValueParts.push(this._cs.currentChar);
+                    this._cs.moveNext();
+                }
+
+                escapedValueParts.push(this._cs.currentChar);
+                this._cs.moveNext();
+                this._addLineRange();
+            } else if (!isTriplicate && this._cs.currentChar === quoteChar) {
+                this._cs.moveNext();
+                break;
+            } else if (
+                isTriplicate &&
+                this._cs.currentChar === quoteChar &&
+                this._cs.nextChar === quoteChar &&
+                this._cs.lookAhead(2) === quoteChar
+            ) {
+                this._cs.advance(3);
+                break;
+            } else {
+                escapedValueParts.push(this._cs.currentChar);
+                this._cs.moveNext();
+            }
+        }
+
+        // String.fromCharCode.apply crashes (stack overflow) if passed an array
+        // that is too long. Cut off the extra characters in this case to avoid
+        // the crash. It's unlikely that the full string value will be used as
+        // a string literal, an f-string, or a docstring, so this should be fine.
+        if (escapedValueParts.length > _maxStringTokenLength) {
+            escapedValueParts = escapedValueParts.slice(0, _maxStringTokenLength);
+            flags |= StringTokenFlags.ExceedsMaxSize;
+        }
+
+        return { escapedValue: String.fromCharCode.apply(undefined, escapedValueParts), flags };
+    }
+
+    private _skipFloatingPointCandidate(): boolean {
+        // Determine end of the potential floating point number
+        const start = this._cs.position;
+        this._skipFractionalNumber();
+        if (this._cs.position > start) {
+            // Optional exponent sign
+            if (this._cs.currentChar === Char.e || this._cs.currentChar === Char.E) {
+                this._cs.moveNext();
+
+                // Skip exponent value
+                this._skipDecimalNumber(/* allowSign */ true);
+            }
+        }
+        return this._cs.position > start;
+    }
+
+    private _skipFractionalNumber(): void {
+        this._skipDecimalNumber(false);
+        if (this._cs.currentChar === Char.Period) {
+            // Optional period
+            this._cs.moveNext();
+        }
+        this._skipDecimalNumber(false);
+    }
+
+    private _skipDecimalNumber(allowSign: boolean): void {
+        if (allowSign && (this._cs.currentChar === Char.Hyphen || this._cs.currentChar === Char.Plus)) {
+            // Optional sign
+            this._cs.moveNext();
+        }
+        while (isDecimal(this._cs.currentChar)) {
+            // Skip integer part
+            this._cs.moveNext();
+        }
+    }
+}