@zzzen/pyright-internal 1.2.0-dev.2022-07-02 → 1.2.0-dev.20220717

package/dist/parser/parser.js

@@ -32,12 +32,14 @@ var __importStar = (this && this.__importStar) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Parser = exports.ParseOptions = void 0;
+const sourceFile_1 = require("../analyzer/sourceFile");
 const collectionUtils_1 = require("../common/collectionUtils");
 const debug_1 = require("../common/debug");
 const diagnostic_1 = require("../common/diagnostic");
 const diagnosticSink_1 = require("../common/diagnosticSink");
 const positionUtils_1 = require("../common/positionUtils");
 const pythonVersion_1 = require("../common/pythonVersion");
+const textRange_1 = require("../common/textRange");
 const timing_1 = require("../common/timing");
 const localize_1 = require("../localization/localize");
 const parseNodes_1 = require("./parseNodes");
@@ -50,7 +52,7 @@ class ParseOptions {
         this.pythonVersion = pythonVersion_1.latestStablePythonVersion;
         this.reportInvalidStringEscapeSequence = false;
         this.skipFunctionAndClassBody = false;
-        this.ipythonMode = false;
+        this.ipythonMode = sourceFile_1.IPythonMode.None;
         this.reportErrorsForParsedStringContents = false;
     }
 }
@@ -184,13 +186,13 @@ class Parser {
         switch (this._peekKeywordType()) {
             case 20 /* If */:
                 return this._parseIfStatement();
-            case 35 /* While */:
+            case 36 /* While */:
                 return this._parseWhileStatement();
             case 17 /* For */:
                 return this._parseForStatement();
             case 34 /* Try */:
                 return this._parseTryStatement();
-            case 36 /* With */:
+            case 37 /* With */:
                 return this._parseWithStatement();
             case 10 /* Def */:
                 return this._parseFunctionDef();
@@ -227,6 +229,7 @@ class Parser {
                         return matchStatement;
                     }
                 }
+                break;
             }
         }
         if (this._peekOperatorType() === 22 /* MatrixMultiply */) {
@@ -240,7 +243,7 @@ class Parser {
         switch (this._peekKeywordType()) {
             case 10 /* Def */:
                 return this._parseFunctionDef(asyncToken);
-            case 36 /* With */:
+            case 37 /* With */:
                 return this._parseWithStatement(asyncToken);
             case 17 /* For */:
                 return this._parseForStatement(asyncToken);
@@ -248,6 +251,87 @@ class Parser {
         this._addError(localize_1.Localizer.Diagnostic.unexpectedAsyncToken(), asyncToken);
         return undefined;
     }
+    // type_alias_stmt: "type" name [type_param_seq] = expr
+    _parseTypeAliasStatement() {
+        const typeToken = this._getKeywordToken(35 /* Type */);
+        if (!this._parseOptions.isStubFile && this._getLanguageVersion() < pythonVersion_1.PythonVersion.V3_12) {
+            this._addError(localize_1.Localizer.Diagnostic.typeAliasStatementIllegal(), typeToken);
+        }
+        const nameToken = this._getTokenIfIdentifier();
+        (0, debug_1.assert)(nameToken !== undefined);
+        const name = parseNodes_1.NameNode.create(nameToken);
+        let typeParameters;
+        if (this._peekToken().type === 15 /* OpenBracket */) {
+            typeParameters = this._parseTypeParameterList();
+        }
+        const assignToken = this._peekToken();
+        if (assignToken.type !== 9 /* Operator */ ||
+            assignToken.operatorType !== 2 /* Assign */) {
+            this._addError(localize_1.Localizer.Diagnostic.expectedEquals(), assignToken);
+        }
+        else {
+            this._getNextToken();
+        }
+        const expression = this._parseOrTest();
+        return parseNodes_1.TypeAliasNode.create(typeToken, name, expression, typeParameters);
+    }
+    // type_param_seq: '[' (type_param ',')+ ']'
+    _parseTypeParameterList() {
+        const typeVariableNodes = [];
+        const openBracketToken = this._getNextToken();
+        (0, debug_1.assert)(openBracketToken.type === 15 /* OpenBracket */);
+        while (true) {
+            const firstToken = this._peekToken();
+            if (firstToken.type === 16 /* CloseBracket */) {
+                if (typeVariableNodes.length === 0) {
+                    this._addError(localize_1.Localizer.Diagnostic.typeParametersMissing(), this._peekToken());
+                }
+                break;
+            }
+            const typeVarNode = this._parseTypeParameter();
+            if (!typeVarNode) {
+                break;
+            }
+            typeVariableNodes.push(typeVarNode);
+            if (!this._consumeTokenIfType(12 /* Comma */)) {
+                break;
+            }
+        }
+        const closingToken = this._peekToken();
+        if (closingToken.type !== 16 /* CloseBracket */) {
+            this._addError(localize_1.Localizer.Diagnostic.expectedCloseBracket(), this._peekToken());
+            this._consumeTokensUntilType([2 /* NewLine */, 16 /* CloseBracket */, 10 /* Colon */]);
+        }
+        else {
+            this._getNextToken();
+        }
+        return parseNodes_1.TypeParameterListNode.create(openBracketToken, closingToken, typeVariableNodes);
+    }
+    // type_param: ['*' | '**'] NAME [':' expr]
+    _parseTypeParameter() {
+        let typeParamCategory = parseNodes_1.TypeParameterCategory.TypeVar;
+        if (this._consumeTokenIfOperator(26 /* Multiply */)) {
+            typeParamCategory = parseNodes_1.TypeParameterCategory.TypeVarTuple;
+        }
+        else if (this._consumeTokenIfOperator(29 /* Power */)) {
+            typeParamCategory = parseNodes_1.TypeParameterCategory.ParamSpec;
+        }
+        const nameToken = this._getTokenIfIdentifier();
+        if (!nameToken) {
+            this._addError(localize_1.Localizer.Diagnostic.expectedTypeParameterName(), this._peekToken());
+            return undefined;
+        }
+        const name = parseNodes_1.NameNode.create(nameToken);
+        let boundExpression;
+        if (this._peekTokenType() === 10 /* Colon */) {
+            this._getNextToken();
+            boundExpression = this._parseTestExpression(/* allowAssignmentExpression */ false);
+            if (typeParamCategory !== parseNodes_1.TypeParameterCategory.TypeVar) {
+                this._addError(localize_1.Localizer.Diagnostic.typeParameterBoundNotAllowed(), boundExpression);
+            }
+        }
+        return parseNodes_1.TypeParameterNode.create(name, typeParamCategory, boundExpression);
+    }
     // match_stmt: "match" subject_expr ':' NEWLINE INDENT case_block+ DEDENT
     // subject_expr:
     //     | star_named_expression ',' star_named_expressions?
@@ -1137,7 +1221,7 @@ class Parser {
     }
     // while_stmt: 'while' test suite ['else' suite]
     _parseWhileStatement() {
-        const whileToken = this._getKeywordToken(35 /* While */);
+        const whileToken = this._getKeywordToken(36 /* While */);
         const whileNode = parseNodes_1.WhileNode.create(whileToken, this._parseTestExpression(/* allowAssignmentExpression */ true), this._parseLoopSuite());
         if (this._consumeTokenIfKeyword(13 /* Else */)) {
             whileNode.elseSuite = this._parseSuite(this._isInFunction);
@@ -1242,6 +1326,14 @@ class Parser {
             this._addError(localize_1.Localizer.Diagnostic.expectedFunctionName(), defToken);
             return parseNodes_1.ErrorNode.create(defToken, 10 /* MissingFunctionParameterList */, undefined, decorators);
         }
+        let typeParameters;
+        const possibleOpenBracket = this._peekToken();
+        if (possibleOpenBracket.type === 15 /* OpenBracket */) {
+            typeParameters = this._parseTypeParameterList();
+            if (!this._parseOptions.isStubFile && this._getLanguageVersion() < pythonVersion_1.PythonVersion.V3_12) {
+                this._addError(localize_1.Localizer.Diagnostic.functionTypeParametersIllegal(), typeParameters);
+            }
+        }
         const openParenToken = this._peekToken();
         if (!this._consumeTokenIfType(13 /* OpenParenthesis */)) {
             this._addError(localize_1.Localizer.Diagnostic.expectedOpenParen(), this._peekToken());
@@ -1262,7 +1354,7 @@ class Parser {
                 functionTypeAnnotationToken = this._getTypeAnnotationCommentText();
             }
         });
-        const functionNode = parseNodes_1.FunctionNode.create(defToken, parseNodes_1.NameNode.create(nameToken), suite);
+        const functionNode = parseNodes_1.FunctionNode.create(defToken, parseNodes_1.NameNode.create(nameToken), suite, typeParameters);
         if (asyncToken) {
             functionNode.isAsync = true;
             (0, parseNodes_1.extendRange)(functionNode, asyncToken);
@@ -1481,7 +1573,7 @@ class Parser {
     // Python 3.10 adds support for optional parentheses around
     // with_item list.
     _parseWithStatement(asyncToken) {
-        const withToken = this._getKeywordToken(36 /* With */);
+        const withToken = this._getKeywordToken(37 /* With */);
         let withItemList = [];
         const possibleParen = this._peekToken();
         // If the expression starts with a paren, parse it as though the
@@ -1642,6 +1734,14 @@ class Parser {
             this._addError(localize_1.Localizer.Diagnostic.expectedClassName(), this._peekToken());
             nameToken = tokenizerTypes_1.IdentifierToken.create(0, 0, '', /* comments */ undefined);
         }
+        let typeParameters;
+        const possibleOpenBracket = this._peekToken();
+        if (possibleOpenBracket.type === 15 /* OpenBracket */) {
+            typeParameters = this._parseTypeParameterList();
+            if (!this._parseOptions.isStubFile && this._getLanguageVersion() < pythonVersion_1.PythonVersion.V3_12) {
+                this._addError(localize_1.Localizer.Diagnostic.classTypeParametersIllegal(), typeParameters);
+            }
+        }
         let argList = [];
         const openParenToken = this._peekToken();
         if (this._consumeTokenIfType(13 /* OpenParenthesis */)) {
@@ -1651,7 +1751,7 @@ class Parser {
             }
         }
         const suite = this._parseSuite(/* isFunction */ false, this._parseOptions.skipFunctionAndClassBody);
-        const classNode = parseNodes_1.ClassNode.create(classToken, parseNodes_1.NameNode.create(nameToken), suite);
+        const classNode = parseNodes_1.ClassNode.create(classToken, parseNodes_1.NameNode.create(nameToken), suite, typeParameters);
         classNode.arguments = argList;
         argList.forEach((arg) => {
             arg.parent = classNode;
@@ -2008,7 +2108,7 @@ class Parser {
     // yield_expr: 'yield' [yield_arg]
     // yield_arg: 'from' test | testlist
     _parseYieldExpression() {
-        const yieldToken = this._getKeywordToken(37 /* Yield */);
+        const yieldToken = this._getKeywordToken(38 /* Yield */);
         const nextToken = this._peekToken();
         if (this._consumeTokenIfKeyword(18 /* From */)) {
             if (this._getLanguageVersion() < pythonVersion_1.PythonVersion.V3_3) {
@@ -2026,7 +2126,7 @@ class Parser {
         return parseNodes_1.YieldNode.create(yieldToken, exprList);
     }
     _tryParseYieldExpression() {
-        if (this._peekKeywordType() !== 37 /* Yield */) {
+        if (this._peekKeywordType() !== 38 /* Yield */) {
             return undefined;
         }
         return this._parseYieldExpression();
@@ -2096,8 +2196,28 @@ class Parser {
                 return this._parseAssertStatement();
             case 11 /* Del */:
                 return this._parseDelStatement();
-            case 37 /* Yield */:
+            case 38 /* Yield */:
                 return this._parseYieldExpression();
+            case 35 /* Type */: {
+                // Type is considered a "soft" keyword, so we will treat it
+                // as an identifier if it is followed by an unexpected token.
+                const peekToken1 = this._peekToken(1);
+                const peekToken2 = this._peekToken(2);
+                let isInvalidTypeToken = true;
+                if (peekToken1.type === 7 /* Identifier */ || peekToken1.type === 8 /* Keyword */) {
+                    if (peekToken2.type === 15 /* OpenBracket */) {
+                        isInvalidTypeToken = false;
+                    }
+                    else if (peekToken2.type === 9 /* Operator */ &&
+                        peekToken2.operatorType === 2 /* Assign */) {
+                        isInvalidTypeToken = false;
+                    }
+                }
+                if (!isInvalidTypeToken) {
+                    return this._parseTypeAliasStatement();
+                }
+                break;
+            }
         }
         return this._parseExpressionStatement();
     }
@@ -2883,12 +3003,19 @@ class Parser {
     // child node can be passed to help the completion provider determine
     // what to do.
     _handleExpressionParseError(category, errorMsg, targetToken, childNode, additionalStopTokens) {
+        var _a;
         this._addError(errorMsg, targetToken !== null && targetToken !== void 0 ? targetToken : this._peekToken());
-        const expr = parseNodes_1.ErrorNode.create(this._peekToken(), category, childNode);
         const stopTokens = [2 /* NewLine */];
         if (additionalStopTokens) {
             (0, collectionUtils_1.appendArray)(stopTokens, additionalStopTokens);
         }
+        // Using token that is not consumed by error node will mess up spans in parse node.
+        // Sibling nodes in parse tree shouldn't overlap each other.
+        const nextToken = this._peekToken();
+        const initialRange = stopTokens.some((k) => nextToken.type === k)
+            ? (_a = targetToken !== null && targetToken !== void 0 ? targetToken : childNode) !== null && _a !== void 0 ? _a : textRange_1.TextRange.create(nextToken.start, /* length */ 0)
+            : nextToken;
+        const expr = parseNodes_1.ErrorNode.create(initialRange, category, childNode);
         this._consumeTokensUntilType(stopTokens);
         return expr;
     }
@@ -3743,8 +3870,7 @@ class Parser {
         // If this is a "soft keyword", it can be converted into an identifier.
         if (nextToken.type === 8 /* Keyword */) {
             const keywordType = this._peekKeywordType();
-            const softKeywords = [9 /* Debug */, 25 /* Match */, 6 /* Case */];
-            if (softKeywords.find((type) => type === keywordType)) {
+            if (tokenizerTypes_1.softKeywords.find((type) => type === keywordType)) {
                 const keywordText = this._fileContents.substr(nextToken.start, nextToken.length);
                 this._getNextToken();
                 return tokenizerTypes_1.IdentifierToken.create(nextToken.start, nextToken.length, keywordText, nextToken.comments);