@zzzen/pyright-internal 1.2.0-dev.20230514 → 1.2.0-dev.20230521

package/dist/parser/parser.d.ts

@@ -33,7 +33,7 @@ export interface ModuleImport {
     nameNode: ModuleNameNode;
     leadingDots: number;
     nameParts: string[];
-    importedSymbols: string[] | undefined;
+    importedSymbols: Set<string> | undefined;
 }
 export interface ArgListResult {
     args: ArgumentNode[];
@@ -166,9 +166,9 @@ export declare class Parser {
     private _getTypeAnnotationCommentText;
     private _parseVariableTypeAnnotationComment;
     private _parseFunctionTypeAnnotationComment;
-    private _parseFormatStringSegment;
+    private _parseFStringReplacementField;
+    private _parseFStringFormatString;
     private _parseFormatString;
-    private _getFormatStringExpressionLength;
     private _createBinaryOperationNode;
     private _createUnaryOperationNode;
     private _parseStringList;

package/dist/parser/parser.js

@@ -813,7 +813,7 @@ class Parser {
             (0, debug_1.assert)(stringList.nodeType === 48 /* StringList */);
             // Check for f-strings, which are not allowed.
             stringList.strings.forEach((stringAtom) => {
-                if (stringAtom.token.flags & 64 /* Format */) {
+                if (stringAtom.nodeType === 27 /* FormatString */) {
                     this._addError(localize_1.Localizer.Diagnostic.formatStringInPattern(), stringAtom);
                 }
             });
@@ -1937,7 +1937,7 @@ class Parser {
             nameNode: importFromNode.module,
             leadingDots: importFromNode.module.leadingDots,
             nameParts: importFromNode.module.nameParts.map((p) => p.value),
-            importedSymbols: importFromNode.imports.map((imp) => imp.name.value),
+            importedSymbols: new Set(importFromNode.imports.map((imp) => imp.name.value)),
         });
         let isTypingImport = false;
         if (importFromNode.module.nameParts.length === 1) {
@@ -2990,7 +2990,7 @@ class Parser {
         if (nextToken.type === 7 /* Identifier */) {
             return parseNodes_1.NameNode.create(this._getNextToken());
         }
-        if (nextToken.type === 5 /* String */) {
+        if (nextToken.type === 5 /* String */ || nextToken.type === 24 /* FStringStart */) {
             return this._parseStringList();
         }
         if (nextToken.type === 22 /* Backtick */) {
@@ -3074,7 +3074,7 @@ class Parser {
         if (additionalStopTokens) {
             (0, collectionUtils_1.appendArray)(stopTokens, additionalStopTokens);
         }
-        // Using token that is not consumed by error node will mess up spans in parse node.
+        // Using a token that is not included in the error node creates problems.
         // Sibling nodes in parse tree shouldn't overlap each other.
         const nextToken = this._peekToken();
         const initialRange = stopTokens.some((k) => nextToken.type === k)
@@ -3356,6 +3356,7 @@ class Parser {
     // augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' |
     //             '<<=' | '>>=' | '**=' | '//=')
     _parseExpressionStatement() {
+        var _a, _b;
         let leftExpr = this._parseTestOrStarListAsExpression(
         /* allowAssignmentExpression */ false, 
         /* allowMultipleUnpack */ false, 2 /* MissingExpression */, () => localize_1.Localizer.Diagnostic.expectedExpr());
@@ -3382,10 +3383,9 @@ class Parser {
             if (isTypeAliasDeclaration) {
                 this._isParsingTypeAnnotation = true;
             }
-            const rightExpr = this._tryParseYieldExpression() ||
-                this._parseTestOrStarListAsExpression(
-                /* allowAssignmentExpression */ false, 
-                /* allowMultipleUnpack */ true, 2 /* MissingExpression */, () => localize_1.Localizer.Diagnostic.expectedAssignRightHandExpr());
+            const rightExpr = (_a = this._tryParseYieldExpression()) !== null && _a !== void 0 ? _a : this._parseTestOrStarListAsExpression(
+            /* allowAssignmentExpression */ false, 
+            /* allowMultipleUnpack */ true, 2 /* MissingExpression */, () => localize_1.Localizer.Diagnostic.expectedAssignRightHandExpr());
             this._isParsingTypeAnnotation = wasParsingTypeAnnotation;
             return parseNodes_1.AssignmentNode.create(leftExpr, rightExpr);
         }
@@ -3395,8 +3395,7 @@ class Parser {
         }
         if (tokenizer_1.Tokenizer.isOperatorAssignment(this._peekOperatorType())) {
             const operatorToken = this._getNextToken();
-            const rightExpr = this._tryParseYieldExpression() ||
-                this._parseTestListAsExpression(2 /* MissingExpression */, () => localize_1.Localizer.Diagnostic.expectedBinaryRightHandExpr());
+            const rightExpr = (_b = this._tryParseYieldExpression()) !== null && _b !== void 0 ? _b : this._parseTestListAsExpression(2 /* MissingExpression */, () => localize_1.Localizer.Diagnostic.expectedBinaryRightHandExpr());
             // Make a shallow copy of the dest expression but give it a new ID.
             const destExpr = Object.assign({}, leftExpr);
             destExpr.id = (0, parseNodes_1.getNextNodeId)();
@@ -3405,15 +3404,15 @@ class Parser {
         return leftExpr;
     }
     _parseChainAssignments(leftExpr) {
+        var _a;
         // Make a list of assignment targets.
         const assignmentTargets = [leftExpr];
         let rightExpr;
         while (true) {
             rightExpr =
-                this._tryParseYieldExpression() ||
-                    this._parseTestOrStarListAsExpression(
-                    /* allowAssignmentExpression */ false, 
-                    /* allowMultipleUnpack */ true, 2 /* MissingExpression */, () => localize_1.Localizer.Diagnostic.expectedAssignRightHandExpr());
+                (_a = this._tryParseYieldExpression()) !== null && _a !== void 0 ? _a : this._parseTestOrStarListAsExpression(
+                /* allowAssignmentExpression */ false, 
+                /* allowMultipleUnpack */ true, 2 /* MissingExpression */, () => localize_1.Localizer.Diagnostic.expectedAssignRightHandExpr());
             if (rightExpr.nodeType === 0 /* Error */) {
                 break;
             }
@@ -3517,7 +3516,7 @@ class Parser {
         if (stringToken.flags & 65536 /* Unterminated */) {
             this._addError(localize_1.Localizer.Diagnostic.stringUnterminated(), stringToken);
         }
-        if (unescapedResult.nonAsciiInBytes) {
+        if (unescapedResult === null || unescapedResult === void 0 ? void 0 : unescapedResult.nonAsciiInBytes) {
             this._addError(localize_1.Localizer.Diagnostic.stringNonAsciiBytes(), stringToken);
         }
         if (stringToken.flags & 64 /* Format */) {
@@ -3535,7 +3534,7 @@ class Parser {
     _makeStringNode(stringToken) {
         const unescapedResult = StringTokenUtils.getUnescapedString(stringToken);
         this._reportStringTokenErrors(stringToken, unescapedResult);
-        return parseNodes_1.StringNode.create(stringToken, unescapedResult.value, unescapedResult.unescapeErrors.length > 0);
+        return parseNodes_1.StringNode.create(stringToken, unescapedResult.value);
     }
     _getTypeAnnotationCommentText() {
         if (this._tokenIndex === 0) {
@@ -3601,154 +3600,123 @@ class Parser {
         functionAnnotation.parent = functionNode;
         (0, parseNodes_1.extendRange)(functionNode, functionAnnotation);
     }
-    _parseFormatStringSegment(stringToken, segment, segmentOffset, segmentLength) {
-        (0, debug_1.assert)(segment.isExpression);
-        const parser = new Parser();
-        const parseResults = parser.parseTextExpression(this._fileContents, stringToken.start + stringToken.prefixLength + stringToken.quoteMarkLength + segment.offset + segmentOffset, segmentLength, this._parseOptions, 0 /* Expression */, 
-        /* initialParenDepth */ 1, this._typingSymbolAliases);
-        parseResults.diagnostics.forEach((diag) => {
-            const textRangeStart = (diag.range ? (0, positionUtils_1.convertPositionToOffset)(diag.range.start, parseResults.lines) : stringToken.start) ||
-                stringToken.start;
-            const textRangeEnd = (diag.range
-                ? ((0, positionUtils_1.convertPositionToOffset)(diag.range.end, parseResults.lines) || 0) + 1
-                : stringToken.start + stringToken.length) || stringToken.start + stringToken.length;
-            const textRange = { start: textRangeStart, length: textRangeEnd - textRangeStart };
-            this._addError(diag.message, textRange);
-        });
-        return parseResults.parseTree;
+    _parseFStringReplacementField(fieldExpressions, middleTokens, formatExpressions, nestingDepth = 0) {
+        var _a;
+        let nextToken = this._getNextToken();
+        // The caller should have already confirmed that the next token is an open brace.
+        (0, debug_1.assert)(nextToken.type === 17 /* OpenCurlyBrace */);
+        // Consume the expression.
+        const expr = (_a = this._tryParseYieldExpression()) !== null && _a !== void 0 ? _a : this._parseTestOrStarListAsExpression(
+        /* allowAssignmentExpression */ true, 
+        /* allowMultipleUnpack */ true, 2 /* MissingExpression */, () => localize_1.Localizer.Diagnostic.expectedExpr());
+        fieldExpressions.push(expr);
+        if (expr.nodeType === 0 /* Error */) {
+            return false;
+        }
+        // Consume an optional "=" token after the expression.
+        nextToken = this._peekToken();
+        if (nextToken.type === 9 /* Operator */ &&
+            nextToken.operatorType === 2 /* Assign */) {
+            // This feature requires Python 3.8 or newer.
+            if (this._parseOptions.pythonVersion < pythonVersion_1.PythonVersion.V3_8) {
+                this._addError(localize_1.Localizer.Diagnostic.formatStringDebuggingIllegal(), nextToken);
+            }
+            this._getNextToken();
+            nextToken = this._peekToken();
+        }
+        // Consume an optional !r, !s, or !a token.
+        if (nextToken.type === 23 /* ExclamationMark */) {
+            this._getNextToken();
+            nextToken = this._peekToken();
+            if (nextToken.type !== 7 /* Identifier */) {
+                this._addError(localize_1.Localizer.Diagnostic.formatStringExpectedConversion(), nextToken);
+            }
+            else {
+                this._getNextToken();
+                nextToken = this._peekToken();
+            }
+        }
+        if (nextToken.type === 10 /* Colon */) {
+            this._getNextToken();
+            this._parseFStringFormatString(fieldExpressions, middleTokens, formatExpressions, nestingDepth);
+            nextToken = this._peekToken();
+        }
+        if (nextToken.type !== 18 /* CloseCurlyBrace */) {
+            this._addError(localize_1.Localizer.Diagnostic.formatStringUnterminated(), nextToken);
+            return false;
+        }
+        else {
+            this._getNextToken();
+        }
+        // Indicate success.
+        return true;
     }
-    _parseFormatString(stringToken) {
-        const unescapedResult = StringTokenUtils.getUnescapedString(stringToken);
-        this._reportStringTokenErrors(stringToken, unescapedResult);
-        const formatExpressions = [];
-        for (const segment of unescapedResult.formatStringSegments) {
-            if (segment.isExpression) {
-                // Determine if we need to truncate the expression because it
-                // contains formatting directives that start with a ! or :.
-                const segmentExprLength = this._getFormatStringExpressionLength(segment.value.trimEnd());
-                const parseTree = this._parseFormatStringSegment(stringToken, segment, 0, segmentExprLength);
-                if (parseTree) {
-                    (0, debug_1.assert)(parseTree.nodeType !== 62 /* FunctionAnnotation */);
-                    formatExpressions.push(parseTree);
-                }
-                // Look for additional expressions within the format directive.
-                const formatDirective = segment.value.substr(segmentExprLength);
-                let braceDepth = 0;
-                let startOfExprOffset = 0;
-                for (let i = 0; i < formatDirective.length; i++) {
-                    if (formatDirective.charCodeAt(i) === 123 /* OpenBrace */) {
-                        if (braceDepth === 0) {
-                            startOfExprOffset = i + 1;
-                        }
-                        braceDepth++;
-                    }
-                    else if (formatDirective.charCodeAt(i) === 125 /* CloseBrace */) {
-                        if (braceDepth > 0) {
-                            braceDepth--;
-                            if (braceDepth === 0) {
-                                const formatSegmentLength = this._getFormatStringExpressionLength(segment.value.substr(segmentExprLength + startOfExprOffset, i - startOfExprOffset));
-                                const parseTree = this._parseFormatStringSegment(stringToken, segment, segmentExprLength + startOfExprOffset, formatSegmentLength);
-                                if (parseTree) {
-                                    (0, debug_1.assert)(parseTree.nodeType !== 62 /* FunctionAnnotation */);
-                                    formatExpressions.push(parseTree);
-                                }
-                            }
-                        }
-                    }
+    _parseFStringFormatString(fieldExpressions, middleTokens, formatExpressions, nestingDepth) {
+        while (true) {
+            const nextToken = this._peekToken();
+            if (nextToken.type === 18 /* CloseCurlyBrace */ || nextToken.type === 26 /* FStringEnd */) {
+                break;
+            }
+            if (nextToken.type === 25 /* FStringMiddle */) {
+                this._getNextToken();
+                continue;
+            }
+            if (nextToken.type === 17 /* OpenCurlyBrace */) {
+                // The Python interpreter reports an error at the point where the
+                // nesting level exceeds 1. Don't report the error again for deeper nestings.
+                if (nestingDepth === 2) {
+                    this._addError(localize_1.Localizer.Diagnostic.formatStringNestedFormatSpecifier(), nextToken);
                 }
+                this._parseFStringReplacementField(fieldExpressions, middleTokens, formatExpressions, nestingDepth + 1);
+                continue;
             }
+            break;
         }
-        return parseNodes_1.FormatStringNode.create(stringToken, unescapedResult.value, unescapedResult.unescapeErrors.length > 0, formatExpressions);
     }
-    _getFormatStringExpressionLength(segmentValue) {
-        let segmentExprLength = 0;
-        // PEP 498 says: Expressions cannot contain ':' or '!' outside of
-        // strings or parentheses, brackets, or braces. The exception is
-        // that the '!=' operator is allowed as a special case.
-        const quoteStack = [];
-        let braceCount = 0;
-        let parenCount = 0;
-        let bracketCount = 0;
-        let indexOfDebugEqual;
-        while (segmentExprLength < segmentValue.length) {
-            const curChar = segmentValue[segmentExprLength];
-            const ignoreSeparator = quoteStack.length > 0 || braceCount > 0 || parenCount > 0 || bracketCount > 0;
-            const inString = quoteStack.length > 0;
-            if (curChar === '=') {
-                indexOfDebugEqual = segmentExprLength;
-            }
-            else {
-                if (curChar === ':') {
-                    if (!ignoreSeparator) {
-                        break;
-                    }
+    _parseFormatString(startToken) {
+        const middleTokens = [];
+        const fieldExpressions = [];
+        const formatExpressions = [];
+        let endToken = undefined;
+        // Consume middle tokens and expressions until we hit a "{" or "}" token.
+        while (true) {
+            const nextToken = this._peekToken();
+            if (nextToken.type === 26 /* FStringEnd */) {
+                endToken = nextToken;
+                if ((endToken.flags & 65536 /* Unterminated */) !== 0) {
+                    this._addError(localize_1.Localizer.Diagnostic.stringUnterminated(), startToken);
                 }
-                else if (curChar === '!') {
-                    if (!ignoreSeparator) {
-                        // Allow !=, as per PEP 498
-                        if (segmentExprLength === segmentValue.length - 1 ||
-                            segmentValue[segmentExprLength + 1] !== '=') {
-                            break;
+                this._getNextToken();
+                break;
+            }
+            if (nextToken.type === 25 /* FStringMiddle */) {
+                middleTokens.push(nextToken);
+                this._getNextToken();
+                continue;
+            }
+            if (nextToken.type === 17 /* OpenCurlyBrace */) {
+                if (!this._parseFStringReplacementField(fieldExpressions, middleTokens, formatExpressions)) {
+                    // An error was reported. Try to recover the parse.
+                    if (this._consumeTokensUntilType([26 /* FStringEnd */, 2 /* NewLine */])) {
+                        if (this._peekToken().type === 26 /* FStringEnd */) {
+                            this._getNextToken();
                         }
                     }
+                    break;
                 }
-                else if (curChar === "'" || curChar === '"') {
-                    let quoteSequence = curChar;
-                    if (segmentExprLength + 2 < segmentValue.length &&
-                        segmentValue[segmentExprLength + 1] === curChar &&
-                        segmentValue[segmentExprLength + 2] === curChar) {
-                        quoteSequence = curChar + curChar + curChar;
-                        segmentExprLength += 2;
-                    }
-                    if (quoteStack.length > 0 && quoteStack[quoteStack.length - 1] === quoteSequence) {
-                        quoteStack.pop();
-                    }
-                    else if (quoteStack.length === 0) {
-                        quoteStack.push(quoteSequence);
-                    }
-                }
-                else if (curChar === '(') {
-                    if (!inString) {
-                        parenCount++;
-                    }
-                }
-                else if (curChar === ')') {
-                    if (!inString && parenCount > 0) {
-                        parenCount--;
-                    }
-                }
-                else if (curChar === '{') {
-                    if (!inString) {
-                        braceCount++;
-                    }
-                }
-                else if (curChar === '}') {
-                    if (!inString && braceCount > 0) {
-                        braceCount--;
-                    }
-                }
-                else if (curChar === '[') {
-                    if (!inString) {
-                        bracketCount++;
-                    }
-                }
-                else if (curChar === ']') {
-                    if (!inString && bracketCount > 0) {
-                        bracketCount--;
-                    }
-                }
-                if (curChar !== ' ') {
-                    indexOfDebugEqual = undefined;
-                }
+                continue;
             }
-            segmentExprLength++;
-        }
-        // Handle Python 3.8 f-string formatting expressions that
-        // end in an "=".
-        if (this._parseOptions.pythonVersion >= pythonVersion_1.PythonVersion.V3_8 && indexOfDebugEqual !== undefined) {
-            segmentExprLength = indexOfDebugEqual;
+            // We've hit an error. Consume tokens until we find the end.
+            if (this._consumeTokensUntilType([26 /* FStringEnd */])) {
+                this._getNextToken();
+            }
+            this._addError(nextToken.type === 18 /* CloseCurlyBrace */
+                ? localize_1.Localizer.Diagnostic.formatStringBrace()
+                : localize_1.Localizer.Diagnostic.stringUnterminated(), nextToken);
+            break;
         }
-        return segmentExprLength;
+        this._reportStringTokenErrors(startToken);
+        return parseNodes_1.FormatStringNode.create(startToken, endToken, middleTokens, fieldExpressions, formatExpressions);
     }
     _createBinaryOperationNode(leftExpression, rightExpression, operatorToken, operator) {
         // Determine if we're exceeding the max parse depth. If so, replace
@@ -3776,13 +3744,16 @@ class Parser {
     }
     _parseStringList() {
         const stringList = [];
-        while (this._peekTokenType() === 5 /* String */) {
-            const stringToken = this._getNextToken();
-            if (stringToken.flags & 64 /* Format */) {
-                stringList.push(this._parseFormatString(stringToken));
+        while (true) {
+            const nextToken = this._peekToken();
+            if (nextToken.type === 5 /* String */) {
+                stringList.push(this._makeStringNode(this._getNextToken()));
+            }
+            else if (nextToken.type === 24 /* FStringStart */) {
+                stringList.push(this._parseFormatString(this._getNextToken()));
             }
             else {
-                stringList.push(this._makeStringNode(stringToken));
+                break;
             }
         }
         const stringNode = parseNodes_1.StringListNode.create(stringList);
@@ -3793,7 +3764,7 @@ class Parser {
             if (stringNode.strings.length > 1) {
                 this._addError(localize_1.Localizer.Diagnostic.annotationSpansStrings(), stringNode);
             }
-            else if (stringNode.strings[0].token.flags & 64 /* Format */) {
+            else if (stringNode.strings[0].nodeType === 27 /* FormatString */) {
                 this._addError(localize_1.Localizer.Diagnostic.annotationFormatString(), stringNode);
             }
             else {
@@ -3889,6 +3860,9 @@ class Parser {
             case 18 /* CloseCurlyBrace */:
             case 12 /* Comma */:
             case 10 /* Colon */:
+            case 23 /* ExclamationMark */:
+            case 25 /* FStringMiddle */:
+            case 26 /* FStringEnd */:
                 return true;
         }
         return false;