@zzzen/pyright-internal 1.1.254 → 1.1.255

package/src/analyzer/codeFlowTypes.ts

@@ -0,0 +1,266 @@
+/*
+ * codeFlowTypes.ts
+ * Copyright (c) Microsoft Corporation.
+ * Licensed under the MIT license.
+ * Author: Eric Traut
+ *
+ * Data structures that track the code flow (or more accurately,
+ * the inverse of code flow) starting with return statements and
+ * working back to the entry. This allows us to work out the
+ * types at each point of the code flow.
+ *
+ * This is largely based on the code flow engine in the
+ * TypeScript compiler.
+ */
+
+import { assert, fail } from '../common/debug';
+import {
+    ArgumentCategory,
+    CallNode,
+    CaseNode,
+    ExpressionNode,
+    ImportFromNode,
+    IndexNode,
+    MatchNode,
+    MemberAccessNode,
+    NameNode,
+    NumberNode,
+    ParseNodeType,
+    StringNode,
+    SuiteNode,
+} from '../parser/parseNodes';
+import { OperatorType } from '../parser/tokenizerTypes';
+
+export enum FlowFlags {
+    Unreachable = 1 << 0, // Unreachable code
+    Start = 1 << 1, // Entry point
+    BranchLabel = 1 << 2, // Junction for forward control flow
+    LoopLabel = 1 << 3, // Junction for backward control flow
+    Assignment = 1 << 4, // Assignment statement
+    Unbind = 1 << 5, // Used with assignment to indicate target should be unbound
+    WildcardImport = 1 << 6, // For "from X import *" statements
+    TrueCondition = 1 << 7, // Condition known to be true
+    FalseCondition = 1 << 9, // Condition known to be false
+    Call = 1 << 10, // Call node
+    PreFinallyGate = 1 << 11, // Injected edge that links pre-finally label and pre-try flow
+    PostFinally = 1 << 12, // Injected edge that links post-finally flow with the rest of the graph
+    VariableAnnotation = 1 << 14, // Separates a variable annotation from its name node
+    PostContextManager = 1 << 15, // Label that's used for context managers that suppress exceptions
+    TrueNeverCondition = 1 << 16, // Condition whose type evaluates to never when narrowed in positive test
+    FalseNeverCondition = 1 << 17, // Condition whose type evaluates to never when narrowed in negative test
+    NarrowForPattern = 1 << 18, // Narrow the type of the subject expression within a case statement
+    ExhaustedMatch = 1 << 19, // Control flow gate that is closed when match is provably exhaustive
+}
+
+let _nextFlowNodeId = 1;
+
+export type CodeFlowReferenceExpressionNode = NameNode | MemberAccessNode | IndexNode;
+
+export function getUniqueFlowNodeId() {
+    return _nextFlowNodeId++;
+}
+
+export interface FlowNode {
+    flags: FlowFlags;
+    id: number;
+}
+
+// FlowLabel represents a junction with multiple possible
+// preceding control flows.
+export interface FlowLabel extends FlowNode {
+    antecedents: FlowNode[];
+
+    // Set of all expressions that require code flow analysis
+    // through the loop or in branch paths to determine their types.
+    // If an expression is not within this map, branch or loop analysis
+    // can be skipped and determined from the first antecedent only.
+    affectedExpressions: Set<string> | undefined;
+}
+
+export interface FlowBranchLabel extends FlowLabel {
+    // If specified, this label represents a flow node that precedes
+    // (i.e. is higher up in the control flow graph) than all of
+    // the antecedents of this branch label. If an expression is
+    // not affected by the branch label, the entire flow node can be
+    // skipped, and processing can proceed at this label.
+    preBranchAntecedent: FlowNode | undefined;
+}
+
+// FlowAssignment represents a node that assigns a value.
+export interface FlowAssignment extends FlowNode {
+    node: CodeFlowReferenceExpressionNode;
+    antecedent: FlowNode;
+    targetSymbolId: number;
+}
+
+// FlowVariableAnnotation separates a variable annotation
+// node from its type annotation. For example, the declaration
+// "foo: bar", the "bar" needs to be associated with a flow
+// node that precedes the "foo". This is important if the
+// same name is used for both (e.g. "foo: foo") and we need
+// to determine that the annotation refers to a symbol within
+// an outer scope.
+export interface FlowVariableAnnotation extends FlowNode {
+    antecedent: FlowNode;
+}
+
+// Similar to FlowAssignment but used specifically for
+// wildcard "from X import *" statements.
+export interface FlowWildcardImport extends FlowNode {
+    node: ImportFromNode;
+    names: string[];
+    antecedent: FlowNode;
+}
+
+// FlowCondition represents a condition that is known to
+// be true or false at the node's location in the control flow.
+export interface FlowCondition extends FlowNode {
+    expression: ExpressionNode;
+    reference?: NameNode | undefined;
+    antecedent: FlowNode;
+}
+
+export interface FlowNarrowForPattern extends FlowNode {
+    subjectExpression: ExpressionNode;
+    statement: CaseNode | MatchNode;
+    antecedent: FlowNode;
+}
+
+// FlowExhaustedMatch represents a control flow gate that is "closed"
+// if a match statement can be statically proven to exhaust all cases
+// (i.e. the narrowed type of the subject expression is Never at the bottom).
+export interface FlowExhaustedMatch extends FlowNode {
+    node: MatchNode;
+    antecedent: FlowNode;
+}
+
+// Records a call, which may raise exceptions, thus affecting
+// the code flow and making subsequent code unreachable.
+export interface FlowCall extends FlowNode {
+    node: CallNode;
+    antecedent: FlowNode;
+}
+
+// See comment in the visitTry method in binder.ts for a full
+// explanation of the FlowPreFinally and FlowPostFinally nodes.
+export interface FlowPreFinallyGate extends FlowNode {
+    antecedent: FlowNode;
+    isGateClosed: boolean;
+}
+
+export interface FlowPostFinally extends FlowNode {
+    antecedent: FlowNode;
+    finallyNode: SuiteNode;
+    preFinallyGate: FlowPreFinallyGate;
+}
+
+export interface FlowPostContextManagerLabel extends FlowLabel {
+    expressions: ExpressionNode[];
+    isAsync: boolean;
+
+    // If the context manager swallows exceptions and this value
+    // is true, block any code flow analysis along this path. Conversely,
+    // if the context manager does not swallow exceptions and this
+    // value is false, block any code flow analysis along this path.
+    blockIfSwallowsExceptions: boolean;
+}
+
+export function isCodeFlowSupportedForReference(reference: ExpressionNode): boolean {
+    if (reference.nodeType === ParseNodeType.Name) {
+        return true;
+    }
+
+    if (reference.nodeType === ParseNodeType.MemberAccess) {
+        return isCodeFlowSupportedForReference(reference.leftExpression);
+    }
+
+    if (reference.nodeType === ParseNodeType.Index) {
+        // Allow index expressions that have a single subscript that is a
+        // literal integer or string value.
+        if (
+            reference.items.length !== 1 ||
+            reference.trailingComma ||
+            reference.items[0].name !== undefined ||
+            reference.items[0].argumentCategory !== ArgumentCategory.Simple
+        ) {
+            return false;
+        }
+
+        const subscriptNode = reference.items[0].valueExpression;
+        const isIntegerIndex =
+            subscriptNode.nodeType === ParseNodeType.Number && !subscriptNode.isImaginary && subscriptNode.isInteger;
+        const isNegativeIntegerIndex =
+            subscriptNode.nodeType === ParseNodeType.UnaryOperation &&
+            subscriptNode.operator === OperatorType.Subtract &&
+            subscriptNode.expression.nodeType === ParseNodeType.Number &&
+            !subscriptNode.expression.isImaginary &&
+            subscriptNode.expression.isInteger;
+        const isStringIndex =
+            subscriptNode.nodeType === ParseNodeType.StringList &&
+            subscriptNode.strings.length === 1 &&
+            subscriptNode.strings[0].nodeType === ParseNodeType.String;
+
+        if (!isIntegerIndex && !isNegativeIntegerIndex && !isStringIndex) {
+            return false;
+        }
+
+        return isCodeFlowSupportedForReference(reference.baseExpression);
+    }
+
+    return false;
+}
+
+export function createKeyForReference(reference: CodeFlowReferenceExpressionNode): string {
+    let key;
+    if (reference.nodeType === ParseNodeType.Name) {
+        key = reference.value;
+    } else if (reference.nodeType === ParseNodeType.MemberAccess) {
+        const leftKey = createKeyForReference(reference.leftExpression as CodeFlowReferenceExpressionNode);
+        key = `${leftKey}.${reference.memberName.value}`;
+    } else if (reference.nodeType === ParseNodeType.Index) {
+        const leftKey = createKeyForReference(reference.baseExpression as CodeFlowReferenceExpressionNode);
+        assert(reference.items.length === 1);
+        const expr = reference.items[0].valueExpression;
+        if (expr.nodeType === ParseNodeType.Number) {
+            key = `${leftKey}[${(expr as NumberNode).value.toString()}]`;
+        } else if (expr.nodeType === ParseNodeType.StringList) {
+            const valExpr = expr;
+            assert(valExpr.strings.length === 1 && valExpr.strings[0].nodeType === ParseNodeType.String);
+            key = `${leftKey}["${(valExpr.strings[0] as StringNode).value}"]`;
+        } else if (
+            expr.nodeType === ParseNodeType.UnaryOperation &&
+            expr.operator === OperatorType.Subtract &&
+            expr.expression.nodeType === ParseNodeType.Number
+        ) {
+            key = `${leftKey}[-${(expr.expression as NumberNode).value.toString()}]`;
+        } else {
+            fail('createKeyForReference received unexpected index type');
+        }
+    } else {
+        fail('createKeyForReference received unexpected expression type');
+    }
+
+    return key;
+}
+
+export function createKeysForReferenceSubexpressions(reference: CodeFlowReferenceExpressionNode): string[] {
+    if (reference.nodeType === ParseNodeType.Name) {
+        return [createKeyForReference(reference)];
+    }
+
+    if (reference.nodeType === ParseNodeType.MemberAccess) {
+        return [
+            ...createKeysForReferenceSubexpressions(reference.leftExpression as CodeFlowReferenceExpressionNode),
+            createKeyForReference(reference),
+        ];
+    }
+
+    if (reference.nodeType === ParseNodeType.Index) {
+        return [
+            ...createKeysForReferenceSubexpressions(reference.baseExpression as CodeFlowReferenceExpressionNode),
+            createKeyForReference(reference),
+        ];
+    }
+
+    fail('createKeyForReference received unexpected expression type');
+}

package/src/analyzer/codeFlowUtils.ts

@@ -0,0 +1,436 @@
+/*
+ * codeFlowUtils.ts
+ * Copyright (c) Microsoft Corporation.
+ * Licensed under the MIT license.
+ * Author: Eric Traut
+ *
+ * Utility functions that operate on code flow nodes and graphs.
+ */
+
+import { convertOffsetToPosition } from '../common/positionUtils';
+import { ParseNode } from '../parser/parseNodes';
+import { getFileInfo } from './analyzerNodeInfo';
+import {
+    FlowAssignment,
+    FlowCall,
+    FlowCondition,
+    FlowExhaustedMatch,
+    FlowFlags,
+    FlowLabel,
+    FlowNarrowForPattern,
+    FlowNode,
+    FlowPostFinally,
+    FlowPreFinallyGate,
+    FlowVariableAnnotation,
+    FlowWildcardImport,
+} from './codeFlowTypes';
+
+export function formatControlFlowGraph(flowNode: FlowNode) {
+    const enum BoxCharacter {
+        lr = '─',
+        ud = '│',
+        dr = '╭',
+        dl = '╮',
+        ul = '╯',
+        ur = '╰',
+        udr = '├',
+        udl = '┤',
+        dlr = '┬',
+        ulr = '┴',
+        udlr = '╫',
+    }
+
+    const enum Connection {
+        Up = 1 << 0,
+        Down = 1 << 1,
+        Left = 1 << 2,
+        Right = 1 << 3,
+
+        UpDown = Up | Down,
+        LeftRight = Left | Right,
+        UpLeft = Up | Left,
+        UpRight = Up | Right,
+        DownLeft = Down | Left,
+        DownRight = Down | Right,
+        UpDownLeft = UpDown | Left,
+        UpDownRight = UpDown | Right,
+        UpLeftRight = Up | LeftRight,
+        DownLeftRight = Down | LeftRight,
+        UpDownLeftRight = UpDown | LeftRight,
+
+        NoChildren = 1 << 4,
+    }
+
+    interface FlowGraphNode {
+        id: number;
+        flowNode: FlowNode;
+        edges: FlowGraphEdge[];
+        text: string;
+        lane: number;
+        endLane: number;
+        level: number;
+        circular: boolean;
+    }
+
+    interface FlowGraphEdge {
+        source: FlowGraphNode;
+        target: FlowGraphNode;
+    }
+
+    const links: Record<number, FlowGraphNode> = Object.create(/* o */ null);
+    const nodes: FlowGraphNode[] = [];
+    const edges: FlowGraphEdge[] = [];
+    const root = buildGraphNode(flowNode, new Set());
+
+    for (const node of nodes) {
+        node.text = renderFlowNode(node.flowNode, node.circular);
+        computeLevel(node);
+    }
+
+    const height = computeHeight(root);
+    const columnWidths = computeColumnWidths(height);
+    computeLanes(root, 0);
+    return renderGraph();
+
+    function getAntecedents(f: FlowNode): FlowNode[] {
+        if (f.flags & (FlowFlags.LoopLabel | FlowFlags.BranchLabel)) {
+            return (f as FlowLabel).antecedents;
+        }
+
+        if (
+            f.flags &
+            (FlowFlags.Assignment |
+                FlowFlags.VariableAnnotation |
+                FlowFlags.WildcardImport |
+                FlowFlags.TrueCondition |
+                FlowFlags.FalseCondition |
+                FlowFlags.NarrowForPattern |
+                FlowFlags.ExhaustedMatch |
+                FlowFlags.Call |
+                FlowFlags.PreFinallyGate |
+                FlowFlags.PostFinally)
+        ) {
+            const typedFlowNode = f as
+                | FlowAssignment
+                | FlowVariableAnnotation
+                | FlowWildcardImport
+                | FlowCondition
+                | FlowCondition
+                | FlowExhaustedMatch
+                | FlowCall
+                | FlowPreFinallyGate
+                | FlowPostFinally;
+            return [typedFlowNode.antecedent];
+        }
+
+        return [];
+    }
+
+    function getChildren(node: FlowGraphNode) {
+        const children: FlowGraphNode[] = [];
+        for (const edge of node.edges) {
+            if (edge.source === node) {
+                children.push(edge.target);
+            }
+        }
+        return children;
+    }
+
+    function getParents(node: FlowGraphNode) {
+        const parents: FlowGraphNode[] = [];
+        for (const edge of node.edges) {
+            if (edge.target === node) {
+                parents.push(edge.source);
+            }
+        }
+        return parents;
+    }
+
+    function buildGraphNode(flowNode: FlowNode, seen: Set<FlowNode>): FlowGraphNode {
+        const id = flowNode.id;
+        let graphNode = links[id];
+
+        if (graphNode && seen.has(flowNode)) {
+            graphNode = {
+                id: -1,
+                flowNode,
+                edges: [],
+                text: '',
+                lane: -1,
+                endLane: -1,
+                level: -1,
+                circular: true,
+            };
+            nodes.push(graphNode);
+            return graphNode;
+        }
+        seen.add(flowNode);
+
+        if (!graphNode) {
+            links[id] = graphNode = {
+                id,
+                flowNode,
+                edges: [],
+                text: '',
+                lane: -1,
+                endLane: -1,
+                level: -1,
+                circular: false,
+            };
+
+            nodes.push(graphNode);
+
+            const antecedents = getAntecedents(flowNode);
+            for (const antecedent of antecedents) {
+                buildGraphEdge(graphNode, antecedent, seen);
+            }
+        }
+
+        seen.delete(flowNode);
+        return graphNode;
+    }
+
+    function buildGraphEdge(source: FlowGraphNode, antecedent: FlowNode, seen: Set<FlowNode>) {
+        const target = buildGraphNode(antecedent, seen);
+        const edge: FlowGraphEdge = { source, target };
+        edges.push(edge);
+        source.edges.push(edge);
+        target.edges.push(edge);
+    }
+
+    function computeLevel(node: FlowGraphNode): number {
+        if (node.level !== -1) {
+            return node.level;
+        }
+        let level = 0;
+        for (const parent of getParents(node)) {
+            level = Math.max(level, computeLevel(parent) + 1);
+        }
+        return (node.level = level);
+    }
+
+    function computeHeight(node: FlowGraphNode): number {
+        let height = 0;
+        for (const child of getChildren(node)) {
+            height = Math.max(height, computeHeight(child));
+        }
+        return height + 1;
+    }
+
+    function computeColumnWidths(height: number) {
+        const columns: number[] = fill(Array(height), 0);
+        for (const node of nodes) {
+            columns[node.level] = Math.max(columns[node.level], node.text.length);
+        }
+        return columns;
+    }
+
+    function computeLanes(node: FlowGraphNode, lane: number) {
+        if (node.lane === -1) {
+            node.lane = lane;
+            node.endLane = lane;
+            const children = getChildren(node);
+            for (let i = 0; i < children.length; i++) {
+                if (i > 0) lane++;
+                const child = children[i];
+                computeLanes(child, lane);
+                if (child.endLane > node.endLane) {
+                    lane = child.endLane;
+                }
+            }
+            node.endLane = lane;
+        }
+    }
+
+    function getHeader(flags: FlowFlags) {
+        if (flags & FlowFlags.Start) return 'Start';
+        if (flags & FlowFlags.BranchLabel) return 'Branch';
+        if (flags & FlowFlags.LoopLabel) return 'Loop';
+        if (flags & FlowFlags.Unbind) return 'Unbind';
+        if (flags & FlowFlags.Assignment) return 'Assign';
+        if (flags & FlowFlags.TrueCondition) return 'True';
+        if (flags & FlowFlags.FalseCondition) return 'False';
+        if (flags & FlowFlags.Call) return 'Call';
+        if (flags & FlowFlags.Unreachable) return 'Unreachable';
+        if (flags & FlowFlags.WildcardImport) return 'Wildcard';
+        if (flags & FlowFlags.PreFinallyGate) return 'PreFinal';
+        if (flags & FlowFlags.PostFinally) return 'PostFinal';
+        if (flags & FlowFlags.VariableAnnotation) return 'Annotate';
+        if (flags & FlowFlags.TrueNeverCondition) return 'TrueNever';
+        if (flags & FlowFlags.FalseNeverCondition) return 'FalseNever';
+        if (flags & FlowFlags.NarrowForPattern) return 'Pattern';
+        if (flags & FlowFlags.ExhaustedMatch) return 'Exhaust';
+        throw new Error();
+    }
+
+    function getParseNode(f: FlowNode): ParseNode | undefined {
+        if (f.flags & FlowFlags.Assignment) {
+            return (f as FlowAssignment).node;
+        }
+
+        if (f.flags & FlowFlags.WildcardImport) {
+            return (f as FlowWildcardImport).node;
+        }
+
+        if (f.flags & (FlowFlags.TrueCondition | FlowFlags.FalseCondition)) {
+            return (f as FlowCondition).expression;
+        }
+
+        if (f.flags & FlowFlags.NarrowForPattern) {
+            return (f as FlowNarrowForPattern).statement;
+        }
+
+        if (f.flags & FlowFlags.Call) {
+            return (f as FlowCall).node;
+        }
+
+        return undefined;
+    }
+
+    function getNodeText(f: FlowNode): string | undefined {
+        const parseNode = getParseNode(f);
+
+        if (!parseNode) {
+            return undefined;
+        }
+
+        const fileInfo = getFileInfo(parseNode);
+        const startPos = convertOffsetToPosition(parseNode.start, fileInfo.lines);
+
+        return `[${startPos.line + 1}:${startPos.character + 1}]`;
+    }
+
+    function renderFlowNode(flowNode: FlowNode, circular: boolean) {
+        const text = `${getHeader(flowNode.flags)}@${flowNode.id}${getNodeText(flowNode) || ''}`;
+        return circular ? `Circular(${text})` : text;
+    }
+
+    function renderGraph() {
+        const columnCount = columnWidths.length;
+        const laneCount = nodes.reduce((x, n) => Math.max(x, n.lane), 0) + 1;
+        const lanes: string[] = fill(Array(laneCount), '');
+        const grid: (FlowGraphNode | undefined)[][] = columnWidths.map(() => Array(laneCount));
+        const connectors: Connection[][] = columnWidths.map(() => fill(Array(laneCount), 0));
+
+        // Build connectors.
+        for (const node of nodes) {
+            grid[node.level][node.lane] = node;
+            const children = getChildren(node);
+            for (let i = 0; i < children.length; i++) {
+                const child = children[i];
+                let connector: Connection = Connection.Right;
+                if (child.lane === node.lane) connector |= Connection.Left;
+                if (i > 0) connector |= Connection.Up;
+                if (i < children.length - 1) connector |= Connection.Down;
+                connectors[node.level][child.lane] |= connector;
+            }
+            if (children.length === 0) {
+                connectors[node.level][node.lane] |= Connection.NoChildren;
+            }
+            const parents = getParents(node);
+            for (let i = 0; i < parents.length; i++) {
+                const parent = parents[i];
+                let connector: Connection = Connection.Left;
+                if (i > 0) connector |= Connection.Up;
+                if (i < parents.length - 1) connector |= Connection.Down;
+                connectors[node.level - 1][parent.lane] |= connector;
+            }
+        }
+
+        // Fill in missing connectors.
+        for (let column = 0; column < columnCount; column++) {
+            for (let lane = 0; lane < laneCount; lane++) {
+                const left = column > 0 ? connectors[column - 1][lane] : 0;
+                const above = lane > 0 ? connectors[column][lane - 1] : 0;
+                let connector = connectors[column][lane];
+                if (!connector) {
+                    if (left & Connection.Right) connector |= Connection.LeftRight;
+                    if (above & Connection.Down) connector |= Connection.UpDown;
+                    connectors[column][lane] = connector;
+                }
+            }
+        }
+
+        for (let column = 0; column < columnCount; column++) {
+            for (let lane = 0; lane < lanes.length; lane++) {
+                const connector = connectors[column][lane];
+                const fill = connector & Connection.Left ? BoxCharacter.lr : ' ';
+                const node = grid[column][lane];
+                if (!node) {
+                    if (column < columnCount - 1) {
+                        writeLane(lane, repeat(fill, columnWidths[column] + 1));
+                    }
+                } else {
+                    writeLane(lane, node.text);
+                    if (column < columnCount - 1) {
+                        writeLane(lane, ' ');
+                        writeLane(lane, repeat(fill, columnWidths[column] - node.text.length));
+                    }
+                }
+                writeLane(lane, getBoxCharacter(connector));
+                writeLane(
+                    lane,
+                    connector & Connection.Right && column < columnCount - 1 && !grid[column + 1][lane]
+                        ? BoxCharacter.lr
+                        : ' '
+                );
+            }
+        }
+
+        return `${lanes.join('\n')}\n`;
+
+        function writeLane(lane: number, text: string) {
+            lanes[lane] += text;
+        }
+    }
+
+    function getBoxCharacter(connector: Connection) {
+        switch (connector) {
+            case Connection.UpDown:
+                return BoxCharacter.ud;
+            case Connection.LeftRight:
+                return BoxCharacter.lr;
+            case Connection.UpLeft:
+                return BoxCharacter.ul;
+            case Connection.UpRight:
+                return BoxCharacter.ur;
+            case Connection.DownLeft:
+                return BoxCharacter.dl;
+            case Connection.DownRight:
+                return BoxCharacter.dr;
+            case Connection.UpDownLeft:
+                return BoxCharacter.udl;
+            case Connection.UpDownRight:
+                return BoxCharacter.udr;
+            case Connection.UpLeftRight:
+                return BoxCharacter.ulr;
+            case Connection.DownLeftRight:
+                return BoxCharacter.dlr;
+            case Connection.UpDownLeftRight:
+                return BoxCharacter.udlr;
+        }
+        return ' ';
+    }
+
+    function fill<T>(array: T[], value: T) {
+        if (array.fill) {
+            array.fill(value);
+        } else {
+            for (let i = 0; i < array.length; i++) {
+                array[i] = value;
+            }
+        }
+        return array;
+    }
+
+    function repeat(ch: string, length: number) {
+        if (ch.repeat) {
+            return length > 0 ? ch.repeat(length) : '';
+        }
+        let s = '';
+        while (s.length < length) {
+            s += ch;
+        }
+        return s;
+    }
+}