@zzzen/pyright-internal 1.2.0-dev.20260816 → 1.2.0-dev.20260830

package/dist/analyzer/program.js

@@ -58,6 +58,7 @@ require("../common/serviceProviderExtensions");
 const textRange_1 = require("../common/textRange");
 const timing_1 = require("../common/timing");
 const uri_1 = require("../common/uri/uri");
+const uriUtils_1 = require("../common/uri/uriUtils");
 const cellChainIndex_1 = require("./cellChainIndex");
 const AnalyzerNodeInfo = __importStar(require("./analyzerNodeInfo"));
 const cacheManager_1 = require("./cacheManager");
@@ -81,15 +82,23 @@ function isTaggedHintDiagnostic(diag) {
 // Track edit mode related information.
 class EditModeTracker {
     constructor() {
+        this.analysisContextOwner = this;
         this._isEditMode = false;
+        this._isClonedProgram = false;
         this._mutatedFiles = [];
     }
     get isEditMode() {
         return this._isEditMode;
     }
+    get isClonedProgram() {
+        return this._isClonedProgram;
+    }
     addMutatedFiles(file) {
         this._mutatedFiles.push(file);
     }
+    markClonedProgram() {
+        this._isClonedProgram = true;
+    }
     enable() {
         this._isEditMode = true;
         this._mutatedFiles = [];
@@ -113,6 +122,16 @@ class Program {
         this._disableChecker = _disableChecker;
         this._sourceFileList = [];
         this._sourceFileMap = new Map();
+        // A filesystem symlink and its target are tracked as separate SourceFileInfo
+        // entries (the source-file map is keyed by uri.key with no realpath dedup).
+        // These indexes let invalidation bridge such aliases so a change routed to one
+        // alias also re-checks consumers of the other.
+        // _realpathAliasMap groups uri.keys that share a realpath (only populated for
+        // symlink-involved groups); _realpathByUriKey caches each user file's realpath
+        // key so removal/lookup don't recompute it.
+        this._realpathAliasMap = new Map();
+        this._realpathByUriKey = new Map();
+        this._analyzerNodeInfoContext = new AnalyzerNodeInfo.AnalyzerNodeInfoContextImpl();
         this._cellChainIndex = new cellChainIndex_1.CellChainIndex(() => this._sourceFileList, (uri) => this.getSourceFileInfo(uri));
         this._disposed = false;
         this._parsedFileCount = 0;
@@ -174,8 +193,8 @@ class Program {
             }
             const parseResults = sourceFileInfo.sourceFile.getParserOutput();
             const moduleNode = parseResults.parseTree;
-            const fileInfo = AnalyzerNodeInfo.getFileInfo(moduleNode);
-            const dunderAllInfo = AnalyzerNodeInfo.getDunderAllInfo(parseResults.parseTree);
+            const fileInfo = AnalyzerNodeInfo.getFileInfo(moduleNode, this._analyzerNodeInfoContext);
+            const dunderAllInfo = AnalyzerNodeInfo.getDunderAllInfo(parseResults.parseTree, this._analyzerNodeInfoContext);
             return {
                 symbolTable,
                 dunderAllNames: dunderAllInfo?.names,
@@ -227,13 +246,21 @@ class Program {
     get cellChainIndex() {
         return this._cellChainIndex;
     }
+    get analyzerNodeInfoContext() {
+        return this._analyzerNodeInfoContext;
+    }
+    get analyzerNodeInfoReader() {
+        return this._analyzerNodeInfoContext;
+    }
     dispose() {
         this.disposeInternal(this._disposed);
+        this._analyzerNodeInfoContext.dispose();
         this._cacheManager.unregisterCacheOwner(this);
         this._disposed = true;
     }
     enterEditMode() {
         this._editModeTracker.enable();
+        this._analyzerNodeInfoContext.enterOverlay();
     }
     exitEditMode() {
         // Stop applying edit mode to new source files.
@@ -278,6 +305,14 @@ class Program {
             // All cache is invalid now.
             this._createNewEvaluator();
         }
+        // Keep overlay-produced bindings only for parse trees that survived source-file restoration.
+        for (const info of this._sourceFileList) {
+            const parseTree = info.sourceFile.getParserOutput()?.parseTree;
+            if (parseTree) {
+                this._analyzerNodeInfoContext.promoteToPreviousLayer(parseTree);
+            }
+        }
+        this._analyzerNodeInfoContext.discardOverlay();
         return edits;
     }
     setConfigOptions(configOptions) {
@@ -349,6 +384,10 @@ class Program {
             // search paths. Clear any cached module name so it is recomputed.
             sourceFileInfo.sourceFile.clearCachedModuleName();
             sourceFileInfo.isTracked = true;
+            // The file may have first been added as an untracked referenced import
+            // (skipped by the user-code-only realpath alias index). Now that it is
+            // tracked, (re-)index it so symlink-twin co-invalidation applies.
+            this._indexRealpathAlias(sourceFileInfo);
             return sourceFileInfo.sourceFile;
         }
         // Detect py.typed status if not explicitly provided. This ensures that
@@ -441,9 +480,21 @@ class Program {
             // We need to mark the file dirty so we can re-analyze next time.
             // This won't matter much for OpenFileOnly users, but it will matter for
             // people who use diagnosticMode Workspace.
+            const markDirtySet = new Set();
             if (sourceFileInfo.sourceFile.didContentsChangeOnDisk()) {
                 sourceFileInfo.sourceFile.markDirty();
-                this._markFileDirtyRecursive(sourceFileInfo, new Set());
+                this._markFileDirtyRecursive(sourceFileInfo, markDirtySet);
+            }
+            // Co-invalidate symlink twins so consumers that imported the other
+            // alias get re-checked as well.
+            this._markRealpathAliasesDirty(sourceFileInfo, markDirtySet);
+            // If anything was marked dirty (the closed file's own dependency
+            // subtree or a symlink twin's), recreate the evaluator so the
+            // re-checked consumers re-resolve against fresh types instead of the
+            // stale type cache. This mirrors markFilesDirty's markDirtySet.size
+            // check so the close path cannot silently diverge from it.
+            if (markDirtySet.size > 0) {
+                this._createNewEvaluator();
             }
         }
         return this._removeUnneededFiles();
@@ -480,13 +531,19 @@ class Program {
                 // If !evenIfContentsAreSame, see if the on-disk contents have
                 // changed. If the file is open, the on-disk contents don't matter
                 // because we'll receive updates directly from the client.
-                if (evenIfContentsAreSame ||
-                    (!sourceFileInfo.isOpenByClient && sourceFileInfo.sourceFile.didContentsChangeOnDisk())) {
+                if (this._shouldContentInvalidate(sourceFileInfo, evenIfContentsAreSame)) {
                     sourceFileInfo.sourceFile.markDirty();
                     // Mark any files that depend on this file as dirty
                     // also. This will retrigger analysis of these other files.
                     this._markFileDirtyRecursive(sourceFileInfo, markDirtySet);
                 }
+                // A filesystem symlink and its target are tracked as separate
+                // SourceFileInfo entries. A change routed to one alias (e.g. the fs
+                // watcher fires for the real target path) must also re-parse the
+                // other alias(es) sharing the same realpath and re-check their
+                // dependents; otherwise stale diagnostics persist on consumers
+                // that imported the twin.
+                this._markRealpathAliasesDirty(sourceFileInfo, markDirtySet);
             }
         });
         if (markDirtySet.size > 0) {
@@ -859,22 +916,28 @@ class Program {
     }
     clone() {
         const program = new Program(this._importResolver, this._configOptions, this.serviceProvider, new logTracker_1.LogTracker(this._console, 'Cloned'), this._disableChecker);
-        // Cloned program will use whatever user files the program currently has.
-        const userFiles = this.getUserFiles();
-        program.setTrackedFiles(userFiles.map((i) => i.uri));
-        program.markAllFilesDirty(/* evenIfContentsAreSame */ true);
-        // Make sure we keep editor content (open file) which could be different than one in the file system.
-        for (const fileInfo of this.getOpened()) {
-            const version = fileInfo.sourceFile.getClientVersion();
-            if (version === undefined) {
-                continue;
+        program._editModeTracker.markClonedProgram();
+        // Clone user and open files in their original order. Open files must be created
+        // with their full construction metadata before they are marked as tracked;
+        // precreating them as plain tracked files loses virtual-document and IPython state.
+        const filesToClone = this._sourceFileList.filter((fileInfo) => (0, sourceFileInfoUtils_1.isUserCode)(fileInfo) || fileInfo.isOpenByClient);
+        for (const fileInfo of filesToClone) {
+            if (fileInfo.isOpenByClient) {
+                program._addClonedOpenFile(fileInfo);
+            }
+            else {
+                program.addTrackedFile(fileInfo.uri);
+            }
+        }
+        program.setTrackedFiles(filesToClone.filter((fileInfo) => fileInfo.isTracked).map((fileInfo) => fileInfo.uri));
+        // Restore chains only after every source exists, since a predecessor can appear
+        // later in source-file order.
+        for (const fileInfo of filesToClone) {
+            if (fileInfo.isOpenByClient && fileInfo.chainedSourceFile) {
+                program.updateChainedUri(fileInfo.uri, fileInfo.chainedSourceFile.uri);
             }
-            program.setFileOpened(fileInfo.uri, version, fileInfo.sourceFile.getOpenFileContents() ?? '', {
-                chainedFileUri: fileInfo.chainedSourceFile?.uri,
-                ipythonMode: fileInfo.ipythonMode,
-                isVirtual: fileInfo.isVirtual,
-            });
         }
+        program.markAllFilesDirty(/* evenIfContentsAreSame */ true);
         return program;
     }
     // Returns a value from 0 to 1 (or more) indicating how "full" the cache is
@@ -914,6 +977,23 @@ class Program {
     createNewEvaluatorInternal() {
         // Empty
     }
+    _addClonedOpenFile(originalFileInfo) {
+        const version = originalFileInfo.sourceFile.getClientVersion();
+        if (version === undefined) {
+            return;
+        }
+        const sourceFile = this._sourceFileFactory.createSourceFile(this.serviceProvider, originalFileInfo.uri, (uri) => this._getModuleName(uri), originalFileInfo.isThirdPartyImport, originalFileInfo.isThirdPartyPyTypedPresent, this._editModeTracker, this._console, this._logTracker, originalFileInfo.ipythonMode);
+        const sourceFileInfo = new sourceFileInfo_1.SourceFileInfo(sourceFile, originalFileInfo.isTypeshedFile, originalFileInfo.isThirdPartyImport, originalFileInfo.isThirdPartyPyTypedPresent, this._editModeTracker, {
+            isTracked: originalFileInfo.isTracked,
+            isVirtual: originalFileInfo.isVirtual,
+            isOpenByClient: true,
+        });
+        this._addToSourceFileListAndMap(sourceFileInfo);
+        if (sourceFileInfo.ipythonMode === sourceFile_1.IPythonMode.CellDocs) {
+            this._cellChainIndex.invalidate();
+        }
+        sourceFile.setClientVersion(version, originalFileInfo.sourceFile.getOpenFileContents() ?? '');
+    }
     _handleMemoryHighUsage() {
         const cacheUsage = this._cacheManager.getCacheUsage();
         const usedHeapRatio = this._cacheManager.getUsedHeapRatio(this._configOptions.verboseOutput ? this._console : undefined);
@@ -937,7 +1017,10 @@ class Program {
     // It does not discard cached index results or diagnostics for files.
     _discardCachedParseResults() {
         for (const sourceFileInfo of this._sourceFileList) {
-            sourceFileInfo.sourceFile.dropParseAndBindInfo();
+            const parseTree = sourceFileInfo.sourceFile.dropParseAndBindInfo();
+            if (parseTree) {
+                this._analyzerNodeInfoContext.retainRemovedStore(parseTree);
+            }
         }
     }
     _runEvaluatorWithCancellationToken(token, callback) {
@@ -1072,7 +1155,7 @@ class Program {
         return false;
     }
     _createSourceMapper(execEnv, token, from, mapCompiled, preferStubs) {
-        const sourceMapper = new sourceMapper_1.SourceMapper(this._importResolver, execEnv, this._evaluator, (stubFileUri, implFileUri) => this.bindShadowFile(stubFileUri, implFileUri), (f) => {
+        const sourceMapper = new sourceMapper_1.SourceMapper(this._importResolver, execEnv, this._evaluator, this._analyzerNodeInfoContext, (stubFileUri, implFileUri) => this.bindShadowFile(stubFileUri, implFileUri), (f) => {
             let fileInfo = this.getBoundSourceFileInfo(f);
             if (!fileInfo) {
                 // make sure uri exits before adding interimFile
@@ -1314,9 +1397,20 @@ class Program {
         return filesAdded;
     }
     _removeSourceFileFromListAndMap(fileUri, indexToRemove) {
+        const sourceFileInfo = this._sourceFileMap.get(fileUri.key);
+        if (sourceFileInfo) {
+            this._dropParseAndBindInfo(sourceFileInfo.sourceFile);
+        }
+        this._unindexRealpathAlias(fileUri);
         this._sourceFileMap.delete(fileUri.key);
         this._sourceFileList.splice(indexToRemove, 1);
     }
+    _dropParseAndBindInfo(sourceFile) {
+        const parseTree = sourceFile.dropParseAndBindInfo();
+        if (parseTree) {
+            this._analyzerNodeInfoContext.remove(parseTree);
+        }
+    }
     _addToSourceFileListAndMap(fileInfo) {
         const fileUri = fileInfo.uri;
         // We should never add a file with the same path twice.
@@ -1325,6 +1419,121 @@ class Program {
         (0, debug_1.assert)(!fileUri.isEmpty());
         this._sourceFileList.push(fileInfo);
         this._sourceFileMap.set(fileUri.key, fileInfo);
+        this._indexRealpathAlias(fileInfo);
+    }
+    // Record the file's realpath so symlink twins can co-invalidate each other.
+    // Only user-code files participate: library/typeshed files are resolved to
+    // their real paths during import resolution and are not diagnostic-checked,
+    // so indexing them would only add realpath syscalls without benefit.
+    _indexRealpathAlias(fileInfo) {
+        if (!(0, sourceFileInfoUtils_1.isUserCode)(fileInfo)) {
+            return;
+        }
+        const fileUri = fileInfo.uri;
+        // Cache hit: this URI's realpath is already indexed, so skip the
+        // filesystem syscall and redundant group bookkeeping. This matters
+        // because `addTrackedFile` re-indexes already-tracked files (e.g. the
+        // untracked -> tracked flip). Watcher-driven changes remove the file
+        // first (clearing this cache via `_unindexRealpathAlias`) before
+        // re-adding, so a genuinely retargeted symlink is recomputed on re-add.
+        if (this._realpathByUriKey.has(fileUri.key)) {
+            return;
+        }
+        const realpathUri = (0, uriUtils_1.tryRealpath)(this.fileSystem, fileUri) ?? fileUri;
+        const realpathKey = realpathUri.key;
+        this._realpathByUriKey.set(fileUri.key, realpathKey);
+        // Add this file to its realpath group when it is either reached through a
+        // symlink (its realpath differs from its own URI) or a realpath target
+        // that already has aliases pointing at it. The latter lets a re-add of
+        // only the realpath target rebuild the group instead of orphaning the
+        // aliases that still reference it.
+        const existingGroup = this._realpathAliasMap.get(realpathKey);
+        if (realpathKey !== fileUri.key || existingGroup) {
+            let aliasKeys = existingGroup;
+            if (!aliasKeys) {
+                aliasKeys = new Set();
+                this._realpathAliasMap.set(realpathKey, aliasKeys);
+            }
+            aliasKeys.add(fileUri.key);
+            aliasKeys.add(realpathKey);
+        }
+    }
+    _unindexRealpathAlias(fileUri) {
+        const realpathKey = this._realpathByUriKey.get(fileUri.key);
+        if (realpathKey === undefined) {
+            return;
+        }
+        this._realpathByUriKey.delete(fileUri.key);
+        const aliasKeys = this._realpathAliasMap.get(realpathKey);
+        if (!aliasKeys) {
+            return;
+        }
+        aliasKeys.delete(fileUri.key);
+        // Retain the group only while at least one member is still indexed
+        // (present in _realpathByUriKey). Keeping a lingering group that still
+        // holds an indexed member lets a later re-add of the realpath target
+        // (or another alias) rebuild the twin relationship instead of orphaning
+        // it. But when the only keys left are unindexed seeds — e.g. a realpath
+        // target that was never added because it resolves outside the workspace
+        // — drop the group so such single-entry remnants can't accumulate over a
+        // long session that churns those links.
+        let hasIndexedMember = false;
+        for (const key of aliasKeys) {
+            if (this._realpathByUriKey.has(key)) {
+                hasIndexedMember = true;
+                break;
+            }
+        }
+        if (!hasIndexedMember) {
+            this._realpathAliasMap.delete(realpathKey);
+        }
+    }
+    // Predicate used by the primary `markFilesDirty`/`markAllFilesDirty` loops
+    // (and, with evenIfContentsAreSame fixed to false, by twin invalidation).
+    // When !evenIfContentsAreSame, only invalidate a file whose on-disk contents
+    // actually changed and that isn't open by the client (open docs receive their
+    // content directly from the client via the normal update path). When
+    // evenIfContentsAreSame is set, always invalidate.
+    _shouldContentInvalidate(fileInfo, evenIfContentsAreSame) {
+        return evenIfContentsAreSame || (!fileInfo.isOpenByClient && fileInfo.sourceFile.didContentsChangeOnDisk());
+    }
+    // Content-invalidate the realpath twin(s) of the given file and re-check their
+    // dependents. The twin must be marked dirty at the CONTENT level (markDirty),
+    // not merely re-check-required, so its symbol table is re-parsed from the
+    // updated backing file before consumers re-resolve against it.
+    //
+    // Twin fan-out ALWAYS uses the disk-change predicate (never the primary's
+    // evenIfContentsAreSame flag): a twin only needs re-parsing when its own
+    // backing file actually changed on disk. In particular, an in-memory edit to
+    // one open alias (updateOpenFileContents -> markFilesDirty(evenIfContentsAreSame=true))
+    // must NOT force the twin and its importer subtree to be re-checked on every
+    // keystroke, since the twin's on-disk contents are unchanged.
+    _markRealpathAliasesDirty(sourceFileInfo, markDirtySet) {
+        const realpathKey = this._realpathByUriKey.get(sourceFileInfo.uri.key);
+        if (realpathKey === undefined) {
+            return;
+        }
+        const aliasKeys = this._realpathAliasMap.get(realpathKey);
+        if (!aliasKeys || aliasKeys.size <= 1) {
+            return;
+        }
+        aliasKeys.forEach((aliasKey) => {
+            if (aliasKey === sourceFileInfo.uri.key || markDirtySet.has(aliasKey)) {
+                return;
+            }
+            const aliasInfo = this._sourceFileMap.get(aliasKey);
+            if (!aliasInfo) {
+                return;
+            }
+            // Only invalidate the twin when its own backing file changed on disk
+            // (and it isn't open by the client). This deliberately ignores the
+            // primary file's evenIfContentsAreSame flag so unsaved edits to one
+            // alias don't repeatedly re-check the other alias's dependents.
+            if (this._shouldContentInvalidate(aliasInfo, /* evenIfContentsAreSame */ false)) {
+                aliasInfo.sourceFile.markDirty();
+                this._markFileDirtyRecursive(aliasInfo, markDirtySet);
+            }
+        });
     }
     _getModuleName(fileUri) {
         const moduleInfo = this._getModuleImportInfoForFile(fileUri);
@@ -1381,8 +1590,9 @@ class Program {
             minimumLoggingThreshold: this._configOptions.typeEvaluationTimeThreshold,
             evaluateUnknownImportsAsAny: !!this._configOptions.evaluateUnknownImportsAsAny,
             verifyTypeCacheEvaluatorFlags: !!this._configOptions.internalTestMode,
+            nodeInfoReader: this._analyzerNodeInfoContext,
         }, this._logTracker, this._configOptions.logTypeEvaluationTime
-            ? (0, tracePrinter_1.createTracePrinter)(this._importResolver.getImportRoots(this._configOptions.findExecEnvironment(this._configOptions.projectRoot)))
+            ? (0, tracePrinter_1.createTracePrinter)(this._importResolver.getImportRoots(this._configOptions.findExecEnvironment(this._configOptions.projectRoot)), this._analyzerNodeInfoContext)
             : undefined);
         return this._evaluator;
     }
@@ -1468,7 +1678,7 @@ class Program {
                 return undefined;
             }
             // File should already be bound because of the chained file binding above.
-            const scope = AnalyzerNodeInfo.getScope(parseResults.parseTree);
+            const scope = AnalyzerNodeInfo.getScope(parseResults.parseTree, this._analyzerNodeInfoContext);
             return scope;
         };
         let builtinsScope;
@@ -1496,8 +1706,7 @@ class Program {
             futureImports = this._getEffectiveFutureImports(futureImports, fileToBind.chainedSourceFile);
         }
         fileToBind.effectiveFutureImports = futureImports.size > 0 ? futureImports : undefined;
-        fileToBind.sourceFile.bind(this._configOptions, this._lookUpImport, builtinsScope, futureImports, fileToBind.ipythonMode === sourceFile_1.IPythonMode.CellDocs ? this._cellChainIndex : undefined);
-        return true;
+        return fileToBind.sourceFile.bind(this._configOptions, this._lookUpImport, builtinsScope, futureImports, fileToBind.ipythonMode === sourceFile_1.IPythonMode.CellDocs ? this._cellChainIndex : undefined, this._analyzerNodeInfoContext);
     }
     _getEffectiveFutureImports(futureImports, chainedSourceFile) {
         const effectiveFutureImports = new Set(futureImports);
@@ -1556,7 +1765,7 @@ class Program {
                     }
                 }
                 if (boundFile) {
-                    fileToCheck.sourceFile.check(this.configOptions, this._lookUpImport, this._importResolver, this._evaluator, dependentFiles);
+                    fileToCheck.sourceFile.check(this.configOptions, this._lookUpImport, this._importResolver, this._evaluator, dependentFiles, this._analyzerNodeInfoContext);
                 }
             }
             // Detect import cycles that involve the file.
@@ -1632,7 +1841,7 @@ class Program {
             if (file.sourceFile.isBindingRequired()) {
                 continue;
             }
-            const fileInfo = AnalyzerNodeInfo.getFileInfo(parseResults.parseTree);
+            const fileInfo = AnalyzerNodeInfo.getFileInfo(parseResults.parseTree, this._analyzerNodeInfoContext);
             if (fileInfo.accessedSymbolSet) {
                 dependentFiles.push(parseResults);
             }
@@ -1738,7 +1947,7 @@ class Program {
         if (markSet.has(fileUri.key)) {
             return;
         }
-        sourceFileInfo.sourceFile.markReanalysisRequired(forceRebinding);
+        sourceFileInfo.sourceFile.markReanalysisRequired(forceRebinding, this._analyzerNodeInfoContext);
         markSet.add(fileUri.key);
         sourceFileInfo.importedBy.forEach((dep) => {
             // Changes on chained source file can change symbols in the symbol table and
@@ -1759,7 +1968,7 @@ class Program {
             }
             reevaluationRequired = true;
             chainedSourceFile.sourceFile.markReanalysisRequired(
-            /* forceRebinding */ sourceFileInfo.ipythonMode === sourceFile_1.IPythonMode.CellDocs);
+            /* forceRebinding */ sourceFileInfo.ipythonMode === sourceFile_1.IPythonMode.CellDocs, this._analyzerNodeInfoContext);
             chainedSourceFile = chainedSourceFile.chainedSourceFile;
         }
         // If the checker is going to run again, we have to recreate the type evaluator so