@zzzen/pyright-internal 1.1.254 → 1.1.255

package/src/common/fileBasedCancellationUtils.ts

@@ -0,0 +1,274 @@
+/*
+ * fileBasedCancellationUtils.ts
+ * Copyright (c) Microsoft Corporation.
+ * Licensed under the MIT license.
+ *
+ * Helper methods relating to file based cancellation.
+ */
+
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { CancellationId, CancellationTokenSource } from 'vscode-jsonrpc';
+import {
+    AbstractCancellationTokenSource,
+    CancellationReceiverStrategy,
+    CancellationSenderStrategy,
+    CancellationStrategy,
+    CancellationToken,
+    Emitter,
+    Event,
+} from 'vscode-languageserver';
+
+import { CancellationProvider, getCancellationFolderName, setCancellationFolderName } from './cancellationUtils';
+
+class CancellationThrottle {
+    private static _lastCheckTimestamp = 0;
+
+    static shouldCheck() {
+        // Throttle cancellation checks to one every 5ms. This value
+        // was selected through empirical testing. If we call the
+        // file system more often than this, type analysis performance
+        // is affected. If we call it less often, performance doesn't
+        // improve much, but responsiveness suffers.
+        const minTimeBetweenChecksInMs = 5;
+        const curTimestamp = Date.now().valueOf();
+        const timeSinceLastCheck = curTimestamp - this._lastCheckTimestamp;
+
+        if (timeSinceLastCheck >= minTimeBetweenChecksInMs) {
+            this._lastCheckTimestamp = curTimestamp;
+            return true;
+        }
+
+        return false;
+    }
+}
+
+class FileBasedToken implements CancellationToken {
+    protected isCancelled = false;
+    private _emitter: Emitter<any> | undefined;
+
+    constructor(readonly cancellationFilePath: string) {}
+
+    cancel() {
+        if (!this.isCancelled) {
+            this.isCancelled = true;
+            if (this._emitter) {
+                this._emitter.fire(undefined);
+                this._disposeEmitter();
+            }
+        }
+    }
+
+    get isCancellationRequested(): boolean {
+        if (this.isCancelled) {
+            return true;
+        }
+
+        if (CancellationThrottle.shouldCheck() && this._pipeExists()) {
+            // The first time it encounters the cancellation file, it will
+            // cancel itself and raise a cancellation event.
+            // In this mode, cancel() might not be called explicitly by
+            // jsonrpc layer.
+            this.cancel();
+        }
+
+        return this.isCancelled;
+    }
+
+    get onCancellationRequested(): Event<any> {
+        if (!this._emitter) {
+            this._emitter = new Emitter<any>();
+        }
+        return this._emitter.event;
+    }
+
+    dispose(): void {
+        this._disposeEmitter();
+    }
+
+    private _disposeEmitter() {
+        if (this._emitter) {
+            this._emitter.dispose();
+            this._emitter = undefined;
+        }
+    }
+
+    private _pipeExists(): boolean {
+        try {
+            fs.statSync(this.cancellationFilePath);
+            return true;
+        } catch (e: any) {
+            return false;
+        }
+    }
+}
+
+class OwningFileToken extends FileBasedToken {
+    private _disposed = false;
+
+    constructor(cancellationFilePath: string) {
+        super(cancellationFilePath);
+    }
+
+    override cancel() {
+        if (!this._disposed && !this.isCancelled) {
+            this._createPipe();
+            super.cancel();
+        }
+    }
+
+    override get isCancellationRequested(): boolean {
+        // Since this object owns the file and it gets created when the
+        // token is cancelled, there's no point in checking the pipe.
+        return this.isCancelled;
+    }
+
+    override dispose(): void {
+        this._disposed = true;
+
+        super.dispose();
+        this._removePipe();
+    }
+
+    private _createPipe() {
+        try {
+            fs.writeFileSync(this.cancellationFilePath, '', { flag: 'w' });
+        } catch {
+            // Ignore the exception.
+        }
+    }
+
+    private _removePipe() {
+        try {
+            fs.unlinkSync(this.cancellationFilePath);
+        } catch {
+            // Ignore the exception.
+        }
+    }
+}
+
+class FileBasedCancellationTokenSource implements AbstractCancellationTokenSource {
+    private _token: CancellationToken | undefined;
+    constructor(private _cancellationFilePath: string, private _ownFile: boolean = false) {}
+
+    get token(): CancellationToken {
+        if (!this._token) {
+            // Be lazy and create the token only when actually needed.
+            this._token = this._ownFile
+                ? new OwningFileToken(this._cancellationFilePath)
+                : new FileBasedToken(this._cancellationFilePath);
+        }
+        return this._token;
+    }
+
+    cancel(): void {
+        if (!this._token) {
+            // Save an object by returning the default
+            // cancelled token when cancellation happens
+            // before someone asks for the token.
+            this._token = CancellationToken.Cancelled;
+        } else {
+            (this._token as FileBasedToken).cancel();
+        }
+    }
+
+    dispose(): void {
+        if (!this._token) {
+            // Make sure to initialize with an empty token if we had none.
+            this._token = CancellationToken.None;
+        } else if (this._token instanceof FileBasedToken) {
+            // Actually dispose.
+            this._token.dispose();
+        }
+    }
+}
+
+function getCancellationFolderPath(folderName: string) {
+    return path.join(os.tmpdir(), 'python-languageserver-cancellation', folderName);
+}
+
+function getCancellationFilePath(folderName: string, id: CancellationId) {
+    return path.join(getCancellationFolderPath(folderName), `cancellation-${String(id)}.tmp`);
+}
+
+class FileCancellationReceiverStrategy implements CancellationReceiverStrategy {
+    constructor(readonly folderName: string) {}
+
+    createCancellationTokenSource(id: CancellationId): AbstractCancellationTokenSource {
+        return new FileBasedCancellationTokenSource(getCancellationFilePath(this.folderName, id));
+    }
+}
+
+export function getCancellationStrategyFromArgv(argv: string[]): CancellationStrategy {
+    let receiver: CancellationReceiverStrategy | undefined;
+
+    for (let i = 0; i < argv.length; i++) {
+        const arg = argv[i];
+        if (arg === '--cancellationReceive') {
+            receiver = createReceiverStrategyFromArgv(argv[i + 1]);
+        } else {
+            const args = arg.split('=');
+            if (args[0] === '--cancellationReceive') {
+                receiver = createReceiverStrategyFromArgv(args[1]);
+            }
+        }
+    }
+
+    if (receiver && !getCancellationFolderName()) {
+        setCancellationFolderName((receiver as FileCancellationReceiverStrategy).folderName);
+    }
+
+    receiver = receiver ? receiver : CancellationReceiverStrategy.Message;
+    return { receiver, sender: CancellationSenderStrategy.Message };
+
+    function createReceiverStrategyFromArgv(arg: string): CancellationReceiverStrategy | undefined {
+        const folderName = extractCancellationFolderName(arg);
+        return folderName ? new FileCancellationReceiverStrategy(folderName) : undefined;
+    }
+
+    function extractCancellationFolderName(arg: string): string | undefined {
+        const fileRegex = /^file:(.+)$/;
+        const folderName = arg.match(fileRegex);
+        return folderName ? folderName[1] : undefined;
+    }
+}
+
+export function disposeCancellationToken(token: CancellationToken) {
+    if (token instanceof FileBasedToken) {
+        token.dispose();
+    }
+}
+
+export function getCancellationTokenFromId(cancellationId: string) {
+    if (!cancellationId) {
+        return CancellationToken.None;
+    }
+
+    return new FileBasedToken(cancellationId);
+}
+
+export function getCancellationTokenId(token: CancellationToken) {
+    return token instanceof FileBasedToken ? token.cancellationFilePath : undefined;
+}
+
+let cancellationSourceId = 0;
+export class FileBasedCancellationProvider implements CancellationProvider {
+    constructor(private _prefix: string) {
+        // empty
+    }
+
+    createCancellationTokenSource(): AbstractCancellationTokenSource {
+        const folderName = getCancellationFolderName();
+        if (!folderName) {
+            // File-based cancellation is not used.
+            // Return regular cancellation token source.
+            return new CancellationTokenSource();
+        }
+
+        return new FileBasedCancellationTokenSource(
+            getCancellationFilePath(folderName, `${this._prefix}-${String(cancellationSourceId++)}`),
+            /* ownFile */ true
+        );
+    }
+}

package/src/common/fileSystem.ts

@@ -0,0 +1,152 @@
+/*
+ * fileSystem.ts
+ * Copyright (c) Microsoft Corporation.
+ * Licensed under the MIT license.
+ *
+ * A "file system provider" abstraction that allows us to swap out a
+ * real file system implementation for a virtual (mocked) implementation
+ * for testing.
+ */
+
+// * NOTE * except tests, this should be only file that import "fs"
+import type * as fs from 'fs';
+
+export type FileWatcherEventType = 'add' | 'addDir' | 'change' | 'unlink' | 'unlinkDir';
+export type FileWatcherEventHandler = (eventName: FileWatcherEventType, path: string, stats?: Stats) => void;
+
+export interface FileWatcher {
+    close(): void;
+}
+
+export interface FileWatcherProvider {
+    createFileWatcher(paths: string[], listener: FileWatcherEventHandler): FileWatcher;
+    onFileChange(eventType: FileWatcherEventType, path: string): void;
+}
+
+export interface Stats {
+    size: number;
+
+    isFile(): boolean;
+    isDirectory(): boolean;
+    isBlockDevice(): boolean;
+    isCharacterDevice(): boolean;
+    isSymbolicLink(): boolean;
+    isFIFO(): boolean;
+    isSocket(): boolean;
+    isZipDirectory?: () => boolean;
+}
+
+export interface MkDirOptions {
+    recursive: boolean;
+    // Not supported on Windows so commented out.
+    // mode: string | number;
+}
+
+export interface TmpfileOptions {
+    postfix?: string;
+    prefix?: string;
+}
+
+export interface FileSystem {
+    existsSync(path: string): boolean;
+    mkdirSync(path: string, options?: MkDirOptions): void;
+    chdir(path: string): void;
+    readdirEntriesSync(path: string): fs.Dirent[];
+    readdirSync(path: string): string[];
+    readFileSync(path: string, encoding?: null): Buffer;
+    readFileSync(path: string, encoding: BufferEncoding): string;
+    readFileSync(path: string, encoding?: BufferEncoding | null): string | Buffer;
+    writeFileSync(path: string, data: string | Buffer, encoding: BufferEncoding | null): void;
+    statSync(path: string): Stats;
+    unlinkSync(path: string): void;
+    realpathSync(path: string): string;
+    getModulePath(): string;
+    createFileSystemWatcher(paths: string[], listener: FileWatcherEventHandler): FileWatcher;
+    createReadStream(path: string): fs.ReadStream;
+    createWriteStream(path: string): fs.WriteStream;
+    copyFileSync(src: string, dst: string): void;
+    // Async I/O
+    readFile(path: string): Promise<Buffer>;
+    readFileText(path: string, encoding?: BufferEncoding): Promise<string>;
+    // The directory returned by tmpdir must exist and be the same each time tmpdir is called.
+    tmpdir(): string;
+    tmpfile(options?: TmpfileOptions): string;
+
+    // Return path in casing on OS.
+    realCasePath(path: string): string;
+
+    // See whether the file is mapped to another location.
+    isMappedFilePath(filepath: string): boolean;
+
+    // Get original filepath if the given filepath is mapped.
+    getOriginalFilePath(mappedFilePath: string): string;
+
+    // Get mapped filepath if the given filepath is mapped.
+    getMappedFilePath(originalFilepath: string): string;
+
+    getUri(path: string): string;
+
+    isInZipOrEgg(path: string): boolean;
+}
+
+// File watchers can give "changed" event even for a file open. but for those cases,
+// it will give relative path rather than absolute path. To get rid of such cases,
+// we will drop any event with relative paths. this trick is copied from VS Code
+// (https://github.com/microsoft/vscode/blob/main/src/vs/platform/files/node/watcher/unix/chokidarWatcherService.ts)
+export function ignoredWatchEventFunction(paths: string[]) {
+    const normalizedPaths = paths.map((p) => p.toLowerCase());
+    return (path: string): boolean => {
+        if (!path || path.indexOf('__pycache__') >= 0) {
+            return true;
+        }
+        const normalizedPath = path.toLowerCase();
+        return normalizedPaths.every((p) => normalizedPath.indexOf(p) < 0);
+    };
+}
+
+const nullFileWatcher: FileWatcher = {
+    close() {
+        // empty;
+    },
+};
+
+export const nullFileWatcherProvider: FileWatcherProvider = {
+    createFileWatcher(_1: string[], _2: FileWatcherEventHandler): FileWatcher {
+        return nullFileWatcher;
+    },
+    onFileChange(_1: FileWatcherEventType, _2: string): void {
+        // do nothing
+    },
+};
+
+export class VirtualDirent implements fs.Dirent {
+    constructor(public name: string, private _file: boolean) {}
+
+    isFile(): boolean {
+        return this._file;
+    }
+
+    isDirectory(): boolean {
+        return !this._file;
+    }
+
+    isBlockDevice(): boolean {
+        return false;
+    }
+
+    isCharacterDevice(): boolean {
+        return false;
+    }
+
+    isSymbolicLink(): boolean {
+        return false;
+    }
+
+    isFIFO(): boolean {
+        return false;
+    }
+
+    isSocket(): boolean {
+        return false;
+    }
+}

package/src/common/fullAccessHost.ts

@@ -0,0 +1,203 @@
+/*
+ * fullAccessHost.ts
+ * Copyright (c) Microsoft Corporation.
+ * Licensed under the MIT license.
+ *
+ * Implementation of host where it is allowed to run external executables.
+ */
+
+import * as child_process from 'child_process';
+
+import { PythonPathResult } from '../analyzer/pythonPathUtils';
+import { PythonPlatform } from './configOptions';
+import { assertNever } from './debug';
+import { FileSystem } from './fileSystem';
+import { HostKind, NoAccessHost } from './host';
+import { isDirectory, normalizePath } from './pathUtils';
+import { PythonVersion, versionFromMajorMinor } from './pythonVersion';
+
+// preventLocalImports removes the working directory from sys.path.
+// The -c flag adds it automatically, which can allow some stdlib
+// modules (like json) to be overridden by other files (like json.py).
+const removeCwdFromSysPath = [
+    'import os, os.path, sys',
+    'normalize = lambda p: os.path.normcase(os.path.normpath(p))',
+    'cwd = normalize(os.getcwd())',
+    'orig_sys_path = [p for p in sys.path if p != ""]',
+    'sys.path[:] = [p for p in sys.path if p != "" and normalize(p) != cwd]',
+];
+
+const extractSys = [
+    ...removeCwdFromSysPath,
+    'import sys, json',
+    'json.dump(dict(path=orig_sys_path, prefix=sys.prefix), sys.stdout)',
+].join('; ');
+
+const extractVersion = [
+    ...removeCwdFromSysPath,
+    'import sys, json',
+    'json.dump(dict(major=sys.version_info[0], minor=sys.version_info[1]), sys.stdout)',
+].join('; ');
+
+export class LimitedAccessHost extends NoAccessHost {
+    override get kind(): HostKind {
+        return HostKind.LimitedAccess;
+    }
+
+    override getPythonPlatform(logInfo?: string[]): PythonPlatform | undefined {
+        if (process.platform === 'darwin') {
+            return PythonPlatform.Darwin;
+        } else if (process.platform === 'linux') {
+            return PythonPlatform.Linux;
+        } else if (process.platform === 'win32') {
+            return PythonPlatform.Windows;
+        }
+
+        return undefined;
+    }
+}
+
+export class FullAccessHost extends LimitedAccessHost {
+    static createHost(kind: HostKind, fs: FileSystem) {
+        switch (kind) {
+            case HostKind.NoAccess:
+                return new NoAccessHost();
+            case HostKind.LimitedAccess:
+                return new LimitedAccessHost();
+            case HostKind.FullAccess:
+                return new FullAccessHost(fs);
+            default:
+                assertNever(kind);
+        }
+    }
+
+    constructor(protected _fs: FileSystem) {
+        super();
+    }
+
+    override get kind(): HostKind {
+        return HostKind.FullAccess;
+    }
+
+    override getPythonSearchPaths(pythonPath?: string, logInfo?: string[]): PythonPathResult {
+        const importFailureInfo = logInfo ?? [];
+        let result = this._executePythonInterpreter(pythonPath, (p) =>
+            this._getSearchPathResultFromInterpreter(this._fs, p, importFailureInfo)
+        );
+
+        if (!result) {
+            result = {
+                paths: [],
+                prefix: '',
+            };
+        }
+
+        importFailureInfo.push(`Received ${result.paths.length} paths from interpreter`);
+        result.paths.forEach((path) => {
+            importFailureInfo.push(`  ${path}`);
+        });
+
+        return result;
+    }
+
+    override getPythonVersion(pythonPath?: string, logInfo?: string[]): PythonVersion | undefined {
+        const importFailureInfo = logInfo ?? [];
+
+        try {
+            const commandLineArgs: string[] = ['-c', extractVersion];
+            const execOutput = this._executePythonInterpreter(pythonPath, (p) =>
+                child_process.execFileSync(p, commandLineArgs, { encoding: 'utf8' })
+            );
+
+            const versionJson: { major: number; minor: number } = JSON.parse(execOutput!);
+            const version = versionFromMajorMinor(versionJson.major, versionJson.minor);
+            if (version === undefined) {
+                importFailureInfo.push(
+                    `Python version ${versionJson.major}.${versionJson.minor} from interpreter is unsupported`
+                );
+                return undefined;
+            }
+
+            return version;
+        } catch {
+            importFailureInfo.push('Unable to get Python version from interpreter');
+            return undefined;
+        }
+    }
+
+    private _executePythonInterpreter<T>(
+        pythonPath: string | undefined,
+        execute: (path: string) => T | undefined
+    ): T | undefined {
+        if (pythonPath) {
+            return execute(pythonPath);
+        } else {
+            let result: T | undefined;
+            try {
+                // On non-Windows platforms, always default to python3 first. We want to
+                // avoid this on Windows because it might invoke a script that displays
+                // a dialog box indicating that python can be downloaded from the app store.
+                if (process.platform !== 'win32') {
+                    result = execute('python3');
+                }
+            } catch {
+                // Ignore failure on python3
+            }
+
+            if (result !== undefined) {
+                return result;
+            }
+
+            // On some platforms, 'python3' might not exist. Try 'python' instead.
+            return execute('python');
+        }
+    }
+
+    private _getSearchPathResultFromInterpreter(
+        fs: FileSystem,
+        interpreter: string,
+        importFailureInfo: string[]
+    ): PythonPathResult | undefined {
+        const result: PythonPathResult = {
+            paths: [],
+            prefix: '',
+        };
+
+        try {
+            const commandLineArgs: string[] = ['-c', extractSys];
+
+            importFailureInfo.push(`Executing interpreter: '${interpreter}'`);
+            const execOutput = child_process.execFileSync(interpreter, commandLineArgs, { encoding: 'utf8' });
+
+            // Parse the execOutput. It should be a JSON-encoded array of paths.
+            try {
+                const execSplit = JSON.parse(execOutput);
+                for (let execSplitEntry of execSplit.path) {
+                    execSplitEntry = execSplitEntry.trim();
+                    if (execSplitEntry) {
+                        const normalizedPath = normalizePath(execSplitEntry);
+                        // Skip non-existent paths and broken zips/eggs.
+                        if (fs.existsSync(normalizedPath) && isDirectory(fs, normalizedPath)) {
+                            result.paths.push(normalizedPath);
+                        } else {
+                            importFailureInfo.push(`Skipping '${normalizedPath}' because it is not a valid directory`);
+                        }
+                    }
+                }
+
+                result.prefix = execSplit.prefix;
+
+                if (result.paths.length === 0) {
+                    importFailureInfo.push(`Found no valid directories`);
+                }
+            } catch (err) {
+                importFailureInfo.push(`Could not parse output: '${execOutput}'`);
+                throw err;
+            }
+        } catch {
+            return undefined;
+        }
+
+        return result;
+    }
+}

package/src/common/host.ts

@@ -0,0 +1,49 @@
+/*
+ * host.ts
+ * Copyright (c) Microsoft Corporation.
+ * Licensed under the MIT license.
+ *
+ * Provides accesses to the host the language service runs on
+ */
+
+import { PythonPathResult } from '../analyzer/pythonPathUtils';
+import { PythonPlatform } from './configOptions';
+import { PythonVersion } from './pythonVersion';
+
+export const enum HostKind {
+    FullAccess,
+    LimitedAccess,
+    NoAccess,
+}
+
+export interface Host {
+    readonly kind: HostKind;
+    getPythonSearchPaths(pythonPath?: string, logInfo?: string[]): PythonPathResult;
+    getPythonVersion(pythonPath?: string, logInfo?: string[]): PythonVersion | undefined;
+    getPythonPlatform(logInfo?: string[]): PythonPlatform | undefined;
+}
+
+export class NoAccessHost implements Host {
+    get kind(): HostKind {
+        return HostKind.NoAccess;
+    }
+
+    getPythonSearchPaths(pythonPath?: string, logInfo?: string[]): PythonPathResult {
+        logInfo?.push('No access to python executable.');
+
+        return {
+            paths: [],
+            prefix: '',
+        };
+    }
+
+    getPythonVersion(pythonPath?: string, logInfo?: string[]): PythonVersion | undefined {
+        return undefined;
+    }
+
+    getPythonPlatform(logInfo?: string[]): PythonPlatform | undefined {
+        return undefined;
+    }
+}
+
+export type HostFactory = () => Host;