@theia/ai-ide 1.72.0 → 1.72.2

package/src/browser/workspace-functions.ts

@@ -20,6 +20,7 @@ import { EnvVariablesServer } from '@theia/core/lib/common/env-variables';
 import { inject, injectable, postConstruct } from '@theia/core/shared/inversify';
 import { FileService } from '@theia/filesystem/lib/browser/file-service';
 import { FileStat, FileOperationError, FileOperationResult } from '@theia/filesystem/lib/common/files';
+import { FileSearchService } from '@theia/file-search/lib/common/file-search-service';
 import { WorkspaceService } from '@theia/workspace/lib/browser';
 import {
     FILE_CONTENT_FUNCTION_ID, GET_FILE_DIAGNOSTICS_ID,
@@ -59,7 +60,7 @@ export class WorkspaceFunctionScope {
     @inject(EnvVariablesServer)
     protected readonly envVariablesServer: EnvVariablesServer;
 
-    private gitignoreMatchers = new Map<string, ReturnType<typeof ignore>>();
+    private gitignoreMatchers = new Map<string, ReturnType<typeof ignore> | undefined>();
     private gitignoreWatchersInitialized = new Set<string>();
 
     private _rootMapping: Map<string, URI> | undefined;
@@ -398,7 +399,9 @@ export class WorkspaceFunctionScope {
             }
             const uri = await this.toExternalUri(trimmed);
             if (uri && uri.scheme === 'file') {
-                result.push(uri.normalizePath());
+                // Strip a trailing separator so an entry like `/foo/` still matches the
+                // directory `/foo` itself (URI.isEqualOrParent compares the last segment exactly).
+                result.push(WorkspaceFunctionScope.withoutTrailingSeparator(uri.normalizePath()));
             }
         }
         return result;
@@ -473,6 +476,20 @@ export class WorkspaceFunctionScope {
         return /^[A-Za-z]:\//.test(normalized);
     }
 
+    /**
+     * Returns the URI without a trailing path separator (except for a root path). This makes
+     * directory comparisons via {@link URI.isEqualOrParent} insensitive to a trailing slash, so
+     * an allow-list entry such as `/foo/` matches the directory `/foo` itself, not only its
+     * children.
+     */
+    static withoutTrailingSeparator(uri: URI): URI {
+        const path = uri.path.toString();
+        if (path.length > 1 && path.endsWith('/')) {
+            return uri.withPath(path.substring(0, path.length - 1));
+        }
+        return uri;
+    }
+
     protected getHomeDirUri(): Promise<URI | undefined> {
         if (!this.homeDirUri) {
             this.homeDirUri = this.envVariablesServer.getHomeDirUri()
@@ -533,33 +550,43 @@ export class WorkspaceFunctionScope {
     }
 
     protected async isGitIgnored(stat: FileStat, workspaceRoot: URI): Promise<boolean> {
+        const matcher = await this.getGitignoreMatcher(workspaceRoot);
+        if (!matcher) {
+            return false;
+        }
+        const relativePath = workspaceRoot.relative(stat.resource);
+        if (!relativePath) {
+            return false;
+        }
+        const relativePathStr = relativePath.toString() + (stat.isDirectory ? '/' : '');
+        return matcher.ignores(relativePathStr);
+    }
+
+    /**
+     * Returns the cached `.gitignore` matcher for the given root, reading the file at most
+     * once per root. A root with no (or unreadable) `.gitignore` is cached as `undefined`.
+     * The cache is invalidated by {@link initializeGitignoreWatcher} when the `.gitignore` is
+     * created, changed, or deleted, so individual exclusion checks need no filesystem RPC.
+     */
+    protected async getGitignoreMatcher(workspaceRoot: URI): Promise<ReturnType<typeof ignore> | undefined> {
         await this.initializeGitignoreWatcher(workspaceRoot);
 
         const rootKey = workspaceRoot.toString();
-        const gitignoreUri = workspaceRoot.resolve(this.GITIGNORE_FILE_NAME);
+        if (this.gitignoreMatchers.has(rootKey)) {
+            return this.gitignoreMatchers.get(rootKey);
+        }
 
+        let matcher: ReturnType<typeof ignore> | undefined;
         try {
-            const fileStat = await this.fileService.resolve(gitignoreUri);
-            if (fileStat) {
-                let matcher = this.gitignoreMatchers.get(rootKey);
-                if (!matcher) {
-                    const gitignoreContent = await this.fileService.read(gitignoreUri);
-                    matcher = ignore().add(gitignoreContent.value);
-                    this.gitignoreMatchers.set(rootKey, matcher);
-                }
-                const relativePath = workspaceRoot.relative(stat.resource);
-                if (relativePath) {
-                    const relativePathStr = relativePath.toString() + (stat.isDirectory ? '/' : '');
-                    if (matcher.ignores(relativePathStr)) {
-                        return true;
-                    }
-                }
-            }
+            const gitignoreUri = workspaceRoot.resolve(this.GITIGNORE_FILE_NAME);
+            const gitignoreContent = await this.fileService.read(gitignoreUri);
+            matcher = ignore().add(gitignoreContent.value);
         } catch {
-            // If .gitignore does not exist or cannot be read, continue without error
+            // No .gitignore (or it cannot be read): cache the absence so we don't retry on every check.
+            matcher = undefined;
         }
-
-        return false;
+        this.gitignoreMatchers.set(rootKey, matcher);
+        return matcher;
     }
 }
 
@@ -653,17 +680,25 @@ export class GetWorkspaceDirectoryStructure implements ToolProvider {
         const result: Record<string, unknown> = {};
 
         if (stat && stat.isDirectory && stat.children) {
+            // Determine which child directories to include (the exclusion check may be async)...
+            const childDirs: URI[] = [];
             for (const child of stat.children) {
                 if (cancellationToken?.isCancellationRequested) {
                     return { error: 'Operation cancelled by user' };
                 }
-
-                if (!child.isDirectory || (await this.workspaceScope.shouldExclude(child))) {
-                    continue;
+                if (child.isDirectory && !(await this.workspaceScope.shouldExclude(child))) {
+                    childDirs.push(child.resource);
                 }
-                const dirName = child.resource.path.base;
-                result[dirName] = await this.buildDirectoryStructure(child.resource, cancellationToken);
             }
+            // ...then resolve their subtrees concurrently, so the traversal costs O(depth)
+            // round-trips instead of one serial round-trip per directory. Empty directories
+            // are preserved (they resolve to an empty object).
+            const subtrees = await Promise.all(
+                childDirs.map(childUri => this.buildDirectoryStructure(childUri, cancellationToken))
+            );
+            childDirs.forEach((childUri, index) => {
+                result[childUri.path.base] = subtrees[index];
+            });
         }
 
         return result;
@@ -1028,37 +1063,32 @@ export class GetWorkspaceFileList implements ToolProvider {
             if (!stat || !stat.isDirectory) {
                 return JSON.stringify({ error: 'Directory not found' });
             }
-            return await this.listFilesDirectly(targetUri, cancellationToken);
+            return await this.listFilesDirectly(stat, cancellationToken);
         } catch (error) {
             return JSON.stringify({ error: 'Directory not found' });
         }
     }
 
-    private async listFilesDirectly(uri: URI, cancellationToken?: CancellationToken): Promise<string> {
+    private async listFilesDirectly(stat: FileStat, cancellationToken?: CancellationToken): Promise<string> {
         if (cancellationToken?.isCancellationRequested) {
             return JSON.stringify({ error: 'Operation cancelled by user' });
         }
 
-        const stat = await this.fileService.resolve(uri);
         const result: Record<string, 'directory' | 'file'> = {};
 
-        if (stat && stat.isDirectory) {
-            if (await this.workspaceScope.shouldExclude(stat)) {
-                return JSON.stringify(result);
+        if (await this.workspaceScope.shouldExclude(stat)) {
+            return JSON.stringify(result);
+        }
+        // `stat` already carries one level of children from the caller's resolve, so no extra RPC.
+        for (const child of stat.children ?? []) {
+            if (cancellationToken?.isCancellationRequested) {
+                return JSON.stringify({ error: 'Operation cancelled by user' });
             }
-            const children = await this.fileService.resolve(uri);
-            if (children.children) {
-                for (const child of children.children) {
-                    if (cancellationToken?.isCancellationRequested) {
-                        return JSON.stringify({ error: 'Operation cancelled by user' });
-                    }
 
-                    if (await this.workspaceScope.shouldExclude(child)) {
-                        continue;
-                    }
-                    result[child.resource.path.base] = child.isDirectory ? 'directory' : 'file';
-                }
+            if (await this.workspaceScope.shouldExclude(child)) {
+                continue;
             }
+            result[child.resource.path.base] = child.isDirectory ? 'directory' : 'file';
         }
 
         return JSON.stringify(result);
@@ -1224,8 +1254,8 @@ export class FindFilesByPattern implements ToolProvider {
     @inject(PreferenceService)
     protected readonly preferences: PreferenceService;
 
-    @inject(FileService)
-    protected readonly fileService: FileService;
+    @inject(FileSearchService)
+    protected readonly fileSearchService: FileSearchService;
 
     getTool(): ToolRequest {
         return {
@@ -1239,8 +1269,6 @@ export class FindFilesByPattern implements ToolProvider {
                 '\'src/**/*.js\' for JavaScript files in the src directory. The function respects gitignore patterns and user exclusions, ' +
                 'returns workspace-relative paths (e.g., "my-project/src/index.ts") or absolute paths for external roots, ' +
                 'and limits results to 200 files maximum. ' +
-                'Performance note: This traverses directories recursively which may be slow in large workspaces. ' +
-                'For better performance, use specific subdirectory patterns (e.g., \'src/**/*.ts\' instead of \'**/*.ts\'). ' +
                 'Use this to find files by name/extension. Do NOT use this for searching file contents - use searchInWorkspace instead.',
             parameters: {
                 type: 'object',
@@ -1249,8 +1277,7 @@ export class FindFilesByPattern implements ToolProvider {
                         type: 'string',
                         description: 'Glob pattern to match files against. ' +
                             'Examples: \'**/*.ts\' (all TypeScript files), \'src/**/*.js\' (JS files in src), ' +
-                            '\'**/*.{js,ts}\' (JS or TS files), \'**/test/**/*.spec.ts\' (test files). ' +
-                            'Use specific subdirectory prefixes for better performance (e.g., \'packages/core/**/*.ts\' instead of \'**/*.ts\').'
+                            '\'**/*.{js,ts}\' (JS or TS files), \'**/test/**/*.spec.ts\' (test files).'
                     },
                     exclude: {
                         type: 'array',
@@ -1303,76 +1330,59 @@ export class FindFilesByPattern implements ToolProvider {
         }
 
         try {
-            const patternMatcher = new Minimatch(pattern, { dot: false });
-            const files: string[] = [];
             const maxResults = 200;
+            const useGitIgnore = this.preferences.get(CONSIDER_GITIGNORE_PREF, true);
+            const userExcludes = this.preferences.get<string[]>(USER_EXCLUDE_PATTERN_PREF, []);
+            const excludes = [...userExcludes, ...(excludePatterns ?? [])];
 
+            // Resolve the set of roots to search and how each root's results should be rendered.
+            const targets: { rootUri: URI; rootName?: string; external: boolean }[] = [];
             if (searchRoot) {
                 const resolved = await this.workspaceScope.resolveToUri(searchRoot);
                 if (!resolved) {
                     return JSON.stringify({ error: `Invalid searchRoot: '${searchRoot}'` });
                 }
-                const rootUri = resolved;
-                const isExternalRoot = !this.workspaceScope.isInWorkspace(rootUri);
-                await this.workspaceScope.ensureAccessible(rootUri);
-
-                const ignorePatterns = isExternalRoot
-                    ? this.preferences.get<string[]>(USER_EXCLUDE_PATTERN_PREF, [])
-                    : await this.buildIgnorePatterns(rootUri);
-                const allExcludes = [...ignorePatterns];
-                if (excludePatterns && excludePatterns.length > 0) {
-                    allExcludes.push(...excludePatterns);
-                }
-
-                if (cancellationToken?.isCancellationRequested) {
-                    return JSON.stringify({ error: 'Operation cancelled by user' });
-                }
-
-                const excludeMatchers = allExcludes.map(excludePattern => new Minimatch(excludePattern, { dot: true }));
-
-                await this.traverseDirectory(
-                    rootUri,
-                    rootUri,
-                    undefined,
-                    patternMatcher,
-                    excludeMatchers,
-                    files,
-                    maxResults,
-                    cancellationToken,
-                    isExternalRoot
-                );
+                await this.workspaceScope.ensureAccessible(resolved);
+                targets.push({ rootUri: resolved, external: !this.workspaceScope.isInWorkspace(resolved) });
             } else {
                 const rootMapping = this.workspaceScope.getRootMapping();
                 if (rootMapping.size === 0) {
                     return JSON.stringify({ error: 'No workspace has been opened yet' });
                 }
-
                 for (const [rootName, rootUri] of rootMapping) {
-                    if (cancellationToken?.isCancellationRequested) {
-                        return JSON.stringify({ error: 'Operation cancelled by user' });
-                    }
-
-                    if (files.length >= maxResults) {
-                        break;
-                    }
+                    targets.push({ rootUri, rootName, external: false });
+                }
+            }
 
-                    const ignorePatterns = await this.buildIgnorePatterns(rootUri);
-                    const allExcludes = [...ignorePatterns];
-                    if (excludePatterns && excludePatterns.length > 0) {
-                        allExcludes.push(...excludePatterns);
+            // Delegate the actual traversal to the backend ripgrep-based file search.
+            // It runs natively on the backend filesystem (no per-directory RPC) and applies
+            // include/exclude globs.
+            const files: string[] = [];
+            for (const target of targets) {
+                if (cancellationToken?.isCancellationRequested) {
+                    return JSON.stringify({ error: 'Operation cancelled by user' });
+                }
+                if (files.length > maxResults) {
+                    break;
+                }
+                // `considerGitIgnore` is scoped to workspace roots (see its preference description),
+                // so external allow-listed roots are searched with user/caller excludes only (plus
+                // `.git`). Applying gitignore there would also leak the user's *global* gitignore
+                // into an explicitly allow-listed directory and silently hide files.
+                // Request one extra result across all roots so we can detect truncation.
+                const matches = await this.fileSearchService.find('', {
+                    rootUris: [target.rootUri.toString()],
+                    includePatterns: [pattern],
+                    excludePatterns: target.external ? [...excludes, '.git'] : excludes,
+                    useGitIgnore: target.external ? false : useGitIgnore,
+                    fuzzyMatch: false,
+                    limit: maxResults - files.length + 1
+                }, cancellationToken);
+                for (const match of matches) {
+                    const display = this.toDisplayPath(new URI(match), target);
+                    if (display !== undefined) {
+                        files.push(display);
                     }
-                    const excludeMatchers = allExcludes.map(excludePattern => new Minimatch(excludePattern, { dot: true }));
-
-                    await this.traverseDirectory(
-                        rootUri,
-                        rootUri,
-                        rootName,
-                        patternMatcher,
-                        excludeMatchers,
-                        files,
-                        maxResults,
-                        cancellationToken
-                    );
                 }
             }
 
@@ -1380,12 +1390,10 @@ export class FindFilesByPattern implements ToolProvider {
                 return JSON.stringify({ error: 'Operation cancelled by user' });
             }
 
-            const result: { files: string[]; totalFound?: number; truncated?: boolean } = {
+            const result: { files: string[]; truncated?: boolean } = {
                 files: files.slice(0, maxResults)
             };
-
             if (files.length > maxResults) {
-                result.totalFound = files.length;
                 result.truncated = true;
             }
 
@@ -1396,94 +1404,19 @@ export class FindFilesByPattern implements ToolProvider {
         }
     }
 
-    private async buildIgnorePatterns(workspaceRoot: URI): Promise<string[]> {
-        const patterns: string[] = [];
-
-        // Get user exclude patterns from preferences
-        const userExcludePatterns = this.preferences.get<string[]>(USER_EXCLUDE_PATTERN_PREF, []);
-        patterns.push(...userExcludePatterns);
-
-        // Add gitignore patterns if enabled
-        const shouldConsiderGitIgnore = this.preferences.get(CONSIDER_GITIGNORE_PREF, false);
-        if (shouldConsiderGitIgnore) {
-            try {
-                const gitignoreUri = workspaceRoot.resolve('.gitignore');
-                const gitignoreContent = await this.fileService.read(gitignoreUri);
-                const gitignoreLines = gitignoreContent.value
-                    .split('\n')
-                    .map(line => line.trim())
-                    .filter(line => line && !line.startsWith('#'));
-                patterns.push(...gitignoreLines);
-            } catch {
-                // Gitignore file doesn't exist or can't be read, continue without it
-            }
-        }
-
-        return patterns;
-    }
-
-    private async traverseDirectory(
-        currentUri: URI,
-        searchRoot: URI,
-        rootName: string | undefined,
-        patternMatcher: Minimatch,
-        excludeMatchers: Minimatch[],
-        results: string[],
-        maxResults: number,
-        cancellationToken?: CancellationToken,
-        emitAbsolutePaths = false
-    ): Promise<void> {
-        if (cancellationToken?.isCancellationRequested || results.length >= maxResults) {
-            return;
+    /**
+     * Renders a search-result URI in the format expected by the caller: an absolute
+     * path for external roots, or a `<rootName>/<relativePath>` (or bare relative
+     * path when no root name is available) for workspace roots.
+     */
+    protected toDisplayPath(match: URI, target: { rootUri: URI; rootName?: string; external: boolean }): string | undefined {
+        if (target.external) {
+            return match.path.toString();
         }
-
-        try {
-            const stat = await this.fileService.resolve(currentUri);
-            if (!stat || !stat.isDirectory || !stat.children) {
-                return;
-            }
-
-            for (const child of stat.children) {
-                if (cancellationToken?.isCancellationRequested || results.length >= maxResults) {
-                    break;
-                }
-
-                const relativePath = searchRoot.relative(child.resource)?.toString();
-                if (!relativePath) {
-                    continue;
-                }
-
-                const shouldExclude = excludeMatchers.some(matcher => matcher.match(relativePath)) ||
-                    (await this.workspaceScope.shouldExclude(child));
-
-                if (shouldExclude) {
-                    continue;
-                }
-
-                if (child.isDirectory) {
-                    await this.traverseDirectory(
-                        child.resource,
-                        searchRoot,
-                        rootName,
-                        patternMatcher,
-                        excludeMatchers,
-                        results,
-                        maxResults,
-                        cancellationToken,
-                        emitAbsolutePaths
-                    );
-                } else if (patternMatcher.match(relativePath)) {
-                    if (emitAbsolutePaths) {
-                        results.push(child.resource.path.toString());
-                    } else if (rootName) {
-                        results.push(`${rootName}/${relativePath}`);
-                    } else {
-                        results.push(relativePath);
-                    }
-                }
-            }
-        } catch {
-            // If we can't access a directory, skip it
+        const relativePath = target.rootUri.relative(match)?.toString();
+        if (relativePath === undefined) {
+            return undefined;
         }
+        return target.rootName ? `${target.rootName}/${relativePath}` : relativePath;
     }
 }