@theia/ai-ide 1.64.0-next.0 → 1.64.0-next.17

package/src/browser/app-tester-chat-functions.ts

@@ -0,0 +1,170 @@
+// *****************************************************************************
+// Copyright (C) 2025 EclipseSource GmbH.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+import { type ToolProvider, type ToolRequest } from '@theia/ai-core';
+import { isLocalMCPServerDescription, MCPServerManager } from '@theia/ai-mcp/lib/common';
+import { inject, injectable } from '@theia/core/shared/inversify';
+import { CLOSE_BROWSER_FUNCTION_ID, IS_BROWSER_RUNNING_FUNCTION_ID, LAUNCH_BROWSER_FUNCTION_ID, QUERY_DOM_FUNCTION_ID } from '../common/app-tester-chat-functions';
+import { BrowserAutomation } from '../common/browser-automation-protocol';
+
+@injectable()
+export abstract class BrowserAutomationToolProvider implements ToolProvider {
+    @inject(BrowserAutomation)
+    protected readonly browser: BrowserAutomation;
+
+    abstract getTool(): ToolRequest;
+}
+
+@injectable()
+export class LaunchBrowserProvider extends BrowserAutomationToolProvider {
+    static ID = LAUNCH_BROWSER_FUNCTION_ID;
+
+    @inject(MCPServerManager)
+    protected readonly mcpServerManager: MCPServerManager;
+
+    getTool(): ToolRequest {
+        return {
+            id: LaunchBrowserProvider.ID,
+            name: LaunchBrowserProvider.ID,
+            description: 'Start the browser.',
+            parameters: {
+                type: 'object',
+                properties: {},
+                required: []
+            }, handler: async () => {
+                try {
+
+                    const mcp = await this.mcpServerManager.getServerDescription('playwright');
+                    if (!mcp) {
+                        throw new Error('No MCP Playwright instance with name playwright found');
+                    }
+                    if (!isLocalMCPServerDescription(mcp)) {
+                        throw new Error('The MCP Playwright instance must run locally.');
+                    }
+
+                    const cdpEndpointIndex = mcp.args?.findIndex(p => p === '--cdp-endpoint');
+                    if (!cdpEndpointIndex) {
+                        throw new Error('No --cdp-endpoint was provided.');
+                    }
+                    const cdpEndpoint = mcp.args?.[cdpEndpointIndex + 1];
+                    if (!cdpEndpoint) {
+                        throw new Error('No --cdp-endpoint argument was provided.');
+                    }
+
+                    let remoteDebuggingPort = 9222;
+                    try {
+                        const uri = new URL(cdpEndpoint);
+                        if (uri.port) {
+                            remoteDebuggingPort = parseInt(uri.port, 10);
+                        } else {
+                            // Default ports if not specified
+                            remoteDebuggingPort = uri.protocol === 'https:' ? 443 : 80;
+                        }
+                    } catch (error) {
+                        throw new Error(`Invalid --cdp-endpoint format, URL expected: ${cdpEndpoint}`);
+                    }
+
+                    const result = await this.browser.launch(remoteDebuggingPort);
+                    return result;
+                } catch (ex) {
+                    return (`Failed to starting the browser: ${ex.message}`);
+                }
+            }
+        };
+    }
+}
+
+@injectable()
+export class CloseBrowserProvider extends BrowserAutomationToolProvider {
+    static ID = CLOSE_BROWSER_FUNCTION_ID;
+
+    getTool(): ToolRequest {
+        return {
+            id: CloseBrowserProvider.ID,
+            name: CloseBrowserProvider.ID,
+            description: 'Close the browser.',
+            parameters: {
+                type: 'object',
+                properties: {},
+                required: []
+            },
+            handler: async () => {
+                try {
+                    await this.browser.close();
+                } catch (ex) {
+                    return (`Failed to close browser: ${ex.message}`);
+                }
+            }
+        };
+    }
+}
+
+@injectable()
+export class IsBrowserRunningProvider extends BrowserAutomationToolProvider {
+    static ID = IS_BROWSER_RUNNING_FUNCTION_ID;
+
+    getTool(): ToolRequest {
+        return {
+            id: IsBrowserRunningProvider.ID,
+            name: IsBrowserRunningProvider.ID,
+            description: 'Check if the browser is running.',
+            parameters: {
+                type: 'object',
+                properties: {},
+                required: []
+            },
+            handler: async () => {
+                try {
+                    const isRunning = await this.browser.isRunning();
+                    return isRunning ? 'Browser is running.' : 'Browser is not running.';
+                } catch (ex) {
+                    return (`Failed to check if browser is running: ${ex.message}`);
+                }
+            }
+        };
+    }
+}
+
+@injectable()
+export class QueryDomProvider extends BrowserAutomationToolProvider {
+    static ID = QUERY_DOM_FUNCTION_ID;
+
+    getTool(): ToolRequest {
+        return {
+            id: QueryDomProvider.ID,
+            name: QueryDomProvider.ID,
+            description: 'Query the DOM of the active page.',
+            parameters: {
+                type: 'object',
+                properties: {
+                    selector: {
+                        type: 'string',
+                        description: `The selector of the element to get the DOM of. The selector is a 
+                        CSS selector that identifies the element. If not provided, the entire DOM will be returned.`
+                    }
+                },
+                required: []
+            },
+            handler: async arg => {
+                try {
+                    const { selector } = JSON.parse(arg);
+                    return await this.browser.queryDom(selector);
+                } catch (ex) {
+                    return (`Failed to get DOM: ${ex.message}`);
+                }
+            }
+        };
+    }
+}

package/src/browser/frontend-module.ts

@@ -24,7 +24,14 @@ import { CoderAgent } from './coder-agent';
 import { SummarizeSessionCommandContribution } from './summarize-session-command-contribution';
 import { FileContentFunction, FileDiagnosticProvider, GetWorkspaceDirectoryStructure, GetWorkspaceFileList, WorkspaceFunctionScope } from './workspace-functions';
 import { WorkspaceSearchProvider } from './workspace-search-provider';
-import { FrontendApplicationContribution, PreferenceContribution, WidgetFactory, bindViewContribution } from '@theia/core/lib/browser';
+import {
+    FrontendApplicationContribution,
+    PreferenceContribution,
+    WidgetFactory,
+    bindViewContribution,
+    RemoteConnectionProvider,
+    ServiceConnectionProvider
+} from '@theia/core/lib/browser';
 import { TaskListProvider, TaskRunnerProvider } from './workspace-task-provider';
 import { WorkspacePreferencesSchema } from './workspace-preferences';
 import {
@@ -62,6 +69,8 @@ import { TaskContextFileStorageService } from './task-context-file-storage-servi
 import { TaskContextStorageService } from '@theia/ai-chat/lib/browser/task-context-service';
 import { CommandContribution } from '@theia/core';
 import { AIPromptFragmentsConfigurationWidget } from './ai-configuration/prompt-fragments-configuration-widget';
+import { BrowserAutomation, browserAutomationPath } from '../common/browser-automation-protocol';
+import { CloseBrowserProvider, IsBrowserRunningProvider, LaunchBrowserProvider, QueryDomProvider } from './app-tester-chat-functions';
 
 export default new ContainerModule((bind, _unbind, _isBound, rebind) => {
     bind(PreferenceContribution).toConstantValue({ schema: WorkspacePreferencesSchema });
@@ -85,6 +94,10 @@ export default new ContainerModule((bind, _unbind, _isBound, rebind) => {
     bind(AppTesterChatAgent).toSelf().inSingletonScope();
     bind(Agent).toService(AppTesterChatAgent);
     bind(ChatAgent).toService(AppTesterChatAgent);
+    bind(BrowserAutomation).toDynamicValue(ctx => {
+        const provider = ctx.container.get<ServiceConnectionProvider>(RemoteConnectionProvider);
+        return provider.createProxy<BrowserAutomation>(browserAutomationPath);
+    }).inSingletonScope();
 
     bind(CommandChatAgent).toSelf().inSingletonScope();
     bind(Agent).toService(CommandChatAgent);
@@ -120,6 +133,11 @@ export default new ContainerModule((bind, _unbind, _isBound, rebind) => {
         }))
         .inSingletonScope();
 
+    bindToolProvider(LaunchBrowserProvider, bind);
+    bindToolProvider(CloseBrowserProvider, bind);
+    bindToolProvider(IsBrowserRunningProvider, bind);
+    bindToolProvider(QueryDomProvider, bind);
+
     bindViewContribution(bind, AIAgentConfigurationViewContribution);
     bind(TabBarToolbarContribution).toService(AIAgentConfigurationViewContribution);
 

package/src/common/app-tester-chat-functions.ts

@@ -0,0 +1,20 @@
+// *****************************************************************************
+// Copyright (C) 2025 EclipseSource GmbH.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+export const LAUNCH_BROWSER_FUNCTION_ID = 'launchBrowser';
+export const IS_BROWSER_RUNNING_FUNCTION_ID = 'isBrowserRunning';
+export const CLOSE_BROWSER_FUNCTION_ID = 'closeBrowser';
+export const QUERY_DOM_FUNCTION_ID = 'queryDom';

package/src/common/browser-automation-protocol.ts

@@ -0,0 +1,32 @@
+// *****************************************************************************
+// Copyright (C) 2025 EclipseSource GmbH.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+export const browserAutomationPath = '/services/automation/browser';
+export const BrowserAutomation = Symbol('BrowserAutomation');
+export interface BrowserAutomation {
+    launch(remoteDebuggingPort: number): Promise<LaunchResult | undefined>;
+    isRunning(): Promise<boolean>;
+    queryDom(selector?: string): Promise<string>;
+    close(): Promise<void>;
+}
+
+export interface LaunchResult {
+    remoteDebuggingPort: number;
+}
+
+export const BrowserAutomationClient = Symbol('BrowserAutomationClient');
+export interface BrowserAutomationClient {
+}

package/src/node/app-tester-agent/browser-automation-impl.ts

@@ -0,0 +1,107 @@
+// *****************************************************************************
+// Copyright (C) 2025 EclipseSource GmbH.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import { type RpcServer } from '@theia/core';
+import { injectable } from '@theia/core/shared/inversify';
+import { Browser, launch, Page } from 'puppeteer-core';
+import type { BrowserAutomation, BrowserAutomationClient, LaunchResult } from '../../common/browser-automation-protocol';
+
+const MAX_DOM_LENGTH = 50000;
+
+@injectable()
+export class BrowserAutomationImpl implements RpcServer<BrowserAutomationClient>, BrowserAutomation {
+    protected _browser?: Browser;
+    protected _page?: Page;
+    protected client?: BrowserAutomationClient;
+
+    protected get browser(): Browser {
+        if (!this._browser) {
+            throw new Error('Browser is not launched');
+        }
+        return this._browser;
+    }
+
+    protected get page(): Page {
+        if (!this._page) {
+            throw new Error('Page is not created');
+        }
+        return this._page;
+    }
+
+    async isRunning(): Promise<boolean> {
+        return this._browser !== undefined && this._browser.connected;
+    }
+
+    async launch(remoteDebuggingPort: number): Promise<LaunchResult | undefined> {
+        if (this._browser) {
+            await this.close();
+        }
+
+        const browser = await launch({
+            headless: false,
+            channel: 'chrome',
+            args: [
+                `--remote-debugging-port=${remoteDebuggingPort}`
+            ],
+        });
+        this._browser = browser;
+        // The initial page will be used per default
+        this._page = (await browser.pages())[0];
+        return {
+            remoteDebuggingPort
+        };
+    }
+
+    async close(): Promise<void> {
+        await this._browser?.close();
+        this._browser = undefined;
+    }
+
+    async queryDom(selector?: string): Promise<string> {
+        const page = this.page;
+        let content = '';
+
+        if (selector) {
+            const element = await page.$(selector);
+            if (!element) {
+                throw new Error(`Element with selector "${selector}" not found`);
+            }
+            content = await page.evaluate(el => el.outerHTML, element);
+        } else {
+            content = await page.content();
+        }
+
+        if (content.length > MAX_DOM_LENGTH) {
+            return 'The queried DOM is too large. Please provide a more specific query.';
+        }
+
+        return content;
+    }
+
+    dispose(): void {
+        this._browser?.close();
+        this._browser = undefined;
+    }
+
+    setClient(client: BrowserAutomationClient | undefined): void {
+        this.client = client;
+    }
+
+    getClient?(): BrowserAutomationClient | undefined {
+        return this.client;
+    }
+
+}

package/src/node/backend-module.ts

@@ -0,0 +1,38 @@
+// *****************************************************************************
+// Copyright (C) 2025 EclipseSource GmbH.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import { ConnectionHandler, RpcConnectionHandler } from '@theia/core';
+import { ContainerModule } from '@theia/core/shared/inversify';
+import { BrowserAutomation, browserAutomationPath, type BrowserAutomationClient } from '../common/browser-automation-protocol';
+import { BrowserAutomationImpl } from './app-tester-agent/browser-automation-impl';
+import { ConnectionContainerModule } from '@theia/core/lib/node/messaging/connection-container-module';
+
+const browserAutomationModule = ConnectionContainerModule.create(({ bind, bindBackendService, bindFrontendService }) => {
+    bind(BrowserAutomation).to(BrowserAutomationImpl).inSingletonScope();
+    bind(ConnectionHandler).toDynamicValue(ctx =>
+        new RpcConnectionHandler<BrowserAutomationClient>(browserAutomationPath, client => {
+            const server = ctx.container.get<BrowserAutomationImpl>(BrowserAutomation);
+            server.setClient(client);
+            client.onDidCloseConnection(() => server.close());
+            return server;
+        })
+    ).inSingletonScope();
+});
+
+export default new ContainerModule(bind => {
+    bind(ConnectionContainerModule).toConstantValue(browserAutomationModule);
+
+});