@theia/ai-ide 1.64.0-next.0 → 1.64.0-next.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/browser/ai-configuration/agent-configuration-widget.d.ts.map +1 -1
- package/lib/browser/ai-configuration/agent-configuration-widget.js +1 -1
- package/lib/browser/ai-configuration/agent-configuration-widget.js.map +1 -1
- package/lib/browser/ai-configuration/mcp-configuration-widget.d.ts +1 -0
- package/lib/browser/ai-configuration/mcp-configuration-widget.d.ts.map +1 -1
- package/lib/browser/ai-configuration/mcp-configuration-widget.js +12 -0
- package/lib/browser/ai-configuration/mcp-configuration-widget.js.map +1 -1
- package/lib/browser/ai-configuration/template-settings-renderer.d.ts +1 -2
- package/lib/browser/ai-configuration/template-settings-renderer.d.ts.map +1 -1
- package/lib/browser/ai-configuration/template-settings-renderer.js +5 -3
- package/lib/browser/ai-configuration/template-settings-renderer.js.map +1 -1
- package/lib/browser/app-tester-chat-agent.d.ts +1 -1
- package/lib/browser/app-tester-chat-agent.d.ts.map +1 -1
- package/lib/browser/app-tester-chat-agent.js +32 -10
- package/lib/browser/app-tester-chat-agent.js.map +1 -1
- package/lib/browser/app-tester-chat-functions.d.ts +25 -0
- package/lib/browser/app-tester-chat-functions.d.ts.map +1 -0
- package/lib/browser/app-tester-chat-functions.js +170 -0
- package/lib/browser/app-tester-chat-functions.js.map +1 -0
- package/lib/browser/frontend-module.d.ts.map +1 -1
- package/lib/browser/frontend-module.js +10 -0
- package/lib/browser/frontend-module.js.map +1 -1
- package/lib/common/app-tester-chat-functions.d.ts +5 -0
- package/lib/common/app-tester-chat-functions.d.ts.map +1 -0
- package/lib/common/app-tester-chat-functions.js +23 -0
- package/lib/common/app-tester-chat-functions.js.map +1 -0
- package/lib/common/browser-automation-protocol.d.ts +15 -0
- package/lib/common/browser-automation-protocol.d.ts.map +1 -0
- package/lib/common/browser-automation-protocol.js +22 -0
- package/lib/common/browser-automation-protocol.js.map +1 -0
- package/lib/node/app-tester-agent/browser-automation-impl.d.ts +18 -0
- package/lib/node/app-tester-agent/browser-automation-impl.d.ts.map +1 -0
- package/lib/node/app-tester-agent/browser-automation-impl.js +96 -0
- package/lib/node/app-tester-agent/browser-automation-impl.js.map +1 -0
- package/lib/node/backend-module.d.ts +4 -0
- package/lib/node/backend-module.d.ts.map +1 -0
- package/lib/node/backend-module.js +35 -0
- package/lib/node/backend-module.js.map +1 -0
- package/package.json +21 -19
- package/src/browser/ai-configuration/agent-configuration-widget.tsx +0 -1
- package/src/browser/ai-configuration/mcp-configuration-widget.tsx +20 -0
- package/src/browser/ai-configuration/template-settings-renderer.tsx +5 -5
- package/src/browser/app-tester-chat-agent.ts +135 -111
- package/src/browser/app-tester-chat-functions.ts +170 -0
- package/src/browser/frontend-module.ts +19 -1
- package/src/common/app-tester-chat-functions.ts +20 -0
- package/src/common/browser-automation-protocol.ts +32 -0
- package/src/node/app-tester-agent/browser-automation-impl.ts +107 -0
- package/src/node/backend-module.ts +38 -0
|
@@ -25,28 +25,33 @@ import { nls } from '@theia/core';
|
|
|
25
25
|
import { inject, injectable } from '@theia/core/shared/inversify';
|
|
26
26
|
import { MCP_SERVERS_PREF } from '@theia/ai-mcp/lib/browser/mcp-preferences';
|
|
27
27
|
import { PreferenceScope, PreferenceService } from '@theia/core/lib/browser';
|
|
28
|
+
import { QUERY_DOM_FUNCTION_ID, LAUNCH_BROWSER_FUNCTION_ID, CLOSE_BROWSER_FUNCTION_ID, IS_BROWSER_RUNNING_FUNCTION_ID } from '../common/app-tester-chat-functions';
|
|
28
29
|
|
|
29
30
|
export const REQUIRED_MCP_SERVERS: MCPServerDescription[] = [
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
31
|
+
{
|
|
32
|
+
name: 'playwright',
|
|
33
|
+
command: 'npx',
|
|
34
|
+
args: ['-y', '@playwright/mcp@latest',
|
|
35
|
+
'--cdp-endpoint',
|
|
36
|
+
'http://localhost:9222/'],
|
|
37
|
+
autostart: false,
|
|
38
|
+
env: {},
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
name: 'playwright-visual',
|
|
42
|
+
command: 'npx',
|
|
43
|
+
args: ['-y', '@playwright/mcp@latest', '--vision',
|
|
44
|
+
'--cdp-endpoint',
|
|
45
|
+
'http://localhost:9222/'],
|
|
46
|
+
autostart: false,
|
|
47
|
+
env: {},
|
|
48
|
+
}
|
|
44
49
|
];
|
|
45
50
|
|
|
46
51
|
// Prompt templates
|
|
47
52
|
export const appTesterTemplate: BasePromptFragment = {
|
|
48
|
-
|
|
49
|
-
|
|
53
|
+
id: 'app-tester-system-default',
|
|
54
|
+
template: `{{!-- This prompt is licensed under the MIT License (https://opensource.org/license/mit).
|
|
50
55
|
Made improvements or adaptations to this prompt template? We'd love for you to share it with the community! Contribute back here:
|
|
51
56
|
https://github.com/eclipse-theia/theia/discussions/new?category=prompt-template-contribution --}}
|
|
52
57
|
|
|
@@ -63,6 +68,14 @@ Your role is to inspect the application for user-specified test scenarios throug
|
|
|
63
68
|
You have access to these powerful automation tools:
|
|
64
69
|
${REQUIRED_MCP_SERVERS.map(server => `{{prompt:mcp_${server.name}_tools}}`)}
|
|
65
70
|
|
|
71
|
+
- **~{${LAUNCH_BROWSER_FUNCTION_ID}}**: Launch the browser. This is required before performing any browser interactions. Always launch a new browser when starting a test session.
|
|
72
|
+
- **~{${IS_BROWSER_RUNNING_FUNCTION_ID}}**: Check if the browser is running. If a tool fails by saying that the connection failed, you can verify the connection by using this tool.
|
|
73
|
+
- **~{${CLOSE_BROWSER_FUNCTION_ID}}**: Close the browser.
|
|
74
|
+
- **~{${QUERY_DOM_FUNCTION_ID}}**: Query the DOM for specific elements and their properties. Only use when explicitly requested by the user.
|
|
75
|
+
- **browser_snapshot**: Capture the current state of the page for verification or debugging purposes.
|
|
76
|
+
|
|
77
|
+
Prefer snapshots for investigating the page.
|
|
78
|
+
|
|
66
79
|
## Workflow Approach
|
|
67
80
|
1. **Understand Requirements**: Ask the user to clearly define what needs to be tested
|
|
68
81
|
2. **Launch Browser**: Start a fresh browser instance for testing
|
|
@@ -77,112 +90,123 @@ Some files and other pieces of data may have been added by the user to the conte
|
|
|
77
90
|
};
|
|
78
91
|
|
|
79
92
|
export const appTesterTemplateVariant: BasePromptFragment = {
|
|
80
|
-
|
|
81
|
-
|
|
93
|
+
id: 'app-tester-system-empty',
|
|
94
|
+
template: '',
|
|
82
95
|
};
|
|
83
96
|
|
|
84
97
|
export const AppTesterChatAgentId = 'AppTester';
|
|
85
98
|
@injectable()
|
|
86
99
|
export class AppTesterChatAgent extends AbstractStreamParsingChatAgent {
|
|
87
100
|
|
|
88
|
-
|
|
89
|
-
|
|
101
|
+
@inject(MCPFrontendService)
|
|
102
|
+
protected readonly mcpService: MCPFrontendService;
|
|
90
103
|
|
|
91
|
-
|
|
92
|
-
|
|
104
|
+
@inject(PreferenceService)
|
|
105
|
+
protected readonly preferenceService: PreferenceService;
|
|
93
106
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
107
|
+
id: string = AppTesterChatAgentId;
|
|
108
|
+
name = AppTesterChatAgentId;
|
|
109
|
+
languageModelRequirements: LanguageModelRequirement[] = [{
|
|
110
|
+
purpose: 'chat',
|
|
111
|
+
identifier: 'openai/gpt-4o',
|
|
112
|
+
}];
|
|
113
|
+
protected defaultLanguageModelPurpose: string = 'chat';
|
|
114
|
+
override description = nls.localize('theia/ai/chat/app-tester/description', 'This agent tests your application user interface to verify user-specified test scenarios through the Playwright MCP server. '
|
|
115
|
+
+ 'It can automate testing workflows and provide detailed feedback on application functionality.');
|
|
103
116
|
|
|
104
|
-
|
|
117
|
+
override iconClass: string = 'codicon codicon-beaker';
|
|
105
118
|
protected override systemPromptId: string = 'app-tester-system';
|
|
106
119
|
override prompts = [{ id: 'app-tester-system', defaultVariant: appTesterTemplate, variants: [appTesterTemplateVariant] }];
|
|
107
120
|
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
121
|
+
/**
|
|
122
|
+
* Override invoke to check if the Playwright MCP server is running, and if not, ask the user if it should be started.
|
|
123
|
+
*/
|
|
124
|
+
override async invoke(request: MutableChatRequestModel): Promise<void> {
|
|
125
|
+
try {
|
|
126
|
+
if (await this.requiresStartingServers()) {
|
|
127
|
+
// Ask the user if they want to start the server
|
|
128
|
+
request.response.response.addContent(new QuestionResponseContentImpl(
|
|
129
|
+
'The Playwright MCP servers are not running. Would you like to start them now? This may install the Playwright MCP servers.',
|
|
130
|
+
[
|
|
131
|
+
{ text: 'Yes, start the servers', value: 'yes' },
|
|
132
|
+
{ text: 'No, cancel', value: 'no' }
|
|
133
|
+
],
|
|
134
|
+
request,
|
|
135
|
+
async selectedOption => {
|
|
136
|
+
if (selectedOption.value === 'yes') {
|
|
137
|
+
// Show progress
|
|
138
|
+
const progress = request.response.addProgressMessage({ content: 'Starting Playwright MCP servers.', show: 'whileIncomplete' });
|
|
139
|
+
try {
|
|
140
|
+
await this.startServers();
|
|
141
|
+
// Remove progress, continue with normal flow
|
|
142
|
+
request.response.updateProgressMessage({ ...progress, show: 'whileIncomplete', status: 'completed' });
|
|
143
|
+
await super.invoke(request);
|
|
144
|
+
} catch (error) {
|
|
145
|
+
request.response.response.addContent(new ErrorChatResponseContentImpl(
|
|
146
|
+
new Error('Failed to start Playwright MCP server: ' + (error instanceof Error ? error.message : String(error)))
|
|
147
|
+
));
|
|
148
|
+
request.response.complete();
|
|
149
|
+
}
|
|
150
|
+
} else {
|
|
151
|
+
// Continue without starting the server
|
|
152
|
+
request.response.response.addContent(new MarkdownChatResponseContentImpl('Please setup the MCP servers.'));
|
|
153
|
+
request.response.complete();
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
));
|
|
157
|
+
request.response.waitForInput();
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
// If already running, continue as normal
|
|
161
|
+
await super.invoke(request);
|
|
162
|
+
} catch (error) {
|
|
163
|
+
request.response.response.addContent(new ErrorChatResponseContentImpl(
|
|
164
|
+
new Error('Error checking Playwright MCP server status: ' + (error instanceof Error ? error.message : String(error)))
|
|
143
165
|
));
|
|
144
|
-
request.response.
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
166
|
+
request.response.complete();
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
protected async requiresStartingServers(): Promise<boolean> {
|
|
171
|
+
const allStarted = await Promise.all(REQUIRED_MCP_SERVERS.map(server => this.mcpService.isServerStarted(server.name)));
|
|
172
|
+
return allStarted.some(started => !started);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
protected async startServers(): Promise<void> {
|
|
176
|
+
await this.ensureServersStarted(...REQUIRED_MCP_SERVERS);
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* Starts the Playwright MCP server if it doesn't exist or isn't running.
|
|
181
|
+
*
|
|
182
|
+
* @returns A promise that resolves when the server is started
|
|
183
|
+
*/
|
|
184
|
+
async ensureServersStarted(...servers: MCPServerDescription[]): Promise<void> {
|
|
185
|
+
try {
|
|
186
|
+
const serversToInstall: MCPServerDescription[] = [];
|
|
187
|
+
const serversToStart: MCPServerDescription[] = [];
|
|
188
|
+
|
|
189
|
+
for (const server of servers) {
|
|
190
|
+
if (!(await this.mcpService.hasServer(server.name))) {
|
|
191
|
+
serversToInstall.push(server);
|
|
192
|
+
}
|
|
193
|
+
if (!(await this.mcpService.isServerStarted(server.name))) {
|
|
194
|
+
serversToStart.push(server);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
for (const server of serversToInstall) {
|
|
199
|
+
const currentServers = this.preferenceService.get<Record<string, MCPServerDescription>>(MCP_SERVERS_PREF, {});
|
|
200
|
+
await this.preferenceService.set(MCP_SERVERS_PREF, { ...currentServers, [server.name]: server }, PreferenceScope.User);
|
|
201
|
+
await this.mcpService.addOrUpdateServer(server);
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
for (const server of serversToStart) {
|
|
205
|
+
await this.mcpService.startServer(server.name);
|
|
206
|
+
}
|
|
207
|
+
} catch (error) {
|
|
208
|
+
this.logger.error(`Error starting MCP servers ${servers.map(s => s.name)}: ${error}`);
|
|
209
|
+
throw error;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
188
212
|
}
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
// *****************************************************************************
|
|
2
|
+
// Copyright (C) 2025 EclipseSource GmbH.
|
|
3
|
+
//
|
|
4
|
+
// This program and the accompanying materials are made available under the
|
|
5
|
+
// terms of the Eclipse Public License v. 2.0 which is available at
|
|
6
|
+
// http://www.eclipse.org/legal/epl-2.0.
|
|
7
|
+
//
|
|
8
|
+
// This Source Code may also be made available under the following Secondary
|
|
9
|
+
// Licenses when the conditions for such availability set forth in the Eclipse
|
|
10
|
+
// Public License v. 2.0 are satisfied: GNU General Public License, version 2
|
|
11
|
+
// with the GNU Classpath Exception which is available at
|
|
12
|
+
// https://www.gnu.org/software/classpath/license.html.
|
|
13
|
+
//
|
|
14
|
+
// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
|
|
15
|
+
// *****************************************************************************
|
|
16
|
+
import { type ToolProvider, type ToolRequest } from '@theia/ai-core';
|
|
17
|
+
import { isLocalMCPServerDescription, MCPServerManager } from '@theia/ai-mcp/lib/common';
|
|
18
|
+
import { inject, injectable } from '@theia/core/shared/inversify';
|
|
19
|
+
import { CLOSE_BROWSER_FUNCTION_ID, IS_BROWSER_RUNNING_FUNCTION_ID, LAUNCH_BROWSER_FUNCTION_ID, QUERY_DOM_FUNCTION_ID } from '../common/app-tester-chat-functions';
|
|
20
|
+
import { BrowserAutomation } from '../common/browser-automation-protocol';
|
|
21
|
+
|
|
22
|
+
@injectable()
|
|
23
|
+
export abstract class BrowserAutomationToolProvider implements ToolProvider {
|
|
24
|
+
@inject(BrowserAutomation)
|
|
25
|
+
protected readonly browser: BrowserAutomation;
|
|
26
|
+
|
|
27
|
+
abstract getTool(): ToolRequest;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
@injectable()
|
|
31
|
+
export class LaunchBrowserProvider extends BrowserAutomationToolProvider {
|
|
32
|
+
static ID = LAUNCH_BROWSER_FUNCTION_ID;
|
|
33
|
+
|
|
34
|
+
@inject(MCPServerManager)
|
|
35
|
+
protected readonly mcpServerManager: MCPServerManager;
|
|
36
|
+
|
|
37
|
+
getTool(): ToolRequest {
|
|
38
|
+
return {
|
|
39
|
+
id: LaunchBrowserProvider.ID,
|
|
40
|
+
name: LaunchBrowserProvider.ID,
|
|
41
|
+
description: 'Start the browser.',
|
|
42
|
+
parameters: {
|
|
43
|
+
type: 'object',
|
|
44
|
+
properties: {},
|
|
45
|
+
required: []
|
|
46
|
+
}, handler: async () => {
|
|
47
|
+
try {
|
|
48
|
+
|
|
49
|
+
const mcp = await this.mcpServerManager.getServerDescription('playwright');
|
|
50
|
+
if (!mcp) {
|
|
51
|
+
throw new Error('No MCP Playwright instance with name playwright found');
|
|
52
|
+
}
|
|
53
|
+
if (!isLocalMCPServerDescription(mcp)) {
|
|
54
|
+
throw new Error('The MCP Playwright instance must run locally.');
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const cdpEndpointIndex = mcp.args?.findIndex(p => p === '--cdp-endpoint');
|
|
58
|
+
if (!cdpEndpointIndex) {
|
|
59
|
+
throw new Error('No --cdp-endpoint was provided.');
|
|
60
|
+
}
|
|
61
|
+
const cdpEndpoint = mcp.args?.[cdpEndpointIndex + 1];
|
|
62
|
+
if (!cdpEndpoint) {
|
|
63
|
+
throw new Error('No --cdp-endpoint argument was provided.');
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
let remoteDebuggingPort = 9222;
|
|
67
|
+
try {
|
|
68
|
+
const uri = new URL(cdpEndpoint);
|
|
69
|
+
if (uri.port) {
|
|
70
|
+
remoteDebuggingPort = parseInt(uri.port, 10);
|
|
71
|
+
} else {
|
|
72
|
+
// Default ports if not specified
|
|
73
|
+
remoteDebuggingPort = uri.protocol === 'https:' ? 443 : 80;
|
|
74
|
+
}
|
|
75
|
+
} catch (error) {
|
|
76
|
+
throw new Error(`Invalid --cdp-endpoint format, URL expected: ${cdpEndpoint}`);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const result = await this.browser.launch(remoteDebuggingPort);
|
|
80
|
+
return result;
|
|
81
|
+
} catch (ex) {
|
|
82
|
+
return (`Failed to starting the browser: ${ex.message}`);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
@injectable()
|
|
90
|
+
export class CloseBrowserProvider extends BrowserAutomationToolProvider {
|
|
91
|
+
static ID = CLOSE_BROWSER_FUNCTION_ID;
|
|
92
|
+
|
|
93
|
+
getTool(): ToolRequest {
|
|
94
|
+
return {
|
|
95
|
+
id: CloseBrowserProvider.ID,
|
|
96
|
+
name: CloseBrowserProvider.ID,
|
|
97
|
+
description: 'Close the browser.',
|
|
98
|
+
parameters: {
|
|
99
|
+
type: 'object',
|
|
100
|
+
properties: {},
|
|
101
|
+
required: []
|
|
102
|
+
},
|
|
103
|
+
handler: async () => {
|
|
104
|
+
try {
|
|
105
|
+
await this.browser.close();
|
|
106
|
+
} catch (ex) {
|
|
107
|
+
return (`Failed to close browser: ${ex.message}`);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
@injectable()
|
|
115
|
+
export class IsBrowserRunningProvider extends BrowserAutomationToolProvider {
|
|
116
|
+
static ID = IS_BROWSER_RUNNING_FUNCTION_ID;
|
|
117
|
+
|
|
118
|
+
getTool(): ToolRequest {
|
|
119
|
+
return {
|
|
120
|
+
id: IsBrowserRunningProvider.ID,
|
|
121
|
+
name: IsBrowserRunningProvider.ID,
|
|
122
|
+
description: 'Check if the browser is running.',
|
|
123
|
+
parameters: {
|
|
124
|
+
type: 'object',
|
|
125
|
+
properties: {},
|
|
126
|
+
required: []
|
|
127
|
+
},
|
|
128
|
+
handler: async () => {
|
|
129
|
+
try {
|
|
130
|
+
const isRunning = await this.browser.isRunning();
|
|
131
|
+
return isRunning ? 'Browser is running.' : 'Browser is not running.';
|
|
132
|
+
} catch (ex) {
|
|
133
|
+
return (`Failed to check if browser is running: ${ex.message}`);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
@injectable()
|
|
141
|
+
export class QueryDomProvider extends BrowserAutomationToolProvider {
|
|
142
|
+
static ID = QUERY_DOM_FUNCTION_ID;
|
|
143
|
+
|
|
144
|
+
getTool(): ToolRequest {
|
|
145
|
+
return {
|
|
146
|
+
id: QueryDomProvider.ID,
|
|
147
|
+
name: QueryDomProvider.ID,
|
|
148
|
+
description: 'Query the DOM of the active page.',
|
|
149
|
+
parameters: {
|
|
150
|
+
type: 'object',
|
|
151
|
+
properties: {
|
|
152
|
+
selector: {
|
|
153
|
+
type: 'string',
|
|
154
|
+
description: `The selector of the element to get the DOM of. The selector is a
|
|
155
|
+
CSS selector that identifies the element. If not provided, the entire DOM will be returned.`
|
|
156
|
+
}
|
|
157
|
+
},
|
|
158
|
+
required: []
|
|
159
|
+
},
|
|
160
|
+
handler: async arg => {
|
|
161
|
+
try {
|
|
162
|
+
const { selector } = JSON.parse(arg);
|
|
163
|
+
return await this.browser.queryDom(selector);
|
|
164
|
+
} catch (ex) {
|
|
165
|
+
return (`Failed to get DOM: ${ex.message}`);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
}
|
|
@@ -24,7 +24,14 @@ import { CoderAgent } from './coder-agent';
|
|
|
24
24
|
import { SummarizeSessionCommandContribution } from './summarize-session-command-contribution';
|
|
25
25
|
import { FileContentFunction, FileDiagnosticProvider, GetWorkspaceDirectoryStructure, GetWorkspaceFileList, WorkspaceFunctionScope } from './workspace-functions';
|
|
26
26
|
import { WorkspaceSearchProvider } from './workspace-search-provider';
|
|
27
|
-
import {
|
|
27
|
+
import {
|
|
28
|
+
FrontendApplicationContribution,
|
|
29
|
+
PreferenceContribution,
|
|
30
|
+
WidgetFactory,
|
|
31
|
+
bindViewContribution,
|
|
32
|
+
RemoteConnectionProvider,
|
|
33
|
+
ServiceConnectionProvider
|
|
34
|
+
} from '@theia/core/lib/browser';
|
|
28
35
|
import { TaskListProvider, TaskRunnerProvider } from './workspace-task-provider';
|
|
29
36
|
import { WorkspacePreferencesSchema } from './workspace-preferences';
|
|
30
37
|
import {
|
|
@@ -62,6 +69,8 @@ import { TaskContextFileStorageService } from './task-context-file-storage-servi
|
|
|
62
69
|
import { TaskContextStorageService } from '@theia/ai-chat/lib/browser/task-context-service';
|
|
63
70
|
import { CommandContribution } from '@theia/core';
|
|
64
71
|
import { AIPromptFragmentsConfigurationWidget } from './ai-configuration/prompt-fragments-configuration-widget';
|
|
72
|
+
import { BrowserAutomation, browserAutomationPath } from '../common/browser-automation-protocol';
|
|
73
|
+
import { CloseBrowserProvider, IsBrowserRunningProvider, LaunchBrowserProvider, QueryDomProvider } from './app-tester-chat-functions';
|
|
65
74
|
|
|
66
75
|
export default new ContainerModule((bind, _unbind, _isBound, rebind) => {
|
|
67
76
|
bind(PreferenceContribution).toConstantValue({ schema: WorkspacePreferencesSchema });
|
|
@@ -85,6 +94,10 @@ export default new ContainerModule((bind, _unbind, _isBound, rebind) => {
|
|
|
85
94
|
bind(AppTesterChatAgent).toSelf().inSingletonScope();
|
|
86
95
|
bind(Agent).toService(AppTesterChatAgent);
|
|
87
96
|
bind(ChatAgent).toService(AppTesterChatAgent);
|
|
97
|
+
bind(BrowserAutomation).toDynamicValue(ctx => {
|
|
98
|
+
const provider = ctx.container.get<ServiceConnectionProvider>(RemoteConnectionProvider);
|
|
99
|
+
return provider.createProxy<BrowserAutomation>(browserAutomationPath);
|
|
100
|
+
}).inSingletonScope();
|
|
88
101
|
|
|
89
102
|
bind(CommandChatAgent).toSelf().inSingletonScope();
|
|
90
103
|
bind(Agent).toService(CommandChatAgent);
|
|
@@ -120,6 +133,11 @@ export default new ContainerModule((bind, _unbind, _isBound, rebind) => {
|
|
|
120
133
|
}))
|
|
121
134
|
.inSingletonScope();
|
|
122
135
|
|
|
136
|
+
bindToolProvider(LaunchBrowserProvider, bind);
|
|
137
|
+
bindToolProvider(CloseBrowserProvider, bind);
|
|
138
|
+
bindToolProvider(IsBrowserRunningProvider, bind);
|
|
139
|
+
bindToolProvider(QueryDomProvider, bind);
|
|
140
|
+
|
|
123
141
|
bindViewContribution(bind, AIAgentConfigurationViewContribution);
|
|
124
142
|
bind(TabBarToolbarContribution).toService(AIAgentConfigurationViewContribution);
|
|
125
143
|
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
// *****************************************************************************
|
|
2
|
+
// Copyright (C) 2025 EclipseSource GmbH.
|
|
3
|
+
//
|
|
4
|
+
// This program and the accompanying materials are made available under the
|
|
5
|
+
// terms of the Eclipse Public License v. 2.0 which is available at
|
|
6
|
+
// http://www.eclipse.org/legal/epl-2.0.
|
|
7
|
+
//
|
|
8
|
+
// This Source Code may also be made available under the following Secondary
|
|
9
|
+
// Licenses when the conditions for such availability set forth in the Eclipse
|
|
10
|
+
// Public License v. 2.0 are satisfied: GNU General Public License, version 2
|
|
11
|
+
// with the GNU Classpath Exception which is available at
|
|
12
|
+
// https://www.gnu.org/software/classpath/license.html.
|
|
13
|
+
//
|
|
14
|
+
// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
|
|
15
|
+
// *****************************************************************************
|
|
16
|
+
|
|
17
|
+
export const LAUNCH_BROWSER_FUNCTION_ID = 'launchBrowser';
|
|
18
|
+
export const IS_BROWSER_RUNNING_FUNCTION_ID = 'isBrowserRunning';
|
|
19
|
+
export const CLOSE_BROWSER_FUNCTION_ID = 'closeBrowser';
|
|
20
|
+
export const QUERY_DOM_FUNCTION_ID = 'queryDom';
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
// *****************************************************************************
|
|
2
|
+
// Copyright (C) 2025 EclipseSource GmbH.
|
|
3
|
+
//
|
|
4
|
+
// This program and the accompanying materials are made available under the
|
|
5
|
+
// terms of the Eclipse Public License v. 2.0 which is available at
|
|
6
|
+
// http://www.eclipse.org/legal/epl-2.0.
|
|
7
|
+
//
|
|
8
|
+
// This Source Code may also be made available under the following Secondary
|
|
9
|
+
// Licenses when the conditions for such availability set forth in the Eclipse
|
|
10
|
+
// Public License v. 2.0 are satisfied: GNU General Public License, version 2
|
|
11
|
+
// with the GNU Classpath Exception which is available at
|
|
12
|
+
// https://www.gnu.org/software/classpath/license.html.
|
|
13
|
+
//
|
|
14
|
+
// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
|
|
15
|
+
// *****************************************************************************
|
|
16
|
+
|
|
17
|
+
export const browserAutomationPath = '/services/automation/browser';
|
|
18
|
+
export const BrowserAutomation = Symbol('BrowserAutomation');
|
|
19
|
+
export interface BrowserAutomation {
|
|
20
|
+
launch(remoteDebuggingPort: number): Promise<LaunchResult | undefined>;
|
|
21
|
+
isRunning(): Promise<boolean>;
|
|
22
|
+
queryDom(selector?: string): Promise<string>;
|
|
23
|
+
close(): Promise<void>;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface LaunchResult {
|
|
27
|
+
remoteDebuggingPort: number;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export const BrowserAutomationClient = Symbol('BrowserAutomationClient');
|
|
31
|
+
export interface BrowserAutomationClient {
|
|
32
|
+
}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
// *****************************************************************************
|
|
2
|
+
// Copyright (C) 2025 EclipseSource GmbH.
|
|
3
|
+
//
|
|
4
|
+
// This program and the accompanying materials are made available under the
|
|
5
|
+
// terms of the Eclipse Public License v. 2.0 which is available at
|
|
6
|
+
// http://www.eclipse.org/legal/epl-2.0.
|
|
7
|
+
//
|
|
8
|
+
// This Source Code may also be made available under the following Secondary
|
|
9
|
+
// Licenses when the conditions for such availability set forth in the Eclipse
|
|
10
|
+
// Public License v. 2.0 are satisfied: GNU General Public License, version 2
|
|
11
|
+
// with the GNU Classpath Exception which is available at
|
|
12
|
+
// https://www.gnu.org/software/classpath/license.html.
|
|
13
|
+
//
|
|
14
|
+
// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
|
|
15
|
+
// *****************************************************************************
|
|
16
|
+
|
|
17
|
+
import { type RpcServer } from '@theia/core';
|
|
18
|
+
import { injectable } from '@theia/core/shared/inversify';
|
|
19
|
+
import { Browser, launch, Page } from 'puppeteer-core';
|
|
20
|
+
import type { BrowserAutomation, BrowserAutomationClient, LaunchResult } from '../../common/browser-automation-protocol';
|
|
21
|
+
|
|
22
|
+
const MAX_DOM_LENGTH = 50000;
|
|
23
|
+
|
|
24
|
+
@injectable()
|
|
25
|
+
export class BrowserAutomationImpl implements RpcServer<BrowserAutomationClient>, BrowserAutomation {
|
|
26
|
+
protected _browser?: Browser;
|
|
27
|
+
protected _page?: Page;
|
|
28
|
+
protected client?: BrowserAutomationClient;
|
|
29
|
+
|
|
30
|
+
protected get browser(): Browser {
|
|
31
|
+
if (!this._browser) {
|
|
32
|
+
throw new Error('Browser is not launched');
|
|
33
|
+
}
|
|
34
|
+
return this._browser;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
protected get page(): Page {
|
|
38
|
+
if (!this._page) {
|
|
39
|
+
throw new Error('Page is not created');
|
|
40
|
+
}
|
|
41
|
+
return this._page;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
async isRunning(): Promise<boolean> {
|
|
45
|
+
return this._browser !== undefined && this._browser.connected;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
async launch(remoteDebuggingPort: number): Promise<LaunchResult | undefined> {
|
|
49
|
+
if (this._browser) {
|
|
50
|
+
await this.close();
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const browser = await launch({
|
|
54
|
+
headless: false,
|
|
55
|
+
channel: 'chrome',
|
|
56
|
+
args: [
|
|
57
|
+
`--remote-debugging-port=${remoteDebuggingPort}`
|
|
58
|
+
],
|
|
59
|
+
});
|
|
60
|
+
this._browser = browser;
|
|
61
|
+
// The initial page will be used per default
|
|
62
|
+
this._page = (await browser.pages())[0];
|
|
63
|
+
return {
|
|
64
|
+
remoteDebuggingPort
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
async close(): Promise<void> {
|
|
69
|
+
await this._browser?.close();
|
|
70
|
+
this._browser = undefined;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
async queryDom(selector?: string): Promise<string> {
|
|
74
|
+
const page = this.page;
|
|
75
|
+
let content = '';
|
|
76
|
+
|
|
77
|
+
if (selector) {
|
|
78
|
+
const element = await page.$(selector);
|
|
79
|
+
if (!element) {
|
|
80
|
+
throw new Error(`Element with selector "${selector}" not found`);
|
|
81
|
+
}
|
|
82
|
+
content = await page.evaluate(el => el.outerHTML, element);
|
|
83
|
+
} else {
|
|
84
|
+
content = await page.content();
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
if (content.length > MAX_DOM_LENGTH) {
|
|
88
|
+
return 'The queried DOM is too large. Please provide a more specific query.';
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
return content;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
dispose(): void {
|
|
95
|
+
this._browser?.close();
|
|
96
|
+
this._browser = undefined;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
setClient(client: BrowserAutomationClient | undefined): void {
|
|
100
|
+
this.client = client;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
getClient?(): BrowserAutomationClient | undefined {
|
|
104
|
+
return this.client;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
}
|