explorbot 0.2.2 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/bin/explorbot-cli.ts +52 -37
- package/boat/api-tester/src/apibot.ts +4 -2
- package/boat/api-tester/src/cli.ts +2 -2
- package/boat/api-tester/src/config.ts +39 -8
- package/boat/doc-collector/src/cli.ts +1 -0
- package/boat/doc-collector/src/docs-renderer.ts +18 -4
- package/boat/doc-collector/src/state-diagram.ts +61 -14
- package/boat/prima/bin/prima-cli.ts +5 -0
- package/boat/prima/package.json +16 -0
- package/boat/prima/src/cli.ts +222 -0
- package/boat/prima/src/envelope.ts +141 -0
- package/boat/prima/src/prima.ts +705 -0
- package/boat/prima/src/pw-parser.ts +17 -0
- package/boat/prima/src/pw-registry.ts +75 -0
- package/dist/bin/explorbot-cli.js +44 -31
- package/dist/boat/api-tester/src/apibot.js +3 -2
- package/dist/boat/api-tester/src/cli.js +2 -2
- package/dist/boat/api-tester/src/config.js +36 -8
- package/dist/boat/doc-collector/src/cli.js +1 -0
- package/dist/boat/doc-collector/src/docs-renderer.js +17 -3
- package/dist/boat/doc-collector/src/state-diagram.js +57 -13
- package/dist/boat/prima/bin/prima-cli.js +4 -0
- package/dist/boat/prima/src/cli.js +200 -0
- package/dist/boat/prima/src/envelope.js +116 -0
- package/dist/boat/prima/src/prima.js +635 -0
- package/dist/boat/prima/src/pw-parser.js +18 -0
- package/dist/boat/prima/src/pw-registry.js +66 -0
- package/dist/models.json +3 -0
- package/dist/package.json +6 -2
- package/dist/src/action.d.ts +5 -2
- package/dist/src/action.js +5 -5
- package/dist/src/ai/captain/mixin.js +3 -4
- package/dist/src/ai/captain/web-mode.js +1 -1
- package/dist/src/ai/navigator.d.ts +4 -0
- package/dist/src/ai/navigator.js +11 -6
- package/dist/src/ai/planner.d.ts +1 -0
- package/dist/src/ai/planner.js +6 -0
- package/dist/src/ai/researcher.js +1 -1
- package/dist/src/ai/task-agent.js +1 -1
- package/dist/src/ai/tester.d.ts +1 -0
- package/dist/src/ai/tester.js +13 -0
- package/dist/src/application-spec-contract.d.ts +8 -0
- package/dist/src/application-spec-contract.js +8 -0
- package/dist/src/application-spec.d.ts +15 -0
- package/dist/src/application-spec.js +71 -0
- package/dist/src/browser-server.d.ts +12 -6
- package/dist/src/browser-server.js +74 -19
- package/dist/src/commands/clean-command.js +2 -7
- package/dist/src/commands/init-command.d.ts +5 -0
- package/dist/src/commands/init-command.js +119 -1
- package/dist/src/commands/navigate-command.js +1 -1
- package/dist/src/commands/research-command.js +1 -1
- package/dist/src/commands/sites-command.d.ts +6 -0
- package/dist/src/commands/sites-command.js +23 -0
- package/dist/src/components/InitWizard.d.ts +10 -0
- package/dist/src/components/InitWizard.js +133 -0
- package/dist/src/components/InputReadline.d.ts +1 -0
- package/dist/src/components/InputReadline.js +7 -4
- package/dist/src/config.d.ts +24 -5
- package/dist/src/config.js +146 -37
- package/dist/src/explorbot.d.ts +9 -0
- package/dist/src/explorbot.js +24 -5
- package/dist/src/explorer.d.ts +4 -1
- package/dist/src/explorer.js +40 -6
- package/dist/src/global-config.d.ts +22 -0
- package/dist/src/global-config.js +117 -0
- package/dist/src/knowledge-tracker.d.ts +5 -1
- package/dist/src/knowledge-tracker.js +14 -1
- package/dist/src/utils/cli-name.js +6 -2
- package/dist/src/utils/test-files.js +1 -2
- package/dist/src/utils/url-matcher.d.ts +1 -0
- package/dist/src/utils/url-matcher.js +9 -0
- package/models.json +3 -0
- package/package.json +6 -2
- package/src/action.ts +9 -5
- package/src/ai/captain/mixin.ts +3 -3
- package/src/ai/captain/web-mode.ts +1 -1
- package/src/ai/navigator.ts +12 -7
- package/src/ai/planner.ts +7 -0
- package/src/ai/researcher.ts +1 -1
- package/src/ai/task-agent.ts +1 -1
- package/src/ai/tester.ts +15 -0
- package/src/application-spec-contract.ts +10 -0
- package/src/application-spec.ts +87 -0
- package/src/browser-server.ts +74 -19
- package/src/commands/clean-command.ts +1 -6
- package/src/commands/init-command.ts +146 -1
- package/src/commands/navigate-command.ts +1 -1
- package/src/commands/research-command.ts +1 -1
- package/src/commands/sites-command.ts +27 -0
- package/src/components/InitWizard.tsx +166 -0
- package/src/components/InputReadline.tsx +8 -4
- package/src/config.ts +162 -39
- package/src/explorbot.ts +30 -5
- package/src/explorer.ts +45 -7
- package/src/global-config.ts +148 -0
- package/src/knowledge-tracker.ts +17 -1
- package/src/utils/cli-name.ts +5 -2
- package/src/utils/test-files.ts +1 -2
- package/src/utils/url-matcher.ts +10 -0
|
@@ -0,0 +1,705 @@
|
|
|
1
|
+
import { createRequire } from 'node:module';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import dedent from 'dedent';
|
|
4
|
+
import * as playwright from 'playwright';
|
|
5
|
+
import type { Browser } from 'playwright';
|
|
6
|
+
import { ActionResult } from '../../../src/action-result.ts';
|
|
7
|
+
import type { Navigator } from '../../../src/ai/navigator.ts';
|
|
8
|
+
import { actionRule, locatorRule } from '../../../src/ai/rules.ts';
|
|
9
|
+
import { createCodeceptJSTools } from '../../../src/ai/tools.ts';
|
|
10
|
+
import { getAliveEndpoint, launchServer, listInstances, stopServer } from '../../../src/browser-server.ts';
|
|
11
|
+
import { ConfigMissingError, ConfigParser, type ExplorbotConfig, outputPath } from '../../../src/config.ts';
|
|
12
|
+
import { ExplorBot } from '../../../src/explorbot.ts';
|
|
13
|
+
import type { WebPageState } from '../../../src/state-manager.ts';
|
|
14
|
+
import { Task } from '../../../src/test-plan.ts';
|
|
15
|
+
import { compactAriaSnapshot } from '../../../src/utils/aria.ts';
|
|
16
|
+
import { browserErrorMessage } from '../../../src/utils/browser-errors.ts';
|
|
17
|
+
import { pluralize } from '../../../src/utils/logger.ts';
|
|
18
|
+
import { type EnvelopeData, type HealAttempt, type InstanceInfo, writeArtifacts } from './envelope.ts';
|
|
19
|
+
import { isFunctionExpression, toCodeceptWrapper } from './pw-parser.ts';
|
|
20
|
+
import { type PwServerDescriptor, readDescriptors, selectDescriptor } from './pw-registry.ts';
|
|
21
|
+
|
|
22
|
+
const MAX_INSTRUCTION_ITERATIONS = 6;
|
|
23
|
+
const MAX_TOOL_ROUNDTRIPS = 5;
|
|
24
|
+
const AI_AGENT_NAME = 'prima';
|
|
25
|
+
const CONNECT_TIMEOUT = 3000;
|
|
26
|
+
const requireLib = createRequire(import.meta.url);
|
|
27
|
+
|
|
28
|
+
export class Prima {
|
|
29
|
+
private options: PrimaOptions;
|
|
30
|
+
private bot: ExplorBot;
|
|
31
|
+
private artifactsDir?: string;
|
|
32
|
+
private server: { close: () => Promise<void> } | null = null;
|
|
33
|
+
private attached: string | null = null;
|
|
34
|
+
|
|
35
|
+
constructor(options: PrimaOptions = {}) {
|
|
36
|
+
this.options = options;
|
|
37
|
+
this.bot = new ExplorBot({
|
|
38
|
+
config: options.config,
|
|
39
|
+
path: options.path,
|
|
40
|
+
baseUrl: this.configBaseUrl(),
|
|
41
|
+
verbose: options.verbose,
|
|
42
|
+
session: options.session,
|
|
43
|
+
instance: options.instance,
|
|
44
|
+
headless: true,
|
|
45
|
+
optionalAi: true,
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
async start(): Promise<void> {
|
|
50
|
+
const config = await this.loadConfig();
|
|
51
|
+
await this.resolveBrowser(config);
|
|
52
|
+
await this.bot.start();
|
|
53
|
+
|
|
54
|
+
if (!this.options.url) return;
|
|
55
|
+
if (this.bot.getCurrentState()) return;
|
|
56
|
+
await this.bot.visit(this.options.url);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
async stop(): Promise<void> {
|
|
60
|
+
await this.bot.stop();
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async pw(expression: string): Promise<EnvelopeData> {
|
|
64
|
+
const command = `pw ${expression}`;
|
|
65
|
+
const validation = isFunctionExpression(expression);
|
|
66
|
+
if (!validation.valid) return this.toolFailureEnvelope(command, validation.error!);
|
|
67
|
+
|
|
68
|
+
const previousState = this.bot.stateManager().getCurrentState();
|
|
69
|
+
let result: ActionResult | null = null;
|
|
70
|
+
let executionError: unknown = null;
|
|
71
|
+
|
|
72
|
+
try {
|
|
73
|
+
const executed = await this.bot.getExplorer().action().execute(toCodeceptWrapper(expression), { verbatim: true });
|
|
74
|
+
result = executed.actionResult;
|
|
75
|
+
} catch (error) {
|
|
76
|
+
executionError = error;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
if (executionError) return this.heal(command, expression, executionError, previousState);
|
|
80
|
+
|
|
81
|
+
result ||= await this.capturedResult(previousState);
|
|
82
|
+
return this.successEnvelope(command, [expression], result, previousState);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
async do(instructions: string[]): Promise<EnvelopeData> {
|
|
86
|
+
const command = `do ${instructions.map((instruction) => `"${instruction}"`).join(' ')}`;
|
|
87
|
+
const guard = await this.aiGuard(command);
|
|
88
|
+
if (guard) return guard;
|
|
89
|
+
|
|
90
|
+
const provider = this.bot.getProvider();
|
|
91
|
+
const previousState = this.bot.stateManager().getCurrentState();
|
|
92
|
+
const conversation = provider.startConversation(this.instructionSystemPrompt(), AI_AGENT_NAME);
|
|
93
|
+
const task = new Task(instructions.join('; '), previousState?.url || '');
|
|
94
|
+
const tools = createCodeceptJSTools({ explorer: this.bot.getExplorer(), stateManager: this.bot.stateManager(), ai: provider }, task);
|
|
95
|
+
conversation.addUserText(this.instructionPrompt(instructions, await this.capturedResult(previousState)));
|
|
96
|
+
|
|
97
|
+
const used: string[] = [];
|
|
98
|
+
let failure: { code: string; message: string } | null = null;
|
|
99
|
+
let aiError: unknown = null;
|
|
100
|
+
let narration = '';
|
|
101
|
+
let contextHash = this.bot.stateManager().getCurrentState()?.hash;
|
|
102
|
+
|
|
103
|
+
for (let iteration = 1; iteration <= Math.min(instructions.length + 2, MAX_INSTRUCTION_ITERATIONS); iteration++) {
|
|
104
|
+
const state = this.bot.stateManager().getCurrentState();
|
|
105
|
+
if (iteration > 1 && state && state.hash !== contextHash) {
|
|
106
|
+
contextHash = state.hash;
|
|
107
|
+
conversation.addUserText(this.pageContext(ActionResult.fromState(state)));
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
const invoked = await provider.invokeConversation(conversation, tools, { maxToolRoundtrips: MAX_TOOL_ROUNDTRIPS, agentName: AI_AGENT_NAME }).catch((error: unknown) => {
|
|
111
|
+
aiError = error;
|
|
112
|
+
return null;
|
|
113
|
+
});
|
|
114
|
+
if (!invoked) break;
|
|
115
|
+
|
|
116
|
+
const executions = invoked.toolExecutions || [];
|
|
117
|
+
if (!executions.length) {
|
|
118
|
+
narration = invoked.response?.text?.trim() || '';
|
|
119
|
+
break;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
for (const execution of executions) {
|
|
123
|
+
if (!execution.wasSuccessful) {
|
|
124
|
+
failure = { code: execution.output?.code || '', message: execution.output?.message || 'action failed' };
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
used.push(...this.executedCodes(execution.output?.code));
|
|
128
|
+
failure = null;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
if (aiError) return this.failureEnvelope(command, aiError, previousState);
|
|
133
|
+
|
|
134
|
+
if (failure) {
|
|
135
|
+
const envelope = await this.heal(command, failure.code || instructions.join('; '), failure.message, previousState);
|
|
136
|
+
envelope.used = [...used, ...(envelope.used || [])];
|
|
137
|
+
return envelope;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
if (!used.length) {
|
|
141
|
+
const reason = ['No action was performed for these instructions on the current page.', narration].filter(Boolean).join(' ');
|
|
142
|
+
return this.failureEnvelope(command, reason, previousState);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
const result = await this.capturedResult(this.bot.stateManager().getCurrentState());
|
|
146
|
+
return this.successEnvelope(command, used, result, previousState);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
async click(target: string): Promise<EnvelopeData> {
|
|
150
|
+
return this.do([`click ${target}`]);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
async fill(field: string, value: string): Promise<EnvelopeData> {
|
|
154
|
+
return this.do([`fill ${field} with value: ${value}`]);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
async ask(question: string): Promise<EnvelopeData> {
|
|
158
|
+
const command = `ask ${question}`;
|
|
159
|
+
const guard = await this.aiGuard(command);
|
|
160
|
+
if (guard) return guard;
|
|
161
|
+
|
|
162
|
+
const previousState = this.bot.stateManager().getCurrentState();
|
|
163
|
+
const result = await this.capturedResult(previousState, { screenshot: this.visionEnabled() });
|
|
164
|
+
return this.reportEnvelope(command, result, previousState, { answer: await this.answer(question, result) });
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
async verify(assertion: string): Promise<EnvelopeData> {
|
|
168
|
+
const command = `verify ${assertion}`;
|
|
169
|
+
const guard = await this.aiGuard(command);
|
|
170
|
+
if (guard) return guard;
|
|
171
|
+
|
|
172
|
+
const previousState = this.bot.stateManager().getCurrentState();
|
|
173
|
+
const result = await this.capturedResult(previousState);
|
|
174
|
+
const verification = await this.bot.agentNavigator().verifyState(assertion, result);
|
|
175
|
+
const codes = verification.successfulCodes || [];
|
|
176
|
+
const verdict = { passed: verification.verified, evidence: this.verdictEvidence(verification.verified, codes), code: codes.join('\n') };
|
|
177
|
+
return this.reportEnvelope(command, result, previousState, { ok: verification.verified, verdict });
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
async research(opts: { data?: boolean; deep?: boolean; fresh?: boolean } = {}): Promise<EnvelopeData> {
|
|
181
|
+
const flags = [opts.data && '--data', opts.deep && '--deep', opts.fresh && '--fresh'].filter(Boolean);
|
|
182
|
+
const command = ['research', ...flags].join(' ');
|
|
183
|
+
const guard = await this.aiGuard(command);
|
|
184
|
+
if (guard) return guard;
|
|
185
|
+
|
|
186
|
+
const previousState = this.bot.stateManager().getCurrentState();
|
|
187
|
+
const result = await this.capturedResult(previousState);
|
|
188
|
+
const uiMap = await this.bot.agentResearcher().research(result, { screenshot: true, data: opts.data, deep: opts.deep, force: opts.fresh });
|
|
189
|
+
return this.reportEnvelope(command, result, previousState, { research: uiMap });
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
async go(target: string): Promise<EnvelopeData> {
|
|
193
|
+
const command = `go ${target}`;
|
|
194
|
+
const code = `I.amOnPage('${target}')`;
|
|
195
|
+
const isUrl = this.isUrlTarget(target);
|
|
196
|
+
|
|
197
|
+
if (!isUrl) {
|
|
198
|
+
const guard = await this.aiGuard(command);
|
|
199
|
+
if (guard) return guard;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
const previousState = this.bot.stateManager().getCurrentState();
|
|
203
|
+
let navigationError: unknown = null;
|
|
204
|
+
|
|
205
|
+
try {
|
|
206
|
+
await this.bot.agentNavigator().visit(target);
|
|
207
|
+
} catch (error) {
|
|
208
|
+
navigationError = error;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
if (navigationError) return this.heal(command, code, navigationError, previousState);
|
|
212
|
+
|
|
213
|
+
const used: string[] = [];
|
|
214
|
+
if (isUrl) used.push(code);
|
|
215
|
+
const result = await this.capturedResult(this.bot.stateManager().getCurrentState());
|
|
216
|
+
return this.successEnvelope(command, used, result, previousState);
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
async browserStart(): Promise<void> {
|
|
220
|
+
const config = await this.loadConfig();
|
|
221
|
+
let show = config.playwright.show || false;
|
|
222
|
+
if (this.options.show) show = true;
|
|
223
|
+
if (this.options.headless) show = false;
|
|
224
|
+
this.server = await this.launchOwnServer({ browser: config.playwright.browser, show }, this.instanceName());
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
async browserStop(all = false): Promise<boolean> {
|
|
228
|
+
await this.loadConfig();
|
|
229
|
+
if (this.attached) return false;
|
|
230
|
+
|
|
231
|
+
if (!all) return this.stopInstance(this.instanceName());
|
|
232
|
+
|
|
233
|
+
let stopped = false;
|
|
234
|
+
for (const instance of listInstances()) {
|
|
235
|
+
if (await this.stopInstance(instance.name)) stopped = true;
|
|
236
|
+
}
|
|
237
|
+
return stopped;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
async browserStatus(): Promise<string> {
|
|
241
|
+
await this.loadConfig();
|
|
242
|
+
const info = await this.instanceInfo();
|
|
243
|
+
const endpoint = await getAliveEndpoint(info.name);
|
|
244
|
+
const others = info.others.map((other) => other.name).join(', ') || 'none';
|
|
245
|
+
const lines = [`instance: ${info.name} (${info.tabs} ${pluralize(info.tabs, 'tab')}) | other instances: ${others}`];
|
|
246
|
+
if (!endpoint) lines.push('browser: not running');
|
|
247
|
+
if (endpoint) lines.push(`browser: running at ${endpoint}`);
|
|
248
|
+
return lines.join('\n');
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
async browserList(): Promise<string> {
|
|
252
|
+
await this.loadConfig();
|
|
253
|
+
|
|
254
|
+
const lines: string[] = [];
|
|
255
|
+
for (const instance of listInstances()) {
|
|
256
|
+
const endpoint = await getAliveEndpoint(instance.name);
|
|
257
|
+
if (!endpoint) continue;
|
|
258
|
+
lines.push(`prima --instance ${instance.name} ${endpoint}`);
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
const discovery = await this.discover();
|
|
262
|
+
await discovery.browser?.close().catch(() => {});
|
|
263
|
+
for (const descriptor of discovery.candidates) {
|
|
264
|
+
lines.push(`playwright-cli --pw-session ${descriptor.title} ${descriptor.endpoint}`);
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
if (!lines.length) return 'no browser instances running';
|
|
268
|
+
return lines.join('\n');
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
async instanceInfo(): Promise<InstanceInfo> {
|
|
272
|
+
const name = this.instanceName();
|
|
273
|
+
const others = listInstances()
|
|
274
|
+
.filter((instance) => instance.name !== name)
|
|
275
|
+
.map((instance) => ({ name: instance.name, tabs: 0 }));
|
|
276
|
+
|
|
277
|
+
const info: InstanceInfo = { name, tabs: this.tabCount(), others };
|
|
278
|
+
if (this.attached) info.attached = this.attached;
|
|
279
|
+
return info;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
async toolFailureEnvelope(command: string, error: unknown): Promise<EnvelopeData> {
|
|
283
|
+
const state = this.bot.getCurrentState();
|
|
284
|
+
const instance = await this.instanceInfo().catch(() => ({ name: this.instanceName(), tabs: 0, others: [] }));
|
|
285
|
+
const failure: EnvelopeData['failure'] = { error: `tool: ${browserErrorMessage(error)}`, attempts: [] };
|
|
286
|
+
if (error instanceof ConfigMissingError) failure.error = browserErrorMessage(error);
|
|
287
|
+
if (state?.ariaSnapshot) failure.compactAria = compactAriaSnapshot(state.ariaSnapshot, true);
|
|
288
|
+
|
|
289
|
+
return {
|
|
290
|
+
ok: false,
|
|
291
|
+
command,
|
|
292
|
+
page: { url: state?.url || '', title: state?.title || '', state: state?.hash || '', visits: 0 },
|
|
293
|
+
failure,
|
|
294
|
+
instance,
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
private async loadConfig(): Promise<ExplorbotConfig> {
|
|
299
|
+
return ConfigParser.getInstance().loadConfig({ config: this.options.config, path: this.options.path, baseUrl: this.configBaseUrl() });
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
private configBaseUrl(): string | undefined {
|
|
303
|
+
const url = this.options.baseUrl || this.options.url;
|
|
304
|
+
if (!url) return undefined;
|
|
305
|
+
if (!URL.canParse(url)) return undefined;
|
|
306
|
+
return url;
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
private async resolveBrowser(config: ExplorbotConfig): Promise<void> {
|
|
310
|
+
if (this.options.endpoint) {
|
|
311
|
+
const endpoint = this.options.endpoint;
|
|
312
|
+
const browserName = config.playwright.browser || 'chromium';
|
|
313
|
+
if (await this.attachToEndpoint({ file: '', title: '', endpoint, workspaceDir: '', browserName, playwrightLib: '' })) return;
|
|
314
|
+
throw new Error(dedent`
|
|
315
|
+
No browser answered at ${endpoint}.
|
|
316
|
+
Check the endpoint of the running session, or drop --endpoint to attach to the
|
|
317
|
+
playwright-cli browser of this workspace.
|
|
318
|
+
`);
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
const { match, candidates, browser } = await this.discover();
|
|
322
|
+
if (match && (await this.attachToEndpoint(match, browser))) return;
|
|
323
|
+
|
|
324
|
+
if (!match && candidates.length) {
|
|
325
|
+
const titles = candidates.map((candidate) => candidate.title).join(', ');
|
|
326
|
+
throw new Error(dedent`
|
|
327
|
+
Several playwright-cli sessions are open for this workspace: ${titles}
|
|
328
|
+
Pick one with --pw-session <title>.
|
|
329
|
+
`);
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
if (await this.connectOwnInstance()) return;
|
|
333
|
+
|
|
334
|
+
throw new Error(dedent`
|
|
335
|
+
No browser to drive for instance "${this.instanceName()}".
|
|
336
|
+
Open one first:
|
|
337
|
+
playwright-cli open <url> prima attaches to this workspace session by default
|
|
338
|
+
prima browser start starts a prima-owned browser instead
|
|
339
|
+
Prima never launches a browser implicitly.
|
|
340
|
+
`);
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
private async discover(descriptors = readDescriptors()): Promise<Discovery> {
|
|
344
|
+
const title = this.options.pwSession ?? process.env.PLAYWRIGHT_CLI_SESSION;
|
|
345
|
+
const opts = { workspaceDir: this.workspaceDir(), title };
|
|
346
|
+
|
|
347
|
+
const alive = new Map<PwServerDescriptor, Browser>();
|
|
348
|
+
for (const candidate of selectDescriptor(descriptors, opts).candidates) {
|
|
349
|
+
const browser = await this.connectDescriptor(candidate);
|
|
350
|
+
if (browser) alive.set(candidate, browser);
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
const selected = selectDescriptor([...alive.keys()], opts);
|
|
354
|
+
const discovery: Discovery = { match: selected.match, candidates: selected.candidates };
|
|
355
|
+
if (selected.match) discovery.browser = alive.get(selected.match);
|
|
356
|
+
|
|
357
|
+
for (const [descriptor, browser] of alive) {
|
|
358
|
+
if (descriptor === selected.match) continue;
|
|
359
|
+
await browser.close().catch(() => {});
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
return discovery;
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
private async attachToEndpoint(descriptor: PwServerDescriptor, probed?: Browser): Promise<boolean> {
|
|
366
|
+
const browser = probed || (await this.connectDescriptor(descriptor));
|
|
367
|
+
if (!browser) return false;
|
|
368
|
+
|
|
369
|
+
this.bot.attachBrowser(browser);
|
|
370
|
+
this.attached = this.attachmentLabel(descriptor);
|
|
371
|
+
return true;
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
private async connectDescriptor(descriptor: PwServerDescriptor): Promise<Browser | null> {
|
|
375
|
+
const connected = await this.connectWith(playwright, descriptor);
|
|
376
|
+
if (connected) return connected;
|
|
377
|
+
return this.connectWith(this.descriptorLib(descriptor), descriptor);
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
private async connectWith(lib: any, descriptor: PwServerDescriptor): Promise<Browser | null> {
|
|
381
|
+
const launcher = lib?.[descriptor.browserName];
|
|
382
|
+
if (!launcher?.connect) return null;
|
|
383
|
+
return launcher.connect(descriptor.endpoint, { timeout: CONNECT_TIMEOUT }).catch(() => null);
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
private descriptorLib(descriptor: PwServerDescriptor): any {
|
|
387
|
+
if (!descriptor.playwrightLib) return null;
|
|
388
|
+
try {
|
|
389
|
+
return requireLib(descriptor.playwrightLib);
|
|
390
|
+
} catch {
|
|
391
|
+
return null;
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
private attachmentLabel(descriptor: PwServerDescriptor): string {
|
|
396
|
+
if (!descriptor.title) return `endpoint ${descriptor.endpoint}`;
|
|
397
|
+
return `playwright-cli session "${descriptor.title}", workspace ${descriptor.workspaceDir}`;
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
private workspaceDir(): string {
|
|
401
|
+
return path.resolve(this.options.path || process.cwd());
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
private async connectOwnInstance(): Promise<boolean> {
|
|
405
|
+
return !!(await getAliveEndpoint(this.instanceName()));
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
private async launchOwnServer(opts: { browser?: string; show?: boolean }, instance: string): Promise<{ close: () => Promise<void> }> {
|
|
409
|
+
return launchServer(opts, instance);
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
private async stopInstance(instance: string): Promise<boolean> {
|
|
413
|
+
if (instance === this.instanceName()) {
|
|
414
|
+
const server = this.server;
|
|
415
|
+
this.server = null;
|
|
416
|
+
await server?.close();
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
return stopServer(instance);
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
private isUrlTarget(target: string): boolean {
|
|
423
|
+
const value = target.trim();
|
|
424
|
+
if (value.startsWith('/')) return true;
|
|
425
|
+
return URL.canParse(value);
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
private async heal(command: string, expression: string, error: unknown, previousState: WebPageState | null): Promise<EnvelopeData> {
|
|
429
|
+
if (this.options.heal === false) return this.failureEnvelope(command, error, previousState);
|
|
430
|
+
|
|
431
|
+
const navigator = this.healNavigator();
|
|
432
|
+
if (!navigator) {
|
|
433
|
+
const envelope = await this.failureEnvelope(command, error, previousState);
|
|
434
|
+
envelope.healed = false;
|
|
435
|
+
envelope.healNote = this.aiUnavailableNote();
|
|
436
|
+
return envelope;
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
const message = dedent`
|
|
440
|
+
I tried to run this command on the page: ${expression}
|
|
441
|
+
But it failed with: ${browserErrorMessage(error)}
|
|
442
|
+
Reach the same outcome on the current page in a different way.
|
|
443
|
+
`;
|
|
444
|
+
|
|
445
|
+
const attempts: Array<{ code: string; error?: string }> = [];
|
|
446
|
+
const failedResult = await this.capturedResult(previousState);
|
|
447
|
+
const resolved = await navigator.resolveState(message, failedResult, { onAttempt: (attempt) => attempts.push(attempt) }).catch(() => false);
|
|
448
|
+
|
|
449
|
+
if (!resolved) {
|
|
450
|
+
const healAttempts = attempts.map((attempt) => ({ code: attempt.code, outcome: attempt.error || 'ok' }));
|
|
451
|
+
return this.failureEnvelope(command, error, previousState, healAttempts);
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
const used = attempts.filter((attempt) => !attempt.error).map((attempt) => attempt.code);
|
|
455
|
+
const result = await this.capturedResult(this.bot.stateManager().getCurrentState());
|
|
456
|
+
const envelope = await this.successEnvelope(command, used, result, previousState);
|
|
457
|
+
envelope.healed = true;
|
|
458
|
+
envelope.healNote = `recovered after ${attempts.length} ${pluralize(attempts.length, 'attempt')}`;
|
|
459
|
+
return envelope;
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
private healNavigator(): Navigator | null {
|
|
463
|
+
if (this.aiUnavailable()) return null;
|
|
464
|
+
try {
|
|
465
|
+
return this.bot.agentNavigator?.() ?? null;
|
|
466
|
+
} catch {
|
|
467
|
+
return null;
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
private aiUnavailable(): string | null {
|
|
472
|
+
try {
|
|
473
|
+
if (this.bot.getProvider?.()) return null;
|
|
474
|
+
} catch (error) {
|
|
475
|
+
return browserErrorMessage(error);
|
|
476
|
+
}
|
|
477
|
+
return this.bot.aiFailureReason?.() || 'no AI model is configured';
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
private aiUnavailableNote(): string {
|
|
481
|
+
const reason = this.aiUnavailable();
|
|
482
|
+
if (!reason) return 'ai unavailable';
|
|
483
|
+
return `ai unavailable: ${reason}`;
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
private async aiGuard(command: string): Promise<EnvelopeData | null> {
|
|
487
|
+
const reason = this.aiUnavailable();
|
|
488
|
+
if (!reason) return null;
|
|
489
|
+
const message = `this command needs an AI model and none is usable: ${reason}. Drive the browser directly with playwright-cli or prima pw, or fix the AI config in ~/.explorbot/config.js or EXPLORBOT_AI_PROVIDER.`;
|
|
490
|
+
return this.toolFailureEnvelope(command, message);
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
private instructionSystemPrompt(): string {
|
|
494
|
+
return dedent`
|
|
495
|
+
<role>
|
|
496
|
+
You are a web automation engineer performing high-level instructions on the page that is already open in the browser.
|
|
497
|
+
</role>
|
|
498
|
+
|
|
499
|
+
<approach>
|
|
500
|
+
1. Read the page context and perform the instructions in the order they are listed.
|
|
501
|
+
2. Interact with the page only through the provided tools.
|
|
502
|
+
3. Pick the smallest interaction that fulfills an instruction, then move to the next one.
|
|
503
|
+
4. After the page changes, work from the updated context you are given, not from the earlier one.
|
|
504
|
+
5. Stop calling tools when every instruction is done, or when an instruction cannot be performed on this page — say what is missing instead.
|
|
505
|
+
</approach>
|
|
506
|
+
|
|
507
|
+
${locatorRule}
|
|
508
|
+
|
|
509
|
+
${actionRule}
|
|
510
|
+
`;
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
private instructionPrompt(instructions: string[], result: ActionResult): string {
|
|
514
|
+
const list = instructions.map((instruction, index) => `${index + 1}. ${instruction}`).join('\n');
|
|
515
|
+
return dedent`
|
|
516
|
+
<instructions>
|
|
517
|
+
${list}
|
|
518
|
+
</instructions>
|
|
519
|
+
|
|
520
|
+
${this.pageContext(result)}
|
|
521
|
+
`;
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
private pageContext(result: ActionResult): string {
|
|
525
|
+
const experience = this.bot.experienceTracker?.()?.renderExperienceTocFor?.(result) || '';
|
|
526
|
+
return dedent`
|
|
527
|
+
<page url="${result.url}" title="${result.title}">
|
|
528
|
+
${compactAriaSnapshot(result.ariaSnapshot, true)}
|
|
529
|
+
</page>
|
|
530
|
+
|
|
531
|
+
${experience}
|
|
532
|
+
`;
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
private executedCodes(code: unknown): string[] {
|
|
536
|
+
if (typeof code !== 'string') return [];
|
|
537
|
+
return code
|
|
538
|
+
.split('\n')
|
|
539
|
+
.map((line) => line.trim())
|
|
540
|
+
.filter((line) => line);
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
private visionEnabled(): boolean {
|
|
544
|
+
if (this.options.noVision) return false;
|
|
545
|
+
return this.bot.getProvider().hasVision?.() === true;
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
private async answer(question: string, result: ActionResult): Promise<string> {
|
|
549
|
+
const seen = await this.visionAnswer(question, result);
|
|
550
|
+
if (seen) return seen;
|
|
551
|
+
|
|
552
|
+
const described = await this.textAnswer(question, result);
|
|
553
|
+
if (this.options.noVision) return described;
|
|
554
|
+
return `${described}\n\n(vision was unavailable, answered from page structure)`;
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
private async visionAnswer(question: string, result: ActionResult): Promise<string | null> {
|
|
558
|
+
if (!this.visionEnabled()) return null;
|
|
559
|
+
return this.bot.agentResearcher().answerQuestionAboutScreenshot(result, question);
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
private async textAnswer(question: string, result: ActionResult): Promise<string> {
|
|
563
|
+
const provider = this.bot.getProvider();
|
|
564
|
+
const summary = await this.bot.agentResearcher().summary(result);
|
|
565
|
+
const prompt = dedent`
|
|
566
|
+
Answer the question about the web page described below.
|
|
567
|
+
Rely only on the page summary and the accessibility tree given here.
|
|
568
|
+
When the answer is not present in them, say which part of the page would be needed instead of guessing.
|
|
569
|
+
Keep the answer under five sentences.
|
|
570
|
+
|
|
571
|
+
<question>
|
|
572
|
+
${question}
|
|
573
|
+
</question>
|
|
574
|
+
|
|
575
|
+
<page_summary>
|
|
576
|
+
${summary}
|
|
577
|
+
</page_summary>
|
|
578
|
+
|
|
579
|
+
<page_aria>
|
|
580
|
+
${compactAriaSnapshot(result.ariaSnapshot, true)}
|
|
581
|
+
</page_aria>
|
|
582
|
+
`;
|
|
583
|
+
|
|
584
|
+
const response = await provider.chat([{ role: 'user', content: prompt }], provider.getModelForAgent?.(AI_AGENT_NAME), { agentName: AI_AGENT_NAME });
|
|
585
|
+
return response?.text || '';
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
private verdictEvidence(verified: boolean, codes: string[]): string {
|
|
589
|
+
if (!verified) return 'no assertion held on the current page';
|
|
590
|
+
if (!codes.length) return 'already verified on this page';
|
|
591
|
+
return `${codes[0]} passed`;
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
private async successEnvelope(command: string, used: string[], result: ActionResult, previousState: WebPageState | null): Promise<EnvelopeData> {
|
|
595
|
+
return {
|
|
596
|
+
ok: true,
|
|
597
|
+
command,
|
|
598
|
+
used,
|
|
599
|
+
page: this.pageBlock(result, previousState),
|
|
600
|
+
changes: await this.pageChanges(result, previousState, used[0]),
|
|
601
|
+
instance: await this.instanceInfo(),
|
|
602
|
+
artifacts: await this.writeSnapshot(result),
|
|
603
|
+
};
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
private async failureEnvelope(command: string, error: unknown, previousState: WebPageState | null, attempts: HealAttempt[] = []): Promise<EnvelopeData> {
|
|
607
|
+
const result = await this.capturedResult(previousState);
|
|
608
|
+
const failure: EnvelopeData['failure'] = { error: browserErrorMessage(error), attempts };
|
|
609
|
+
if (result.ariaSnapshot) failure.compactAria = compactAriaSnapshot(result.ariaSnapshot, true);
|
|
610
|
+
if (attempts.length) failure.reasoning = [...new Set(attempts.map((attempt) => attempt.outcome))].join('; ');
|
|
611
|
+
|
|
612
|
+
return {
|
|
613
|
+
ok: false,
|
|
614
|
+
command,
|
|
615
|
+
page: this.pageBlock(result, previousState),
|
|
616
|
+
failure,
|
|
617
|
+
instance: await this.instanceInfo(),
|
|
618
|
+
artifacts: await this.writeSnapshot(result),
|
|
619
|
+
};
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
private async reportEnvelope(command: string, result: ActionResult, previousState: WebPageState | null, outcome: Partial<EnvelopeData>): Promise<EnvelopeData> {
|
|
623
|
+
return {
|
|
624
|
+
ok: true,
|
|
625
|
+
command,
|
|
626
|
+
page: this.pageBlock(result, previousState),
|
|
627
|
+
...outcome,
|
|
628
|
+
instance: await this.instanceInfo(),
|
|
629
|
+
artifacts: await this.writeSnapshot(result),
|
|
630
|
+
};
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
private async capturedResult(previousState: WebPageState | null, opts: { screenshot?: boolean } = {}): Promise<ActionResult> {
|
|
634
|
+
const captured = await this.bot
|
|
635
|
+
.getExplorer()
|
|
636
|
+
?.capture(opts)
|
|
637
|
+
.catch(() => null);
|
|
638
|
+
if (captured) return captured;
|
|
639
|
+
if (previousState) return ActionResult.fromState(previousState);
|
|
640
|
+
return new ActionResult({ url: '' });
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
private pageBlock(result: ActionResult, previousState: WebPageState | null): EnvelopeData['page'] {
|
|
644
|
+
return {
|
|
645
|
+
url: result.url,
|
|
646
|
+
previousUrl: previousState?.url,
|
|
647
|
+
title: result.title,
|
|
648
|
+
state: result.getStateHash(),
|
|
649
|
+
visits: this.bot.stateManager().getVisitCount(result.url),
|
|
650
|
+
};
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
private async pageChanges(result: ActionResult, previousState: WebPageState | null, code: string): Promise<string | null> {
|
|
654
|
+
if (!previousState) return null;
|
|
655
|
+
const toolResult = await result.toToolResult(ActionResult.fromState(previousState), code);
|
|
656
|
+
return toolResult.pageDiff?.ariaChanges ?? null;
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
private async writeSnapshot(result: ActionResult): Promise<EnvelopeData['artifacts']> {
|
|
660
|
+
return writeArtifacts(this.nextArtifactDir(), {
|
|
661
|
+
aria: result.ariaSnapshot,
|
|
662
|
+
html: await result.combinedHtml(),
|
|
663
|
+
requests: this.bot.requestStore().getRequests(),
|
|
664
|
+
});
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
private nextArtifactDir(): string {
|
|
668
|
+
this.artifactsDir ||= outputPath('prima');
|
|
669
|
+
return path.join(this.artifactsDir, new Date().toISOString().replace(/[:.]/g, '-'));
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
private tabCount(): number {
|
|
673
|
+
const page = this.bot.getExplorer()?.page;
|
|
674
|
+
if (!page) return 0;
|
|
675
|
+
return page.context().pages().length;
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
private instanceName(): string {
|
|
679
|
+
return this.options.instance || 'default';
|
|
680
|
+
}
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
interface Discovery {
|
|
684
|
+
match?: PwServerDescriptor;
|
|
685
|
+
candidates: PwServerDescriptor[];
|
|
686
|
+
browser?: Browser;
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
export interface PrimaOptions {
|
|
690
|
+
verbose?: boolean;
|
|
691
|
+
config?: string;
|
|
692
|
+
path?: string;
|
|
693
|
+
instance?: string;
|
|
694
|
+
session?: string | boolean;
|
|
695
|
+
heal?: boolean;
|
|
696
|
+
ephemeral?: boolean;
|
|
697
|
+
framework?: 'codeceptjs' | 'playwright';
|
|
698
|
+
noVision?: boolean;
|
|
699
|
+
url?: string;
|
|
700
|
+
baseUrl?: string;
|
|
701
|
+
show?: boolean;
|
|
702
|
+
headless?: boolean;
|
|
703
|
+
endpoint?: string;
|
|
704
|
+
pwSession?: string;
|
|
705
|
+
}
|