explorbot 0.1.27 → 0.1.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/package.json +1 -1
- package/dist/src/action-result.js +2 -0
- package/dist/src/action.js +88 -7
- package/dist/src/ai/captain/file-tools.js +100 -0
- package/dist/src/ai/captain/idle-mode.js +70 -6
- package/dist/src/ai/captain/web-mode.js +36 -6
- package/dist/src/ai/captain.js +87 -19
- package/dist/src/ai/historian/screencast.js +11 -2
- package/dist/src/ai/navigator.js +5 -2
- package/dist/src/ai/pilot.js +33 -5
- package/dist/src/ai/researcher/coordinates.js +2 -3
- package/dist/src/ai/researcher/deep-analysis.js +3 -4
- package/dist/src/ai/researcher/locators.js +1 -2
- package/dist/src/ai/researcher.js +17 -18
- package/dist/src/ai/task-agent.js +1 -0
- package/dist/src/ai/tester.js +91 -39
- package/dist/src/ai/tools.js +17 -7
- package/dist/src/commands/explore-command.js +6 -1
- package/dist/src/components/LogPane.js +4 -3
- package/dist/src/explorer.js +270 -35
- package/dist/src/utils/browser-errors.js +23 -0
- package/dist/src/utils/error-page.js +17 -2
- package/dist/src/utils/logger.js +2 -2
- package/package.json +1 -1
- package/src/action-result.ts +2 -0
- package/src/action.ts +83 -7
- package/src/ai/captain/file-tools.ts +126 -0
- package/src/ai/captain/idle-mode.ts +72 -6
- package/src/ai/captain/mixin.ts +1 -1
- package/src/ai/captain/web-mode.ts +40 -5
- package/src/ai/captain.ts +94 -20
- package/src/ai/historian/screencast.ts +11 -2
- package/src/ai/navigator.ts +6 -2
- package/src/ai/pilot.ts +34 -5
- package/src/ai/researcher/coordinates.ts +2 -3
- package/src/ai/researcher/deep-analysis.ts +3 -4
- package/src/ai/researcher/locators.ts +1 -2
- package/src/ai/researcher.ts +17 -18
- package/src/ai/task-agent.ts +1 -1
- package/src/ai/tester.ts +101 -41
- package/src/ai/tools.ts +17 -7
- package/src/commands/explore-command.ts +6 -1
- package/src/components/LogPane.tsx +4 -3
- package/src/explorer.ts +295 -38
- package/src/state-manager.ts +2 -0
- package/src/utils/browser-errors.ts +25 -0
- package/src/utils/error-page.ts +16 -3
- package/src/utils/logger.ts +3 -3
package/src/ai/captain.ts
CHANGED
|
@@ -26,7 +26,6 @@ const MAX_STEPS = 15;
|
|
|
26
26
|
const CaptainBase = WithTestMode(WithWebMode(WithIdleMode(TaskAgent as unknown as new (...args: any[]) => TaskAgent)));
|
|
27
27
|
|
|
28
28
|
export class Captain extends CaptainBase implements Agent {
|
|
29
|
-
protected readonly ACTION_TOOLS = ['click', 'pressKey', 'form', 'navigate'];
|
|
30
29
|
emoji = '🧑✈️';
|
|
31
30
|
private explorBot: ExplorBot;
|
|
32
31
|
private conversation: Conversation | null = null;
|
|
@@ -72,6 +71,12 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
72
71
|
|
|
73
72
|
protected trackToolExecutions(toolExecutions: any[]): void {
|
|
74
73
|
super.trackToolExecutions(toolExecutions);
|
|
74
|
+
if (toolExecutions.length > 0) {
|
|
75
|
+
this.recentToolCalls.push(...toolExecutions);
|
|
76
|
+
if (this.recentToolCalls.length > 20) {
|
|
77
|
+
this.recentToolCalls = this.recentToolCalls.slice(-20);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
75
80
|
for (const exec of toolExecutions) {
|
|
76
81
|
const label = toolExecutionLabel(exec.input);
|
|
77
82
|
if (!label) continue;
|
|
@@ -80,14 +85,19 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
80
85
|
}
|
|
81
86
|
}
|
|
82
87
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
88
|
+
getMode(): CaptainMode {
|
|
89
|
+
const explorer = this.explorBot.getExplorer();
|
|
90
|
+
const activeTest = explorer.activeTest;
|
|
91
|
+
const page = explorer.playwrightHelper?.page;
|
|
92
|
+
|
|
93
|
+
if (activeTest && (!page || page.isClosed?.())) return 'heal';
|
|
94
|
+
if (activeTest) return 'test';
|
|
95
|
+
if (explorer.getStateManager().getCurrentState()) return 'web';
|
|
86
96
|
return 'idle';
|
|
87
97
|
}
|
|
88
98
|
|
|
89
99
|
private systemPrompt(): string {
|
|
90
|
-
const mode = this.
|
|
100
|
+
const mode = this.getMode();
|
|
91
101
|
const currentUrl = this.explorBot.getExplorer().getStateManager().getCurrentState()?.url;
|
|
92
102
|
const customPrompt = this.explorBot.getProvider().getSystemPromptForAgent('captain', currentUrl);
|
|
93
103
|
|
|
@@ -101,18 +111,21 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
101
111
|
- idle: plan management, file operations, knowledge. Always available.
|
|
102
112
|
- web: page interaction, navigation, browser diagnostics. When working with a web page.
|
|
103
113
|
- test: test analysis, state inspection. When a test is running or analyzing results.
|
|
114
|
+
- heal: browser/test recovery. When a test is running and browser state is broken or unavailable.
|
|
104
115
|
</modes>
|
|
105
116
|
|
|
106
117
|
${this.idleModePrompt()}
|
|
107
|
-
${mode === 'web' ? this.webModePrompt() : ''}
|
|
108
|
-
${mode === 'test' ? this.testModePrompt() : ''}
|
|
118
|
+
${mode === 'web' || mode === 'heal' ? this.webModePrompt() : ''}
|
|
119
|
+
${mode === 'test' || mode === 'heal' ? this.testModePrompt() : ''}
|
|
109
120
|
|
|
110
121
|
<rules>
|
|
111
122
|
- After a successful action, if the pageDiff confirms the goal, call done() immediately — do not verify with see() or context() unless the user explicitly asked for verification
|
|
112
123
|
- Prefer completing in fewer tool calls over thoroughness
|
|
113
124
|
- NEVER run tests unless the user explicitly asks
|
|
114
|
-
|
|
115
|
-
${mode === '
|
|
125
|
+
- If you are answering with information rather than completing a browser action, include the actual user-facing answer in done({ details }). Do not only say that it was shown or explained.
|
|
126
|
+
${mode === 'web' || mode === 'heal' ? this.webModeRules() : ''}
|
|
127
|
+
${mode === 'test' || mode === 'heal' ? this.testModeRules() : ''}
|
|
128
|
+
${mode === 'heal' ? '- First diagnose browser availability, then recover the browser/page before continuing test analysis.' : ''}
|
|
116
129
|
</rules>
|
|
117
130
|
|
|
118
131
|
${customPrompt || ''}
|
|
@@ -250,9 +263,20 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
250
263
|
description: 'Call when the user request is fulfilled.',
|
|
251
264
|
inputSchema: z.object({
|
|
252
265
|
summary: z.string().describe('What was done'),
|
|
266
|
+
details: z.string().optional().describe('Actual user-facing content. Required when the user asked to show, display, explain, summarize, compare, or diagnose information.'),
|
|
253
267
|
}),
|
|
254
|
-
execute: async ({ summary }) => {
|
|
268
|
+
execute: async ({ summary, details }) => {
|
|
255
269
|
debugLog('done', summary);
|
|
270
|
+
if (!details?.trim() && !this.canCompleteWithoutDetails()) {
|
|
271
|
+
return {
|
|
272
|
+
success: false,
|
|
273
|
+
message: 'No user-facing result was provided. Call done() again with the actual answer in details, or complete a browser action first.',
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
if (details?.trim()) {
|
|
277
|
+
tag('details').log(details);
|
|
278
|
+
task.addNote(details);
|
|
279
|
+
}
|
|
256
280
|
task.addNote(summary);
|
|
257
281
|
onDone(summary);
|
|
258
282
|
return { success: true, summary };
|
|
@@ -261,6 +285,9 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
261
285
|
runCommand: tool({
|
|
262
286
|
description: dedent`
|
|
263
287
|
Execute a TUI command. Returns log output from command execution.
|
|
288
|
+
Use only when the user explicitly asks to run a slash command.
|
|
289
|
+
Never use this to analyze files, reports, logs, plans, generated tests, knowledge, or experience.
|
|
290
|
+
Never run a slash command unless the user request itself starts with that slash command.
|
|
264
291
|
${this.commandDescriptions
|
|
265
292
|
.map((c) => {
|
|
266
293
|
const opts = c.options ? ` (${c.options})` : '';
|
|
@@ -274,6 +301,13 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
274
301
|
execute: async ({ command }) => {
|
|
275
302
|
if (!this.commandExecutor) return { success: false, message: 'Command executor not available' };
|
|
276
303
|
const cmd = command.startsWith('/') ? command : `/${command}`;
|
|
304
|
+
if (!isExplicitSlashRequest(task.description, cmd)) {
|
|
305
|
+
return {
|
|
306
|
+
success: false,
|
|
307
|
+
command: cmd,
|
|
308
|
+
message: 'Command blocked: slash commands require an explicit matching slash-command request from the user.',
|
|
309
|
+
};
|
|
310
|
+
}
|
|
277
311
|
startLogCapture();
|
|
278
312
|
try {
|
|
279
313
|
await this.commandExecutor(cmd);
|
|
@@ -286,11 +320,12 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
286
320
|
}
|
|
287
321
|
|
|
288
322
|
private async tools(task: Task, onDone: (summary: string) => void) {
|
|
289
|
-
const mode = this.
|
|
323
|
+
const mode = this.getMode();
|
|
290
324
|
const ctx: ModeContext = { explorBot: this.explorBot, task };
|
|
291
325
|
const core = this.coreTools(task, onDone);
|
|
292
326
|
const idle = await this.idleModeTools(ctx);
|
|
293
327
|
|
|
328
|
+
if (mode === 'heal') return { ...core, ...idle, ...this.testModeTools(ctx), ...this.webModeTools(ctx) };
|
|
294
329
|
if (mode === 'test') return { ...core, ...idle, ...this.testModeTools(ctx) };
|
|
295
330
|
if (mode === 'web') return { ...core, ...idle, ...this.webModeTools(ctx) };
|
|
296
331
|
return { ...core, ...idle };
|
|
@@ -365,20 +400,28 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
365
400
|
return result.object;
|
|
366
401
|
}
|
|
367
402
|
|
|
403
|
+
async processExecutionError(error: Error, activeTest: Test): Promise<ExecutionRecoveryAction> {
|
|
404
|
+
const explorer = this.explorBot.getExplorer();
|
|
405
|
+
const result = await explorer.handleExecutionError(error);
|
|
406
|
+
return {
|
|
407
|
+
...result,
|
|
408
|
+
message: result.recovered ? `${result.message}\nContinue the test "${activeTest.scenario}" from the restored page.` : result.message,
|
|
409
|
+
};
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
private canCompleteWithoutDetails(): boolean {
|
|
413
|
+
return (this.recentToolCalls || []).some(hasBrowserCompletionEvidence);
|
|
414
|
+
}
|
|
415
|
+
|
|
368
416
|
async handle(input: string, options: { reset?: boolean } = {}): Promise<string | null> {
|
|
369
417
|
const stateManager = this.explorBot.getExplorer().getStateManager();
|
|
370
418
|
const initialState = stateManager.getCurrentState();
|
|
371
419
|
|
|
372
|
-
if (!initialState) {
|
|
373
|
-
tag('warning').log('No page loaded. Use /navigate or I.amOnPage() first.');
|
|
374
|
-
return null;
|
|
375
|
-
}
|
|
376
|
-
|
|
377
420
|
const conversation = options.reset ? this.resetConversation() : this.ensureConversation();
|
|
378
421
|
let isDone = false;
|
|
379
422
|
let finalSummary: string | null = null;
|
|
380
423
|
|
|
381
|
-
const startUrl = initialState
|
|
424
|
+
const startUrl = initialState?.url || '';
|
|
382
425
|
const task = new Task(input, startUrl);
|
|
383
426
|
const onDone = (summary: string) => {
|
|
384
427
|
isDone = true;
|
|
@@ -421,12 +464,14 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
421
464
|
}
|
|
422
465
|
|
|
423
466
|
const currentState = stateManager.getCurrentState();
|
|
424
|
-
if (!currentState) {
|
|
467
|
+
if (!currentState && this.getMode() !== 'idle') {
|
|
425
468
|
stop();
|
|
426
469
|
return;
|
|
427
470
|
}
|
|
428
471
|
|
|
429
|
-
|
|
472
|
+
if (currentState) {
|
|
473
|
+
await this.reinjectContextIfNeeded(conversation, currentState);
|
|
474
|
+
}
|
|
430
475
|
|
|
431
476
|
if (userInput) {
|
|
432
477
|
const newContext = await this.getPageContext();
|
|
@@ -463,7 +508,7 @@ export class Captain extends CaptainBase implements Agent {
|
|
|
463
508
|
|
|
464
509
|
if (result?.toolExecutions?.length) {
|
|
465
510
|
const lastExec = result.toolExecutions[result.toolExecutions.length - 1];
|
|
466
|
-
if (
|
|
511
|
+
if (hasBrowserCompletionEvidence(lastExec)) {
|
|
467
512
|
conversation.addUserText('Action succeeded. If the goal is achieved, call done() now with a brief summary.');
|
|
468
513
|
}
|
|
469
514
|
}
|
|
@@ -500,3 +545,32 @@ interface SupervisorAction {
|
|
|
500
545
|
action: 'inject' | 'stop' | 'pass' | 'skip';
|
|
501
546
|
message: string;
|
|
502
547
|
}
|
|
548
|
+
|
|
549
|
+
interface ExecutionRecoveryAction {
|
|
550
|
+
action: 'continue' | 'stop';
|
|
551
|
+
message: string;
|
|
552
|
+
recovered?: boolean;
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
function isExplicitSlashRequest(input: string, command: string): boolean {
|
|
556
|
+
const requested = slashCommandToken(input);
|
|
557
|
+
const actual = slashCommandToken(command);
|
|
558
|
+
if (!requested || !actual) return false;
|
|
559
|
+
return requested === actual;
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
function slashCommandToken(value: string): string | null {
|
|
563
|
+
const trimmed = value.trim();
|
|
564
|
+
if (!trimmed.startsWith('/')) return null;
|
|
565
|
+
|
|
566
|
+
for (let i = 1; i < trimmed.length; i++) {
|
|
567
|
+
if (trimmed[i] <= ' ') return trimmed.slice(0, i);
|
|
568
|
+
}
|
|
569
|
+
return trimmed;
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
function hasBrowserCompletionEvidence(execution: any): boolean {
|
|
573
|
+
if (!execution?.wasSuccessful) return false;
|
|
574
|
+
const output = execution.output || {};
|
|
575
|
+
return Boolean(output.pageDiff || output.code || output.playwrightGroupId);
|
|
576
|
+
}
|
|
@@ -10,6 +10,8 @@ import { relativeToCwd } from '../../utils/next-steps.ts';
|
|
|
10
10
|
import { safeFilename } from '../../utils/strings.ts';
|
|
11
11
|
import { type Constructor, debugLog } from './mixin.ts';
|
|
12
12
|
|
|
13
|
+
const FATAL_SCREENCAST_STOP_ERRORS = /Target page, context or browser has been closed|Target closed|Session closed|Protocol error/i;
|
|
14
|
+
|
|
13
15
|
export interface ScreencastMethods {
|
|
14
16
|
attachScreencast(): void;
|
|
15
17
|
isScreencastActive(): boolean;
|
|
@@ -113,17 +115,24 @@ export function WithScreencast<T extends Constructor>(Base: T) {
|
|
|
113
115
|
if (!this.screencastActive) return;
|
|
114
116
|
const path = this.screencastPath;
|
|
115
117
|
const task = this.screencastTask;
|
|
118
|
+
let stopped = false;
|
|
116
119
|
try {
|
|
117
120
|
await this.screencastPage.screencast.stop();
|
|
121
|
+
stopped = true;
|
|
118
122
|
} catch (err) {
|
|
119
|
-
|
|
123
|
+
const message = (err as Error).message;
|
|
124
|
+
if (FATAL_SCREENCAST_STOP_ERRORS.test(message)) {
|
|
125
|
+
tag('operation').log('Screencast skipped: browser was closed before recording could be finalized');
|
|
126
|
+
} else {
|
|
127
|
+
tag('operation').log(`Screencast stop failed: ${message}`);
|
|
128
|
+
}
|
|
120
129
|
}
|
|
121
130
|
this.screencastActive = false;
|
|
122
131
|
this.screencastPage = null;
|
|
123
132
|
this.screencastPath = null;
|
|
124
133
|
this.screencastTask = null;
|
|
125
134
|
this.screencastLastChapter = null;
|
|
126
|
-
if (path) {
|
|
135
|
+
if (path && stopped) {
|
|
127
136
|
this.savedFiles.add(path);
|
|
128
137
|
task?.addArtifact?.(path);
|
|
129
138
|
tag('operation').log(`Saved screencast: ${relativeToCwd(path)}`);
|
package/src/ai/navigator.ts
CHANGED
|
@@ -136,6 +136,10 @@ class Navigator implements Agent {
|
|
|
136
136
|
}
|
|
137
137
|
|
|
138
138
|
async visit(url: string): Promise<void> {
|
|
139
|
+
return this.explorer.runWithBrowserRecovery('navigator.visit', () => this.visitOnce(url));
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
private async visitOnce(url: string): Promise<void> {
|
|
139
143
|
try {
|
|
140
144
|
const action = this.explorer.createAction();
|
|
141
145
|
|
|
@@ -170,7 +174,7 @@ class Navigator implements Agent {
|
|
|
170
174
|
throw new Error(`Navigation to ${url} failed: ${action.lastError?.message}`);
|
|
171
175
|
}
|
|
172
176
|
}
|
|
173
|
-
await
|
|
177
|
+
await this.explorer.capturePageWithScreenshot();
|
|
174
178
|
await this.hooksRunner.runAfterHook('navigator', url);
|
|
175
179
|
} catch (error) {
|
|
176
180
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
@@ -382,7 +386,7 @@ class Navigator implements Agent {
|
|
|
382
386
|
// URL did not transition to expectedUrl within timeout
|
|
383
387
|
}
|
|
384
388
|
}
|
|
385
|
-
const freshState = await
|
|
389
|
+
const freshState = await this.explorer.capturePageState();
|
|
386
390
|
const currentUrl = /^https?:\/\//i.test(expectedUrl) ? freshState.fullUrl || freshState.url || '' : freshState.url || '';
|
|
387
391
|
const urlMatches = this.isSameExpectedOrigin(expectedUrl, action.stateManager) && normalizeUrl(currentUrl) === normalizeUrl(expectedUrl);
|
|
388
392
|
const stateChanged = freshState.getStateHash() !== actionResult.getStateHash();
|
package/src/ai/pilot.ts
CHANGED
|
@@ -67,7 +67,7 @@ export class Pilot implements Agent {
|
|
|
67
67
|
}
|
|
68
68
|
|
|
69
69
|
async reviewCompletion(task: Test, currentState: ActionResult, testerConversation: Conversation, navigator?: Navigator): Promise<boolean> {
|
|
70
|
-
const verdictType =
|
|
70
|
+
const verdictType = this.hasCompletionEvidence(task, currentState, testerConversation) ? 'finish' : 'stop';
|
|
71
71
|
return this.reviewDecision(verdictType, task, currentState, testerConversation, navigator);
|
|
72
72
|
}
|
|
73
73
|
|
|
@@ -86,14 +86,14 @@ export class Pilot implements Agent {
|
|
|
86
86
|
|
|
87
87
|
const sessionLog = this.formatSessionLog(testerConversation);
|
|
88
88
|
const stateContext = this.buildStateContext(currentState);
|
|
89
|
+
const successfulAssertions = this.formatSuccessfulAssertions(currentState, testerConversation);
|
|
89
90
|
const notes = task.notesToString() || 'No notes recorded.';
|
|
90
91
|
|
|
91
92
|
let visualAnalysis = '';
|
|
92
93
|
let screenshotState: ActionResult | null = null;
|
|
93
94
|
if (this.provider.hasVision()) {
|
|
94
95
|
try {
|
|
95
|
-
|
|
96
|
-
screenshotState = await action.caputrePageWithScreenshot();
|
|
96
|
+
screenshotState = await this.explorer.capturePageWithScreenshot();
|
|
97
97
|
if (screenshotState.screenshot) {
|
|
98
98
|
visualAnalysis = (await this.researcher.answerQuestionAboutScreenshot(screenshotState, `Describe current page state relevant to: ${task.scenario}`)) || '';
|
|
99
99
|
}
|
|
@@ -125,6 +125,10 @@ export class Pilot implements Agent {
|
|
|
125
125
|
|
|
126
126
|
${this.formatExpectations(task)}
|
|
127
127
|
|
|
128
|
+
<successful_assertions>
|
|
129
|
+
${successfulAssertions || 'None'}
|
|
130
|
+
</successful_assertions>
|
|
131
|
+
|
|
128
132
|
<notes>
|
|
129
133
|
${notes}
|
|
130
134
|
</notes>
|
|
@@ -658,8 +662,7 @@ export class Pilot implements Agent {
|
|
|
658
662
|
private async checkDataAvailability(task: Test, requestedData: string, fishermanReason: string | undefined): Promise<string | null> {
|
|
659
663
|
if (!this.provider.hasVision()) return null;
|
|
660
664
|
|
|
661
|
-
const
|
|
662
|
-
const screenshotState = await action.caputrePageWithScreenshot().catch(() => null);
|
|
665
|
+
const screenshotState = await this.explorer.capturePageWithScreenshot().catch(() => null);
|
|
663
666
|
if (!screenshotState?.screenshot) return null;
|
|
664
667
|
|
|
665
668
|
const question = dedent`
|
|
@@ -882,6 +885,32 @@ export class Pilot implements Agent {
|
|
|
882
885
|
return parts.join('\n\n');
|
|
883
886
|
}
|
|
884
887
|
|
|
888
|
+
private hasCompletionEvidence(task: Test, currentState: ActionResult, testerConversation: Conversation): boolean {
|
|
889
|
+
if (task.hasAchievedAny()) return true;
|
|
890
|
+
return this.hasSuccessfulCheckEvidence(currentState, testerConversation);
|
|
891
|
+
}
|
|
892
|
+
|
|
893
|
+
private hasSuccessfulCheckEvidence(currentState: ActionResult, testerConversation: Conversation): boolean {
|
|
894
|
+
if (Object.values(currentState.verifications ?? {}).some(Boolean)) return true;
|
|
895
|
+
return testerConversation.getToolExecutions().some((t) => CHECK_TOOLS.includes(t.toolName) && t.wasSuccessful);
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
private formatSuccessfulAssertions(currentState: ActionResult, testerConversation: Conversation): string {
|
|
899
|
+
const lines: string[] = [];
|
|
900
|
+
for (const [assertion, passed] of Object.entries(currentState.verifications ?? {})) {
|
|
901
|
+
if (passed) lines.push(`PASS state verification: ${assertion}`);
|
|
902
|
+
}
|
|
903
|
+
|
|
904
|
+
for (const exec of testerConversation.getToolExecutions()) {
|
|
905
|
+
if (!CHECK_TOOLS.includes(exec.toolName) || !exec.wasSuccessful) continue;
|
|
906
|
+
const description = exec.input?.assertion || exec.input?.request || truncateJson(exec.input);
|
|
907
|
+
const result = exec.output?.message || exec.output?.analysis || exec.output?.result;
|
|
908
|
+
lines.push(`PASS ${exec.toolName}: ${description}${result ? ` -> ${result}` : ''}`);
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
return [...new Set(lines)].join('\n');
|
|
912
|
+
}
|
|
913
|
+
|
|
885
914
|
private formatActions(toolCalls: any[]): string {
|
|
886
915
|
return toolCalls
|
|
887
916
|
.map((t) => {
|
|
@@ -81,7 +81,7 @@ export function WithCoordinates<T extends Constructor>(Base: T) {
|
|
|
81
81
|
}
|
|
82
82
|
|
|
83
83
|
async visuallyAnnotateElements(opts?: { containers?: Array<{ css: string; label: string }> }): Promise<number> {
|
|
84
|
-
return
|
|
84
|
+
return this.explorer.visuallyAnnotateElements(opts);
|
|
85
85
|
}
|
|
86
86
|
|
|
87
87
|
private async _analyzeScreenshotForVisualProps(): Promise<VisualAnalysisResult> {
|
|
@@ -193,7 +193,6 @@ export function WithCoordinates<T extends Constructor>(Base: T) {
|
|
|
193
193
|
}
|
|
194
194
|
|
|
195
195
|
async backfillCoordinates(result: ResearchResult): Promise<void> {
|
|
196
|
-
const page = this.explorer.playwrightHelper.page;
|
|
197
196
|
const sections = parseResearchSections(result.text);
|
|
198
197
|
const eidxWithoutCoords: string[] = [];
|
|
199
198
|
for (const section of sections) {
|
|
@@ -203,7 +202,7 @@ export function WithCoordinates<T extends Constructor>(Base: T) {
|
|
|
203
202
|
}
|
|
204
203
|
if (eidxWithoutCoords.length === 0) return;
|
|
205
204
|
|
|
206
|
-
const webElements = await WebElement.fromEidxList(page, eidxWithoutCoords);
|
|
205
|
+
const webElements = await this.explorer.runWithBrowserRecovery('backfillCoordinates', () => WebElement.fromEidxList(this.explorer.playwrightHelper.page, eidxWithoutCoords));
|
|
207
206
|
if (webElements.length === 0) return;
|
|
208
207
|
|
|
209
208
|
const rectMap = new Map(webElements.map((w) => [w.eidx!, w]));
|
|
@@ -359,11 +359,10 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
|
|
|
359
359
|
const isCoordinateClick = el.commands[0].startsWith('I.clickXY(');
|
|
360
360
|
if (!isCoordinateClick) {
|
|
361
361
|
const hoverCmd = el.commands[0].replace('I.click(', 'I.moveCursorTo(');
|
|
362
|
-
|
|
363
|
-
await hoverAction.attempt(hoverCmd, undefined, false);
|
|
362
|
+
await this.explorer.attemptAction(hoverCmd, undefined, false);
|
|
364
363
|
await new Promise((r) => setTimeout(r, 500));
|
|
365
364
|
|
|
366
|
-
await this.explorer.
|
|
365
|
+
await this.explorer.capturePageState();
|
|
367
366
|
const hoverAR = ActionResult.fromState(this.stateManager.getCurrentState()!);
|
|
368
367
|
const hoverDiff = await hoverAR.diff(previousState);
|
|
369
368
|
await hoverDiff.calculate();
|
|
@@ -452,7 +451,7 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
|
|
|
452
451
|
private async _restorePageState(url: string, originalAria: string): Promise<void> {
|
|
453
452
|
try {
|
|
454
453
|
await (this as any).cancelInUi();
|
|
455
|
-
await this.explorer.
|
|
454
|
+
await this.explorer.capturePageState();
|
|
456
455
|
const currentAria = this.stateManager.getCurrentState()?.ariaSnapshot || '';
|
|
457
456
|
if (!diffAriaSnapshots(originalAria, currentAria)) return;
|
|
458
457
|
} catch (err) {
|
|
@@ -194,8 +194,7 @@ export function WithLocators<T extends Constructor>(Base: T) {
|
|
|
194
194
|
}
|
|
195
195
|
|
|
196
196
|
if (needsXpath.length > 0) {
|
|
197
|
-
const
|
|
198
|
-
const webElements = await WebElement.fromEidxList(page, needsXpath);
|
|
197
|
+
const webElements = await this.explorer.runWithBrowserRecovery('backfillBrokenLocators', () => WebElement.fromEidxList(this.explorer.playwrightHelper.page, needsXpath));
|
|
199
198
|
const changedSections = new Set<(typeof sections)[0]>();
|
|
200
199
|
for (const w of webElements) {
|
|
201
200
|
const entry = needsXpathEls.get(w.eidx!);
|
package/src/ai/researcher.ts
CHANGED
|
@@ -130,12 +130,12 @@ export class Researcher extends ResearcherBase implements Agent {
|
|
|
130
130
|
|
|
131
131
|
const annotatedElements = await this.explorer.annotateElements();
|
|
132
132
|
debugLog(`Annotated ${annotatedElements.length} interactive elements with eidx`);
|
|
133
|
-
this.actionResult = await this.explorer.
|
|
133
|
+
this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot && this.provider.hasVision() });
|
|
134
134
|
|
|
135
135
|
const condition = detectPageCondition(this.actionResult!);
|
|
136
136
|
if (condition === 'error') {
|
|
137
137
|
tag('warning').log(`Detected error page at ${state.url}`);
|
|
138
|
-
throw new ErrorPageError(state.url, this.actionResult!.title);
|
|
138
|
+
throw new ErrorPageError(state.url, this.actionResult!.title, this.actionResult!.httpStatus);
|
|
139
139
|
}
|
|
140
140
|
if (condition === 'loading') {
|
|
141
141
|
const settled = await this.waitUntilSettled(screenshot);
|
|
@@ -239,7 +239,7 @@ export class Researcher extends ResearcherBase implements Agent {
|
|
|
239
239
|
// Must run BEFORE visuallyAnnotateContainers — annotation overlays inject z-index 99998+ which would pollute the scoring.
|
|
240
240
|
if (!interrupted() && this.hasScreenshotToAnalyze) {
|
|
241
241
|
const sections = parseResearchSections(result.text);
|
|
242
|
-
const focused = await detectFocusedSection(this.explorer.playwrightHelper.page, sections);
|
|
242
|
+
const focused = await this.explorer.runWithBrowserRecovery('detectFocusedSection', () => detectFocusedSection(this.explorer.playwrightHelper.page, sections));
|
|
243
243
|
if (focused) markSectionAsFocused(result, focused);
|
|
244
244
|
}
|
|
245
245
|
|
|
@@ -252,7 +252,7 @@ export class Researcher extends ResearcherBase implements Agent {
|
|
|
252
252
|
const freshBroken = freshContainerLocs.filter((l) => l.valid === false).map((l) => l.locator);
|
|
253
253
|
const containers = validContainers.filter((c) => !freshBroken.includes(c.css));
|
|
254
254
|
await this.visuallyAnnotateElements({ containers });
|
|
255
|
-
this.actionResult = await this.explorer.
|
|
255
|
+
this.actionResult = await this.explorer.capturePageWithScreenshot();
|
|
256
256
|
const visualResult = await this.analyzeScreenshotForVisualProps();
|
|
257
257
|
if (visualResult.elements.size > 0) {
|
|
258
258
|
await this.mergeVisualData(result, visualResult.elements);
|
|
@@ -331,7 +331,7 @@ export class Researcher extends ResearcherBase implements Agent {
|
|
|
331
331
|
if (!this.actionResult) {
|
|
332
332
|
debugLog('No action result, navigating to URL');
|
|
333
333
|
await this.explorer.visit(url);
|
|
334
|
-
this.actionResult = await this.explorer.
|
|
334
|
+
this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot });
|
|
335
335
|
return;
|
|
336
336
|
}
|
|
337
337
|
|
|
@@ -341,7 +341,7 @@ export class Researcher extends ResearcherBase implements Agent {
|
|
|
341
341
|
|
|
342
342
|
if (!isEmpty && isOnCurrentState) {
|
|
343
343
|
if ((!this.actionResult.screenshot && screenshot) || !this.actionResult.ariaSnapshot) {
|
|
344
|
-
this.actionResult = await this.explorer.
|
|
344
|
+
this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot });
|
|
345
345
|
}
|
|
346
346
|
return;
|
|
347
347
|
}
|
|
@@ -349,6 +349,8 @@ export class Researcher extends ResearcherBase implements Agent {
|
|
|
349
349
|
if (isEmpty && isOnCurrentState) {
|
|
350
350
|
debugLog('HTML body empty on current URL, waiting for content');
|
|
351
351
|
tag('step').log('Page body is empty, waiting for content...');
|
|
352
|
+
await this.explorer.visit(url);
|
|
353
|
+
this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot ?? false });
|
|
352
354
|
await this.waitUntilSettled(screenshot ?? false);
|
|
353
355
|
return;
|
|
354
356
|
}
|
|
@@ -357,36 +359,35 @@ export class Researcher extends ResearcherBase implements Agent {
|
|
|
357
359
|
tag('step').log('Navigating to URL...');
|
|
358
360
|
|
|
359
361
|
await this.explorer.visit(url);
|
|
360
|
-
this.actionResult = await this.explorer.
|
|
362
|
+
this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot ?? false });
|
|
361
363
|
}
|
|
362
364
|
|
|
363
365
|
private async waitUntilSettled(screenshot: boolean): Promise<boolean> {
|
|
364
366
|
const errorPageTimeout = (this.explorer.getConfig().ai?.agents?.researcher as any)?.errorPageTimeout ?? 10;
|
|
365
367
|
if (errorPageTimeout <= 0) return false;
|
|
366
368
|
|
|
367
|
-
const page = this.explorer.playwrightHelper.page;
|
|
368
369
|
const includeScreenshot = screenshot && this.provider.hasVision();
|
|
369
370
|
|
|
370
371
|
try {
|
|
371
|
-
await page?.waitForLoadState('networkidle', { timeout: errorPageTimeout * 1000 });
|
|
372
|
+
await this.explorer.runWithBrowserRecovery('waitUntilSettled', () => this.explorer.playwrightHelper.page?.waitForLoadState('networkidle', { timeout: errorPageTimeout * 1000 }));
|
|
372
373
|
} catch {}
|
|
373
374
|
|
|
374
375
|
await this.explorer.annotateElements();
|
|
375
|
-
this.actionResult = await this.explorer.
|
|
376
|
+
this.actionResult = await this.explorer.capturePageState({ includeScreenshot });
|
|
376
377
|
|
|
377
378
|
let condition = detectPageCondition(this.actionResult!);
|
|
378
379
|
if (condition === 'error') {
|
|
379
|
-
throw new ErrorPageError(this.actionResult!.url, this.actionResult!.title);
|
|
380
|
+
throw new ErrorPageError(this.actionResult!.url, this.actionResult!.title, this.actionResult!.httpStatus);
|
|
380
381
|
}
|
|
381
382
|
if (condition === 'ok') return true;
|
|
382
383
|
|
|
383
384
|
for (let i = 0; i < 3; i++) {
|
|
384
385
|
await new Promise((r) => setTimeout(r, 1000));
|
|
385
386
|
await this.explorer.annotateElements();
|
|
386
|
-
this.actionResult = await this.explorer.
|
|
387
|
+
this.actionResult = await this.explorer.capturePageState({ includeScreenshot });
|
|
387
388
|
condition = detectPageCondition(this.actionResult!);
|
|
388
389
|
if (condition === 'error') {
|
|
389
|
-
throw new ErrorPageError(this.actionResult!.url, this.actionResult!.title);
|
|
390
|
+
throw new ErrorPageError(this.actionResult!.url, this.actionResult!.title, this.actionResult!.httpStatus);
|
|
390
391
|
}
|
|
391
392
|
if (condition === 'ok') return true;
|
|
392
393
|
}
|
|
@@ -762,17 +763,15 @@ export class Researcher extends ResearcherBase implements Agent {
|
|
|
762
763
|
}
|
|
763
764
|
|
|
764
765
|
async navigateTo(url: string): Promise<void> {
|
|
765
|
-
|
|
766
|
-
await action.execute(`I.amOnPage("${url}")`);
|
|
766
|
+
await this.explorer.visit(url);
|
|
767
767
|
}
|
|
768
768
|
|
|
769
769
|
async cancelInUi() {
|
|
770
770
|
const beforeAria = this.stateManager.getCurrentState()?.ariaSnapshot || null;
|
|
771
|
-
const action = this.explorer.createAction();
|
|
772
771
|
|
|
773
|
-
await
|
|
772
|
+
await this.explorer.executeAction('I.clickXY(0, 0)');
|
|
774
773
|
if (diffAriaSnapshots(beforeAria, this.stateManager.getCurrentState()?.ariaSnapshot || null)) return;
|
|
775
774
|
|
|
776
|
-
await
|
|
775
|
+
await this.explorer.executeAction(`I.pressKey('Escape')`);
|
|
777
776
|
}
|
|
778
777
|
}
|
package/src/ai/task-agent.ts
CHANGED
|
@@ -24,7 +24,7 @@ export abstract class TaskAgent {
|
|
|
24
24
|
protected consecutiveFailures = 0;
|
|
25
25
|
protected consecutiveEmptyResults = 0;
|
|
26
26
|
protected recentToolCalls: any[] = [];
|
|
27
|
-
protected
|
|
27
|
+
protected readonly ACTION_TOOLS: string[] = [];
|
|
28
28
|
|
|
29
29
|
private _historian: Historian | null = null;
|
|
30
30
|
private _quartermaster: Quartermaster | null = null;
|