explorbot 0.1.27 → 0.1.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/package.json +1 -1
- package/dist/src/action-result.js +2 -0
- package/dist/src/action.js +88 -7
- package/dist/src/ai/captain/file-tools.js +100 -0
- package/dist/src/ai/captain/idle-mode.js +70 -6
- package/dist/src/ai/captain/web-mode.js +36 -6
- package/dist/src/ai/captain.js +87 -19
- package/dist/src/ai/historian/screencast.js +11 -2
- package/dist/src/ai/navigator.js +5 -2
- package/dist/src/ai/pilot.js +33 -5
- package/dist/src/ai/researcher/coordinates.js +2 -3
- package/dist/src/ai/researcher/deep-analysis.js +3 -4
- package/dist/src/ai/researcher/locators.js +1 -2
- package/dist/src/ai/researcher.js +17 -18
- package/dist/src/ai/task-agent.js +1 -0
- package/dist/src/ai/tester.js +91 -39
- package/dist/src/ai/tools.js +17 -7
- package/dist/src/commands/explore-command.js +6 -1
- package/dist/src/components/LogPane.js +4 -3
- package/dist/src/explorer.js +270 -35
- package/dist/src/utils/browser-errors.js +23 -0
- package/dist/src/utils/error-page.js +17 -2
- package/dist/src/utils/logger.js +2 -2
- package/package.json +1 -1
- package/src/action-result.ts +2 -0
- package/src/action.ts +83 -7
- package/src/ai/captain/file-tools.ts +126 -0
- package/src/ai/captain/idle-mode.ts +72 -6
- package/src/ai/captain/mixin.ts +1 -1
- package/src/ai/captain/web-mode.ts +40 -5
- package/src/ai/captain.ts +94 -20
- package/src/ai/historian/screencast.ts +11 -2
- package/src/ai/navigator.ts +6 -2
- package/src/ai/pilot.ts +34 -5
- package/src/ai/researcher/coordinates.ts +2 -3
- package/src/ai/researcher/deep-analysis.ts +3 -4
- package/src/ai/researcher/locators.ts +1 -2
- package/src/ai/researcher.ts +17 -18
- package/src/ai/task-agent.ts +1 -1
- package/src/ai/tester.ts +101 -41
- package/src/ai/tools.ts +17 -7
- package/src/commands/explore-command.ts +6 -1
- package/src/components/LogPane.tsx +4 -3
- package/src/explorer.ts +295 -38
- package/src/state-manager.ts +2 -0
- package/src/utils/browser-errors.ts +25 -0
- package/src/utils/error-page.ts +16 -3
- package/src/utils/logger.ts +3 -3
package/dist/src/ai/captain.js
CHANGED
|
@@ -18,7 +18,6 @@ import { TaskAgent } from "./task-agent.js";
|
|
|
18
18
|
const MAX_STEPS = 15;
|
|
19
19
|
const CaptainBase = WithTestMode(WithWebMode(WithIdleMode(TaskAgent)));
|
|
20
20
|
export class Captain extends CaptainBase {
|
|
21
|
-
ACTION_TOOLS = ['click', 'pressKey', 'form', 'navigate'];
|
|
22
21
|
emoji = '🧑✈️';
|
|
23
22
|
explorBot;
|
|
24
23
|
conversation = null;
|
|
@@ -56,6 +55,12 @@ export class Captain extends CaptainBase {
|
|
|
56
55
|
}
|
|
57
56
|
trackToolExecutions(toolExecutions) {
|
|
58
57
|
super.trackToolExecutions(toolExecutions);
|
|
58
|
+
if (toolExecutions.length > 0) {
|
|
59
|
+
this.recentToolCalls.push(...toolExecutions);
|
|
60
|
+
if (this.recentToolCalls.length > 20) {
|
|
61
|
+
this.recentToolCalls = this.recentToolCalls.slice(-20);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
59
64
|
for (const exec of toolExecutions) {
|
|
60
65
|
const label = toolExecutionLabel(exec.input);
|
|
61
66
|
if (!label)
|
|
@@ -64,15 +69,20 @@ export class Captain extends CaptainBase {
|
|
|
64
69
|
tag('substep').log(`${icon} ${label}`);
|
|
65
70
|
}
|
|
66
71
|
}
|
|
67
|
-
|
|
68
|
-
|
|
72
|
+
getMode() {
|
|
73
|
+
const explorer = this.explorBot.getExplorer();
|
|
74
|
+
const activeTest = explorer.activeTest;
|
|
75
|
+
const page = explorer.playwrightHelper?.page;
|
|
76
|
+
if (activeTest && (!page || page.isClosed?.()))
|
|
77
|
+
return 'heal';
|
|
78
|
+
if (activeTest)
|
|
69
79
|
return 'test';
|
|
70
|
-
if (
|
|
80
|
+
if (explorer.getStateManager().getCurrentState())
|
|
71
81
|
return 'web';
|
|
72
82
|
return 'idle';
|
|
73
83
|
}
|
|
74
84
|
systemPrompt() {
|
|
75
|
-
const mode = this.
|
|
85
|
+
const mode = this.getMode();
|
|
76
86
|
const currentUrl = this.explorBot.getExplorer().getStateManager().getCurrentState()?.url;
|
|
77
87
|
const customPrompt = this.explorBot.getProvider().getSystemPromptForAgent('captain', currentUrl);
|
|
78
88
|
return dedent `
|
|
@@ -85,18 +95,21 @@ export class Captain extends CaptainBase {
|
|
|
85
95
|
- idle: plan management, file operations, knowledge. Always available.
|
|
86
96
|
- web: page interaction, navigation, browser diagnostics. When working with a web page.
|
|
87
97
|
- test: test analysis, state inspection. When a test is running or analyzing results.
|
|
98
|
+
- heal: browser/test recovery. When a test is running and browser state is broken or unavailable.
|
|
88
99
|
</modes>
|
|
89
100
|
|
|
90
101
|
${this.idleModePrompt()}
|
|
91
|
-
${mode === 'web' ? this.webModePrompt() : ''}
|
|
92
|
-
${mode === 'test' ? this.testModePrompt() : ''}
|
|
102
|
+
${mode === 'web' || mode === 'heal' ? this.webModePrompt() : ''}
|
|
103
|
+
${mode === 'test' || mode === 'heal' ? this.testModePrompt() : ''}
|
|
93
104
|
|
|
94
105
|
<rules>
|
|
95
106
|
- After a successful action, if the pageDiff confirms the goal, call done() immediately — do not verify with see() or context() unless the user explicitly asked for verification
|
|
96
107
|
- Prefer completing in fewer tool calls over thoroughness
|
|
97
108
|
- NEVER run tests unless the user explicitly asks
|
|
98
|
-
|
|
99
|
-
${mode === '
|
|
109
|
+
- If you are answering with information rather than completing a browser action, include the actual user-facing answer in done({ details }). Do not only say that it was shown or explained.
|
|
110
|
+
${mode === 'web' || mode === 'heal' ? this.webModeRules() : ''}
|
|
111
|
+
${mode === 'test' || mode === 'heal' ? this.testModeRules() : ''}
|
|
112
|
+
${mode === 'heal' ? '- First diagnose browser availability, then recover the browser/page before continuing test analysis.' : ''}
|
|
100
113
|
</rules>
|
|
101
114
|
|
|
102
115
|
${customPrompt || ''}
|
|
@@ -228,9 +241,20 @@ export class Captain extends CaptainBase {
|
|
|
228
241
|
description: 'Call when the user request is fulfilled.',
|
|
229
242
|
inputSchema: z.object({
|
|
230
243
|
summary: z.string().describe('What was done'),
|
|
244
|
+
details: z.string().optional().describe('Actual user-facing content. Required when the user asked to show, display, explain, summarize, compare, or diagnose information.'),
|
|
231
245
|
}),
|
|
232
|
-
execute: async ({ summary }) => {
|
|
246
|
+
execute: async ({ summary, details }) => {
|
|
233
247
|
debugLog('done', summary);
|
|
248
|
+
if (!details?.trim() && !this.canCompleteWithoutDetails()) {
|
|
249
|
+
return {
|
|
250
|
+
success: false,
|
|
251
|
+
message: 'No user-facing result was provided. Call done() again with the actual answer in details, or complete a browser action first.',
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
if (details?.trim()) {
|
|
255
|
+
tag('details').log(details);
|
|
256
|
+
task.addNote(details);
|
|
257
|
+
}
|
|
234
258
|
task.addNote(summary);
|
|
235
259
|
onDone(summary);
|
|
236
260
|
return { success: true, summary };
|
|
@@ -239,6 +263,9 @@ export class Captain extends CaptainBase {
|
|
|
239
263
|
runCommand: tool({
|
|
240
264
|
description: dedent `
|
|
241
265
|
Execute a TUI command. Returns log output from command execution.
|
|
266
|
+
Use only when the user explicitly asks to run a slash command.
|
|
267
|
+
Never use this to analyze files, reports, logs, plans, generated tests, knowledge, or experience.
|
|
268
|
+
Never run a slash command unless the user request itself starts with that slash command.
|
|
242
269
|
${this.commandDescriptions
|
|
243
270
|
.map((c) => {
|
|
244
271
|
const opts = c.options ? ` (${c.options})` : '';
|
|
@@ -253,6 +280,13 @@ export class Captain extends CaptainBase {
|
|
|
253
280
|
if (!this.commandExecutor)
|
|
254
281
|
return { success: false, message: 'Command executor not available' };
|
|
255
282
|
const cmd = command.startsWith('/') ? command : `/${command}`;
|
|
283
|
+
if (!isExplicitSlashRequest(task.description, cmd)) {
|
|
284
|
+
return {
|
|
285
|
+
success: false,
|
|
286
|
+
command: cmd,
|
|
287
|
+
message: 'Command blocked: slash commands require an explicit matching slash-command request from the user.',
|
|
288
|
+
};
|
|
289
|
+
}
|
|
256
290
|
startLogCapture();
|
|
257
291
|
try {
|
|
258
292
|
await this.commandExecutor(cmd);
|
|
@@ -265,10 +299,12 @@ export class Captain extends CaptainBase {
|
|
|
265
299
|
};
|
|
266
300
|
}
|
|
267
301
|
async tools(task, onDone) {
|
|
268
|
-
const mode = this.
|
|
302
|
+
const mode = this.getMode();
|
|
269
303
|
const ctx = { explorBot: this.explorBot, task };
|
|
270
304
|
const core = this.coreTools(task, onDone);
|
|
271
305
|
const idle = await this.idleModeTools(ctx);
|
|
306
|
+
if (mode === 'heal')
|
|
307
|
+
return { ...core, ...idle, ...this.testModeTools(ctx), ...this.webModeTools(ctx) };
|
|
272
308
|
if (mode === 'test')
|
|
273
309
|
return { ...core, ...idle, ...this.testModeTools(ctx) };
|
|
274
310
|
if (mode === 'web')
|
|
@@ -333,17 +369,24 @@ export class Captain extends CaptainBase {
|
|
|
333
369
|
}
|
|
334
370
|
return result.object;
|
|
335
371
|
}
|
|
372
|
+
async processExecutionError(error, activeTest) {
|
|
373
|
+
const explorer = this.explorBot.getExplorer();
|
|
374
|
+
const result = await explorer.handleExecutionError(error);
|
|
375
|
+
return {
|
|
376
|
+
...result,
|
|
377
|
+
message: result.recovered ? `${result.message}\nContinue the test "${activeTest.scenario}" from the restored page.` : result.message,
|
|
378
|
+
};
|
|
379
|
+
}
|
|
380
|
+
canCompleteWithoutDetails() {
|
|
381
|
+
return (this.recentToolCalls || []).some(hasBrowserCompletionEvidence);
|
|
382
|
+
}
|
|
336
383
|
async handle(input, options = {}) {
|
|
337
384
|
const stateManager = this.explorBot.getExplorer().getStateManager();
|
|
338
385
|
const initialState = stateManager.getCurrentState();
|
|
339
|
-
if (!initialState) {
|
|
340
|
-
tag('warning').log('No page loaded. Use /navigate or I.amOnPage() first.');
|
|
341
|
-
return null;
|
|
342
|
-
}
|
|
343
386
|
const conversation = options.reset ? this.resetConversation() : this.ensureConversation();
|
|
344
387
|
let isDone = false;
|
|
345
388
|
let finalSummary = null;
|
|
346
|
-
const startUrl = initialState
|
|
389
|
+
const startUrl = initialState?.url || '';
|
|
347
390
|
const task = new Task(input, startUrl);
|
|
348
391
|
const onDone = (summary) => {
|
|
349
392
|
isDone = true;
|
|
@@ -376,11 +419,13 @@ export class Captain extends CaptainBase {
|
|
|
376
419
|
return;
|
|
377
420
|
}
|
|
378
421
|
const currentState = stateManager.getCurrentState();
|
|
379
|
-
if (!currentState) {
|
|
422
|
+
if (!currentState && this.getMode() !== 'idle') {
|
|
380
423
|
stop();
|
|
381
424
|
return;
|
|
382
425
|
}
|
|
383
|
-
|
|
426
|
+
if (currentState) {
|
|
427
|
+
await this.reinjectContextIfNeeded(conversation, currentState);
|
|
428
|
+
}
|
|
384
429
|
if (userInput) {
|
|
385
430
|
const newContext = await this.getPageContext();
|
|
386
431
|
conversation.addUserText(dedent `
|
|
@@ -410,7 +455,7 @@ export class Captain extends CaptainBase {
|
|
|
410
455
|
}
|
|
411
456
|
if (result?.toolExecutions?.length) {
|
|
412
457
|
const lastExec = result.toolExecutions[result.toolExecutions.length - 1];
|
|
413
|
-
if (
|
|
458
|
+
if (hasBrowserCompletionEvidence(lastExec)) {
|
|
414
459
|
conversation.addUserText('Action succeeded. If the goal is achieved, call done() now with a brief summary.');
|
|
415
460
|
}
|
|
416
461
|
}
|
|
@@ -437,3 +482,26 @@ export class Captain extends CaptainBase {
|
|
|
437
482
|
}
|
|
438
483
|
}
|
|
439
484
|
export default Captain;
|
|
485
|
+
function isExplicitSlashRequest(input, command) {
|
|
486
|
+
const requested = slashCommandToken(input);
|
|
487
|
+
const actual = slashCommandToken(command);
|
|
488
|
+
if (!requested || !actual)
|
|
489
|
+
return false;
|
|
490
|
+
return requested === actual;
|
|
491
|
+
}
|
|
492
|
+
function slashCommandToken(value) {
|
|
493
|
+
const trimmed = value.trim();
|
|
494
|
+
if (!trimmed.startsWith('/'))
|
|
495
|
+
return null;
|
|
496
|
+
for (let i = 1; i < trimmed.length; i++) {
|
|
497
|
+
if (trimmed[i] <= ' ')
|
|
498
|
+
return trimmed.slice(0, i);
|
|
499
|
+
}
|
|
500
|
+
return trimmed;
|
|
501
|
+
}
|
|
502
|
+
function hasBrowserCompletionEvidence(execution) {
|
|
503
|
+
if (!execution?.wasSuccessful)
|
|
504
|
+
return false;
|
|
505
|
+
const output = execution.output || {};
|
|
506
|
+
return Boolean(output.pageDiff || output.code || output.playwrightGroupId);
|
|
507
|
+
}
|
|
@@ -7,6 +7,7 @@ import { tag } from "../../utils/logger.js";
|
|
|
7
7
|
import { relativeToCwd } from "../../utils/next-steps.js";
|
|
8
8
|
import { safeFilename } from "../../utils/strings.js";
|
|
9
9
|
import { debugLog } from "./mixin.js";
|
|
10
|
+
const FATAL_SCREENCAST_STOP_ERRORS = /Target page, context or browser has been closed|Target closed|Session closed|Protocol error/i;
|
|
10
11
|
export function WithScreencast(Base) {
|
|
11
12
|
return class extends Base {
|
|
12
13
|
screencastPage = null;
|
|
@@ -100,18 +101,26 @@ export function WithScreencast(Base) {
|
|
|
100
101
|
return;
|
|
101
102
|
const path = this.screencastPath;
|
|
102
103
|
const task = this.screencastTask;
|
|
104
|
+
let stopped = false;
|
|
103
105
|
try {
|
|
104
106
|
await this.screencastPage.screencast.stop();
|
|
107
|
+
stopped = true;
|
|
105
108
|
}
|
|
106
109
|
catch (err) {
|
|
107
|
-
|
|
110
|
+
const message = err.message;
|
|
111
|
+
if (FATAL_SCREENCAST_STOP_ERRORS.test(message)) {
|
|
112
|
+
tag('operation').log('Screencast skipped: browser was closed before recording could be finalized');
|
|
113
|
+
}
|
|
114
|
+
else {
|
|
115
|
+
tag('operation').log(`Screencast stop failed: ${message}`);
|
|
116
|
+
}
|
|
108
117
|
}
|
|
109
118
|
this.screencastActive = false;
|
|
110
119
|
this.screencastPage = null;
|
|
111
120
|
this.screencastPath = null;
|
|
112
121
|
this.screencastTask = null;
|
|
113
122
|
this.screencastLastChapter = null;
|
|
114
|
-
if (path) {
|
|
123
|
+
if (path && stopped) {
|
|
115
124
|
this.savedFiles.add(path);
|
|
116
125
|
task?.addArtifact?.(path);
|
|
117
126
|
tag('operation').log(`Saved screencast: ${relativeToCwd(path)}`);
|
package/dist/src/ai/navigator.js
CHANGED
|
@@ -121,6 +121,9 @@ class Navigator {
|
|
|
121
121
|
return normalizeUrl(currentUrl) === normalizeUrl(expectedUrl);
|
|
122
122
|
}
|
|
123
123
|
async visit(url) {
|
|
124
|
+
return this.explorer.runWithBrowserRecovery('navigator.visit', () => this.visitOnce(url));
|
|
125
|
+
}
|
|
126
|
+
async visitOnce(url) {
|
|
124
127
|
try {
|
|
125
128
|
const action = this.explorer.createAction();
|
|
126
129
|
await action.execute(`I.amOnPage('${url}')`);
|
|
@@ -150,7 +153,7 @@ class Navigator {
|
|
|
150
153
|
throw new Error(`Navigation to ${url} failed: ${action.lastError?.message}`);
|
|
151
154
|
}
|
|
152
155
|
}
|
|
153
|
-
await
|
|
156
|
+
await this.explorer.capturePageWithScreenshot();
|
|
154
157
|
await this.hooksRunner.runAfterHook('navigator', url);
|
|
155
158
|
}
|
|
156
159
|
catch (error) {
|
|
@@ -349,7 +352,7 @@ class Navigator {
|
|
|
349
352
|
// URL did not transition to expectedUrl within timeout
|
|
350
353
|
}
|
|
351
354
|
}
|
|
352
|
-
const freshState = await
|
|
355
|
+
const freshState = await this.explorer.capturePageState();
|
|
353
356
|
const currentUrl = /^https?:\/\//i.test(expectedUrl) ? freshState.fullUrl || freshState.url || '' : freshState.url || '';
|
|
354
357
|
const urlMatches = this.isSameExpectedOrigin(expectedUrl, action.stateManager) && normalizeUrl(currentUrl) === normalizeUrl(expectedUrl);
|
|
355
358
|
const stateChanged = freshState.getStateHash() !== actionResult.getStateHash();
|
package/dist/src/ai/pilot.js
CHANGED
|
@@ -50,7 +50,7 @@ export class Pilot {
|
|
|
50
50
|
return this.reviewDecision('finish', task, currentState, testerConversation, navigator);
|
|
51
51
|
}
|
|
52
52
|
async reviewCompletion(task, currentState, testerConversation, navigator) {
|
|
53
|
-
const verdictType =
|
|
53
|
+
const verdictType = this.hasCompletionEvidence(task, currentState, testerConversation) ? 'finish' : 'stop';
|
|
54
54
|
return this.reviewDecision(verdictType, task, currentState, testerConversation, navigator);
|
|
55
55
|
}
|
|
56
56
|
async finalReview(task, currentState, testerConversation, navigator) {
|
|
@@ -67,13 +67,13 @@ export class Pilot {
|
|
|
67
67
|
tag('substep').log(`Pilot reviewing ${type} verdict...`);
|
|
68
68
|
const sessionLog = this.formatSessionLog(testerConversation);
|
|
69
69
|
const stateContext = this.buildStateContext(currentState);
|
|
70
|
+
const successfulAssertions = this.formatSuccessfulAssertions(currentState, testerConversation);
|
|
70
71
|
const notes = task.notesToString() || 'No notes recorded.';
|
|
71
72
|
let visualAnalysis = '';
|
|
72
73
|
let screenshotState = null;
|
|
73
74
|
if (this.provider.hasVision()) {
|
|
74
75
|
try {
|
|
75
|
-
|
|
76
|
-
screenshotState = await action.caputrePageWithScreenshot();
|
|
76
|
+
screenshotState = await this.explorer.capturePageWithScreenshot();
|
|
77
77
|
if (screenshotState.screenshot) {
|
|
78
78
|
visualAnalysis = (await this.researcher.answerQuestionAboutScreenshot(screenshotState, `Describe current page state relevant to: ${task.scenario}`)) || '';
|
|
79
79
|
}
|
|
@@ -102,6 +102,10 @@ export class Pilot {
|
|
|
102
102
|
|
|
103
103
|
${this.formatExpectations(task)}
|
|
104
104
|
|
|
105
|
+
<successful_assertions>
|
|
106
|
+
${successfulAssertions || 'None'}
|
|
107
|
+
</successful_assertions>
|
|
108
|
+
|
|
105
109
|
<notes>
|
|
106
110
|
${notes}
|
|
107
111
|
</notes>
|
|
@@ -587,8 +591,7 @@ export class Pilot {
|
|
|
587
591
|
async checkDataAvailability(task, requestedData, fishermanReason) {
|
|
588
592
|
if (!this.provider.hasVision())
|
|
589
593
|
return null;
|
|
590
|
-
const
|
|
591
|
-
const screenshotState = await action.caputrePageWithScreenshot().catch(() => null);
|
|
594
|
+
const screenshotState = await this.explorer.capturePageWithScreenshot().catch(() => null);
|
|
592
595
|
if (!screenshotState?.screenshot)
|
|
593
596
|
return null;
|
|
594
597
|
const question = dedent `
|
|
@@ -791,6 +794,31 @@ export class Pilot {
|
|
|
791
794
|
}
|
|
792
795
|
return parts.join('\n\n');
|
|
793
796
|
}
|
|
797
|
+
hasCompletionEvidence(task, currentState, testerConversation) {
|
|
798
|
+
if (task.hasAchievedAny())
|
|
799
|
+
return true;
|
|
800
|
+
return this.hasSuccessfulCheckEvidence(currentState, testerConversation);
|
|
801
|
+
}
|
|
802
|
+
hasSuccessfulCheckEvidence(currentState, testerConversation) {
|
|
803
|
+
if (Object.values(currentState.verifications ?? {}).some(Boolean))
|
|
804
|
+
return true;
|
|
805
|
+
return testerConversation.getToolExecutions().some((t) => CHECK_TOOLS.includes(t.toolName) && t.wasSuccessful);
|
|
806
|
+
}
|
|
807
|
+
formatSuccessfulAssertions(currentState, testerConversation) {
|
|
808
|
+
const lines = [];
|
|
809
|
+
for (const [assertion, passed] of Object.entries(currentState.verifications ?? {})) {
|
|
810
|
+
if (passed)
|
|
811
|
+
lines.push(`PASS state verification: ${assertion}`);
|
|
812
|
+
}
|
|
813
|
+
for (const exec of testerConversation.getToolExecutions()) {
|
|
814
|
+
if (!CHECK_TOOLS.includes(exec.toolName) || !exec.wasSuccessful)
|
|
815
|
+
continue;
|
|
816
|
+
const description = exec.input?.assertion || exec.input?.request || truncateJson(exec.input);
|
|
817
|
+
const result = exec.output?.message || exec.output?.analysis || exec.output?.result;
|
|
818
|
+
lines.push(`PASS ${exec.toolName}: ${description}${result ? ` -> ${result}` : ''}`);
|
|
819
|
+
}
|
|
820
|
+
return [...new Set(lines)].join('\n');
|
|
821
|
+
}
|
|
794
822
|
formatActions(toolCalls) {
|
|
795
823
|
return toolCalls
|
|
796
824
|
.map((t) => {
|
|
@@ -67,7 +67,7 @@ export function WithCoordinates(Base) {
|
|
|
67
67
|
return this._analyzeScreenshotForVisualProps();
|
|
68
68
|
}
|
|
69
69
|
async visuallyAnnotateElements(opts) {
|
|
70
|
-
return
|
|
70
|
+
return this.explorer.visuallyAnnotateElements(opts);
|
|
71
71
|
}
|
|
72
72
|
async _analyzeScreenshotForVisualProps() {
|
|
73
73
|
const elements = new Map();
|
|
@@ -177,7 +177,6 @@ export function WithCoordinates(Base) {
|
|
|
177
177
|
debugLog(`Merged visual props for ${merged} elements`);
|
|
178
178
|
}
|
|
179
179
|
async backfillCoordinates(result) {
|
|
180
|
-
const page = this.explorer.playwrightHelper.page;
|
|
181
180
|
const sections = parseResearchSections(result.text);
|
|
182
181
|
const eidxWithoutCoords = [];
|
|
183
182
|
for (const section of sections) {
|
|
@@ -188,7 +187,7 @@ export function WithCoordinates(Base) {
|
|
|
188
187
|
}
|
|
189
188
|
if (eidxWithoutCoords.length === 0)
|
|
190
189
|
return;
|
|
191
|
-
const webElements = await WebElement.fromEidxList(page, eidxWithoutCoords);
|
|
190
|
+
const webElements = await this.explorer.runWithBrowserRecovery('backfillCoordinates', () => WebElement.fromEidxList(this.explorer.playwrightHelper.page, eidxWithoutCoords));
|
|
192
191
|
if (webElements.length === 0)
|
|
193
192
|
return;
|
|
194
193
|
const rectMap = new Map(webElements.map((w) => [w.eidx, w]));
|
|
@@ -312,10 +312,9 @@ export function WithDeepAnalysis(Base) {
|
|
|
312
312
|
const isCoordinateClick = el.commands[0].startsWith('I.clickXY(');
|
|
313
313
|
if (!isCoordinateClick) {
|
|
314
314
|
const hoverCmd = el.commands[0].replace('I.click(', 'I.moveCursorTo(');
|
|
315
|
-
|
|
316
|
-
await hoverAction.attempt(hoverCmd, undefined, false);
|
|
315
|
+
await this.explorer.attemptAction(hoverCmd, undefined, false);
|
|
317
316
|
await new Promise((r) => setTimeout(r, 500));
|
|
318
|
-
await this.explorer.
|
|
317
|
+
await this.explorer.capturePageState();
|
|
319
318
|
const hoverAR = ActionResult.fromState(this.stateManager.getCurrentState());
|
|
320
319
|
const hoverDiff = await hoverAR.diff(previousState);
|
|
321
320
|
await hoverDiff.calculate();
|
|
@@ -398,7 +397,7 @@ export function WithDeepAnalysis(Base) {
|
|
|
398
397
|
async _restorePageState(url, originalAria) {
|
|
399
398
|
try {
|
|
400
399
|
await this.cancelInUi();
|
|
401
|
-
await this.explorer.
|
|
400
|
+
await this.explorer.capturePageState();
|
|
402
401
|
const currentAria = this.stateManager.getCurrentState()?.ariaSnapshot || '';
|
|
403
402
|
if (!diffAriaSnapshots(originalAria, currentAria))
|
|
404
403
|
return;
|
|
@@ -173,8 +173,7 @@ export function WithLocators(Base) {
|
|
|
173
173
|
}
|
|
174
174
|
}
|
|
175
175
|
if (needsXpath.length > 0) {
|
|
176
|
-
const
|
|
177
|
-
const webElements = await WebElement.fromEidxList(page, needsXpath);
|
|
176
|
+
const webElements = await this.explorer.runWithBrowserRecovery('backfillBrokenLocators', () => WebElement.fromEidxList(this.explorer.playwrightHelper.page, needsXpath));
|
|
178
177
|
const changedSections = new Set();
|
|
179
178
|
for (const w of webElements) {
|
|
180
179
|
const entry = needsXpathEls.get(w.eidx);
|
|
@@ -97,11 +97,11 @@ export class Researcher extends ResearcherBase {
|
|
|
97
97
|
await this.hooksRunner.runBeforeHook('researcher', state.url);
|
|
98
98
|
const annotatedElements = await this.explorer.annotateElements();
|
|
99
99
|
debugLog(`Annotated ${annotatedElements.length} interactive elements with eidx`);
|
|
100
|
-
this.actionResult = await this.explorer.
|
|
100
|
+
this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot && this.provider.hasVision() });
|
|
101
101
|
const condition = detectPageCondition(this.actionResult);
|
|
102
102
|
if (condition === 'error') {
|
|
103
103
|
tag('warning').log(`Detected error page at ${state.url}`);
|
|
104
|
-
throw new ErrorPageError(state.url, this.actionResult.title);
|
|
104
|
+
throw new ErrorPageError(state.url, this.actionResult.title, this.actionResult.httpStatus);
|
|
105
105
|
}
|
|
106
106
|
if (condition === 'loading') {
|
|
107
107
|
const settled = await this.waitUntilSettled(screenshot);
|
|
@@ -191,7 +191,7 @@ export class Researcher extends ResearcherBase {
|
|
|
191
191
|
// Must run BEFORE visuallyAnnotateContainers — annotation overlays inject z-index 99998+ which would pollute the scoring.
|
|
192
192
|
if (!interrupted() && this.hasScreenshotToAnalyze) {
|
|
193
193
|
const sections = parseResearchSections(result.text);
|
|
194
|
-
const focused = await detectFocusedSection(this.explorer.playwrightHelper.page, sections);
|
|
194
|
+
const focused = await this.explorer.runWithBrowserRecovery('detectFocusedSection', () => detectFocusedSection(this.explorer.playwrightHelper.page, sections));
|
|
195
195
|
if (focused)
|
|
196
196
|
markSectionAsFocused(result, focused);
|
|
197
197
|
}
|
|
@@ -204,7 +204,7 @@ export class Researcher extends ResearcherBase {
|
|
|
204
204
|
const freshBroken = freshContainerLocs.filter((l) => l.valid === false).map((l) => l.locator);
|
|
205
205
|
const containers = validContainers.filter((c) => !freshBroken.includes(c.css));
|
|
206
206
|
await this.visuallyAnnotateElements({ containers });
|
|
207
|
-
this.actionResult = await this.explorer.
|
|
207
|
+
this.actionResult = await this.explorer.capturePageWithScreenshot();
|
|
208
208
|
const visualResult = await this.analyzeScreenshotForVisualProps();
|
|
209
209
|
if (visualResult.elements.size > 0) {
|
|
210
210
|
await this.mergeVisualData(result, visualResult.elements);
|
|
@@ -277,7 +277,7 @@ export class Researcher extends ResearcherBase {
|
|
|
277
277
|
if (!this.actionResult) {
|
|
278
278
|
debugLog('No action result, navigating to URL');
|
|
279
279
|
await this.explorer.visit(url);
|
|
280
|
-
this.actionResult = await this.explorer.
|
|
280
|
+
this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot });
|
|
281
281
|
return;
|
|
282
282
|
}
|
|
283
283
|
const isOnCurrentState = this.actionResult.getStateHash() === this.stateManager.getCurrentState()?.hash;
|
|
@@ -285,46 +285,47 @@ export class Researcher extends ResearcherBase {
|
|
|
285
285
|
const isEmpty = isBodyEmpty(stateHtml);
|
|
286
286
|
if (!isEmpty && isOnCurrentState) {
|
|
287
287
|
if ((!this.actionResult.screenshot && screenshot) || !this.actionResult.ariaSnapshot) {
|
|
288
|
-
this.actionResult = await this.explorer.
|
|
288
|
+
this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot });
|
|
289
289
|
}
|
|
290
290
|
return;
|
|
291
291
|
}
|
|
292
292
|
if (isEmpty && isOnCurrentState) {
|
|
293
293
|
debugLog('HTML body empty on current URL, waiting for content');
|
|
294
294
|
tag('step').log('Page body is empty, waiting for content...');
|
|
295
|
+
await this.explorer.visit(url);
|
|
296
|
+
this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot ?? false });
|
|
295
297
|
await this.waitUntilSettled(screenshot ?? false);
|
|
296
298
|
return;
|
|
297
299
|
}
|
|
298
300
|
debugLog('Not on current state, navigating to URL');
|
|
299
301
|
tag('step').log('Navigating to URL...');
|
|
300
302
|
await this.explorer.visit(url);
|
|
301
|
-
this.actionResult = await this.explorer.
|
|
303
|
+
this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot ?? false });
|
|
302
304
|
}
|
|
303
305
|
async waitUntilSettled(screenshot) {
|
|
304
306
|
const errorPageTimeout = this.explorer.getConfig().ai?.agents?.researcher?.errorPageTimeout ?? 10;
|
|
305
307
|
if (errorPageTimeout <= 0)
|
|
306
308
|
return false;
|
|
307
|
-
const page = this.explorer.playwrightHelper.page;
|
|
308
309
|
const includeScreenshot = screenshot && this.provider.hasVision();
|
|
309
310
|
try {
|
|
310
|
-
await page?.waitForLoadState('networkidle', { timeout: errorPageTimeout * 1000 });
|
|
311
|
+
await this.explorer.runWithBrowserRecovery('waitUntilSettled', () => this.explorer.playwrightHelper.page?.waitForLoadState('networkidle', { timeout: errorPageTimeout * 1000 }));
|
|
311
312
|
}
|
|
312
313
|
catch { }
|
|
313
314
|
await this.explorer.annotateElements();
|
|
314
|
-
this.actionResult = await this.explorer.
|
|
315
|
+
this.actionResult = await this.explorer.capturePageState({ includeScreenshot });
|
|
315
316
|
let condition = detectPageCondition(this.actionResult);
|
|
316
317
|
if (condition === 'error') {
|
|
317
|
-
throw new ErrorPageError(this.actionResult.url, this.actionResult.title);
|
|
318
|
+
throw new ErrorPageError(this.actionResult.url, this.actionResult.title, this.actionResult.httpStatus);
|
|
318
319
|
}
|
|
319
320
|
if (condition === 'ok')
|
|
320
321
|
return true;
|
|
321
322
|
for (let i = 0; i < 3; i++) {
|
|
322
323
|
await new Promise((r) => setTimeout(r, 1000));
|
|
323
324
|
await this.explorer.annotateElements();
|
|
324
|
-
this.actionResult = await this.explorer.
|
|
325
|
+
this.actionResult = await this.explorer.capturePageState({ includeScreenshot });
|
|
325
326
|
condition = detectPageCondition(this.actionResult);
|
|
326
327
|
if (condition === 'error') {
|
|
327
|
-
throw new ErrorPageError(this.actionResult.url, this.actionResult.title);
|
|
328
|
+
throw new ErrorPageError(this.actionResult.url, this.actionResult.title, this.actionResult.httpStatus);
|
|
328
329
|
}
|
|
329
330
|
if (condition === 'ok')
|
|
330
331
|
return true;
|
|
@@ -681,15 +682,13 @@ export class Researcher extends ResearcherBase {
|
|
|
681
682
|
.join('\n\n');
|
|
682
683
|
}
|
|
683
684
|
async navigateTo(url) {
|
|
684
|
-
|
|
685
|
-
await action.execute(`I.amOnPage("${url}")`);
|
|
685
|
+
await this.explorer.visit(url);
|
|
686
686
|
}
|
|
687
687
|
async cancelInUi() {
|
|
688
688
|
const beforeAria = this.stateManager.getCurrentState()?.ariaSnapshot || null;
|
|
689
|
-
|
|
690
|
-
await action.execute('I.clickXY(0, 0)');
|
|
689
|
+
await this.explorer.executeAction('I.clickXY(0, 0)');
|
|
691
690
|
if (diffAriaSnapshots(beforeAria, this.stateManager.getCurrentState()?.ariaSnapshot || null))
|
|
692
691
|
return;
|
|
693
|
-
await
|
|
692
|
+
await this.explorer.executeAction(`I.pressKey('Escape')`);
|
|
694
693
|
}
|
|
695
694
|
}
|