explorbot 0.1.27 → 0.1.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/dist/package.json +1 -1
  2. package/dist/src/action-result.js +2 -0
  3. package/dist/src/action.js +88 -7
  4. package/dist/src/ai/captain/file-tools.js +100 -0
  5. package/dist/src/ai/captain/idle-mode.js +70 -6
  6. package/dist/src/ai/captain/web-mode.js +36 -6
  7. package/dist/src/ai/captain.js +87 -19
  8. package/dist/src/ai/historian/screencast.js +11 -2
  9. package/dist/src/ai/navigator.js +5 -2
  10. package/dist/src/ai/pilot.js +33 -5
  11. package/dist/src/ai/researcher/coordinates.js +2 -3
  12. package/dist/src/ai/researcher/deep-analysis.js +3 -4
  13. package/dist/src/ai/researcher/locators.js +1 -2
  14. package/dist/src/ai/researcher.js +17 -18
  15. package/dist/src/ai/task-agent.js +1 -0
  16. package/dist/src/ai/tester.js +91 -39
  17. package/dist/src/ai/tools.js +17 -7
  18. package/dist/src/commands/explore-command.js +6 -1
  19. package/dist/src/components/LogPane.js +4 -3
  20. package/dist/src/explorer.js +270 -35
  21. package/dist/src/utils/browser-errors.js +23 -0
  22. package/dist/src/utils/error-page.js +17 -2
  23. package/dist/src/utils/logger.js +2 -2
  24. package/package.json +1 -1
  25. package/src/action-result.ts +2 -0
  26. package/src/action.ts +83 -7
  27. package/src/ai/captain/file-tools.ts +126 -0
  28. package/src/ai/captain/idle-mode.ts +72 -6
  29. package/src/ai/captain/mixin.ts +1 -1
  30. package/src/ai/captain/web-mode.ts +40 -5
  31. package/src/ai/captain.ts +94 -20
  32. package/src/ai/historian/screencast.ts +11 -2
  33. package/src/ai/navigator.ts +6 -2
  34. package/src/ai/pilot.ts +34 -5
  35. package/src/ai/researcher/coordinates.ts +2 -3
  36. package/src/ai/researcher/deep-analysis.ts +3 -4
  37. package/src/ai/researcher/locators.ts +1 -2
  38. package/src/ai/researcher.ts +17 -18
  39. package/src/ai/task-agent.ts +1 -1
  40. package/src/ai/tester.ts +101 -41
  41. package/src/ai/tools.ts +17 -7
  42. package/src/commands/explore-command.ts +6 -1
  43. package/src/components/LogPane.tsx +4 -3
  44. package/src/explorer.ts +295 -38
  45. package/src/state-manager.ts +2 -0
  46. package/src/utils/browser-errors.ts +25 -0
  47. package/src/utils/error-page.ts +16 -3
  48. package/src/utils/logger.ts +3 -3
package/src/ai/captain.ts CHANGED
@@ -26,7 +26,6 @@ const MAX_STEPS = 15;
26
26
  const CaptainBase = WithTestMode(WithWebMode(WithIdleMode(TaskAgent as unknown as new (...args: any[]) => TaskAgent)));
27
27
 
28
28
  export class Captain extends CaptainBase implements Agent {
29
- protected readonly ACTION_TOOLS = ['click', 'pressKey', 'form', 'navigate'];
30
29
  emoji = '🧑‍✈️';
31
30
  private explorBot: ExplorBot;
32
31
  private conversation: Conversation | null = null;
@@ -72,6 +71,12 @@ export class Captain extends CaptainBase implements Agent {
72
71
 
73
72
  protected trackToolExecutions(toolExecutions: any[]): void {
74
73
  super.trackToolExecutions(toolExecutions);
74
+ if (toolExecutions.length > 0) {
75
+ this.recentToolCalls.push(...toolExecutions);
76
+ if (this.recentToolCalls.length > 20) {
77
+ this.recentToolCalls = this.recentToolCalls.slice(-20);
78
+ }
79
+ }
75
80
  for (const exec of toolExecutions) {
76
81
  const label = toolExecutionLabel(exec.input);
77
82
  if (!label) continue;
@@ -80,14 +85,19 @@ export class Captain extends CaptainBase implements Agent {
80
85
  }
81
86
  }
82
87
 
83
- private detectMode(): CaptainMode {
84
- if (this.explorBot.getExplorer().activeTest) return 'test';
85
- if (this.explorBot.getExplorer().getStateManager().getCurrentState()) return 'web';
88
+ getMode(): CaptainMode {
89
+ const explorer = this.explorBot.getExplorer();
90
+ const activeTest = explorer.activeTest;
91
+ const page = explorer.playwrightHelper?.page;
92
+
93
+ if (activeTest && (!page || page.isClosed?.())) return 'heal';
94
+ if (activeTest) return 'test';
95
+ if (explorer.getStateManager().getCurrentState()) return 'web';
86
96
  return 'idle';
87
97
  }
88
98
 
89
99
  private systemPrompt(): string {
90
- const mode = this.detectMode();
100
+ const mode = this.getMode();
91
101
  const currentUrl = this.explorBot.getExplorer().getStateManager().getCurrentState()?.url;
92
102
  const customPrompt = this.explorBot.getProvider().getSystemPromptForAgent('captain', currentUrl);
93
103
 
@@ -101,18 +111,21 @@ export class Captain extends CaptainBase implements Agent {
101
111
  - idle: plan management, file operations, knowledge. Always available.
102
112
  - web: page interaction, navigation, browser diagnostics. When working with a web page.
103
113
  - test: test analysis, state inspection. When a test is running or analyzing results.
114
+ - heal: browser/test recovery. When a test is running and browser state is broken or unavailable.
104
115
  </modes>
105
116
 
106
117
  ${this.idleModePrompt()}
107
- ${mode === 'web' ? this.webModePrompt() : ''}
108
- ${mode === 'test' ? this.testModePrompt() : ''}
118
+ ${mode === 'web' || mode === 'heal' ? this.webModePrompt() : ''}
119
+ ${mode === 'test' || mode === 'heal' ? this.testModePrompt() : ''}
109
120
 
110
121
  <rules>
111
122
  - After a successful action, if the pageDiff confirms the goal, call done() immediately — do not verify with see() or context() unless the user explicitly asked for verification
112
123
  - Prefer completing in fewer tool calls over thoroughness
113
124
  - NEVER run tests unless the user explicitly asks
114
- ${mode === 'web' ? this.webModeRules() : ''}
115
- ${mode === 'test' ? this.testModeRules() : ''}
125
+ - If you are answering with information rather than completing a browser action, include the actual user-facing answer in done({ details }). Do not only say that it was shown or explained.
126
+ ${mode === 'web' || mode === 'heal' ? this.webModeRules() : ''}
127
+ ${mode === 'test' || mode === 'heal' ? this.testModeRules() : ''}
128
+ ${mode === 'heal' ? '- First diagnose browser availability, then recover the browser/page before continuing test analysis.' : ''}
116
129
  </rules>
117
130
 
118
131
  ${customPrompt || ''}
@@ -250,9 +263,20 @@ export class Captain extends CaptainBase implements Agent {
250
263
  description: 'Call when the user request is fulfilled.',
251
264
  inputSchema: z.object({
252
265
  summary: z.string().describe('What was done'),
266
+ details: z.string().optional().describe('Actual user-facing content. Required when the user asked to show, display, explain, summarize, compare, or diagnose information.'),
253
267
  }),
254
- execute: async ({ summary }) => {
268
+ execute: async ({ summary, details }) => {
255
269
  debugLog('done', summary);
270
+ if (!details?.trim() && !this.canCompleteWithoutDetails()) {
271
+ return {
272
+ success: false,
273
+ message: 'No user-facing result was provided. Call done() again with the actual answer in details, or complete a browser action first.',
274
+ };
275
+ }
276
+ if (details?.trim()) {
277
+ tag('details').log(details);
278
+ task.addNote(details);
279
+ }
256
280
  task.addNote(summary);
257
281
  onDone(summary);
258
282
  return { success: true, summary };
@@ -261,6 +285,9 @@ export class Captain extends CaptainBase implements Agent {
261
285
  runCommand: tool({
262
286
  description: dedent`
263
287
  Execute a TUI command. Returns log output from command execution.
288
+ Use only when the user explicitly asks to run a slash command.
289
+ Never use this to analyze files, reports, logs, plans, generated tests, knowledge, or experience.
290
+ Never run a slash command unless the user request itself starts with that slash command.
264
291
  ${this.commandDescriptions
265
292
  .map((c) => {
266
293
  const opts = c.options ? ` (${c.options})` : '';
@@ -274,6 +301,13 @@ export class Captain extends CaptainBase implements Agent {
274
301
  execute: async ({ command }) => {
275
302
  if (!this.commandExecutor) return { success: false, message: 'Command executor not available' };
276
303
  const cmd = command.startsWith('/') ? command : `/${command}`;
304
+ if (!isExplicitSlashRequest(task.description, cmd)) {
305
+ return {
306
+ success: false,
307
+ command: cmd,
308
+ message: 'Command blocked: slash commands require an explicit matching slash-command request from the user.',
309
+ };
310
+ }
277
311
  startLogCapture();
278
312
  try {
279
313
  await this.commandExecutor(cmd);
@@ -286,11 +320,12 @@ export class Captain extends CaptainBase implements Agent {
286
320
  }
287
321
 
288
322
  private async tools(task: Task, onDone: (summary: string) => void) {
289
- const mode = this.detectMode();
323
+ const mode = this.getMode();
290
324
  const ctx: ModeContext = { explorBot: this.explorBot, task };
291
325
  const core = this.coreTools(task, onDone);
292
326
  const idle = await this.idleModeTools(ctx);
293
327
 
328
+ if (mode === 'heal') return { ...core, ...idle, ...this.testModeTools(ctx), ...this.webModeTools(ctx) };
294
329
  if (mode === 'test') return { ...core, ...idle, ...this.testModeTools(ctx) };
295
330
  if (mode === 'web') return { ...core, ...idle, ...this.webModeTools(ctx) };
296
331
  return { ...core, ...idle };
@@ -365,20 +400,28 @@ export class Captain extends CaptainBase implements Agent {
365
400
  return result.object;
366
401
  }
367
402
 
403
+ async processExecutionError(error: Error, activeTest: Test): Promise<ExecutionRecoveryAction> {
404
+ const explorer = this.explorBot.getExplorer();
405
+ const result = await explorer.handleExecutionError(error);
406
+ return {
407
+ ...result,
408
+ message: result.recovered ? `${result.message}\nContinue the test "${activeTest.scenario}" from the restored page.` : result.message,
409
+ };
410
+ }
411
+
412
+ private canCompleteWithoutDetails(): boolean {
413
+ return (this.recentToolCalls || []).some(hasBrowserCompletionEvidence);
414
+ }
415
+
368
416
  async handle(input: string, options: { reset?: boolean } = {}): Promise<string | null> {
369
417
  const stateManager = this.explorBot.getExplorer().getStateManager();
370
418
  const initialState = stateManager.getCurrentState();
371
419
 
372
- if (!initialState) {
373
- tag('warning').log('No page loaded. Use /navigate or I.amOnPage() first.');
374
- return null;
375
- }
376
-
377
420
  const conversation = options.reset ? this.resetConversation() : this.ensureConversation();
378
421
  let isDone = false;
379
422
  let finalSummary: string | null = null;
380
423
 
381
- const startUrl = initialState.url || '';
424
+ const startUrl = initialState?.url || '';
382
425
  const task = new Task(input, startUrl);
383
426
  const onDone = (summary: string) => {
384
427
  isDone = true;
@@ -421,12 +464,14 @@ export class Captain extends CaptainBase implements Agent {
421
464
  }
422
465
 
423
466
  const currentState = stateManager.getCurrentState();
424
- if (!currentState) {
467
+ if (!currentState && this.getMode() !== 'idle') {
425
468
  stop();
426
469
  return;
427
470
  }
428
471
 
429
- await this.reinjectContextIfNeeded(conversation, currentState);
472
+ if (currentState) {
473
+ await this.reinjectContextIfNeeded(conversation, currentState);
474
+ }
430
475
 
431
476
  if (userInput) {
432
477
  const newContext = await this.getPageContext();
@@ -463,7 +508,7 @@ export class Captain extends CaptainBase implements Agent {
463
508
 
464
509
  if (result?.toolExecutions?.length) {
465
510
  const lastExec = result.toolExecutions[result.toolExecutions.length - 1];
466
- if (lastExec.wasSuccessful && this.ACTION_TOOLS.includes(lastExec.toolName)) {
511
+ if (hasBrowserCompletionEvidence(lastExec)) {
467
512
  conversation.addUserText('Action succeeded. If the goal is achieved, call done() now with a brief summary.');
468
513
  }
469
514
  }
@@ -500,3 +545,32 @@ interface SupervisorAction {
500
545
  action: 'inject' | 'stop' | 'pass' | 'skip';
501
546
  message: string;
502
547
  }
548
+
549
+ interface ExecutionRecoveryAction {
550
+ action: 'continue' | 'stop';
551
+ message: string;
552
+ recovered?: boolean;
553
+ }
554
+
555
+ function isExplicitSlashRequest(input: string, command: string): boolean {
556
+ const requested = slashCommandToken(input);
557
+ const actual = slashCommandToken(command);
558
+ if (!requested || !actual) return false;
559
+ return requested === actual;
560
+ }
561
+
562
+ function slashCommandToken(value: string): string | null {
563
+ const trimmed = value.trim();
564
+ if (!trimmed.startsWith('/')) return null;
565
+
566
+ for (let i = 1; i < trimmed.length; i++) {
567
+ if (trimmed[i] <= ' ') return trimmed.slice(0, i);
568
+ }
569
+ return trimmed;
570
+ }
571
+
572
+ function hasBrowserCompletionEvidence(execution: any): boolean {
573
+ if (!execution?.wasSuccessful) return false;
574
+ const output = execution.output || {};
575
+ return Boolean(output.pageDiff || output.code || output.playwrightGroupId);
576
+ }
@@ -10,6 +10,8 @@ import { relativeToCwd } from '../../utils/next-steps.ts';
10
10
  import { safeFilename } from '../../utils/strings.ts';
11
11
  import { type Constructor, debugLog } from './mixin.ts';
12
12
 
13
+ const FATAL_SCREENCAST_STOP_ERRORS = /Target page, context or browser has been closed|Target closed|Session closed|Protocol error/i;
14
+
13
15
  export interface ScreencastMethods {
14
16
  attachScreencast(): void;
15
17
  isScreencastActive(): boolean;
@@ -113,17 +115,24 @@ export function WithScreencast<T extends Constructor>(Base: T) {
113
115
  if (!this.screencastActive) return;
114
116
  const path = this.screencastPath;
115
117
  const task = this.screencastTask;
118
+ let stopped = false;
116
119
  try {
117
120
  await this.screencastPage.screencast.stop();
121
+ stopped = true;
118
122
  } catch (err) {
119
- tag('operation').log(`Screencast stop failed: ${(err as Error).message}`);
123
+ const message = (err as Error).message;
124
+ if (FATAL_SCREENCAST_STOP_ERRORS.test(message)) {
125
+ tag('operation').log('Screencast skipped: browser was closed before recording could be finalized');
126
+ } else {
127
+ tag('operation').log(`Screencast stop failed: ${message}`);
128
+ }
120
129
  }
121
130
  this.screencastActive = false;
122
131
  this.screencastPage = null;
123
132
  this.screencastPath = null;
124
133
  this.screencastTask = null;
125
134
  this.screencastLastChapter = null;
126
- if (path) {
135
+ if (path && stopped) {
127
136
  this.savedFiles.add(path);
128
137
  task?.addArtifact?.(path);
129
138
  tag('operation').log(`Saved screencast: ${relativeToCwd(path)}`);
@@ -136,6 +136,10 @@ class Navigator implements Agent {
136
136
  }
137
137
 
138
138
  async visit(url: string): Promise<void> {
139
+ return this.explorer.runWithBrowserRecovery('navigator.visit', () => this.visitOnce(url));
140
+ }
141
+
142
+ private async visitOnce(url: string): Promise<void> {
139
143
  try {
140
144
  const action = this.explorer.createAction();
141
145
 
@@ -170,7 +174,7 @@ class Navigator implements Agent {
170
174
  throw new Error(`Navigation to ${url} failed: ${action.lastError?.message}`);
171
175
  }
172
176
  }
173
- await action.caputrePageWithScreenshot();
177
+ await this.explorer.capturePageWithScreenshot();
174
178
  await this.hooksRunner.runAfterHook('navigator', url);
175
179
  } catch (error) {
176
180
  const errorMessage = error instanceof Error ? error.message : String(error);
@@ -382,7 +386,7 @@ class Navigator implements Agent {
382
386
  // URL did not transition to expectedUrl within timeout
383
387
  }
384
388
  }
385
- const freshState = await action.capturePageState();
389
+ const freshState = await this.explorer.capturePageState();
386
390
  const currentUrl = /^https?:\/\//i.test(expectedUrl) ? freshState.fullUrl || freshState.url || '' : freshState.url || '';
387
391
  const urlMatches = this.isSameExpectedOrigin(expectedUrl, action.stateManager) && normalizeUrl(currentUrl) === normalizeUrl(expectedUrl);
388
392
  const stateChanged = freshState.getStateHash() !== actionResult.getStateHash();
package/src/ai/pilot.ts CHANGED
@@ -67,7 +67,7 @@ export class Pilot implements Agent {
67
67
  }
68
68
 
69
69
  async reviewCompletion(task: Test, currentState: ActionResult, testerConversation: Conversation, navigator?: Navigator): Promise<boolean> {
70
- const verdictType = task.hasAchievedAny() ? 'finish' : 'stop';
70
+ const verdictType = this.hasCompletionEvidence(task, currentState, testerConversation) ? 'finish' : 'stop';
71
71
  return this.reviewDecision(verdictType, task, currentState, testerConversation, navigator);
72
72
  }
73
73
 
@@ -86,14 +86,14 @@ export class Pilot implements Agent {
86
86
 
87
87
  const sessionLog = this.formatSessionLog(testerConversation);
88
88
  const stateContext = this.buildStateContext(currentState);
89
+ const successfulAssertions = this.formatSuccessfulAssertions(currentState, testerConversation);
89
90
  const notes = task.notesToString() || 'No notes recorded.';
90
91
 
91
92
  let visualAnalysis = '';
92
93
  let screenshotState: ActionResult | null = null;
93
94
  if (this.provider.hasVision()) {
94
95
  try {
95
- const action = this.explorer.createAction();
96
- screenshotState = await action.caputrePageWithScreenshot();
96
+ screenshotState = await this.explorer.capturePageWithScreenshot();
97
97
  if (screenshotState.screenshot) {
98
98
  visualAnalysis = (await this.researcher.answerQuestionAboutScreenshot(screenshotState, `Describe current page state relevant to: ${task.scenario}`)) || '';
99
99
  }
@@ -125,6 +125,10 @@ export class Pilot implements Agent {
125
125
 
126
126
  ${this.formatExpectations(task)}
127
127
 
128
+ <successful_assertions>
129
+ ${successfulAssertions || 'None'}
130
+ </successful_assertions>
131
+
128
132
  <notes>
129
133
  ${notes}
130
134
  </notes>
@@ -658,8 +662,7 @@ export class Pilot implements Agent {
658
662
  private async checkDataAvailability(task: Test, requestedData: string, fishermanReason: string | undefined): Promise<string | null> {
659
663
  if (!this.provider.hasVision()) return null;
660
664
 
661
- const action = this.explorer.createAction();
662
- const screenshotState = await action.caputrePageWithScreenshot().catch(() => null);
665
+ const screenshotState = await this.explorer.capturePageWithScreenshot().catch(() => null);
663
666
  if (!screenshotState?.screenshot) return null;
664
667
 
665
668
  const question = dedent`
@@ -882,6 +885,32 @@ export class Pilot implements Agent {
882
885
  return parts.join('\n\n');
883
886
  }
884
887
 
888
+ private hasCompletionEvidence(task: Test, currentState: ActionResult, testerConversation: Conversation): boolean {
889
+ if (task.hasAchievedAny()) return true;
890
+ return this.hasSuccessfulCheckEvidence(currentState, testerConversation);
891
+ }
892
+
893
+ private hasSuccessfulCheckEvidence(currentState: ActionResult, testerConversation: Conversation): boolean {
894
+ if (Object.values(currentState.verifications ?? {}).some(Boolean)) return true;
895
+ return testerConversation.getToolExecutions().some((t) => CHECK_TOOLS.includes(t.toolName) && t.wasSuccessful);
896
+ }
897
+
898
+ private formatSuccessfulAssertions(currentState: ActionResult, testerConversation: Conversation): string {
899
+ const lines: string[] = [];
900
+ for (const [assertion, passed] of Object.entries(currentState.verifications ?? {})) {
901
+ if (passed) lines.push(`PASS state verification: ${assertion}`);
902
+ }
903
+
904
+ for (const exec of testerConversation.getToolExecutions()) {
905
+ if (!CHECK_TOOLS.includes(exec.toolName) || !exec.wasSuccessful) continue;
906
+ const description = exec.input?.assertion || exec.input?.request || truncateJson(exec.input);
907
+ const result = exec.output?.message || exec.output?.analysis || exec.output?.result;
908
+ lines.push(`PASS ${exec.toolName}: ${description}${result ? ` -> ${result}` : ''}`);
909
+ }
910
+
911
+ return [...new Set(lines)].join('\n');
912
+ }
913
+
885
914
  private formatActions(toolCalls: any[]): string {
886
915
  return toolCalls
887
916
  .map((t) => {
@@ -81,7 +81,7 @@ export function WithCoordinates<T extends Constructor>(Base: T) {
81
81
  }
82
82
 
83
83
  async visuallyAnnotateElements(opts?: { containers?: Array<{ css: string; label: string }> }): Promise<number> {
84
- return visuallyAnnotateContainers(this.explorer.playwrightHelper.page, opts?.containers || []);
84
+ return this.explorer.visuallyAnnotateElements(opts);
85
85
  }
86
86
 
87
87
  private async _analyzeScreenshotForVisualProps(): Promise<VisualAnalysisResult> {
@@ -193,7 +193,6 @@ export function WithCoordinates<T extends Constructor>(Base: T) {
193
193
  }
194
194
 
195
195
  async backfillCoordinates(result: ResearchResult): Promise<void> {
196
- const page = this.explorer.playwrightHelper.page;
197
196
  const sections = parseResearchSections(result.text);
198
197
  const eidxWithoutCoords: string[] = [];
199
198
  for (const section of sections) {
@@ -203,7 +202,7 @@ export function WithCoordinates<T extends Constructor>(Base: T) {
203
202
  }
204
203
  if (eidxWithoutCoords.length === 0) return;
205
204
 
206
- const webElements = await WebElement.fromEidxList(page, eidxWithoutCoords);
205
+ const webElements = await this.explorer.runWithBrowserRecovery('backfillCoordinates', () => WebElement.fromEidxList(this.explorer.playwrightHelper.page, eidxWithoutCoords));
207
206
  if (webElements.length === 0) return;
208
207
 
209
208
  const rectMap = new Map(webElements.map((w) => [w.eidx!, w]));
@@ -359,11 +359,10 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
359
359
  const isCoordinateClick = el.commands[0].startsWith('I.clickXY(');
360
360
  if (!isCoordinateClick) {
361
361
  const hoverCmd = el.commands[0].replace('I.click(', 'I.moveCursorTo(');
362
- const hoverAction = this.explorer.createAction();
363
- await hoverAction.attempt(hoverCmd, undefined, false);
362
+ await this.explorer.attemptAction(hoverCmd, undefined, false);
364
363
  await new Promise((r) => setTimeout(r, 500));
365
364
 
366
- await this.explorer.createAction().capturePageState();
365
+ await this.explorer.capturePageState();
367
366
  const hoverAR = ActionResult.fromState(this.stateManager.getCurrentState()!);
368
367
  const hoverDiff = await hoverAR.diff(previousState);
369
368
  await hoverDiff.calculate();
@@ -452,7 +451,7 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
452
451
  private async _restorePageState(url: string, originalAria: string): Promise<void> {
453
452
  try {
454
453
  await (this as any).cancelInUi();
455
- await this.explorer.createAction().capturePageState();
454
+ await this.explorer.capturePageState();
456
455
  const currentAria = this.stateManager.getCurrentState()?.ariaSnapshot || '';
457
456
  if (!diffAriaSnapshots(originalAria, currentAria)) return;
458
457
  } catch (err) {
@@ -194,8 +194,7 @@ export function WithLocators<T extends Constructor>(Base: T) {
194
194
  }
195
195
 
196
196
  if (needsXpath.length > 0) {
197
- const page = this.explorer.playwrightHelper.page;
198
- const webElements = await WebElement.fromEidxList(page, needsXpath);
197
+ const webElements = await this.explorer.runWithBrowserRecovery('backfillBrokenLocators', () => WebElement.fromEidxList(this.explorer.playwrightHelper.page, needsXpath));
199
198
  const changedSections = new Set<(typeof sections)[0]>();
200
199
  for (const w of webElements) {
201
200
  const entry = needsXpathEls.get(w.eidx!);
@@ -130,12 +130,12 @@ export class Researcher extends ResearcherBase implements Agent {
130
130
 
131
131
  const annotatedElements = await this.explorer.annotateElements();
132
132
  debugLog(`Annotated ${annotatedElements.length} interactive elements with eidx`);
133
- this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot: screenshot && this.provider.hasVision() });
133
+ this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot && this.provider.hasVision() });
134
134
 
135
135
  const condition = detectPageCondition(this.actionResult!);
136
136
  if (condition === 'error') {
137
137
  tag('warning').log(`Detected error page at ${state.url}`);
138
- throw new ErrorPageError(state.url, this.actionResult!.title);
138
+ throw new ErrorPageError(state.url, this.actionResult!.title, this.actionResult!.httpStatus);
139
139
  }
140
140
  if (condition === 'loading') {
141
141
  const settled = await this.waitUntilSettled(screenshot);
@@ -239,7 +239,7 @@ export class Researcher extends ResearcherBase implements Agent {
239
239
  // Must run BEFORE visuallyAnnotateContainers — annotation overlays inject z-index 99998+ which would pollute the scoring.
240
240
  if (!interrupted() && this.hasScreenshotToAnalyze) {
241
241
  const sections = parseResearchSections(result.text);
242
- const focused = await detectFocusedSection(this.explorer.playwrightHelper.page, sections);
242
+ const focused = await this.explorer.runWithBrowserRecovery('detectFocusedSection', () => detectFocusedSection(this.explorer.playwrightHelper.page, sections));
243
243
  if (focused) markSectionAsFocused(result, focused);
244
244
  }
245
245
 
@@ -252,7 +252,7 @@ export class Researcher extends ResearcherBase implements Agent {
252
252
  const freshBroken = freshContainerLocs.filter((l) => l.valid === false).map((l) => l.locator);
253
253
  const containers = validContainers.filter((c) => !freshBroken.includes(c.css));
254
254
  await this.visuallyAnnotateElements({ containers });
255
- this.actionResult = await this.explorer.createAction().caputrePageWithScreenshot();
255
+ this.actionResult = await this.explorer.capturePageWithScreenshot();
256
256
  const visualResult = await this.analyzeScreenshotForVisualProps();
257
257
  if (visualResult.elements.size > 0) {
258
258
  await this.mergeVisualData(result, visualResult.elements);
@@ -331,7 +331,7 @@ export class Researcher extends ResearcherBase implements Agent {
331
331
  if (!this.actionResult) {
332
332
  debugLog('No action result, navigating to URL');
333
333
  await this.explorer.visit(url);
334
- this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot: screenshot });
334
+ this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot });
335
335
  return;
336
336
  }
337
337
 
@@ -341,7 +341,7 @@ export class Researcher extends ResearcherBase implements Agent {
341
341
 
342
342
  if (!isEmpty && isOnCurrentState) {
343
343
  if ((!this.actionResult.screenshot && screenshot) || !this.actionResult.ariaSnapshot) {
344
- this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot: screenshot });
344
+ this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot });
345
345
  }
346
346
  return;
347
347
  }
@@ -349,6 +349,8 @@ export class Researcher extends ResearcherBase implements Agent {
349
349
  if (isEmpty && isOnCurrentState) {
350
350
  debugLog('HTML body empty on current URL, waiting for content');
351
351
  tag('step').log('Page body is empty, waiting for content...');
352
+ await this.explorer.visit(url);
353
+ this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot ?? false });
352
354
  await this.waitUntilSettled(screenshot ?? false);
353
355
  return;
354
356
  }
@@ -357,36 +359,35 @@ export class Researcher extends ResearcherBase implements Agent {
357
359
  tag('step').log('Navigating to URL...');
358
360
 
359
361
  await this.explorer.visit(url);
360
- this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot: screenshot ?? false });
362
+ this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot ?? false });
361
363
  }
362
364
 
363
365
  private async waitUntilSettled(screenshot: boolean): Promise<boolean> {
364
366
  const errorPageTimeout = (this.explorer.getConfig().ai?.agents?.researcher as any)?.errorPageTimeout ?? 10;
365
367
  if (errorPageTimeout <= 0) return false;
366
368
 
367
- const page = this.explorer.playwrightHelper.page;
368
369
  const includeScreenshot = screenshot && this.provider.hasVision();
369
370
 
370
371
  try {
371
- await page?.waitForLoadState('networkidle', { timeout: errorPageTimeout * 1000 });
372
+ await this.explorer.runWithBrowserRecovery('waitUntilSettled', () => this.explorer.playwrightHelper.page?.waitForLoadState('networkidle', { timeout: errorPageTimeout * 1000 }));
372
373
  } catch {}
373
374
 
374
375
  await this.explorer.annotateElements();
375
- this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot });
376
+ this.actionResult = await this.explorer.capturePageState({ includeScreenshot });
376
377
 
377
378
  let condition = detectPageCondition(this.actionResult!);
378
379
  if (condition === 'error') {
379
- throw new ErrorPageError(this.actionResult!.url, this.actionResult!.title);
380
+ throw new ErrorPageError(this.actionResult!.url, this.actionResult!.title, this.actionResult!.httpStatus);
380
381
  }
381
382
  if (condition === 'ok') return true;
382
383
 
383
384
  for (let i = 0; i < 3; i++) {
384
385
  await new Promise((r) => setTimeout(r, 1000));
385
386
  await this.explorer.annotateElements();
386
- this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot });
387
+ this.actionResult = await this.explorer.capturePageState({ includeScreenshot });
387
388
  condition = detectPageCondition(this.actionResult!);
388
389
  if (condition === 'error') {
389
- throw new ErrorPageError(this.actionResult!.url, this.actionResult!.title);
390
+ throw new ErrorPageError(this.actionResult!.url, this.actionResult!.title, this.actionResult!.httpStatus);
390
391
  }
391
392
  if (condition === 'ok') return true;
392
393
  }
@@ -762,17 +763,15 @@ export class Researcher extends ResearcherBase implements Agent {
762
763
  }
763
764
 
764
765
  async navigateTo(url: string): Promise<void> {
765
- const action = this.explorer.createAction();
766
- await action.execute(`I.amOnPage("${url}")`);
766
+ await this.explorer.visit(url);
767
767
  }
768
768
 
769
769
  async cancelInUi() {
770
770
  const beforeAria = this.stateManager.getCurrentState()?.ariaSnapshot || null;
771
- const action = this.explorer.createAction();
772
771
 
773
- await action.execute('I.clickXY(0, 0)');
772
+ await this.explorer.executeAction('I.clickXY(0, 0)');
774
773
  if (diffAriaSnapshots(beforeAria, this.stateManager.getCurrentState()?.ariaSnapshot || null)) return;
775
774
 
776
- await action.execute(`I.pressKey('Escape')`);
775
+ await this.explorer.executeAction(`I.pressKey('Escape')`);
777
776
  }
778
777
  }
@@ -24,7 +24,7 @@ export abstract class TaskAgent {
24
24
  protected consecutiveFailures = 0;
25
25
  protected consecutiveEmptyResults = 0;
26
26
  protected recentToolCalls: any[] = [];
27
- protected abstract readonly ACTION_TOOLS: string[];
27
+ protected readonly ACTION_TOOLS: string[] = [];
28
28
 
29
29
  private _historian: Historian | null = null;
30
30
  private _quartermaster: Quartermaster | null = null;