explorbot 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. package/README.md +21 -5
  2. package/bin/explorbot-cli.ts +29 -7
  3. package/boat/api-tester/src/config.ts +45 -3
  4. package/boat/doc-collector/src/ai/documentarian.ts +8 -2
  5. package/boat/doc-collector/src/ai/tools.ts +29 -16
  6. package/boat/doc-collector/src/cli.ts +2 -6
  7. package/boat/doc-collector/src/config.ts +2 -0
  8. package/boat/doc-collector/src/docbot.ts +42 -14
  9. package/boat/doc-collector/src/interaction-screenshots.ts +160 -0
  10. package/boat/doc-collector/src/screenshots.ts +24 -16
  11. package/dist/bin/explorbot-cli.js +26 -6
  12. package/dist/boat/api-tester/src/config.js +43 -4
  13. package/dist/boat/doc-collector/src/ai/documentarian.js +7 -2
  14. package/dist/boat/doc-collector/src/ai/tools.js +21 -15
  15. package/dist/boat/doc-collector/src/cli.js +2 -5
  16. package/dist/boat/doc-collector/src/config.js +1 -0
  17. package/dist/boat/doc-collector/src/docbot.js +39 -13
  18. package/dist/boat/doc-collector/src/interaction-screenshots.js +156 -0
  19. package/dist/boat/doc-collector/src/screenshots.js +25 -16
  20. package/dist/models.json +30 -0
  21. package/dist/package.json +8 -2
  22. package/dist/src/action.d.ts +11 -3
  23. package/dist/src/action.js +22 -5
  24. package/dist/src/ai/agent.d.ts +17 -0
  25. package/dist/src/ai/captain/idle-mode.js +1 -1
  26. package/dist/src/ai/captain/test-mode.js +1 -1
  27. package/dist/src/ai/captain/web-mode.js +18 -22
  28. package/dist/src/ai/captain.js +13 -26
  29. package/dist/src/ai/driller.d.ts +2 -13
  30. package/dist/src/ai/driller.js +23 -38
  31. package/dist/src/ai/historian/codeceptjs.js +1 -1
  32. package/dist/src/ai/historian/playwright.d.ts +3 -2
  33. package/dist/src/ai/historian/playwright.js +1 -1
  34. package/dist/src/ai/historian/screencast.d.ts +2 -1
  35. package/dist/src/ai/historian/screencast.js +2 -2
  36. package/dist/src/ai/historian.d.ts +5 -4
  37. package/dist/src/ai/navigator.d.ts +6 -2
  38. package/dist/src/ai/navigator.js +26 -22
  39. package/dist/src/ai/pilot.d.ts +13 -6
  40. package/dist/src/ai/pilot.js +49 -44
  41. package/dist/src/ai/planner.d.ts +3 -2
  42. package/dist/src/ai/planner.js +19 -17
  43. package/dist/src/ai/provider.js +0 -1
  44. package/dist/src/ai/quartermaster.d.ts +3 -2
  45. package/dist/src/ai/quartermaster.js +4 -4
  46. package/dist/src/ai/rerunner.d.ts +2 -11
  47. package/dist/src/ai/rerunner.js +9 -22
  48. package/dist/src/ai/researcher/coordinates.js +4 -3
  49. package/dist/src/ai/researcher/deep-analysis.d.ts +2 -0
  50. package/dist/src/ai/researcher/deep-analysis.js +6 -6
  51. package/dist/src/ai/researcher/locators.d.ts +9 -2
  52. package/dist/src/ai/researcher/locators.js +65 -8
  53. package/dist/src/ai/researcher/sections.d.ts +2 -0
  54. package/dist/src/ai/researcher/sections.js +2 -2
  55. package/dist/src/ai/researcher.js +29 -54
  56. package/dist/src/ai/task-agent.d.ts +18 -5
  57. package/dist/src/ai/task-agent.js +35 -6
  58. package/dist/src/ai/tester.d.ts +6 -12
  59. package/dist/src/ai/tester.js +37 -51
  60. package/dist/src/ai/tools.d.ts +11 -10
  61. package/dist/src/ai/tools.js +24 -36
  62. package/dist/src/command-handler.js +1 -1
  63. package/dist/src/commands/context-aria-command.js +1 -1
  64. package/dist/src/commands/context-command.js +7 -5
  65. package/dist/src/commands/context-data-command.js +2 -2
  66. package/dist/src/commands/context-experience-command.js +2 -2
  67. package/dist/src/commands/context-html-command.js +2 -2
  68. package/dist/src/commands/context-knowledge-command.js +1 -1
  69. package/dist/src/commands/drill-command.js +1 -1
  70. package/dist/src/commands/explore-command.js +4 -3
  71. package/dist/src/commands/freesail-command.js +2 -2
  72. package/dist/src/commands/learn-command.js +4 -4
  73. package/dist/src/commands/path-command.js +1 -1
  74. package/dist/src/commands/research-command.js +1 -1
  75. package/dist/src/commands/test-command.js +1 -1
  76. package/dist/src/components/App.js +1 -1
  77. package/dist/src/config.d.ts +17 -0
  78. package/dist/src/config.js +144 -10
  79. package/dist/src/experience-tracker.js +1 -1
  80. package/dist/src/explorbot.d.ts +17 -8
  81. package/dist/src/explorbot.js +74 -36
  82. package/dist/src/explorer.d.ts +67 -91
  83. package/dist/src/explorer.js +327 -567
  84. package/dist/src/state-manager.d.ts +6 -1
  85. package/dist/src/state-manager.js +1 -0
  86. package/dist/src/utils/hooks-runner.js +2 -4
  87. package/dist/src/utils/test-files.js +1 -1
  88. package/dist/src/utils/web-annotate.d.ts +5 -0
  89. package/dist/src/utils/web-annotate.js +58 -0
  90. package/dist/src/utils/web-eidx.d.ts +2 -0
  91. package/dist/src/utils/web-eidx.js +20 -0
  92. package/dist/src/utils/web-element.d.ts +1 -0
  93. package/dist/src/utils/web-element.js +29 -0
  94. package/models.json +30 -0
  95. package/package.json +8 -2
  96. package/src/action.ts +25 -5
  97. package/src/ai/agent.ts +20 -0
  98. package/src/ai/captain/idle-mode.ts +1 -1
  99. package/src/ai/captain/test-mode.ts +1 -1
  100. package/src/ai/captain/web-mode.ts +18 -23
  101. package/src/ai/captain.ts +12 -25
  102. package/src/ai/driller.ts +50 -67
  103. package/src/ai/historian/codeceptjs.ts +1 -1
  104. package/src/ai/historian/playwright.ts +4 -3
  105. package/src/ai/historian/screencast.ts +4 -3
  106. package/src/ai/historian.ts +3 -2
  107. package/src/ai/navigator.ts +30 -25
  108. package/src/ai/pilot.ts +55 -47
  109. package/src/ai/planner.ts +22 -20
  110. package/src/ai/provider.ts +0 -1
  111. package/src/ai/quartermaster.ts +5 -4
  112. package/src/ai/rerunner.ts +10 -29
  113. package/src/ai/researcher/coordinates.ts +4 -3
  114. package/src/ai/researcher/deep-analysis.ts +8 -6
  115. package/src/ai/researcher/locators.ts +66 -9
  116. package/src/ai/researcher/sections.ts +4 -2
  117. package/src/ai/researcher.ts +32 -61
  118. package/src/ai/task-agent.ts +45 -11
  119. package/src/ai/tester.ts +40 -57
  120. package/src/ai/tools.ts +32 -53
  121. package/src/command-handler.ts +1 -1
  122. package/src/commands/context-aria-command.ts +1 -1
  123. package/src/commands/context-command.ts +7 -5
  124. package/src/commands/context-data-command.ts +2 -2
  125. package/src/commands/context-experience-command.ts +2 -2
  126. package/src/commands/context-html-command.ts +2 -2
  127. package/src/commands/context-knowledge-command.ts +1 -1
  128. package/src/commands/drill-command.ts +1 -1
  129. package/src/commands/explore-command.ts +4 -3
  130. package/src/commands/freesail-command.ts +2 -2
  131. package/src/commands/learn-command.ts +4 -4
  132. package/src/commands/path-command.ts +1 -1
  133. package/src/commands/research-command.ts +1 -1
  134. package/src/commands/test-command.ts +1 -1
  135. package/src/components/App.tsx +1 -1
  136. package/src/config.ts +173 -11
  137. package/src/experience-tracker.ts +1 -1
  138. package/src/explorbot.ts +78 -39
  139. package/src/explorer.ts +388 -626
  140. package/src/state-manager.ts +7 -1
  141. package/src/utils/hooks-runner.ts +2 -4
  142. package/src/utils/test-files.ts +1 -1
  143. package/src/utils/web-annotate.ts +64 -0
  144. package/src/utils/web-eidx.ts +21 -0
  145. package/src/utils/web-element.ts +30 -0
package/src/ai/tester.ts CHANGED
@@ -5,18 +5,17 @@ import dedent from 'dedent';
5
5
  import { z } from 'zod';
6
6
  import { ActionResult } from '../action-result.ts';
7
7
  import { clearActivity, setActivity } from '../activity.ts';
8
- import type { ExperienceTracker } from '../experience-tracker.ts';
9
- import type Explorer from '../explorer.ts';
8
+ import type { RequestStore } from '../api/request-store.ts';
9
+ import type { TestRun } from '../explorer.ts';
10
10
  import { Observability } from '../observability.ts';
11
11
  import type { StateTransition } from '../state-manager.ts';
12
12
  import { Stats } from '../stats.ts';
13
13
  import { type Test, TestResult, type TestResultType } from '../test-plan.ts';
14
14
  import { detectFocusArea, extractFocusedElement } from '../utils/aria.ts';
15
15
  import { ErrorPageError, isErrorPage } from '../utils/error-page.ts';
16
- import { HooksRunner } from '../utils/hooks-runner.ts';
17
16
  import { createDebug, tag } from '../utils/logger.ts';
18
17
  import { loop } from '../utils/loop.ts';
19
- import type { Agent } from './agent.ts';
18
+ import type { Agent, AgentDeps } from './agent.ts';
20
19
  import type { Captain } from './captain.ts';
21
20
  import type { Conversation } from './conversation.ts';
22
21
  import { Navigator } from './navigator.ts';
@@ -44,8 +43,8 @@ export class Tester extends TaskAgent implements Agent {
44
43
  protected readonly ACTION_TOOLS = ['click', 'hover', 'pressKey', 'form'];
45
44
  protected readonly SPECIAL_CONTEXT_ACTION_TOOLS = ['exitIframe'];
46
45
  emoji = '🧪';
47
- private explorer: Explorer;
48
- private provider: Provider;
46
+ private requestStore: RequestStore;
47
+ private testRun: TestRun | null = null;
49
48
  private currentConversation: Conversation | null = null;
50
49
  private pilot: Pilot | null = null;
51
50
  private captain: Captain | null = null;
@@ -60,38 +59,23 @@ export class Tester extends TaskAgent implements Agent {
60
59
  private previousStateHash: string | null = null;
61
60
  private pageStateHash: string | null = null;
62
61
  private pageActionResult: ActionResult | null = null;
63
- private hooksRunner: HooksRunner;
64
62
  private seenUiMapUrls = new Set<string>();
65
63
  private lastAnalyzedStateHash: string | null = null;
66
64
  private stalledIterations = 0;
67
65
  private readonly MAX_STALLED_ITERATIONS = 3;
68
66
 
69
- constructor(explorer: Explorer, provider: Provider, researcher: Researcher, navigator: Navigator, agentTools?: any) {
70
- super();
71
- this.explorer = explorer;
72
- this.provider = provider;
67
+ constructor(deps: AgentDeps, researcher: Researcher, navigator: Navigator, agentTools?: any) {
68
+ super(deps);
69
+ this.requestStore = deps.requestStore;
73
70
  this.researcher = researcher;
74
71
  this.navigator = navigator;
75
72
  this.agentTools = agentTools;
76
- this.hooksRunner = new HooksRunner(explorer, explorer.getConfig());
77
73
  }
78
74
 
79
75
  protected getNavigator(): Navigator {
80
76
  return this.navigator;
81
77
  }
82
78
 
83
- protected getExperienceTracker(): ExperienceTracker {
84
- return this.explorer.getStateManager().getExperienceTracker();
85
- }
86
-
87
- protected getKnowledgeTracker() {
88
- return this.explorer.getKnowledgeTracker();
89
- }
90
-
91
- protected getProvider(): Provider {
92
- return this.provider;
93
- }
94
-
95
79
  setPilot(pilot: Pilot): void {
96
80
  this.pilot = pilot;
97
81
  }
@@ -101,11 +85,11 @@ export class Tester extends TaskAgent implements Agent {
101
85
  }
102
86
 
103
87
  private getCurrentState(): ActionResult {
104
- return ActionResult.fromState(this.explorer.getStateManager().getCurrentState()!);
88
+ return ActionResult.fromState(this.stateManager.getCurrentState()!);
105
89
  }
106
90
 
107
91
  private get progressCheckInterval(): number {
108
- return (this.explorer.getConfig().ai?.agents?.tester as any)?.progressCheckInterval ?? 3;
92
+ return (this.config.ai?.agents?.tester as any)?.progressCheckInterval ?? 3;
109
93
  }
110
94
 
111
95
  getConversation(): Conversation | null {
@@ -114,7 +98,7 @@ export class Tester extends TaskAgent implements Agent {
114
98
 
115
99
  async test(task: Test): Promise<{ success: boolean }> {
116
100
  Stats.tests++;
117
- const state = this.explorer.getStateManager().getCurrentState();
101
+ const state = this.stateManager.getCurrentState();
118
102
  if (!state) throw new Error('No state found');
119
103
 
120
104
  setActivity(`🧪 Testing: ${task.scenario}`, 'action');
@@ -126,20 +110,20 @@ export class Tester extends TaskAgent implements Agent {
126
110
  this.seenUiMapUrls.clear();
127
111
  this.lastAnalyzedStateHash = null;
128
112
  this.stalledIterations = 0;
129
- this.explorer.getStateManager().clearHistory();
113
+ this.stateManager.clearHistory();
130
114
  this.resetFailureCount();
131
115
  this.pilot?.reset();
132
116
 
133
- const requestStore = this.explorer.getRequestStore();
134
- requestStore?.clear();
135
- const offFailedRequest = requestStore?.onFailedRequest((r) => {
117
+ const requestStore = this.requestStore;
118
+ requestStore.clear();
119
+ const offFailedRequest = requestStore.onFailedRequest((r) => {
136
120
  task.addNote(`Network error: ${r.method} ${r.path} → ${r.status}`, TestResult.FAILED);
137
121
  });
138
122
 
139
123
  const initialState = ActionResult.fromState(state);
140
124
  if (isErrorPage(initialState)) {
141
125
  task.start();
142
- await this.explorer.startTest(task);
126
+ this.testRun = await this.explorer.beginTest(task);
143
127
  offFailedRequest?.();
144
128
  return await this.abortStartedTestOnErrorPage(task, initialState);
145
129
  }
@@ -196,7 +180,8 @@ export class Tester extends TaskAgent implements Agent {
196
180
 
197
181
  debugLog('Starting test execution with tools');
198
182
 
199
- if (!(await this.explorer.startTest(task))) {
183
+ this.testRun = await this.explorer.beginTest(task);
184
+ if (!this.testRun.started) {
200
185
  offFailedRequest?.();
201
186
  await this.cleanupStartedTest(task);
202
187
  return { success: task.isSuccessful };
@@ -214,7 +199,7 @@ export class Tester extends TaskAgent implements Agent {
214
199
  }
215
200
  }
216
201
 
217
- const startState = this.explorer.getStateManager().getCurrentState();
202
+ const startState = this.stateManager.getCurrentState();
218
203
  if (startState) {
219
204
  task.addUrlNote(startState);
220
205
  const startActionResult = ActionResult.fromState(startState);
@@ -226,14 +211,14 @@ export class Tester extends TaskAgent implements Agent {
226
211
  const currentUrl = startState?.url || task.startUrl || '';
227
212
  await this.hooksRunner.runBeforeHook('tester', currentUrl);
228
213
 
229
- const offStateChange = this.explorer.getStateManager().onStateChange((event: StateTransition) => {
214
+ const offStateChange = this.stateManager.onStateChange((event: StateTransition) => {
230
215
  if (task.hasFinished) return;
231
216
  if (event.toState?.url === event.fromState?.url) return;
232
217
  if (event.toState) task.addUrlNote(event.toState, event.fromState || undefined);
233
218
  task.states.push(event.toState);
234
219
  });
235
220
 
236
- const codeceptjsTools = createCodeceptJSTools(this.explorer, task);
221
+ const codeceptjsTools = createCodeceptJSTools(this.toolDeps, task);
237
222
  let assertionPerformed = false;
238
223
  let extensions = 0;
239
224
  let shouldContinue = true;
@@ -244,7 +229,7 @@ export class Tester extends TaskAgent implements Agent {
244
229
  await loop(
245
230
  async ({ stop, pause, iteration, userInput }) => {
246
231
  debugLog('iteration', iteration);
247
- if (!(await this.explorer.ensurePageAvailable())) {
232
+ if (!(await this.explorer.recover()).ok) {
248
233
  task.addNote('Browser page is unavailable');
249
234
  task.finish(TestResult.FAILED);
250
235
  stop();
@@ -258,12 +243,12 @@ export class Tester extends TaskAgent implements Agent {
258
243
  ...this.agentTools,
259
244
  };
260
245
  if (currentState.isInsideIframe) {
261
- Object.assign(tools, createIframeTools(this.explorer));
246
+ Object.assign(tools, createIframeTools(this.toolDeps));
262
247
  }
263
248
 
264
249
  debugLog(`Test ${task.scenario} iteration ${iteration}`);
265
250
 
266
- if (this.explorer.getStateManager().isInDeadLoop()) {
251
+ if (this.stateManager.isInDeadLoop()) {
267
252
  task.addNote('Dead loop detected. Stopped');
268
253
  stop();
269
254
  return;
@@ -412,7 +397,7 @@ export class Tester extends TaskAgent implements Agent {
412
397
 
413
398
  if (task.hasFinished) break;
414
399
 
415
- if (!(await this.explorer.ensurePageAvailable())) break;
400
+ if (!(await this.explorer.recover()).ok) break;
416
401
 
417
402
  const finalState = this.getCurrentState();
418
403
  const wantsContinue = await this.pilot!.finalReview(task, finalState, conversation, this.navigator);
@@ -431,7 +416,7 @@ export class Tester extends TaskAgent implements Agent {
431
416
  shouldContinue = true;
432
417
  }
433
418
 
434
- const finalUrl = this.explorer.getStateManager().getCurrentState()?.url || currentUrl;
419
+ const finalUrl = this.stateManager.getCurrentState()?.url || currentUrl;
435
420
  await this.hooksRunner.runAfterHook('tester', finalUrl);
436
421
 
437
422
  await this.getHistorian().saveSession(task, initialState, conversation);
@@ -443,7 +428,7 @@ export class Tester extends TaskAgent implements Agent {
443
428
  offStateChange();
444
429
  offFailedRequest?.();
445
430
  await this.finishTest(task);
446
- await this.explorer.stopTest(task, this.buildStopTestMeta(task));
431
+ await this.testRun?.stop(this.buildStopTestMeta(task));
447
432
 
448
433
  return {
449
434
  success: task.isSuccessful,
@@ -559,7 +544,7 @@ export class Tester extends TaskAgent implements Agent {
559
544
  }
560
545
 
561
546
  if (currentState.isInsideIframe) {
562
- const iframeInfo = currentState.iframeURL || this.explorer.getCurrentIframeInfo() || 'iframe context active';
547
+ const iframeInfo = currentState.iframeURL || 'iframe context active';
563
548
  context += dedent`
564
549
  <iframe_context>
565
550
  INSIDE IFRAME: ${iframeInfo}
@@ -568,10 +553,10 @@ export class Tester extends TaskAgent implements Agent {
568
553
  `;
569
554
  }
570
555
 
571
- if (this.explorer.hasOtherTabs()) {
572
- const otherTabs = this.explorer.getOtherTabsInfo();
556
+ const otherTabs = this.stateManager.otherTabs;
557
+ if (otherTabs.length > 0) {
573
558
  context += multipleTabsRule(otherTabs);
574
- this.explorer.clearOtherTabsInfo();
559
+ this.stateManager.otherTabs = [];
575
560
  }
576
561
 
577
562
  if (isNewUrl) {
@@ -675,7 +660,7 @@ export class Tester extends TaskAgent implements Agent {
675
660
  task.addNote(error.message, TestResult.FAILED, actionResult.screenshotFile, actionResult.fullUrl || actionResult.url);
676
661
  task.finish(TestResult.FAILED);
677
662
  this.finishTest(task);
678
- await this.explorer.stopTest(task, this.buildStopTestMeta(task));
663
+ await this.testRun?.stop(this.buildStopTestMeta(task));
679
664
  clearActivity(true);
680
665
  return { success: false };
681
666
  }
@@ -790,7 +775,7 @@ export class Tester extends TaskAgent implements Agent {
790
775
 
791
776
  ${dataProtectionRules}
792
777
 
793
- ${this.provider.getSystemPromptForAgent('tester', this.explorer.getStateManager().getCurrentState()?.url) || ''}
778
+ ${this.provider.getSystemPromptForAgent('tester', this.stateManager.getCurrentState()?.url) || ''}
794
779
  `;
795
780
  }
796
781
 
@@ -837,7 +822,7 @@ export class Tester extends TaskAgent implements Agent {
837
822
  }
838
823
 
839
824
  private buildAvailableFiles(): string {
840
- const userFiles = this.explorer.getConfig().files || {};
825
+ const userFiles = this.config.files || {};
841
826
  const codeceptDir = (global as any).codecept_dir || process.cwd();
842
827
  const lines: string[] = [];
843
828
 
@@ -895,11 +880,9 @@ export class Tester extends TaskAgent implements Agent {
895
880
  reason: z.string().optional().describe('Explanation why reset is the only option'),
896
881
  }),
897
882
  execute: async ({ reason }) => {
898
- if (this.getCurrentState().isInsideIframe) {
899
- await this.explorer.switchToMainFrame();
900
- }
883
+ await this.explorer.action().exitIframe();
901
884
 
902
- const currentState = this.explorer.getStateManager().getCurrentState();
885
+ const currentState = this.stateManager.getCurrentState();
903
886
  const currentUrl = currentState?.fullUrl || currentState?.url;
904
887
  if (currentUrl === resetUrl!) {
905
888
  return {
@@ -927,7 +910,7 @@ export class Tester extends TaskAgent implements Agent {
927
910
  const explanation = reason ? `${reason} (RESET)` : 'Resetting to initial page';
928
911
  const targetUrl = resetUrl!;
929
912
  task.addNote(explanation);
930
- const resetAction = this.explorer.createAction();
913
+ const resetAction = this.explorer.action();
931
914
  const success = await resetAction.attempt(`I.amOnPage(${JSON.stringify(targetUrl)})`, explanation);
932
915
 
933
916
  if (success) {
@@ -1076,7 +1059,7 @@ export class Tester extends TaskAgent implements Agent {
1076
1059
  mappedStatus = TestResult.FAILED;
1077
1060
  }
1078
1061
 
1079
- const screenshotFile = this.explorer.getStateManager().getCurrentState()?.screenshotFile;
1062
+ const screenshotFile = this.stateManager.getCurrentState()?.screenshotFile;
1080
1063
 
1081
1064
  for (const noteText of input.notes) {
1082
1065
  task.addNote(noteText, mappedStatus, screenshotFile);
@@ -1117,7 +1100,7 @@ export class Tester extends TaskAgent implements Agent {
1117
1100
  const message = error instanceof Error ? error.message : String(error);
1118
1101
  if (!task.hasFinished) task.addNote(`Execution error: ${message}`);
1119
1102
 
1120
- const result = await this.explorer.handleExecutionError(error);
1103
+ const result = await this.explorer.recover(error);
1121
1104
  tag('info').log(`Browser supervisor: ${result.action} - ${result.message}`);
1122
1105
  task.addNote(result.message);
1123
1106
 
@@ -1152,7 +1135,7 @@ export class Tester extends TaskAgent implements Agent {
1152
1135
 
1153
1136
  private async cleanupStartedTest(task: Test): Promise<void> {
1154
1137
  await this.finishTest(task);
1155
- await this.explorer.stopTest(task, {
1138
+ await this.testRun?.stop({
1156
1139
  startUrl: task.startUrl,
1157
1140
  style: task.style,
1158
1141
  sessionName: task.sessionName,
package/src/ai/tools.ts CHANGED
@@ -3,13 +3,13 @@ import dedent from 'dedent';
3
3
  import { z } from 'zod';
4
4
  import { ActionResult, type PageDiff, type ToolResultMetadata } from '../action-result.ts';
5
5
  import type { ExperienceTracker } from '../experience-tracker.ts';
6
- import type Explorer from '../explorer.ts';
7
6
  import { type Task, TestResult } from '../test-plan.js';
8
7
  import { LARGE_ARIA_CHANGE_THRESHOLD, extractFocusedElement } from '../utils/aria.ts';
9
8
  import { isFatalBrowserError } from '../utils/browser-errors.ts';
10
9
  import { createDebug, tag } from '../utils/logger.js';
11
10
  import { pause } from '../utils/loop.js';
12
11
  import { WebElement } from '../utils/web-element.ts';
12
+ import type { ToolDeps } from './agent.ts';
13
13
  import { Navigator } from './navigator.ts';
14
14
  import type { AIProvider } from './provider.ts';
15
15
  import { Researcher } from './researcher.ts';
@@ -18,11 +18,16 @@ import { isInteractive } from './task-agent.ts';
18
18
 
19
19
  const debugLog = createDebug('explorbot:tools');
20
20
 
21
- export const ASSERTION_TOOLS = ['verify'] as const;
21
+ interface AgentToolDeps extends ToolDeps {
22
+ researcher: Researcher;
23
+ navigator: Navigator;
24
+ supervisor?: boolean;
25
+ withExperience?: boolean;
26
+ }
22
27
 
23
- export function createCodeceptJSTools(explorer: Explorer, task: Task) {
24
- const stateManager = explorer.getStateManager();
28
+ export const ASSERTION_TOOLS = ['verify'] as const;
25
29
 
30
+ export function createCodeceptJSTools({ explorer, stateManager, ai }: ToolDeps, task: Task) {
26
31
  return {
27
32
  click: tool({
28
33
  description: dedent`
@@ -81,7 +86,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
81
86
  });
82
87
 
83
88
  const previousState = ActionResult.fromState(stateManager.getCurrentState()!);
84
- const action = explorer.createAction();
89
+ const action = explorer.action();
85
90
  const attempts: Array<{ command: string; success: boolean; error?: string }> = [];
86
91
 
87
92
  for (let i = 0; i < commands.length; i++) {
@@ -101,7 +106,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
101
106
 
102
107
  let disambiguated = null;
103
108
  if (attempts.some((a) => a.error?.toLowerCase().includes(MULTIPLE_ELEMENTS_PATTERN))) {
104
- disambiguated = await disambiguateElements(action.lastError, explanation, explorer.getAIProvider());
109
+ disambiguated = await disambiguateElements(action.lastError, explanation, ai);
105
110
  }
106
111
 
107
112
  if (disambiguated) {
@@ -193,7 +198,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
193
198
  });
194
199
 
195
200
  const previousState = ActionResult.fromState(stateManager.getCurrentState()!);
196
- const action = explorer.createAction();
201
+ const action = explorer.action();
197
202
  const attempts: Array<{ command: string; success: boolean; error?: string }> = [];
198
203
 
199
204
  for (const command of commands) {
@@ -263,7 +268,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
263
268
 
264
269
  if (!isSingleChar && !isStandardKey) {
265
270
  const previousState = ActionResult.fromState(stateManager.getCurrentState()!);
266
- const action = explorer.createAction();
271
+ const action = explorer.action();
267
272
  const typeCommand = `I.type(${JSON.stringify(key)})`;
268
273
  await action.attempt(typeCommand, explanation);
269
274
  const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, key);
@@ -306,7 +311,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
306
311
  }
307
312
 
308
313
  const previousState = ActionResult.fromState(stateManager.getCurrentState()!);
309
- const action = explorer.createAction();
314
+ const action = explorer.action();
310
315
 
311
316
  let pressKeyCommand: string;
312
317
  if (modifier) {
@@ -406,14 +411,9 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
406
411
 
407
412
  const previousState = ActionResult.fromState(stateManager.getCurrentState()!);
408
413
  const formLocator = codeLines[0] || 'form';
409
- const action = explorer.createAction();
410
- const wasInIframe = await explorer.isInsideIframe();
414
+ const action = explorer.action();
411
415
  await action.attempt(codeBlock, explanation);
412
416
 
413
- if (action.lastError && !wasInIframe && (await explorer.isInsideIframe())) {
414
- await explorer.switchToMainFrame();
415
- }
416
-
417
417
  const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, formLocator);
418
418
 
419
419
  if (action.lastError) {
@@ -422,7 +422,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
422
422
 
423
423
  let formSuggestion = 'Look into error message and identify which commands passed and which failed. Continue execution using step-by-step approach using click() and form() tools.';
424
424
  if (message.toLowerCase().includes(MULTIPLE_ELEMENTS_PATTERN)) {
425
- const disambiguated = await disambiguateElements(action.lastError, explanation, explorer.getAIProvider());
425
+ const disambiguated = await disambiguateElements(action.lastError, explanation, ai);
426
426
  if (disambiguated) {
427
427
  formSuggestion = `Multiple elements matched. Add step.opts({ elementIndex: ${disambiguated.position} }) to the failing command. Fallback locator: ${disambiguated.xpath}`;
428
428
  }
@@ -471,9 +471,7 @@ export function createCodeceptJSTools(explorer: Explorer, task: Task) {
471
471
  };
472
472
  }
473
473
 
474
- export function createIframeTools(explorer: Explorer) {
475
- const stateManager = explorer.getStateManager();
476
-
474
+ export function createIframeTools({ explorer, stateManager }: ToolDeps) {
477
475
  return {
478
476
  exitIframe: tool({
479
477
  description: dedent`
@@ -495,9 +493,9 @@ export function createIframeTools(explorer: Explorer) {
495
493
  });
496
494
  }
497
495
 
498
- await explorer.switchToMainFrame();
499
-
500
- const nextState = await explorer.capturePageState();
496
+ const action = explorer.action();
497
+ await action.execute('I.switchTo()');
498
+ const nextState = action.getActionResult()!;
501
499
  const toolResult = await nextState.toToolResult(previousState, 'I.switchTo()');
502
500
 
503
501
  return successToolResult('exitIframe', {
@@ -540,19 +538,7 @@ export function createLearnExperienceTool({ getExperienceTracker, getState }: {
540
538
  });
541
539
  }
542
540
 
543
- export function createAgentTools({
544
- explorer,
545
- researcher,
546
- navigator,
547
- supervisor,
548
- withExperience,
549
- }: {
550
- explorer: Explorer;
551
- researcher: Researcher;
552
- navigator: Navigator;
553
- supervisor?: boolean;
554
- withExperience?: boolean;
555
- }): any {
541
+ export function createAgentTools({ explorer, stateManager, ai, researcher, navigator, supervisor, withExperience }: AgentToolDeps): any {
556
542
  let visionDisabled = false;
557
543
 
558
544
  const tools: Record<string, any> = {
@@ -577,7 +563,7 @@ export function createAgentTools({
577
563
  }
578
564
 
579
565
  try {
580
- const actionResult = await explorer.capturePageWithScreenshot();
566
+ const actionResult = await explorer.capture({ screenshot: true });
581
567
 
582
568
  if (!actionResult.screenshot) {
583
569
  return failedToolResult('see', 'Failed to capture screenshot for analysis');
@@ -625,7 +611,6 @@ export function createAgentTools({
625
611
  }),
626
612
  execute: async ({ reason }) => {
627
613
  try {
628
- const stateManager = explorer.getStateManager();
629
614
  const currentState = stateManager.getCurrentState();
630
615
 
631
616
  if (!currentState) {
@@ -666,7 +651,7 @@ export function createAgentTools({
666
651
  }),
667
652
  execute: async ({ assertion }) => {
668
653
  try {
669
- const currentState = explorer.getStateManager().getCurrentState();
654
+ const currentState = stateManager.getCurrentState();
670
655
  const verifications = currentState?.verifications;
671
656
 
672
657
  if (verifications?.[assertion] !== undefined) {
@@ -677,7 +662,7 @@ export function createAgentTools({
677
662
  });
678
663
  }
679
664
 
680
- const actionResult = await explorer.capturePageState();
665
+ const actionResult = await explorer.capture();
681
666
  const result = await navigator.verifyState(assertion, actionResult);
682
667
 
683
668
  if (result.verified) {
@@ -727,7 +712,6 @@ export function createAgentTools({
727
712
  }),
728
713
  execute: async ({ reason }) => {
729
714
  try {
730
- const stateManager = explorer.getStateManager();
731
715
  const currentState = stateManager.getCurrentState();
732
716
 
733
717
  if (!currentState) {
@@ -771,7 +755,6 @@ export function createAgentTools({
771
755
  }),
772
756
  execute: async ({ instruction }) => {
773
757
  try {
774
- const stateManager = explorer.getStateManager();
775
758
  const currentState = stateManager.getCurrentState();
776
759
 
777
760
  if (!currentState) {
@@ -826,7 +809,6 @@ export function createAgentTools({
826
809
  }
827
810
 
828
811
  try {
829
- const stateManager = explorer.getStateManager();
830
812
  const currentState = stateManager.getCurrentState();
831
813
 
832
814
  if (!currentState) {
@@ -834,8 +816,8 @@ export function createAgentTools({
834
816
  }
835
817
 
836
818
  const previousState = ActionResult.fromState(currentState);
837
- const action = explorer.createAction();
838
- const actionResult = await explorer.capturePageWithScreenshot();
819
+ const action = explorer.action();
820
+ const actionResult = await explorer.capture({ screenshot: true });
839
821
 
840
822
  if (!actionResult.screenshot) {
841
823
  return failedToolResult('visualClick', 'Failed to capture screenshot for visual analysis');
@@ -897,7 +879,6 @@ export function createAgentTools({
897
879
  reason: z.string().describe('Why you need to go back'),
898
880
  }),
899
881
  execute: async ({ reason }) => {
900
- const stateManager = explorer.getStateManager();
901
882
  const currentState = stateManager.getCurrentState();
902
883
  const currentUrl = currentState?.fullUrl || currentState?.url;
903
884
  const history = stateManager.getStateHistory();
@@ -920,7 +901,7 @@ export function createAgentTools({
920
901
  let previousState: ActionResult | null = null;
921
902
  if (currentState) previousState = ActionResult.fromState(currentState);
922
903
 
923
- const action = explorer.createAction();
904
+ const action = explorer.action();
924
905
  const success = await action.attempt(`I.amOnPage(${JSON.stringify(targetUrl)})`, `${reason} (BACK to ${targetUrl})`);
925
906
 
926
907
  if (success) {
@@ -942,7 +923,7 @@ export function createAgentTools({
942
923
  description: 'List all previously visited page states (deduped by URL). Use to find pages to navigate back to.',
943
924
  inputSchema: z.object({}),
944
925
  execute: async () => {
945
- const history = explorer.getStateManager().getStateHistory();
926
+ const history = stateManager.getStateHistory();
946
927
  const seen = new Set<string>();
947
928
  const states = history
948
929
  .map((t) => t.toState)
@@ -975,7 +956,6 @@ export function createAgentTools({
975
956
  reason: z.string().describe('What element you are looking for and why'),
976
957
  }),
977
958
  execute: async ({ xpath, reason }) => {
978
- const stateManager = explorer.getStateManager();
979
959
  const currentState = stateManager.getCurrentState();
980
960
 
981
961
  if (!currentState) {
@@ -1001,9 +981,9 @@ export function createAgentTools({
1001
981
  });
1002
982
  }
1003
983
 
1004
- const action = explorer.createAction();
984
+ const action = explorer.action();
1005
985
  const visible = await action.attempt(`I.seeElement(${JSON.stringify(xpath)})`, 'xpathCheck visibility');
1006
- const liveElement = await WebElement.fromPlaywrightLocator(explorer.playwrightHelper.page.locator(`xpath=${xpath}`));
986
+ const liveElement = await explorer.withPage((page) => WebElement.fromPlaywrightLocator(page.locator(`xpath=${xpath}`)));
1007
987
 
1008
988
  const matchesSummary = result.elements.map((el, i) => `${i + 1}. <${el.tag} ${el.keyAttrs}> text="${el.text}" html: ${el.outerHTML}`).join('\n');
1009
989
 
@@ -1033,9 +1013,8 @@ export function createAgentTools({
1033
1013
 
1034
1014
  if (withExperience !== false) {
1035
1015
  tools.learnExperience = createLearnExperienceTool({
1036
- getExperienceTracker: () => explorer.getStateManager().getExperienceTracker(),
1016
+ getExperienceTracker: () => stateManager.getExperienceTracker(),
1037
1017
  getState: () => {
1038
- const stateManager = explorer.getStateManager();
1039
1018
  const currentState = stateManager.getCurrentState();
1040
1019
  return currentState ? ActionResult.fromState(currentState) : null;
1041
1020
  },
@@ -1270,7 +1249,7 @@ async function disambiguateElements(error: Error | null | undefined, explanation
1270
1249
  },
1271
1250
  ],
1272
1251
  schema,
1273
- provider.getAgenticModel(),
1252
+ provider.getModelForAgent(),
1274
1253
  { agentName: 'disambiguator', timeout: 15000 }
1275
1254
  );
1276
1255
 
@@ -178,7 +178,7 @@ export class CommandHandler implements InputManager {
178
178
  }
179
179
 
180
180
  private async executeBrowserCommand(input: string): Promise<void> {
181
- const action = this.explorBot.getExplorer().createAction();
181
+ const action = this.explorBot.getExplorer().action();
182
182
  await action.execute(input);
183
183
  }
184
184
 
@@ -6,7 +6,7 @@ export class ContextAriaCommand extends BaseCommand {
6
6
  description = 'Print full ARIA snapshot for current page';
7
7
 
8
8
  async execute(_args: string): Promise<void> {
9
- const state = this.explorBot.getExplorer().getStateManager().getCurrentState();
9
+ const state = this.explorBot.stateManager().getCurrentState();
10
10
 
11
11
  if (!state) {
12
12
  throw new Error('No active page to snapshot');
@@ -1,9 +1,11 @@
1
1
  import { ActionResult } from '../action-result.js';
2
2
  import { Researcher } from '../ai/researcher.js';
3
+ import { visuallyAnnotateContainers } from '../ai/researcher/coordinates.js';
3
4
  import { outputPath } from '../config.js';
4
5
  import { type ContextData, type ContextMode, formatContextSummary } from '../utils/context-formatter.js';
5
6
  import { tag } from '../utils/logger.js';
6
7
  import { extractValidContainers } from '../utils/research-parser.js';
8
+ import { annotatePageElements } from '../utils/web-annotate.js';
7
9
  import { BaseCommand, type Suggestion } from './base-command.js';
8
10
 
9
11
  export class ContextCommand extends BaseCommand {
@@ -25,7 +27,7 @@ export class ContextCommand extends BaseCommand {
25
27
 
26
28
  async execute(args: string): Promise<void> {
27
29
  const explorer = this.explorBot.getExplorer();
28
- const state = explorer.getStateManager().getCurrentState();
30
+ const state = this.explorBot.stateManager().getCurrentState();
29
31
 
30
32
  if (!state) {
31
33
  throw new Error('No active page to show context for');
@@ -34,16 +36,16 @@ export class ContextCommand extends BaseCommand {
34
36
  const { opts } = this.parseArgs(args);
35
37
  const isVisual = !!(opts.visual || opts.screenshot);
36
38
 
37
- await explorer.annotateElements();
39
+ await explorer.withPage(annotatePageElements);
38
40
 
39
41
  if (isVisual) {
40
42
  const cachedResearch = Researcher.getCachedResearch(state);
41
43
  const containers = cachedResearch ? extractValidContainers(cachedResearch) : [];
42
- await explorer.visuallyAnnotateElements({ containers });
44
+ await explorer.withPage((page) => visuallyAnnotateContainers(page, containers));
43
45
  }
44
46
 
45
- const actionResult = await explorer.createAction().capturePageState({ includeScreenshot: isVisual });
46
- const experienceTracker = explorer.getStateManager().getExperienceTracker();
47
+ const actionResult = await explorer.capture({ screenshot: isVisual });
48
+ const experienceTracker = this.explorBot.experienceTracker();
47
49
  const knowledgeTracker = this.explorBot.knowledgeTracker();
48
50
 
49
51
  let mode: ContextMode = 'compact';
@@ -8,7 +8,7 @@ export class ContextDataCommand extends BaseCommand {
8
8
 
9
9
  async execute(_args: string): Promise<void> {
10
10
  const explorer = this.explorBot.getExplorer();
11
- const state = explorer.getStateManager().getCurrentState();
11
+ const state = this.explorBot.stateManager().getCurrentState();
12
12
 
13
13
  if (!state) {
14
14
  throw new Error('No active page to extract data from');
@@ -18,7 +18,7 @@ export class ContextDataCommand extends BaseCommand {
18
18
 
19
19
  if (!actionResult.html || actionResult.html.trim().length < 100) {
20
20
  tag('info').log('Capturing fresh page content...');
21
- const freshResult = await explorer.createAction().capturePageState();
21
+ const freshResult = await explorer.capture();
22
22
  const table = await this.explorBot.agentResearcher().extractData(freshResult);
23
23
  tag('multiline').log(table);
24
24
  return;