explorbot 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. package/README.md +21 -5
  2. package/bin/explorbot-cli.ts +29 -7
  3. package/boat/api-tester/src/config.ts +45 -3
  4. package/boat/doc-collector/src/ai/documentarian.ts +8 -2
  5. package/boat/doc-collector/src/ai/tools.ts +29 -16
  6. package/boat/doc-collector/src/cli.ts +2 -6
  7. package/boat/doc-collector/src/config.ts +2 -0
  8. package/boat/doc-collector/src/docbot.ts +42 -14
  9. package/boat/doc-collector/src/interaction-screenshots.ts +160 -0
  10. package/boat/doc-collector/src/screenshots.ts +24 -16
  11. package/dist/bin/explorbot-cli.js +26 -6
  12. package/dist/boat/api-tester/src/config.js +43 -4
  13. package/dist/boat/doc-collector/src/ai/documentarian.js +7 -2
  14. package/dist/boat/doc-collector/src/ai/tools.js +21 -15
  15. package/dist/boat/doc-collector/src/cli.js +2 -5
  16. package/dist/boat/doc-collector/src/config.js +1 -0
  17. package/dist/boat/doc-collector/src/docbot.js +39 -13
  18. package/dist/boat/doc-collector/src/interaction-screenshots.js +156 -0
  19. package/dist/boat/doc-collector/src/screenshots.js +25 -16
  20. package/dist/models.json +30 -0
  21. package/dist/package.json +8 -2
  22. package/dist/src/action.d.ts +11 -3
  23. package/dist/src/action.js +22 -5
  24. package/dist/src/ai/agent.d.ts +17 -0
  25. package/dist/src/ai/captain/idle-mode.js +1 -1
  26. package/dist/src/ai/captain/test-mode.js +1 -1
  27. package/dist/src/ai/captain/web-mode.js +18 -22
  28. package/dist/src/ai/captain.js +13 -26
  29. package/dist/src/ai/driller.d.ts +2 -13
  30. package/dist/src/ai/driller.js +23 -38
  31. package/dist/src/ai/historian/codeceptjs.js +1 -1
  32. package/dist/src/ai/historian/playwright.d.ts +3 -2
  33. package/dist/src/ai/historian/playwright.js +1 -1
  34. package/dist/src/ai/historian/screencast.d.ts +2 -1
  35. package/dist/src/ai/historian/screencast.js +2 -2
  36. package/dist/src/ai/historian.d.ts +5 -4
  37. package/dist/src/ai/navigator.d.ts +6 -2
  38. package/dist/src/ai/navigator.js +26 -22
  39. package/dist/src/ai/pilot.d.ts +13 -6
  40. package/dist/src/ai/pilot.js +49 -44
  41. package/dist/src/ai/planner.d.ts +3 -2
  42. package/dist/src/ai/planner.js +19 -17
  43. package/dist/src/ai/provider.js +0 -1
  44. package/dist/src/ai/quartermaster.d.ts +3 -2
  45. package/dist/src/ai/quartermaster.js +4 -4
  46. package/dist/src/ai/rerunner.d.ts +2 -11
  47. package/dist/src/ai/rerunner.js +9 -22
  48. package/dist/src/ai/researcher/coordinates.js +4 -3
  49. package/dist/src/ai/researcher/deep-analysis.d.ts +2 -0
  50. package/dist/src/ai/researcher/deep-analysis.js +6 -6
  51. package/dist/src/ai/researcher/locators.d.ts +9 -2
  52. package/dist/src/ai/researcher/locators.js +65 -8
  53. package/dist/src/ai/researcher/sections.d.ts +2 -0
  54. package/dist/src/ai/researcher/sections.js +2 -2
  55. package/dist/src/ai/researcher.js +29 -54
  56. package/dist/src/ai/task-agent.d.ts +18 -5
  57. package/dist/src/ai/task-agent.js +35 -6
  58. package/dist/src/ai/tester.d.ts +6 -12
  59. package/dist/src/ai/tester.js +37 -51
  60. package/dist/src/ai/tools.d.ts +11 -10
  61. package/dist/src/ai/tools.js +24 -36
  62. package/dist/src/command-handler.js +1 -1
  63. package/dist/src/commands/context-aria-command.js +1 -1
  64. package/dist/src/commands/context-command.js +7 -5
  65. package/dist/src/commands/context-data-command.js +2 -2
  66. package/dist/src/commands/context-experience-command.js +2 -2
  67. package/dist/src/commands/context-html-command.js +2 -2
  68. package/dist/src/commands/context-knowledge-command.js +1 -1
  69. package/dist/src/commands/drill-command.js +1 -1
  70. package/dist/src/commands/explore-command.js +4 -3
  71. package/dist/src/commands/freesail-command.js +2 -2
  72. package/dist/src/commands/learn-command.js +4 -4
  73. package/dist/src/commands/path-command.js +1 -1
  74. package/dist/src/commands/research-command.js +1 -1
  75. package/dist/src/commands/test-command.js +1 -1
  76. package/dist/src/components/App.js +1 -1
  77. package/dist/src/config.d.ts +17 -0
  78. package/dist/src/config.js +144 -10
  79. package/dist/src/experience-tracker.js +1 -1
  80. package/dist/src/explorbot.d.ts +17 -8
  81. package/dist/src/explorbot.js +74 -36
  82. package/dist/src/explorer.d.ts +67 -91
  83. package/dist/src/explorer.js +327 -567
  84. package/dist/src/state-manager.d.ts +6 -1
  85. package/dist/src/state-manager.js +1 -0
  86. package/dist/src/utils/hooks-runner.js +2 -4
  87. package/dist/src/utils/test-files.js +1 -1
  88. package/dist/src/utils/web-annotate.d.ts +5 -0
  89. package/dist/src/utils/web-annotate.js +58 -0
  90. package/dist/src/utils/web-eidx.d.ts +2 -0
  91. package/dist/src/utils/web-eidx.js +20 -0
  92. package/dist/src/utils/web-element.d.ts +1 -0
  93. package/dist/src/utils/web-element.js +29 -0
  94. package/models.json +30 -0
  95. package/package.json +8 -2
  96. package/src/action.ts +25 -5
  97. package/src/ai/agent.ts +20 -0
  98. package/src/ai/captain/idle-mode.ts +1 -1
  99. package/src/ai/captain/test-mode.ts +1 -1
  100. package/src/ai/captain/web-mode.ts +18 -23
  101. package/src/ai/captain.ts +12 -25
  102. package/src/ai/driller.ts +50 -67
  103. package/src/ai/historian/codeceptjs.ts +1 -1
  104. package/src/ai/historian/playwright.ts +4 -3
  105. package/src/ai/historian/screencast.ts +4 -3
  106. package/src/ai/historian.ts +3 -2
  107. package/src/ai/navigator.ts +30 -25
  108. package/src/ai/pilot.ts +55 -47
  109. package/src/ai/planner.ts +22 -20
  110. package/src/ai/provider.ts +0 -1
  111. package/src/ai/quartermaster.ts +5 -4
  112. package/src/ai/rerunner.ts +10 -29
  113. package/src/ai/researcher/coordinates.ts +4 -3
  114. package/src/ai/researcher/deep-analysis.ts +8 -6
  115. package/src/ai/researcher/locators.ts +66 -9
  116. package/src/ai/researcher/sections.ts +4 -2
  117. package/src/ai/researcher.ts +32 -61
  118. package/src/ai/task-agent.ts +45 -11
  119. package/src/ai/tester.ts +40 -57
  120. package/src/ai/tools.ts +32 -53
  121. package/src/command-handler.ts +1 -1
  122. package/src/commands/context-aria-command.ts +1 -1
  123. package/src/commands/context-command.ts +7 -5
  124. package/src/commands/context-data-command.ts +2 -2
  125. package/src/commands/context-experience-command.ts +2 -2
  126. package/src/commands/context-html-command.ts +2 -2
  127. package/src/commands/context-knowledge-command.ts +1 -1
  128. package/src/commands/drill-command.ts +1 -1
  129. package/src/commands/explore-command.ts +4 -3
  130. package/src/commands/freesail-command.ts +2 -2
  131. package/src/commands/learn-command.ts +4 -4
  132. package/src/commands/path-command.ts +1 -1
  133. package/src/commands/research-command.ts +1 -1
  134. package/src/commands/test-command.ts +1 -1
  135. package/src/components/App.tsx +1 -1
  136. package/src/config.ts +173 -11
  137. package/src/experience-tracker.ts +1 -1
  138. package/src/explorbot.ts +78 -39
  139. package/src/explorer.ts +388 -626
  140. package/src/state-manager.ts +7 -1
  141. package/src/utils/hooks-runner.ts +2 -4
  142. package/src/utils/test-files.ts +1 -1
  143. package/src/utils/web-annotate.ts +64 -0
  144. package/src/utils/web-eidx.ts +21 -0
  145. package/src/utils/web-element.ts +30 -0
@@ -10,7 +10,6 @@ import { Stats } from "../stats.js";
10
10
  import { TestResult } from "../test-plan.js";
11
11
  import { detectFocusArea, extractFocusedElement } from "../utils/aria.js";
12
12
  import { ErrorPageError, isErrorPage } from "../utils/error-page.js";
13
- import { HooksRunner } from "../utils/hooks-runner.js";
14
13
  import { createDebug, tag } from "../utils/logger.js";
15
14
  import { loop } from "../utils/loop.js";
16
15
  import { actionRule, capabilityGroundingRule, dataProtectionRules, focusedElementRule, formRequirementsRule, locatorRule, multipleTabsRule, sectionContextRule } from "./rules.js";
@@ -31,8 +30,8 @@ export class Tester extends TaskAgent {
31
30
  ACTION_TOOLS = ['click', 'hover', 'pressKey', 'form'];
32
31
  SPECIAL_CONTEXT_ACTION_TOOLS = ['exitIframe'];
33
32
  emoji = '🧪';
34
- explorer;
35
- provider;
33
+ requestStore;
34
+ testRun = null;
36
35
  currentConversation = null;
37
36
  pilot = null;
38
37
  captain = null;
@@ -46,32 +45,20 @@ export class Tester extends TaskAgent {
46
45
  previousStateHash = null;
47
46
  pageStateHash = null;
48
47
  pageActionResult = null;
49
- hooksRunner;
50
48
  seenUiMapUrls = new Set();
51
49
  lastAnalyzedStateHash = null;
52
50
  stalledIterations = 0;
53
51
  MAX_STALLED_ITERATIONS = 3;
54
- constructor(explorer, provider, researcher, navigator, agentTools) {
55
- super();
56
- this.explorer = explorer;
57
- this.provider = provider;
52
+ constructor(deps, researcher, navigator, agentTools) {
53
+ super(deps);
54
+ this.requestStore = deps.requestStore;
58
55
  this.researcher = researcher;
59
56
  this.navigator = navigator;
60
57
  this.agentTools = agentTools;
61
- this.hooksRunner = new HooksRunner(explorer, explorer.getConfig());
62
58
  }
63
59
  getNavigator() {
64
60
  return this.navigator;
65
61
  }
66
- getExperienceTracker() {
67
- return this.explorer.getStateManager().getExperienceTracker();
68
- }
69
- getKnowledgeTracker() {
70
- return this.explorer.getKnowledgeTracker();
71
- }
72
- getProvider() {
73
- return this.provider;
74
- }
75
62
  setPilot(pilot) {
76
63
  this.pilot = pilot;
77
64
  }
@@ -79,17 +66,17 @@ export class Tester extends TaskAgent {
79
66
  this.captain = captain;
80
67
  }
81
68
  getCurrentState() {
82
- return ActionResult.fromState(this.explorer.getStateManager().getCurrentState());
69
+ return ActionResult.fromState(this.stateManager.getCurrentState());
83
70
  }
84
71
  get progressCheckInterval() {
85
- return this.explorer.getConfig().ai?.agents?.tester?.progressCheckInterval ?? 3;
72
+ return this.config.ai?.agents?.tester?.progressCheckInterval ?? 3;
86
73
  }
87
74
  getConversation() {
88
75
  return this.currentConversation;
89
76
  }
90
77
  async test(task) {
91
78
  Stats.tests++;
92
- const state = this.explorer.getStateManager().getCurrentState();
79
+ const state = this.stateManager.getCurrentState();
93
80
  if (!state)
94
81
  throw new Error('No state found');
95
82
  setActivity(`🧪 Testing: ${task.scenario}`, 'action');
@@ -100,18 +87,18 @@ export class Tester extends TaskAgent {
100
87
  this.seenUiMapUrls.clear();
101
88
  this.lastAnalyzedStateHash = null;
102
89
  this.stalledIterations = 0;
103
- this.explorer.getStateManager().clearHistory();
90
+ this.stateManager.clearHistory();
104
91
  this.resetFailureCount();
105
92
  this.pilot?.reset();
106
- const requestStore = this.explorer.getRequestStore();
107
- requestStore?.clear();
108
- const offFailedRequest = requestStore?.onFailedRequest((r) => {
93
+ const requestStore = this.requestStore;
94
+ requestStore.clear();
95
+ const offFailedRequest = requestStore.onFailedRequest((r) => {
109
96
  task.addNote(`Network error: ${r.method} ${r.path} → ${r.status}`, TestResult.FAILED);
110
97
  });
111
98
  const initialState = ActionResult.fromState(state);
112
99
  if (isErrorPage(initialState)) {
113
100
  task.start();
114
- await this.explorer.startTest(task);
101
+ this.testRun = await this.explorer.beginTest(task);
115
102
  offFailedRequest?.();
116
103
  return await this.abortStartedTestOnErrorPage(task, initialState);
117
104
  }
@@ -158,7 +145,8 @@ export class Tester extends TaskAgent {
158
145
  }
159
146
  }
160
147
  debugLog('Starting test execution with tools');
161
- if (!(await this.explorer.startTest(task))) {
148
+ this.testRun = await this.explorer.beginTest(task);
149
+ if (!this.testRun.started) {
162
150
  offFailedRequest?.();
163
151
  await this.cleanupStartedTest(task);
164
152
  return { success: task.isSuccessful };
@@ -175,7 +163,7 @@ export class Tester extends TaskAgent {
175
163
  return { success: task.isSuccessful };
176
164
  }
177
165
  }
178
- const startState = this.explorer.getStateManager().getCurrentState();
166
+ const startState = this.stateManager.getCurrentState();
179
167
  if (startState) {
180
168
  task.addUrlNote(startState);
181
169
  const startActionResult = ActionResult.fromState(startState);
@@ -186,7 +174,7 @@ export class Tester extends TaskAgent {
186
174
  }
187
175
  const currentUrl = startState?.url || task.startUrl || '';
188
176
  await this.hooksRunner.runBeforeHook('tester', currentUrl);
189
- const offStateChange = this.explorer.getStateManager().onStateChange((event) => {
177
+ const offStateChange = this.stateManager.onStateChange((event) => {
190
178
  if (task.hasFinished)
191
179
  return;
192
180
  if (event.toState?.url === event.fromState?.url)
@@ -195,7 +183,7 @@ export class Tester extends TaskAgent {
195
183
  task.addUrlNote(event.toState, event.fromState || undefined);
196
184
  task.states.push(event.toState);
197
185
  });
198
- const codeceptjsTools = createCodeceptJSTools(this.explorer, task);
186
+ const codeceptjsTools = createCodeceptJSTools(this.toolDeps, task);
199
187
  let assertionPerformed = false;
200
188
  let extensions = 0;
201
189
  let shouldContinue = true;
@@ -203,7 +191,7 @@ export class Tester extends TaskAgent {
203
191
  shouldContinue = false;
204
192
  await loop(async ({ stop, pause, iteration, userInput }) => {
205
193
  debugLog('iteration', iteration);
206
- if (!(await this.explorer.ensurePageAvailable())) {
194
+ if (!(await this.explorer.recover()).ok) {
207
195
  task.addNote('Browser page is unavailable');
208
196
  task.finish(TestResult.FAILED);
209
197
  stop();
@@ -216,10 +204,10 @@ export class Tester extends TaskAgent {
216
204
  ...this.agentTools,
217
205
  };
218
206
  if (currentState.isInsideIframe) {
219
- Object.assign(tools, createIframeTools(this.explorer));
207
+ Object.assign(tools, createIframeTools(this.toolDeps));
220
208
  }
221
209
  debugLog(`Test ${task.scenario} iteration ${iteration}`);
222
- if (this.explorer.getStateManager().isInDeadLoop()) {
210
+ if (this.stateManager.isInDeadLoop()) {
223
211
  task.addNote('Dead loop detected. Stopped');
224
212
  stop();
225
213
  return;
@@ -354,7 +342,7 @@ export class Tester extends TaskAgent {
354
342
  });
355
343
  if (task.hasFinished)
356
344
  break;
357
- if (!(await this.explorer.ensurePageAvailable()))
345
+ if (!(await this.explorer.recover()).ok)
358
346
  break;
359
347
  const finalState = this.getCurrentState();
360
348
  const wantsContinue = await this.pilot.finalReview(task, finalState, conversation, this.navigator);
@@ -372,7 +360,7 @@ export class Tester extends TaskAgent {
372
360
  conversation.compactToolResults(1);
373
361
  shouldContinue = true;
374
362
  }
375
- const finalUrl = this.explorer.getStateManager().getCurrentState()?.url || currentUrl;
363
+ const finalUrl = this.stateManager.getCurrentState()?.url || currentUrl;
376
364
  await this.hooksRunner.runAfterHook('tester', finalUrl);
377
365
  await this.getHistorian().saveSession(task, initialState, conversation);
378
366
  if (task.plan) {
@@ -382,7 +370,7 @@ export class Tester extends TaskAgent {
382
370
  offStateChange();
383
371
  offFailedRequest?.();
384
372
  await this.finishTest(task);
385
- await this.explorer.stopTest(task, this.buildStopTestMeta(task));
373
+ await this.testRun?.stop(this.buildStopTestMeta(task));
386
374
  return {
387
375
  success: task.isSuccessful,
388
376
  ...task,
@@ -486,7 +474,7 @@ export class Tester extends TaskAgent {
486
474
  `;
487
475
  }
488
476
  if (currentState.isInsideIframe) {
489
- const iframeInfo = currentState.iframeURL || this.explorer.getCurrentIframeInfo() || 'iframe context active';
477
+ const iframeInfo = currentState.iframeURL || 'iframe context active';
490
478
  context += dedent `
491
479
  <iframe_context>
492
480
  INSIDE IFRAME: ${iframeInfo}
@@ -494,10 +482,10 @@ export class Tester extends TaskAgent {
494
482
  </iframe_context>
495
483
  `;
496
484
  }
497
- if (this.explorer.hasOtherTabs()) {
498
- const otherTabs = this.explorer.getOtherTabsInfo();
485
+ const otherTabs = this.stateManager.otherTabs;
486
+ if (otherTabs.length > 0) {
499
487
  context += multipleTabsRule(otherTabs);
500
- this.explorer.clearOtherTabsInfo();
488
+ this.stateManager.otherTabs = [];
501
489
  }
502
490
  if (isNewUrl) {
503
491
  const alreadySeenUiMap = this.seenUiMapUrls.has(currentUrl);
@@ -600,7 +588,7 @@ export class Tester extends TaskAgent {
600
588
  task.addNote(error.message, TestResult.FAILED, actionResult.screenshotFile, actionResult.fullUrl || actionResult.url);
601
589
  task.finish(TestResult.FAILED);
602
590
  this.finishTest(task);
603
- await this.explorer.stopTest(task, this.buildStopTestMeta(task));
591
+ await this.testRun?.stop(this.buildStopTestMeta(task));
604
592
  clearActivity(true);
605
593
  return { success: false };
606
594
  }
@@ -715,7 +703,7 @@ export class Tester extends TaskAgent {
715
703
 
716
704
  ${dataProtectionRules}
717
705
 
718
- ${this.provider.getSystemPromptForAgent('tester', this.explorer.getStateManager().getCurrentState()?.url) || ''}
706
+ ${this.provider.getSystemPromptForAgent('tester', this.stateManager.getCurrentState()?.url) || ''}
719
707
  `;
720
708
  }
721
709
  buildScenarioBlock(task, actionResult) {
@@ -759,7 +747,7 @@ export class Tester extends TaskAgent {
759
747
  .map((t) => t.sessionName);
760
748
  }
761
749
  buildAvailableFiles() {
762
- const userFiles = this.explorer.getConfig().files || {};
750
+ const userFiles = this.config.files || {};
763
751
  const codeceptDir = global.codecept_dir || process.cwd();
764
752
  const lines = [];
765
753
  for (const [description, filename] of Object.entries(SAMPLE_FILES)) {
@@ -813,10 +801,8 @@ export class Tester extends TaskAgent {
813
801
  reason: z.string().optional().describe('Explanation why reset is the only option'),
814
802
  }),
815
803
  execute: async ({ reason }) => {
816
- if (this.getCurrentState().isInsideIframe) {
817
- await this.explorer.switchToMainFrame();
818
- }
819
- const currentState = this.explorer.getStateManager().getCurrentState();
804
+ await this.explorer.action().exitIframe();
805
+ const currentState = this.stateManager.getCurrentState();
820
806
  const currentUrl = currentState?.fullUrl || currentState?.url;
821
807
  if (currentUrl === resetUrl) {
822
808
  return {
@@ -841,7 +827,7 @@ export class Tester extends TaskAgent {
841
827
  const explanation = reason ? `${reason} (RESET)` : 'Resetting to initial page';
842
828
  const targetUrl = resetUrl;
843
829
  task.addNote(explanation);
844
- const resetAction = this.explorer.createAction();
830
+ const resetAction = this.explorer.action();
845
831
  const success = await resetAction.attempt(`I.amOnPage(${JSON.stringify(targetUrl)})`, explanation);
846
832
  if (success) {
847
833
  return {
@@ -984,7 +970,7 @@ export class Tester extends TaskAgent {
984
970
  else if (input.status === 'fail') {
985
971
  mappedStatus = TestResult.FAILED;
986
972
  }
987
- const screenshotFile = this.explorer.getStateManager().getCurrentState()?.screenshotFile;
973
+ const screenshotFile = this.stateManager.getCurrentState()?.screenshotFile;
988
974
  for (const noteText of input.notes) {
989
975
  task.addNote(noteText, mappedStatus, screenshotFile);
990
976
  if (input.status === 'success') {
@@ -1021,7 +1007,7 @@ export class Tester extends TaskAgent {
1021
1007
  const message = error instanceof Error ? error.message : String(error);
1022
1008
  if (!task.hasFinished)
1023
1009
  task.addNote(`Execution error: ${message}`);
1024
- const result = await this.explorer.handleExecutionError(error);
1010
+ const result = await this.explorer.recover(error);
1025
1011
  tag('info').log(`Browser supervisor: ${result.action} - ${result.message}`);
1026
1012
  task.addNote(result.message);
1027
1013
  if (result.action === 'stop') {
@@ -1052,7 +1038,7 @@ export class Tester extends TaskAgent {
1052
1038
  }
1053
1039
  async cleanupStartedTest(task) {
1054
1040
  await this.finishTest(task);
1055
- await this.explorer.stopTest(task, {
1041
+ await this.testRun?.stop({
1056
1042
  startUrl: task.startUrl,
1057
1043
  style: task.style,
1058
1044
  sessionName: task.sessionName,
@@ -1,11 +1,17 @@
1
1
  import { ActionResult, type PageDiff } from '../action-result.js';
2
2
  import type { ExperienceTracker } from '../experience-tracker.js';
3
- import type Explorer from '../explorer.js';
4
3
  import { type Task } from '../test-plan.js';
4
+ import type { ToolDeps } from './agent.js';
5
5
  import { Navigator } from './navigator.js';
6
6
  import { Researcher } from './researcher.js';
7
+ interface AgentToolDeps extends ToolDeps {
8
+ researcher: Researcher;
9
+ navigator: Navigator;
10
+ supervisor?: boolean;
11
+ withExperience?: boolean;
12
+ }
7
13
  export declare const ASSERTION_TOOLS: readonly ["verify"];
8
- export declare function createCodeceptJSTools(explorer: Explorer, task: Task): {
14
+ export declare function createCodeceptJSTools({ explorer, stateManager, ai }: ToolDeps, task: Task): {
9
15
  click: import("@ai-sdk/provider-utils").ExecutableTool<import("ai").Tool<{
10
16
  commands: any;
11
17
  explanation: any;
@@ -24,7 +30,7 @@ export declare function createCodeceptJSTools(explorer: Explorer, task: Task): {
24
30
  explanation: any;
25
31
  }, Record<string, any>, import("@ai-sdk/provider-utils").Context>>;
26
32
  };
27
- export declare function createIframeTools(explorer: Explorer): {
33
+ export declare function createIframeTools({ explorer, stateManager }: ToolDeps): {
28
34
  exitIframe: import("@ai-sdk/provider-utils").ExecutableTool<import("ai").Tool<{
29
35
  reason?: string;
30
36
  }, Record<string, any>, import("@ai-sdk/provider-utils").Context>>;
@@ -42,11 +48,6 @@ export declare function createLearnExperienceTool({ getExperienceTracker, getSta
42
48
  } | {
43
49
  error: string;
44
50
  }, import("@ai-sdk/provider-utils").Context>>;
45
- export declare function createAgentTools({ explorer, researcher, navigator, supervisor, withExperience, }: {
46
- explorer: Explorer;
47
- researcher: Researcher;
48
- navigator: Navigator;
49
- supervisor?: boolean;
50
- withExperience?: boolean;
51
- }): any;
51
+ export declare function createAgentTools({ explorer, stateManager, ai, researcher, navigator, supervisor, withExperience }: AgentToolDeps): any;
52
52
  export declare function isMajorPageChange(pageDiff: PageDiff): boolean;
53
+ export {};
@@ -12,8 +12,7 @@ import { sectionContextRule } from "./rules.js";
12
12
  import { isInteractive } from "./task-agent.js";
13
13
  const debugLog = createDebug('explorbot:tools');
14
14
  export const ASSERTION_TOOLS = ['verify'];
15
- export function createCodeceptJSTools(explorer, task) {
16
- const stateManager = explorer.getStateManager();
15
+ export function createCodeceptJSTools({ explorer, stateManager, ai }, task) {
17
16
  return {
18
17
  click: tool({
19
18
  description: dedent `
@@ -68,7 +67,7 @@ export function createCodeceptJSTools(explorer, task) {
68
67
  return `I.click(${JSON.stringify(trimmed)})`;
69
68
  });
70
69
  const previousState = ActionResult.fromState(stateManager.getCurrentState());
71
- const action = explorer.createAction();
70
+ const action = explorer.action();
72
71
  const attempts = [];
73
72
  for (let i = 0; i < commands.length; i++) {
74
73
  const command = transformContainsCommand(commands[i]);
@@ -85,7 +84,7 @@ export function createCodeceptJSTools(explorer, task) {
85
84
  }
86
85
  let disambiguated = null;
87
86
  if (attempts.some((a) => a.error?.toLowerCase().includes(MULTIPLE_ELEMENTS_PATTERN))) {
88
- disambiguated = await disambiguateElements(action.lastError, explanation, explorer.getAIProvider());
87
+ disambiguated = await disambiguateElements(action.lastError, explanation, ai);
89
88
  }
90
89
  if (disambiguated) {
91
90
  debugLog('Disambiguation picked element %d', disambiguated.position);
@@ -163,7 +162,7 @@ export function createCodeceptJSTools(explorer, task) {
163
162
  return `I.moveCursorTo(${JSON.stringify(trimmed)})`;
164
163
  });
165
164
  const previousState = ActionResult.fromState(stateManager.getCurrentState());
166
- const action = explorer.createAction();
165
+ const action = explorer.action();
167
166
  const attempts = [];
168
167
  for (const command of commands) {
169
168
  const success = await action.attempt(command, explanation);
@@ -223,7 +222,7 @@ export function createCodeceptJSTools(explorer, task) {
223
222
  const keyToUse = matchingStandardKey || key;
224
223
  if (!isSingleChar && !isStandardKey) {
225
224
  const previousState = ActionResult.fromState(stateManager.getCurrentState());
226
- const action = explorer.createAction();
225
+ const action = explorer.action();
227
226
  const typeCommand = `I.type(${JSON.stringify(key)})`;
228
227
  await action.attempt(typeCommand, explanation);
229
228
  const toolResult = await ActionResult.fromState(stateManager.getCurrentState()).toToolResult(previousState, key);
@@ -257,7 +256,7 @@ export function createCodeceptJSTools(explorer, task) {
257
256
  }
258
257
  }
259
258
  const previousState = ActionResult.fromState(stateManager.getCurrentState());
260
- const action = explorer.createAction();
259
+ const action = explorer.action();
261
260
  let pressKeyCommand;
262
261
  if (modifier) {
263
262
  const modifiers = Array.isArray(modifier) ? modifier : [modifier];
@@ -347,19 +346,15 @@ export function createCodeceptJSTools(explorer, task) {
347
346
  }
348
347
  const previousState = ActionResult.fromState(stateManager.getCurrentState());
349
348
  const formLocator = codeLines[0] || 'form';
350
- const action = explorer.createAction();
351
- const wasInIframe = await explorer.isInsideIframe();
349
+ const action = explorer.action();
352
350
  await action.attempt(codeBlock, explanation);
353
- if (action.lastError && !wasInIframe && (await explorer.isInsideIframe())) {
354
- await explorer.switchToMainFrame();
355
- }
356
351
  const toolResult = await ActionResult.fromState(stateManager.getCurrentState()).toToolResult(previousState, formLocator);
357
352
  if (action.lastError) {
358
353
  const message = action.lastError ? String(action.lastError) : 'Unknown error';
359
354
  await commitNote(activeNote, TestResult.FAILED, toolResult, action);
360
355
  let formSuggestion = 'Look into error message and identify which commands passed and which failed. Continue execution using step-by-step approach using click() and form() tools.';
361
356
  if (message.toLowerCase().includes(MULTIPLE_ELEMENTS_PATTERN)) {
362
- const disambiguated = await disambiguateElements(action.lastError, explanation, explorer.getAIProvider());
357
+ const disambiguated = await disambiguateElements(action.lastError, explanation, ai);
363
358
  if (disambiguated) {
364
359
  formSuggestion = `Multiple elements matched. Add step.opts({ elementIndex: ${disambiguated.position} }) to the failing command. Fallback locator: ${disambiguated.xpath}`;
365
360
  }
@@ -397,8 +392,7 @@ export function createCodeceptJSTools(explorer, task) {
397
392
  }),
398
393
  };
399
394
  }
400
- export function createIframeTools(explorer) {
401
- const stateManager = explorer.getStateManager();
395
+ export function createIframeTools({ explorer, stateManager }) {
402
396
  return {
403
397
  exitIframe: tool({
404
398
  description: dedent `
@@ -418,8 +412,9 @@ export function createIframeTools(explorer) {
418
412
  suggestion: 'Continue interacting with the current page context.',
419
413
  });
420
414
  }
421
- await explorer.switchToMainFrame();
422
- const nextState = await explorer.capturePageState();
415
+ const action = explorer.action();
416
+ await action.execute('I.switchTo()');
417
+ const nextState = action.getActionResult();
423
418
  const toolResult = await nextState.toToolResult(previousState, 'I.switchTo()');
424
419
  return successToolResult('exitIframe', {
425
420
  ...toolResult,
@@ -460,7 +455,7 @@ export function createLearnExperienceTool({ getExperienceTracker, getState }) {
460
455
  },
461
456
  });
462
457
  }
463
- export function createAgentTools({ explorer, researcher, navigator, supervisor, withExperience, }) {
458
+ export function createAgentTools({ explorer, stateManager, ai, researcher, navigator, supervisor, withExperience }) {
464
459
  let visionDisabled = false;
465
460
  const tools = {
466
461
  see: tool({
@@ -483,7 +478,7 @@ export function createAgentTools({ explorer, researcher, navigator, supervisor,
483
478
  return failedToolResult('see', 'Vision tools are disabled for this session. Use context() to get fresh ARIA snapshot and analyze page state from ARIA data.');
484
479
  }
485
480
  try {
486
- const actionResult = await explorer.capturePageWithScreenshot();
481
+ const actionResult = await explorer.capture({ screenshot: true });
487
482
  if (!actionResult.screenshot) {
488
483
  return failedToolResult('see', 'Failed to capture screenshot for analysis');
489
484
  }
@@ -527,7 +522,6 @@ export function createAgentTools({ explorer, researcher, navigator, supervisor,
527
522
  }),
528
523
  execute: async ({ reason }) => {
529
524
  try {
530
- const stateManager = explorer.getStateManager();
531
525
  const currentState = stateManager.getCurrentState();
532
526
  if (!currentState) {
533
527
  return failedToolResult('context', 'No current page state available.');
@@ -565,7 +559,7 @@ export function createAgentTools({ explorer, researcher, navigator, supervisor,
565
559
  }),
566
560
  execute: async ({ assertion }) => {
567
561
  try {
568
- const currentState = explorer.getStateManager().getCurrentState();
562
+ const currentState = stateManager.getCurrentState();
569
563
  const verifications = currentState?.verifications;
570
564
  if (verifications?.[assertion] !== undefined) {
571
565
  return failedToolResult('verify', `Already verified: "${assertion}" → ${verifications[assertion] ? 'PASS' : 'FAIL'}`, {
@@ -574,7 +568,7 @@ export function createAgentTools({ explorer, researcher, navigator, supervisor,
574
568
  suggestion: verifications[assertion] ? 'This verification already passed. Call finish() to complete the test.' : 'This verification already failed. Perform actions to change the page state, then try again.',
575
569
  });
576
570
  }
577
- const actionResult = await explorer.capturePageState();
571
+ const actionResult = await explorer.capture();
578
572
  const result = await navigator.verifyState(assertion, actionResult);
579
573
  if (result.verified) {
580
574
  return successToolResult('verify', {
@@ -618,7 +612,6 @@ export function createAgentTools({ explorer, researcher, navigator, supervisor,
618
612
  }),
619
613
  execute: async ({ reason }) => {
620
614
  try {
621
- const stateManager = explorer.getStateManager();
622
615
  const currentState = stateManager.getCurrentState();
623
616
  if (!currentState) {
624
617
  return failedToolResult('research', 'No current page state available. Navigate to a page first.');
@@ -659,7 +652,6 @@ export function createAgentTools({ explorer, researcher, navigator, supervisor,
659
652
  }),
660
653
  execute: async ({ instruction }) => {
661
654
  try {
662
- const stateManager = explorer.getStateManager();
663
655
  const currentState = stateManager.getCurrentState();
664
656
  if (!currentState) {
665
657
  return failedToolResult('interact', 'No current page state available. Navigate to a page first.');
@@ -708,14 +700,13 @@ export function createAgentTools({ explorer, researcher, navigator, supervisor,
708
700
  return failedToolResult('visualClick', 'Vision tools are disabled for this session. Use xpathCheck() to find the element, then click() with the discovered locator.');
709
701
  }
710
702
  try {
711
- const stateManager = explorer.getStateManager();
712
703
  const currentState = stateManager.getCurrentState();
713
704
  if (!currentState) {
714
705
  return failedToolResult('visualClick', 'No current page state available.');
715
706
  }
716
707
  const previousState = ActionResult.fromState(currentState);
717
- const action = explorer.createAction();
718
- const actionResult = await explorer.capturePageWithScreenshot();
708
+ const action = explorer.action();
709
+ const actionResult = await explorer.capture({ screenshot: true });
719
710
  if (!actionResult.screenshot) {
720
711
  return failedToolResult('visualClick', 'Failed to capture screenshot for visual analysis');
721
712
  }
@@ -768,7 +759,6 @@ export function createAgentTools({ explorer, researcher, navigator, supervisor,
768
759
  reason: z.string().describe('Why you need to go back'),
769
760
  }),
770
761
  execute: async ({ reason }) => {
771
- const stateManager = explorer.getStateManager();
772
762
  const currentState = stateManager.getCurrentState();
773
763
  const currentUrl = currentState?.fullUrl || currentState?.url;
774
764
  const history = stateManager.getStateHistory();
@@ -788,7 +778,7 @@ export function createAgentTools({ explorer, researcher, navigator, supervisor,
788
778
  let previousState = null;
789
779
  if (currentState)
790
780
  previousState = ActionResult.fromState(currentState);
791
- const action = explorer.createAction();
781
+ const action = explorer.action();
792
782
  const success = await action.attempt(`I.amOnPage(${JSON.stringify(targetUrl)})`, `${reason} (BACK to ${targetUrl})`);
793
783
  if (success) {
794
784
  const newState = ActionResult.fromState(stateManager.getCurrentState());
@@ -808,7 +798,7 @@ export function createAgentTools({ explorer, researcher, navigator, supervisor,
808
798
  description: 'List all previously visited page states (deduped by URL). Use to find pages to navigate back to.',
809
799
  inputSchema: z.object({}),
810
800
  execute: async () => {
811
- const history = explorer.getStateManager().getStateHistory();
801
+ const history = stateManager.getStateHistory();
812
802
  const seen = new Set();
813
803
  const states = history
814
804
  .map((t) => t.toState)
@@ -841,7 +831,6 @@ export function createAgentTools({ explorer, researcher, navigator, supervisor,
841
831
  reason: z.string().describe('What element you are looking for and why'),
842
832
  }),
843
833
  execute: async ({ xpath, reason }) => {
844
- const stateManager = explorer.getStateManager();
845
834
  const currentState = stateManager.getCurrentState();
846
835
  if (!currentState) {
847
836
  return failedToolResult('xpathCheck', 'No current page state available.');
@@ -861,9 +850,9 @@ export function createAgentTools({ explorer, researcher, navigator, supervisor,
861
850
  suggestion: 'Try a broader expression. Examples: //*[contains(@class, "btn")], //button, //*[contains(text(), "keyword")]',
862
851
  });
863
852
  }
864
- const action = explorer.createAction();
853
+ const action = explorer.action();
865
854
  const visible = await action.attempt(`I.seeElement(${JSON.stringify(xpath)})`, 'xpathCheck visibility');
866
- const liveElement = await WebElement.fromPlaywrightLocator(explorer.playwrightHelper.page.locator(`xpath=${xpath}`));
855
+ const liveElement = await explorer.withPage((page) => WebElement.fromPlaywrightLocator(page.locator(`xpath=${xpath}`)));
867
856
  const matchesSummary = result.elements.map((el, i) => `${i + 1}. <${el.tag} ${el.keyAttrs}> text="${el.text}" html: ${el.outerHTML}`).join('\n');
868
857
  const visibilityNote = visible ? 'Element IS visible in browser — Tester can use this XPath as locator.' : 'Element exists in DOM but is NOT visible. May need scrolling, a click to reveal, or is hidden.';
869
858
  const hit = liveElement?.ourAttr('hit') || null;
@@ -888,9 +877,8 @@ export function createAgentTools({ explorer, researcher, navigator, supervisor,
888
877
  };
889
878
  if (withExperience !== false) {
890
879
  tools.learnExperience = createLearnExperienceTool({
891
- getExperienceTracker: () => explorer.getStateManager().getExperienceTracker(),
880
+ getExperienceTracker: () => stateManager.getExperienceTracker(),
892
881
  getState: () => {
893
- const stateManager = explorer.getStateManager();
894
882
  const currentState = stateManager.getCurrentState();
895
883
  return currentState ? ActionResult.fromState(currentState) : null;
896
884
  },
@@ -1103,7 +1091,7 @@ async function disambiguateElements(error, explanation, provider) {
1103
1091
  Return the position number, or null if none of them match.
1104
1092
  `,
1105
1093
  },
1106
- ], schema, provider.getAgenticModel(), { agentName: 'disambiguator', timeout: 15000 });
1094
+ ], schema, provider.getModelForAgent(), { agentName: 'disambiguator', timeout: 15000 });
1107
1095
  const position = result?.object?.position;
1108
1096
  if (position && position >= 1 && position <= elementDetails.length) {
1109
1097
  return { position, xpath: elementDetails[position - 1].xpath };
@@ -131,7 +131,7 @@ export class CommandHandler {
131
131
  }
132
132
  }
133
133
  async executeBrowserCommand(input) {
134
- const action = this.explorBot.getExplorer().createAction();
134
+ const action = this.explorBot.getExplorer().action();
135
135
  await action.execute(input);
136
136
  }
137
137
  isCommand(input) {
@@ -4,7 +4,7 @@ export class ContextAriaCommand extends BaseCommand {
4
4
  name = 'context:aria';
5
5
  description = 'Print full ARIA snapshot for current page';
6
6
  async execute(_args) {
7
- const state = this.explorBot.getExplorer().getStateManager().getCurrentState();
7
+ const state = this.explorBot.stateManager().getCurrentState();
8
8
  if (!state) {
9
9
  throw new Error('No active page to snapshot');
10
10
  }
@@ -1,8 +1,10 @@
1
1
  import { Researcher } from '../ai/researcher.js';
2
+ import { visuallyAnnotateContainers } from '../ai/researcher/coordinates.js';
2
3
  import { outputPath } from '../config.js';
3
4
  import { formatContextSummary } from '../utils/context-formatter.js';
4
5
  import { tag } from '../utils/logger.js';
5
6
  import { extractValidContainers } from '../utils/research-parser.js';
7
+ import { annotatePageElements } from '../utils/web-annotate.js';
6
8
  import { BaseCommand } from './base-command.js';
7
9
  export class ContextCommand extends BaseCommand {
8
10
  name = 'context';
@@ -22,20 +24,20 @@ export class ContextCommand extends BaseCommand {
22
24
  ];
23
25
  async execute(args) {
24
26
  const explorer = this.explorBot.getExplorer();
25
- const state = explorer.getStateManager().getCurrentState();
27
+ const state = this.explorBot.stateManager().getCurrentState();
26
28
  if (!state) {
27
29
  throw new Error('No active page to show context for');
28
30
  }
29
31
  const { opts } = this.parseArgs(args);
30
32
  const isVisual = !!(opts.visual || opts.screenshot);
31
- await explorer.annotateElements();
33
+ await explorer.withPage(annotatePageElements);
32
34
  if (isVisual) {
33
35
  const cachedResearch = Researcher.getCachedResearch(state);
34
36
  const containers = cachedResearch ? extractValidContainers(cachedResearch) : [];
35
- await explorer.visuallyAnnotateElements({ containers });
37
+ await explorer.withPage((page) => visuallyAnnotateContainers(page, containers));
36
38
  }
37
- const actionResult = await explorer.createAction().capturePageState({ includeScreenshot: isVisual });
38
- const experienceTracker = explorer.getStateManager().getExperienceTracker();
39
+ const actionResult = await explorer.capture({ screenshot: isVisual });
40
+ const experienceTracker = this.explorBot.experienceTracker();
39
41
  const knowledgeTracker = this.explorBot.knowledgeTracker();
40
42
  let mode = 'compact';
41
43
  if (opts.full) {
@@ -6,14 +6,14 @@ export class ContextDataCommand extends BaseCommand {
6
6
  description = 'Extract structured data from current page';
7
7
  async execute(_args) {
8
8
  const explorer = this.explorBot.getExplorer();
9
- const state = explorer.getStateManager().getCurrentState();
9
+ const state = this.explorBot.stateManager().getCurrentState();
10
10
  if (!state) {
11
11
  throw new Error('No active page to extract data from');
12
12
  }
13
13
  const actionResult = ActionResult.fromState(state);
14
14
  if (!actionResult.html || actionResult.html.trim().length < 100) {
15
15
  tag('info').log('Capturing fresh page content...');
16
- const freshResult = await explorer.createAction().capturePageState();
16
+ const freshResult = await explorer.capture();
17
17
  const table = await this.explorBot.agentResearcher().extractData(freshResult);
18
18
  tag('multiline').log(table);
19
19
  return;
@@ -8,12 +8,12 @@ export class ContextExperienceCommand extends BaseCommand {
8
8
  description = 'Print all matching experience for current page';
9
9
  async execute(_args) {
10
10
  const explorer = this.explorBot.getExplorer();
11
- const state = explorer.getStateManager().getCurrentState();
11
+ const state = this.explorBot.stateManager().getCurrentState();
12
12
  if (!state) {
13
13
  throw new Error('No active page');
14
14
  }
15
15
  const actionResult = ActionResult.fromState(state);
16
- const experienceTracker = explorer.getStateManager().getExperienceTracker();
16
+ const experienceTracker = this.explorBot.experienceTracker();
17
17
  const experience = experienceTracker.getRelevantExperience(actionResult);
18
18
  if (experience.length === 0) {
19
19
  tag('info').log(`No experience found for: ${actionResult.url}`);