explorbot 0.1.32-beta.1 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (222) hide show
  1. package/README.md +35 -19
  2. package/bin/explorbot-cli.ts +33 -10
  3. package/boat/api-tester/src/config.ts +45 -3
  4. package/boat/doc-collector/src/ai/documentarian.ts +26 -28
  5. package/boat/doc-collector/src/ai/tools.ts +74 -16
  6. package/boat/doc-collector/src/cli.ts +1 -6
  7. package/boat/doc-collector/src/docbot.ts +30 -9
  8. package/boat/doc-collector/src/docs-renderer.ts +19 -24
  9. package/boat/doc-collector/src/screenshots.ts +34 -2
  10. package/boat/doc-collector/src/state-diagram.ts +281 -0
  11. package/dist/bin/explorbot-cli.js +29 -9
  12. package/dist/boat/api-tester/src/config.js +43 -4
  13. package/dist/boat/doc-collector/src/ai/documentarian.js +19 -7
  14. package/dist/boat/doc-collector/src/ai/tools.js +53 -16
  15. package/dist/boat/doc-collector/src/cli.js +1 -5
  16. package/dist/boat/doc-collector/src/docbot.js +27 -8
  17. package/dist/boat/doc-collector/src/docs-renderer.js +18 -3
  18. package/dist/boat/doc-collector/src/screenshots.js +30 -1
  19. package/dist/boat/doc-collector/src/state-diagram.js +200 -0
  20. package/dist/models.json +30 -0
  21. package/dist/package.json +23 -3
  22. package/dist/src/action-result.d.ts +8 -10
  23. package/dist/src/action-result.js +72 -164
  24. package/dist/src/action.d.ts +13 -12
  25. package/dist/src/action.js +37 -113
  26. package/dist/src/ai/agent.d.ts +17 -0
  27. package/dist/src/ai/captain/idle-mode.js +1 -1
  28. package/dist/src/ai/captain/test-mode.js +1 -1
  29. package/dist/src/ai/captain/web-mode.js +18 -29
  30. package/dist/src/ai/captain.d.ts +1 -3
  31. package/dist/src/ai/captain.js +17 -45
  32. package/dist/src/ai/conversation.d.ts +2 -5
  33. package/dist/src/ai/conversation.js +9 -28
  34. package/dist/src/ai/driller.d.ts +2 -17
  35. package/dist/src/ai/driller.js +26 -50
  36. package/dist/src/ai/experience-compactor.d.ts +2 -2
  37. package/dist/src/ai/experience-compactor.js +16 -33
  38. package/dist/src/ai/fisherman-tools.js +1 -1
  39. package/dist/src/ai/fisherman.d.ts +0 -1
  40. package/dist/src/ai/fisherman.js +0 -3
  41. package/dist/src/ai/historian/codeceptjs.d.ts +2 -0
  42. package/dist/src/ai/historian/codeceptjs.js +4 -3
  43. package/dist/src/ai/historian/playwright.d.ts +4 -1
  44. package/dist/src/ai/historian/playwright.js +4 -3
  45. package/dist/src/ai/historian/screencast.d.ts +2 -1
  46. package/dist/src/ai/historian/screencast.js +2 -2
  47. package/dist/src/ai/historian/utils.d.ts +0 -1
  48. package/dist/src/ai/historian/utils.js +0 -1
  49. package/dist/src/ai/historian.d.ts +10 -8
  50. package/dist/src/ai/historian.js +1 -5
  51. package/dist/src/ai/navigator.d.ts +8 -6
  52. package/dist/src/ai/navigator.js +34 -65
  53. package/dist/src/ai/pilot.d.ts +13 -8
  54. package/dist/src/ai/pilot.js +53 -59
  55. package/dist/src/ai/planner.d.ts +5 -4
  56. package/dist/src/ai/planner.js +66 -52
  57. package/dist/src/ai/provider.d.ts +4 -1
  58. package/dist/src/ai/provider.js +77 -137
  59. package/dist/src/ai/quartermaster.d.ts +22 -22
  60. package/dist/src/ai/quartermaster.js +8 -14
  61. package/dist/src/ai/rerunner.d.ts +3 -13
  62. package/dist/src/ai/rerunner.js +23 -44
  63. package/dist/src/ai/researcher/cache.js +11 -18
  64. package/dist/src/ai/researcher/coordinates.js +4 -3
  65. package/dist/src/ai/researcher/deep-analysis.d.ts +2 -0
  66. package/dist/src/ai/researcher/deep-analysis.js +8 -11
  67. package/dist/src/ai/researcher/locators.d.ts +9 -2
  68. package/dist/src/ai/researcher/locators.js +65 -8
  69. package/dist/src/ai/researcher/parser.js +1 -1
  70. package/dist/src/ai/researcher/research-result.d.ts +0 -1
  71. package/dist/src/ai/researcher/research-result.js +0 -19
  72. package/dist/src/ai/researcher/sections.d.ts +2 -0
  73. package/dist/src/ai/researcher/sections.js +7 -3
  74. package/dist/src/ai/researcher.js +40 -106
  75. package/dist/src/ai/rules.d.ts +1 -0
  76. package/dist/src/ai/rules.js +22 -15
  77. package/dist/src/ai/session-analyst.js +8 -5
  78. package/dist/src/ai/task-agent.d.ts +19 -6
  79. package/dist/src/ai/task-agent.js +37 -33
  80. package/dist/src/ai/tester.d.ts +6 -14
  81. package/dist/src/ai/tester.js +46 -111
  82. package/dist/src/ai/tools.d.ts +26 -14
  83. package/dist/src/ai/tools.js +138 -132
  84. package/dist/src/command-handler.js +1 -1
  85. package/dist/src/commands/compact-command.d.ts +1 -1
  86. package/dist/src/commands/compact-command.js +6 -6
  87. package/dist/src/commands/context-aria-command.js +1 -1
  88. package/dist/src/commands/context-command.js +8 -6
  89. package/dist/src/commands/context-data-command.js +2 -2
  90. package/dist/src/commands/context-experience-command.js +2 -2
  91. package/dist/src/commands/context-html-command.js +2 -2
  92. package/dist/src/commands/context-knowledge-command.js +2 -2
  93. package/dist/src/commands/drill-command.js +1 -1
  94. package/dist/src/commands/experience-command.js +1 -1
  95. package/dist/src/commands/explore-command.js +4 -3
  96. package/dist/src/commands/freesail-command.js +2 -2
  97. package/dist/src/commands/knows-command.js +1 -1
  98. package/dist/src/commands/learn-command.js +5 -3
  99. package/dist/src/commands/path-command.js +1 -1
  100. package/dist/src/commands/research-command.js +1 -1
  101. package/dist/src/commands/test-command.js +1 -1
  102. package/dist/src/components/AddKnowledge.d.ts +2 -0
  103. package/dist/src/components/AddKnowledge.js +3 -7
  104. package/dist/src/components/App.js +3 -3
  105. package/dist/src/config.d.ts +19 -0
  106. package/dist/src/config.js +156 -10
  107. package/dist/src/experience-tracker.d.ts +11 -6
  108. package/dist/src/experience-tracker.js +82 -155
  109. package/dist/src/explorbot.d.ts +21 -13
  110. package/dist/src/explorbot.js +84 -80
  111. package/dist/src/explorer.d.ts +66 -102
  112. package/dist/src/explorer.js +332 -623
  113. package/dist/src/knowledge-tracker.d.ts +2 -2
  114. package/dist/src/knowledge-tracker.js +38 -45
  115. package/dist/src/state-manager.d.ts +10 -30
  116. package/dist/src/state-manager.js +11 -129
  117. package/dist/src/utils/aria.d.ts +6 -1
  118. package/dist/src/utils/aria.js +4 -1
  119. package/dist/src/utils/cache.d.ts +15 -0
  120. package/dist/src/utils/cache.js +34 -0
  121. package/dist/src/utils/context-formatter.d.ts +6 -0
  122. package/dist/src/utils/context-formatter.js +15 -27
  123. package/dist/src/utils/hooks-runner.js +2 -4
  124. package/dist/src/utils/html.d.ts +5 -0
  125. package/dist/src/utils/html.js +71 -0
  126. package/dist/src/utils/markdown-files.d.ts +10 -0
  127. package/dist/src/utils/markdown-files.js +21 -0
  128. package/dist/src/utils/markdown-query.d.ts +6 -0
  129. package/dist/src/utils/markdown-query.js +14 -1
  130. package/dist/src/utils/page-readiness.d.ts +1 -0
  131. package/dist/src/utils/page-readiness.js +1 -1
  132. package/dist/src/utils/secrets.js +2 -2
  133. package/dist/src/utils/strings.d.ts +2 -0
  134. package/dist/src/utils/strings.js +10 -0
  135. package/dist/src/utils/test-files.js +1 -1
  136. package/dist/src/utils/web-annotate.d.ts +5 -0
  137. package/dist/src/utils/web-annotate.js +58 -0
  138. package/dist/src/utils/web-eidx.d.ts +2 -0
  139. package/dist/src/utils/web-eidx.js +20 -0
  140. package/dist/src/utils/web-element.d.ts +3 -1
  141. package/dist/src/utils/web-element.js +32 -0
  142. package/dist/src/utils/web-sandbox.d.ts +4 -0
  143. package/dist/src/utils/web-sandbox.js +37 -0
  144. package/models.json +30 -0
  145. package/package.json +23 -3
  146. package/src/action-result.ts +70 -173
  147. package/src/action.ts +38 -132
  148. package/src/ai/agent.ts +20 -0
  149. package/src/ai/captain/idle-mode.ts +1 -1
  150. package/src/ai/captain/test-mode.ts +1 -1
  151. package/src/ai/captain/web-mode.ts +18 -30
  152. package/src/ai/captain.ts +17 -42
  153. package/src/ai/conversation.ts +9 -32
  154. package/src/ai/driller.ts +53 -83
  155. package/src/ai/experience-compactor.ts +16 -35
  156. package/src/ai/fisherman-tools.ts +1 -1
  157. package/src/ai/fisherman.ts +0 -4
  158. package/src/ai/historian/codeceptjs.ts +5 -3
  159. package/src/ai/historian/playwright.ts +7 -4
  160. package/src/ai/historian/screencast.ts +4 -3
  161. package/src/ai/historian/utils.ts +0 -1
  162. package/src/ai/historian.ts +6 -8
  163. package/src/ai/navigator.ts +39 -70
  164. package/src/ai/pilot.ts +59 -61
  165. package/src/ai/planner.ts +76 -62
  166. package/src/ai/provider.ts +85 -146
  167. package/src/ai/quartermaster.ts +39 -45
  168. package/src/ai/rerunner.ts +27 -50
  169. package/src/ai/researcher/cache.ts +10 -16
  170. package/src/ai/researcher/coordinates.ts +4 -3
  171. package/src/ai/researcher/deep-analysis.ts +10 -11
  172. package/src/ai/researcher/locators.ts +66 -9
  173. package/src/ai/researcher/parser.ts +1 -1
  174. package/src/ai/researcher/research-result.ts +0 -18
  175. package/src/ai/researcher/sections.ts +9 -3
  176. package/src/ai/researcher.ts +42 -119
  177. package/src/ai/rules.ts +24 -13
  178. package/src/ai/session-analyst.ts +8 -5
  179. package/src/ai/task-agent.ts +45 -38
  180. package/src/ai/tester.ts +51 -129
  181. package/src/ai/tools.ts +145 -153
  182. package/src/command-handler.ts +1 -1
  183. package/src/commands/compact-command.ts +7 -8
  184. package/src/commands/context-aria-command.ts +1 -1
  185. package/src/commands/context-command.ts +8 -6
  186. package/src/commands/context-data-command.ts +2 -2
  187. package/src/commands/context-experience-command.ts +2 -2
  188. package/src/commands/context-html-command.ts +2 -2
  189. package/src/commands/context-knowledge-command.ts +2 -2
  190. package/src/commands/drill-command.ts +1 -1
  191. package/src/commands/experience-command.ts +1 -1
  192. package/src/commands/explore-command.ts +4 -3
  193. package/src/commands/freesail-command.ts +2 -2
  194. package/src/commands/knows-command.ts +1 -1
  195. package/src/commands/learn-command.ts +5 -3
  196. package/src/commands/path-command.ts +1 -1
  197. package/src/commands/research-command.ts +1 -1
  198. package/src/commands/test-command.ts +1 -1
  199. package/src/components/AddKnowledge.tsx +5 -7
  200. package/src/components/App.tsx +3 -3
  201. package/src/config.ts +185 -11
  202. package/src/experience-tracker.ts +89 -152
  203. package/src/explorbot.ts +88 -83
  204. package/src/explorer.ts +402 -693
  205. package/src/knowledge-tracker.ts +42 -49
  206. package/src/state-manager.ts +20 -155
  207. package/src/utils/aria.ts +11 -2
  208. package/src/utils/cache.ts +40 -0
  209. package/src/utils/context-formatter.ts +12 -15
  210. package/src/utils/hooks-runner.ts +2 -4
  211. package/src/utils/html.ts +79 -0
  212. package/src/utils/markdown-files.ts +30 -0
  213. package/src/utils/markdown-query.ts +15 -1
  214. package/src/utils/page-readiness.ts +1 -1
  215. package/src/utils/secrets.ts +2 -3
  216. package/src/utils/strings.ts +12 -0
  217. package/src/utils/test-files.ts +1 -1
  218. package/src/utils/web-annotate.ts +64 -0
  219. package/src/utils/web-eidx.ts +21 -0
  220. package/src/utils/web-element.ts +34 -0
  221. package/src/utils/web-sandbox.ts +43 -0
  222. package/dist/boat/api-tester/example/apibot.config.js +0 -30
@@ -7,11 +7,11 @@ import { Observability } from "../observability.js";
7
7
  import { Stats } from "../stats.js";
8
8
  import { diffAriaSnapshots } from "../utils/aria.js";
9
9
  import { ErrorPageError, detectPageCondition } from "../utils/error-page.js";
10
- import { HooksRunner } from "../utils/hooks-runner.js";
11
10
  import { isBodyEmpty } from "../utils/html.js";
12
- import { createDebug, pluralize, tag } from '../utils/logger.js';
11
+ import { createDebug, tag } from '../utils/logger.js';
13
12
  import { mdq } from "../utils/markdown-query.js";
14
13
  import { RulesLoader } from "../utils/rules-loader.js";
14
+ import { annotatePageElements } from "../utils/web-annotate.js";
15
15
  import { ContextLengthError } from './provider.js';
16
16
  import { findSimilarResearch, getCachedResearch, saveResearch } from "./researcher/cache.js";
17
17
  import { WithCoordinates } from "./researcher/coordinates.js";
@@ -39,15 +39,10 @@ export class Researcher extends ResearcherBase {
39
39
  emoji = '🔍';
40
40
  experienceTracker;
41
41
  hasScreenshotToAnalyze = false;
42
- hooksRunner;
43
- constructor(explorer, provider) {
44
- super();
45
- this.explorer = explorer;
46
- this.provider = provider;
47
- this.stateManager = explorer.getStateManager();
48
- this.experienceTracker = this.stateManager.getExperienceTracker();
49
- this.hooksRunner = new HooksRunner(explorer, explorer.getConfig());
50
- const ai = explorer.getConfig().ai;
42
+ constructor(deps) {
43
+ super(deps);
44
+ this.experienceTracker = deps.stateManager.getExperienceTracker();
45
+ const ai = deps.config.ai;
51
46
  if (ai) {
52
47
  ai.agents ??= {};
53
48
  ai.agents.researcher ??= {};
@@ -57,15 +52,6 @@ export class Researcher extends ResearcherBase {
57
52
  getNavigator() {
58
53
  throw new Error('not implemented');
59
54
  }
60
- getExperienceTracker() {
61
- return this.experienceTracker;
62
- }
63
- getKnowledgeTracker() {
64
- return this.explorer.getKnowledgeTracker();
65
- }
66
- getProvider() {
67
- return this.provider;
68
- }
69
55
  static getCachedResearch(state) {
70
56
  return getCachedResearch(state.hash || '');
71
57
  }
@@ -82,7 +68,7 @@ export class Researcher extends ResearcherBase {
82
68
  }
83
69
  async research(state, opts = {}) {
84
70
  const { screenshot = false, force = false, deep = false, data = false, fix = true } = opts;
85
- const maxRetries = this.explorer.getConfig().ai?.agents?.researcher?.retries ?? 2;
71
+ const maxRetries = this.config.ai?.agents?.researcher?.retries ?? 2;
86
72
  let retriesLeft = opts._retriesLeft ?? maxRetries;
87
73
  this.actionResult = ActionResult.fromState(state);
88
74
  const stateHash = state.hash || this.actionResult.getStateHash();
@@ -101,9 +87,9 @@ export class Researcher extends ResearcherBase {
101
87
  setActivity(`${this.emoji} Researching...`, 'action');
102
88
  await this.ensureNavigated(displayUrl, screenshot && this.provider.hasVision());
103
89
  await this.hooksRunner.runBeforeHook('researcher', state.url);
104
- const annotatedElements = await this.explorer.annotateElements();
90
+ const { elements: annotatedElements } = await this.explorer.withPage(annotatePageElements);
105
91
  debugLog(`Annotated ${annotatedElements.length} interactive elements with eidx`);
106
- this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot && this.provider.hasVision() });
92
+ this.actionResult = await this.explorer.capture({ screenshot: screenshot && this.provider.hasVision() });
107
93
  const condition = detectPageCondition(this.actionResult);
108
94
  if (condition === 'error') {
109
95
  tag('warning').log(`Detected error page at ${state.url}`);
@@ -165,11 +151,10 @@ export class Researcher extends ResearcherBase {
165
151
  return this.research(state, { ...opts, force: true, _retriesLeft: retriesLeft - 1 });
166
152
  }
167
153
  if (!interrupted()) {
168
- const containerLocs = result.containerLocators;
169
- await this.testLocators(containerLocs);
170
- const brokenContainers = containerLocs.filter((l) => l.valid === false);
171
- if (containerLocs.length > 0 && brokenContainers.length === containerLocs.length && retriesLeft > 0) {
172
- tag('warning').log(`All ${containerLocs.length} containers broken, retrying research (${maxRetries - retriesLeft + 1}/${maxRetries})...`);
154
+ const brokenContainers = await this.resolveContainers(result);
155
+ const containerCount = result.containers.length;
156
+ if (containerCount > 0 && brokenContainers.length === containerCount && retriesLeft > 0) {
157
+ tag('warning').log(`All ${containerCount} containers broken, retrying research (${maxRetries - retriesLeft + 1}/${maxRetries})...`);
173
158
  await new Promise((r) => setTimeout(r, 2000));
174
159
  return this.research(state, { ...opts, force: true, _retriesLeft: retriesLeft - 1 });
175
160
  }
@@ -181,13 +166,6 @@ export class Researcher extends ResearcherBase {
181
166
  }
182
167
  const toTest = result.locators.filter((l) => l.valid === null);
183
168
  await this.testLocators(toTest);
184
- const brokenCount = result.locators.filter((l) => l.valid === false).length;
185
- const brokenRatio = result.locators.length > 0 ? brokenCount / result.locators.length : 0;
186
- if (brokenRatio > 0.8 && retriesLeft > 0) {
187
- tag('warning').log(`${Math.round(brokenRatio * 100)}% locators broken, waiting 3s and retrying research (${maxRetries - retriesLeft + 1}/${maxRetries})...`);
188
- await new Promise((r) => setTimeout(r, 3000));
189
- return this.research(state, { ...opts, force: true, _retriesLeft: retriesLeft - 1 });
190
- }
191
169
  }
192
170
  // Stage 3: Fix broken sections via AI conversation continuation
193
171
  if (!interrupted() && fix && result.locators.some((l) => l.valid === false)) {
@@ -197,7 +175,7 @@ export class Researcher extends ResearcherBase {
197
175
  // Must run BEFORE visuallyAnnotateContainers — annotation overlays inject z-index 99998+ which would pollute the scoring.
198
176
  if (!interrupted() && this.hasScreenshotToAnalyze) {
199
177
  const sections = parseResearchSections(result.text);
200
- const focused = await this.explorer.runWithBrowserRecovery('detectFocusedSection', () => detectFocusedSection(this.explorer.playwrightHelper.page, sections));
178
+ const focused = await this.explorer.withPage((page) => detectFocusedSection(page, sections));
201
179
  if (focused)
202
180
  markSectionAsFocused(result, focused);
203
181
  }
@@ -206,11 +184,11 @@ export class Researcher extends ResearcherBase {
206
184
  const validContainers = extractValidContainers(result.text);
207
185
  result.parseLocators();
208
186
  const freshContainerLocs = result.containerLocators;
209
- await this.testLocators(freshContainerLocs);
187
+ await this.testLocators(freshContainerLocs, { scope: true });
210
188
  const freshBroken = freshContainerLocs.filter((l) => l.valid === false).map((l) => l.locator);
211
189
  const containers = validContainers.filter((c) => !freshBroken.includes(c.css));
212
190
  await this.visuallyAnnotateElements({ containers });
213
- this.actionResult = await this.explorer.capturePageWithScreenshot();
191
+ this.actionResult = await this.explorer.capture({ screenshot: true });
214
192
  const visualResult = await this.analyzeScreenshotForVisualProps();
215
193
  if (visualResult.elements.size > 0) {
216
194
  await this.mergeVisualData(result, visualResult.elements);
@@ -261,12 +239,10 @@ export class Researcher extends ResearcherBase {
261
239
  if (stateHash) {
262
240
  researchFile = saveResearch(stateHash, result.text, combinedHtml);
263
241
  }
264
- const summaryMatch = result.text.match(/## Summary\s*\n+([\s\S]*?)(?=\n##|$)/i);
265
- if (summaryMatch) {
266
- const summaryLine = summaryMatch[1].trim().split('\n')[0].trim().slice(0, 200);
267
- if (summaryLine)
268
- this.experienceTracker.updateSummary(this.actionResult, summaryLine);
269
- }
242
+ const summaryText = mdq(result.text).query('section2(/^summary/)').query('paragraph[0]').text().trim();
243
+ const summaryLine = summaryText.split('\n')[0]?.trim().slice(0, 200);
244
+ if (summaryLine)
245
+ this.experienceTracker.updateSummary(this.actionResult, summaryLine);
270
246
  tag('multiline').log(formatResearchSummary(result.text, { visionUsed: this.hasScreenshotToAnalyze }));
271
247
  tag('success').log('Research complete');
272
248
  if (researchFile)
@@ -282,8 +258,7 @@ export class Researcher extends ResearcherBase {
282
258
  async ensureNavigated(url, screenshot) {
283
259
  if (!this.actionResult) {
284
260
  debugLog('No action result, navigating to URL');
285
- await this.explorer.visit(url);
286
- this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot });
261
+ this.actionResult = await this.explorer.visit(url, { screenshot });
287
262
  return;
288
263
  }
289
264
  const isOnCurrentState = this.actionResult.getStateHash() === this.stateManager.getCurrentState()?.hash;
@@ -291,34 +266,32 @@ export class Researcher extends ResearcherBase {
291
266
  const isEmpty = isBodyEmpty(stateHtml);
292
267
  if (!isEmpty && isOnCurrentState) {
293
268
  if ((!this.actionResult.screenshot && screenshot) || !this.actionResult.ariaSnapshot) {
294
- this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot });
269
+ this.actionResult = await this.explorer.capture({ screenshot });
295
270
  }
296
271
  return;
297
272
  }
298
273
  if (isEmpty && isOnCurrentState) {
299
274
  debugLog('HTML body empty on current URL, waiting for content');
300
275
  tag('step').log('Page body is empty, waiting for content...');
301
- await this.explorer.visit(url);
302
- this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot ?? false });
276
+ this.actionResult = await this.explorer.visit(url, { screenshot: screenshot ?? false });
303
277
  await this.waitUntilSettled(screenshot ?? false);
304
278
  return;
305
279
  }
306
280
  debugLog('Not on current state, navigating to URL');
307
281
  tag('step').log('Navigating to URL...');
308
- await this.explorer.visit(url);
309
- this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot ?? false });
282
+ this.actionResult = await this.explorer.visit(url, { screenshot: screenshot ?? false });
310
283
  }
311
284
  async waitUntilSettled(screenshot) {
312
- const errorPageTimeout = this.explorer.getConfig().ai?.agents?.researcher?.errorPageTimeout ?? 10;
285
+ const errorPageTimeout = this.config.ai?.agents?.researcher?.errorPageTimeout ?? 10;
313
286
  if (errorPageTimeout <= 0)
314
287
  return false;
315
288
  const includeScreenshot = screenshot && this.provider.hasVision();
316
289
  try {
317
- await this.explorer.runWithBrowserRecovery('waitUntilSettled', () => this.explorer.playwrightHelper.page?.waitForLoadState('networkidle', { timeout: errorPageTimeout * 1000 }));
290
+ await this.explorer.withPage((page) => page.waitForLoadState('networkidle', { timeout: errorPageTimeout * 1000 }));
318
291
  }
319
292
  catch { }
320
- await this.explorer.annotateElements();
321
- this.actionResult = await this.explorer.capturePageState({ includeScreenshot });
293
+ await this.explorer.withPage(annotatePageElements);
294
+ this.actionResult = await this.explorer.capture({ screenshot: includeScreenshot });
322
295
  let condition = detectPageCondition(this.actionResult);
323
296
  if (condition === 'error') {
324
297
  throw new ErrorPageError(this.actionResult.url, this.actionResult.title, this.actionResult.httpStatus);
@@ -327,8 +300,8 @@ export class Researcher extends ResearcherBase {
327
300
  return true;
328
301
  for (let i = 0; i < 3; i++) {
329
302
  await new Promise((r) => setTimeout(r, 1000));
330
- await this.explorer.annotateElements();
331
- this.actionResult = await this.explorer.capturePageState({ includeScreenshot });
303
+ await this.explorer.withPage(annotatePageElements);
304
+ this.actionResult = await this.explorer.capture({ screenshot: includeScreenshot });
332
305
  condition = detectPageCondition(this.actionResult);
333
306
  if (condition === 'error') {
334
307
  throw new ErrorPageError(this.actionResult.url, this.actionResult.title, this.actionResult.httpStatus);
@@ -339,7 +312,7 @@ export class Researcher extends ResearcherBase {
339
312
  return false;
340
313
  }
341
314
  getConfiguredSections() {
342
- const configSections = this.explorer.getConfig().ai?.agents?.researcher?.sections;
315
+ const configSections = this.config.ai?.agents?.researcher?.sections;
343
316
  if (!configSections?.length)
344
317
  return POSSIBLE_SECTIONS;
345
318
  const filtered = {};
@@ -365,7 +338,10 @@ export class Researcher extends ResearcherBase {
365
338
  - Research all menus and navigational areas.
366
339
  - Ignore decorative sidebars, footer-only links, and external links.
367
340
  - Detect layout patterns: list/detail split, 2-pane, or 3-pane layouts.
368
- - Every element with an eidx attribute MUST appear in the UI map — describe icon-only buttons by their visual role.
341
+ - In split-pane layouts, keep global toolbar, list pane, and detail pane as separate sections. Do not copy toolbar/list elements into the detail section unless they are physically inside that detail container.
342
+ - If the URL or page state opens an entity detail panel, describe that panel as the active detail context and include its close/back/pin controls when present.
343
+ - If an element has data-explorbot-hit="covered" or "offscreen", do not present it as directly actionable. Prefer the overlay, drawer, dialog, or focused section covering it, and mention what must be dismissed or revealed first.
344
+ - Every element with an eidx attribute MUST appear in exactly one matching UI map section — describe icon-only buttons by their visual role.
369
345
  - Every UI map row needs a CSS selector; ARIA may be "-" for icon-only buttons, CSS must never be "-".
370
346
  - ARIA locator JSON uses keys "role" and "text" (NOT "name").
371
347
  - Mark elements with likely hover interactions (title, aria-describedby, menu items with submenus) as "(hover)".
@@ -382,6 +358,7 @@ export class Researcher extends ResearcherBase {
382
358
  .join('\n')}
383
359
 
384
360
  - Sections can overlap; prefer more detailed sections over broader ones.
361
+ - Section tables must list only elements physically inside the declared container. If a control belongs to a global toolbar, list it only in the toolbar section.
385
362
  - Never name a section "Focus" or "Focused" — use what it contains (Detail, Modal, Form, Content, List).
386
363
  - Omit sections that are not present or not relevant.
387
364
  - Each section needs a container CSS locator; UI map CSS locators are relative to it.
@@ -402,21 +379,7 @@ export class Researcher extends ResearcherBase {
402
379
  if (!this.actionResult)
403
380
  throw new Error('actionResult is not set');
404
381
  const html = await this.actionResult.combinedHtml();
405
- const knowledgeFiles = this.stateManager.getRelevantKnowledge();
406
- let knowledge = '';
407
- if (knowledgeFiles.length > 0) {
408
- const knowledgeContent = knowledgeFiles
409
- .map((k) => k.content)
410
- .filter((k) => !!k)
411
- .join('\n\n');
412
- tag('operation').log(`Found ${knowledgeFiles.length} relevant knowledge ${pluralize(knowledgeFiles.length, 'file')} for: ${this.actionResult.url}`);
413
- knowledge = `
414
- <hint>
415
- Here is relevant knowledge for this page:
416
-
417
- ${knowledgeContent}
418
- </hint>`;
419
- }
382
+ const knowledge = this.knowledgeTracker.renderRelevantKnowledge(this.actionResult);
420
383
  const ariaSnapshot = this.actionResult.getCompactARIA();
421
384
  return dedent `
422
385
  Analyze this web page and provide a comprehensive research report in markdown format.
@@ -471,35 +434,6 @@ export class Researcher extends ResearcherBase {
471
434
 
472
435
  `;
473
436
  }
474
- async textContent(state) {
475
- const actionResult = ActionResult.fromState(state);
476
- const html = await actionResult.combinedHtml();
477
- const prompt = dedent `
478
- Transform into markdown.
479
- Identify headers, footers, asides, special application parts and main contant.
480
- Content should be in markdown format. If it is content: tables must be tables, lists must be lists.
481
- Navigation elements should be represented as standalone blocks after the content.
482
- Do not summarize content, just transform it into markdown.
483
- It is important to list all the content text
484
- If it is link it must be linked
485
- You can summarize footers/navigation/aside elements.
486
- But main conteint should be kept as text and formatted as markdown based on its current markup.
487
- Links to external web sites should be avoided in output.
488
-
489
- Break down into sections:
490
-
491
- ## Content Area
492
-
493
- ## Navigation Area
494
-
495
- <page_html>
496
- ${html}
497
- </page_html>
498
- `;
499
- const model = this.provider.getModelForAgent('researcher');
500
- const r = await this.provider.chat([{ role: 'user', content: prompt }], model, { agentName: 'researcher', telemetryFunctionId: 'researcher.textContent' });
501
- return r.text;
502
- }
503
437
  getScreenshotFromState(state) {
504
438
  const actionResult = ActionResult.fromState(state);
505
439
  const image = actionResult.screenshot;
@@ -692,9 +626,9 @@ export class Researcher extends ResearcherBase {
692
626
  }
693
627
  async cancelInUi() {
694
628
  const beforeAria = this.stateManager.getCurrentState()?.ariaSnapshot || null;
695
- await this.explorer.executeAction('I.clickXY(0, 0)');
696
- if (diffAriaSnapshots(beforeAria, this.stateManager.getCurrentState()?.ariaSnapshot || null))
629
+ await this.explorer.action().execute('I.clickXY(0, 0)');
630
+ if (diffAriaSnapshots(beforeAria, this.stateManager.getCurrentState()?.ariaSnapshot || null).text)
697
631
  return;
698
- await this.explorer.executeAction(`I.pressKey('Escape')`);
632
+ await this.explorer.action().execute(`I.pressKey('Escape')`);
699
633
  }
700
634
  }
@@ -1,5 +1,6 @@
1
1
  export declare const recommendedCodeceptCommands: readonly ["I.click", "I.type", "I.fillField", "I.see", "I.seeElement"];
2
2
  export declare const locatorRule: string;
3
+ export declare const drillLocatorRule: string;
3
4
  export declare const fileUploadRule: string;
4
5
  export declare const formRequirementsRule: string;
5
6
  export declare const protectionRule: string;
@@ -1,6 +1,6 @@
1
1
  import dedent from 'dedent';
2
2
  export const recommendedCodeceptCommands = ['I.click', 'I.type', 'I.fillField', 'I.see', 'I.seeElement'];
3
- export const locatorRule = dedent `
3
+ const locatorPriorityRule = dedent `
4
4
  <locator_priority>
5
5
  Use the following priority when selecting locators:
6
6
 
@@ -17,10 +17,11 @@ export const locatorRule = dedent `
17
17
  Example: '#login-btn', '[data-testid="submit"]', 'form#login input[name="email"]'
18
18
 
19
19
  4. XPath (last resort) - for complex hierarchy or when CSS can't express the path
20
- Always start with //, never use positional indices like [1], [2]
20
+ Always start with //. Avoid positional indices like [1], [2] except as a last-resort disambiguator
21
21
  Example: '//form[@id="login"]//input[@name="email"]'
22
22
  </locator_priority>
23
-
23
+ `;
24
+ const contextSimplificationRule = dedent `
24
25
  <context_simplification>
25
26
  When container is available from UI map sections:
26
27
  - Text + container is simplest and PREFERRED: I.click('Save', '.modal')
@@ -28,7 +29,8 @@ export const locatorRule = dedent `
28
29
  - ALWAYS use context parameter unless locator is XPath or unique ID
29
30
  - No need for complex ARIA when container narrows scope sufficiently
30
31
  </context_simplification>
31
-
32
+ `;
33
+ const locatorStrategyRule = dedent `
32
34
  <disambiguation>
33
35
  When multiple elements could match the request, select based on intent:
34
36
  1. Match the context of recent actions - if filling a form, use elements in that same form
@@ -46,8 +48,9 @@ export const locatorRule = dedent `
46
48
  - Use aria-label value if present: { "role": "button", "text": "Close" } (from aria-label="Close")
47
49
  - Use title attribute if present: { "role": "button", "text": "Settings" } (from title="Settings")
48
50
  - If no accessible name exists, mark ARIA as "-" and use CSS/XPath:
49
- * CSS: use partial href a[href*="settings"] or SVG icon class a:has(svg.md-icon-cog)
50
- * XPath: use contains(@href,"settings") or SVG class //a[.//svg[contains(@class,"md-icon-cog")]]
51
+ * CSS: use partial href a[href*="settings"] or SVG icon class a:has(svg.icon-settings)
52
+ * XPath: use contains(@href,"settings") or SVG class //a[.//svg[contains(@class,"icon-settings")]]
53
+ - In inline create/edit rows, confirmation can be an icon-only control near the edited field instead of a text Save button. Anchor the locator to the same row/form as the field and target the adjacent confirm icon/control.
51
54
  - NEVER use empty text: { "role": "button", "text": "" } is INVALID and useless
52
55
 
53
56
  <good_aria_locator_example>
@@ -78,9 +81,9 @@ export const locatorRule = dedent `
78
81
  - Vue: data-v-* attributes
79
82
  Avoid locators that seem to have generated ids or class names (long random numbers, uuids, hashes, etc)
80
83
  Prefer text or ARIA locators over href-based ones. But for icon-only links with no accessible name, use:
81
- - Partial href match: a[href*="settings"], a[href*="requirements"] (use path segments, not full URLs)
82
- - SVG icon class: a:has(svg.md-icon-cog), button:has(svg.md-icon-plus) (target the SVG class inside the link/button)
83
- Avoid full absolute href like a[href="/projects/imr_manual12/settings"] — use generic path segments instead
84
+ - Partial href match: a[href*="settings"], a[href*="reports"] (use path segments, not full URLs)
85
+ - SVG icon class: a:has(svg.icon-settings), button:has(svg.icon-add) (target the SVG class inside the link/button)
86
+ Avoid full absolute href like a[href="/items/12345/settings"] — use generic path segments instead
84
87
  Avoid CSS framework utility classes as containers (Tailwind: flex, grid, space-x-*, justify-*, items-*, w-*, h-*, p-*, m-*, etc; Bootstrap: col-*, row, d-flex, etc)
85
88
  Prefer semantic class names, roles, data attributes, or element hierarchy for containers
86
89
 
@@ -92,7 +95,7 @@ export const locatorRule = dedent `
92
95
 
93
96
  <xpath_rules>
94
97
  XPath locators must start with //.
95
- XPath should use positional indices [1], [2], [3] and contains(., "text") for disambiguation.
98
+ XPath may use positional indices [1], [2] and contains(., "text") as a last-resort disambiguator when attribute/text strategies are exhausted.
96
99
  XPath should rely less on class names — prefer element hierarchy, position, and text content.
97
100
  XPath and CSS MUST provide different strategies for finding the same element.
98
101
  </xpath_rules>
@@ -104,21 +107,23 @@ export const locatorRule = dedent `
104
107
  '#content-top #user_name'
105
108
  '#content-top form input[name="name"]'
106
109
  'a.nav-item[href*="settings"]' // icon-only link matched by partial href
107
- 'a.nav-item:has(svg.md-icon-cog)' // icon-only link matched by SVG icon class
110
+ 'a.nav-item:has(svg.icon-settings)' // icon-only link matched by SVG icon class
108
111
  '//nav//a[contains(@href,"settings")]' // XPath for icon-only nav link
109
112
  </good locator example>
110
113
 
111
114
  <bad locator example>
112
- 'a.filter-tab:nth-of-type(1)' // WRONG: positional in CSS, use :has-text("Manual") instead
113
- '//a[contains(@class,"filter-tab") and contains(@class,"active")]' // WRONG: XPath repeats CSS approach, use positional //a[contains(@class,"filter-tab")][1]
115
+ 'a.filter-tab:nth-of-type(1)' // WRONG: positional in CSS, use :has-text("Active") instead
116
+ '//a[contains(@class,"filter-tab") and contains(@class,"active")]' // WRONG: XPath repeats CSS approach, provide a different strategy e.g. //a[contains(@class,"filter-tab")][1]
114
117
  '//table//tbody/tr[1]//button[contains(@onclick,'fn()')]' // onclick is not semantic attribute
115
118
  '//html/body/vue-button-123' // vue-framework specific locator
116
- 'link "New Template"' // WRONG: malformed string, use {"role":"link","text":"New Template"}
117
- 'a[href="/projects/imr_manual12/settings"]' // WRONG: full absolute href, use a[href*="settings"] instead
119
+ 'link "New Item"' // WRONG: malformed string, use {"role":"link","text":"New Item"}
120
+ 'a[href="/items/12345/settings"]' // WRONG: full absolute href, use a[href*="settings"] instead
118
121
  </bad locator example>
119
122
 
120
123
  HTML locators must be valid JS strings
121
124
  `;
125
+ export const locatorRule = [locatorPriorityRule, contextSimplificationRule, locatorStrategyRule].join('\n\n');
126
+ export const drillLocatorRule = [locatorPriorityRule, locatorStrategyRule].join('\n\n');
122
127
  export const fileUploadRule = dedent `
123
128
  <file_upload>
124
129
  Explorbot CAN upload files using I.attachFile() via form() tool.
@@ -294,6 +299,7 @@ export const actionRule = dedent `
294
299
  Use context parameter (second argument) to narrow click area when:
295
300
  - The same text/button appears multiple times on page
296
301
  - You need to click inside a specific form, modal, or section
302
+ - You need an icon-only confirm/save control next to a field in an inline create/edit row
297
303
  Context should be a CSS selector pointing to a unique container.
298
304
 
299
305
  <example>
@@ -306,6 +312,7 @@ export const actionRule = dedent `
306
312
  </example>
307
313
 
308
314
  Prefer text/ARIA locators with context over complex CSS/XPath selectors.
315
+ For inline create/edit flows, after filling a field verify it contains the value, then confirm using the nearest explicit button/link, an adjacent icon-only confirm control in the same row/form, or Enter if the field remains focused.
309
316
  If locator doesn't work, try CSS or XPath locators.
310
317
  If nothing works, use I.clickXY(x, y) as last resort.
311
318
 
@@ -42,6 +42,9 @@ export class SessionAnalyst {
42
42
  ## Severity (defects only)
43
43
  [High] blocks a core flow · [Medium] degrades a flow but workaround exists · [Low] cosmetic / edge case
44
44
 
45
+ ## Naming tests
46
+ Reference every test by its full label wrapped in backticks: \`\`\`ET-<number> <test title>\`\`\` — the \`ref\` and \`scenario\` from its \`<test>\` block. Backticks only, never bold. Never write a bare \`#2\`.
47
+
45
48
  ## Format
46
49
 
47
50
  # Session Analysis
@@ -53,22 +56,22 @@ export class SessionAnalyst {
53
56
  - Features: <capabilities>
54
57
 
55
58
  ## What works
56
- - **<feature>** — #2, #7, #8
59
+ - **<feature>** — \`ET-2 <title>\`, \`ET-7 <title>\`
57
60
 
58
61
  ## Defects
59
62
 
60
63
  ### [Medium] <plain-English bug title>
61
- Affects: #3, #5
64
+ Affects: \`ET-3 <title>\`, \`ET-5 <title>\`
62
65
  Reproduce:
63
66
  1. <concrete UI step>
64
67
  2. <next>
65
68
  Evidence: <one short observation>
66
69
 
67
70
  ## UX issues
68
- - **<feature>** — <what's confusing> (#7)
71
+ - **<feature>** — <what's confusing> (\`ET-7 <title>\`)
69
72
 
70
73
  ## Execution Issues
71
- - **#2 <scenario>** — <≤10 words, what was unreliable>
74
+ - \`ET-2 <title>\` — <≤10 words, what was unreliable>
72
75
 
73
76
  ## Brevity rules
74
77
 
@@ -127,7 +130,7 @@ export class SessionAnalyst {
127
130
  `
128
131
  : 'verification_status: none';
129
132
  return dedent `
130
- <test ref="#${ref}">
133
+ <test ref="ET-${ref}">
131
134
  url: ${test.startUrl || '/'}
132
135
  scenario: ${test.scenario}
133
136
  result: ${test.result || 'unknown'}
@@ -1,27 +1,40 @@
1
1
  import type { ActionResult } from '../action-result.js';
2
- import { type ExperienceTracker } from '../experience-tracker.js';
2
+ import type { ExplorbotConfig } from '../config.js';
3
+ import type { ExperienceTracker } from '../experience-tracker.js';
4
+ import type Explorer from '../explorer.js';
3
5
  import type { KnowledgeTracker } from '../knowledge-tracker.js';
6
+ import type { StateManager } from '../state-manager.js';
7
+ import { HooksRunner } from '../utils/hooks-runner.js';
8
+ import type { AgentDeps, ToolDeps } from './agent.js';
4
9
  import { Historian } from './historian.js';
5
10
  import type { Navigator } from './navigator.js';
6
11
  import type { Provider } from './provider.js';
7
12
  import { Quartermaster } from './quartermaster.js';
8
13
  export declare function isInteractive(): boolean;
9
14
  export declare abstract class TaskAgent {
15
+ explorer: Explorer;
16
+ provider: Provider;
17
+ config: ExplorbotConfig;
18
+ stateManager: StateManager;
19
+ knowledgeTracker: KnowledgeTracker;
20
+ hooksRunner: HooksRunner;
10
21
  consecutiveFailures: number;
11
22
  consecutiveEmptyResults: number;
12
23
  recentToolCalls: any[];
13
24
  readonly ACTION_TOOLS: string[];
14
25
  _historian: Historian | null;
15
26
  _quartermaster: Quartermaster | null;
27
+ constructor(deps?: AgentDeps);
28
+ setHistorian(historian: Historian): void;
29
+ setQuartermaster(quartermaster: Quartermaster): void;
16
30
  abstract getNavigator(): Navigator;
17
- abstract getExperienceTracker(): ExperienceTracker;
18
- abstract getKnowledgeTracker(): KnowledgeTracker;
19
- abstract getProvider(): Provider;
31
+ get toolDeps(): ToolDeps;
32
+ getExperienceTracker(): ExperienceTracker;
33
+ getKnowledgeTracker(): KnowledgeTracker;
34
+ getProvider(): Provider;
20
35
  getKnowledge(actionResult: ActionResult): string;
21
36
  getExperience(actionResult: ActionResult): string;
22
- setHistorian(historian: Historian): void;
23
37
  getHistorian(): Historian;
24
- setQuartermaster(quartermaster: Quartermaster): void;
25
38
  getQuartermaster(): Quartermaster;
26
39
  trackToolExecutions(toolExecutions: any[]): void;
27
40
  resetFailureCount(): void;
@@ -1,7 +1,4 @@
1
- import dedent from 'dedent';
2
- import { renderExperienceToc } from '../experience-tracker.js';
3
- import { createDebug, pluralize, tag } from '../utils/logger.js';
4
- const debugLog = createDebug('explorbot:task-agent');
1
+ import { HooksRunner } from "../utils/hooks-runner.js";
5
2
  export function isInteractive() {
6
3
  return process.env.INK_RUNNING === 'true';
7
4
  }
@@ -11,50 +8,57 @@ function createNullProxy() {
11
8
  });
12
9
  }
13
10
  export class TaskAgent {
11
+ explorer;
12
+ provider;
13
+ config;
14
+ stateManager;
15
+ knowledgeTracker;
16
+ hooksRunner;
14
17
  consecutiveFailures = 0;
15
18
  consecutiveEmptyResults = 0;
16
19
  recentToolCalls = [];
17
20
  ACTION_TOOLS = [];
18
21
  _historian = null;
19
22
  _quartermaster = null;
20
- getKnowledge(actionResult) {
21
- const knowledgeFiles = this.getKnowledgeTracker().getRelevantKnowledge(actionResult);
22
- if (knowledgeFiles.length === 0)
23
- return '';
24
- const knowledgeContent = knowledgeFiles
25
- .map((k) => k.content)
26
- .filter((k) => !!k)
27
- .join('\n\n');
28
- tag('operation').log(`Found ${knowledgeFiles.length} relevant knowledge ${pluralize(knowledgeFiles.length, 'file')}`);
29
- return dedent `
30
- <knowledge>
31
- Here is relevant knowledge for this page:
32
-
33
- ${knowledgeContent}
34
- </knowledge>
35
- `;
36
- }
37
- getExperience(actionResult) {
38
- const tracker = this.getExperienceTracker();
39
- const toc = tracker.getExperienceTableOfContents(actionResult);
40
- if (toc.length === 0)
41
- return '';
42
- const totalSections = toc.reduce((sum, entry) => sum + entry.sections.length, 0);
43
- debugLog(`injecting experience TOC (${toc.length} files, ${totalSections} sections)`);
44
- tag('operation').log(`Found ${toc.length} experience ${pluralize(toc.length, 'file')} (${totalSections} sections)`);
45
- return renderExperienceToc(toc);
23
+ constructor(deps) {
24
+ if (!deps)
25
+ return;
26
+ this.explorer = deps.explorer;
27
+ this.provider = deps.ai;
28
+ this.config = deps.config;
29
+ this.stateManager = deps.stateManager;
30
+ this.knowledgeTracker = deps.knowledgeTracker;
31
+ this.hooksRunner = new HooksRunner(deps.explorer, deps.config);
46
32
  }
47
33
  setHistorian(historian) {
48
34
  this._historian = historian;
49
35
  }
36
+ setQuartermaster(quartermaster) {
37
+ this._quartermaster = quartermaster;
38
+ }
39
+ get toolDeps() {
40
+ return { explorer: this.explorer, stateManager: this.stateManager, ai: this.provider };
41
+ }
42
+ getExperienceTracker() {
43
+ return this.stateManager.getExperienceTracker();
44
+ }
45
+ getKnowledgeTracker() {
46
+ return this.knowledgeTracker;
47
+ }
48
+ getProvider() {
49
+ return this.provider;
50
+ }
51
+ getKnowledge(actionResult) {
52
+ return this.getKnowledgeTracker().renderRelevantKnowledge(actionResult);
53
+ }
54
+ getExperience(actionResult) {
55
+ return this.getExperienceTracker().renderExperienceTocFor(actionResult);
56
+ }
50
57
  getHistorian() {
51
58
  if (this._historian)
52
59
  return this._historian;
53
60
  return createNullProxy();
54
61
  }
55
- setQuartermaster(quartermaster) {
56
- this._quartermaster = quartermaster;
57
- }
58
62
  getQuartermaster() {
59
63
  if (this._quartermaster)
60
64
  return this._quartermaster;