explorbot 0.1.32-beta.1 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (222) hide show
  1. package/README.md +35 -19
  2. package/bin/explorbot-cli.ts +33 -10
  3. package/boat/api-tester/src/config.ts +45 -3
  4. package/boat/doc-collector/src/ai/documentarian.ts +26 -28
  5. package/boat/doc-collector/src/ai/tools.ts +74 -16
  6. package/boat/doc-collector/src/cli.ts +1 -6
  7. package/boat/doc-collector/src/docbot.ts +30 -9
  8. package/boat/doc-collector/src/docs-renderer.ts +19 -24
  9. package/boat/doc-collector/src/screenshots.ts +34 -2
  10. package/boat/doc-collector/src/state-diagram.ts +281 -0
  11. package/dist/bin/explorbot-cli.js +29 -9
  12. package/dist/boat/api-tester/src/config.js +43 -4
  13. package/dist/boat/doc-collector/src/ai/documentarian.js +19 -7
  14. package/dist/boat/doc-collector/src/ai/tools.js +53 -16
  15. package/dist/boat/doc-collector/src/cli.js +1 -5
  16. package/dist/boat/doc-collector/src/docbot.js +27 -8
  17. package/dist/boat/doc-collector/src/docs-renderer.js +18 -3
  18. package/dist/boat/doc-collector/src/screenshots.js +30 -1
  19. package/dist/boat/doc-collector/src/state-diagram.js +200 -0
  20. package/dist/models.json +30 -0
  21. package/dist/package.json +23 -3
  22. package/dist/src/action-result.d.ts +8 -10
  23. package/dist/src/action-result.js +72 -164
  24. package/dist/src/action.d.ts +13 -12
  25. package/dist/src/action.js +37 -113
  26. package/dist/src/ai/agent.d.ts +17 -0
  27. package/dist/src/ai/captain/idle-mode.js +1 -1
  28. package/dist/src/ai/captain/test-mode.js +1 -1
  29. package/dist/src/ai/captain/web-mode.js +18 -29
  30. package/dist/src/ai/captain.d.ts +1 -3
  31. package/dist/src/ai/captain.js +17 -45
  32. package/dist/src/ai/conversation.d.ts +2 -5
  33. package/dist/src/ai/conversation.js +9 -28
  34. package/dist/src/ai/driller.d.ts +2 -17
  35. package/dist/src/ai/driller.js +26 -50
  36. package/dist/src/ai/experience-compactor.d.ts +2 -2
  37. package/dist/src/ai/experience-compactor.js +16 -33
  38. package/dist/src/ai/fisherman-tools.js +1 -1
  39. package/dist/src/ai/fisherman.d.ts +0 -1
  40. package/dist/src/ai/fisherman.js +0 -3
  41. package/dist/src/ai/historian/codeceptjs.d.ts +2 -0
  42. package/dist/src/ai/historian/codeceptjs.js +4 -3
  43. package/dist/src/ai/historian/playwright.d.ts +4 -1
  44. package/dist/src/ai/historian/playwright.js +4 -3
  45. package/dist/src/ai/historian/screencast.d.ts +2 -1
  46. package/dist/src/ai/historian/screencast.js +2 -2
  47. package/dist/src/ai/historian/utils.d.ts +0 -1
  48. package/dist/src/ai/historian/utils.js +0 -1
  49. package/dist/src/ai/historian.d.ts +10 -8
  50. package/dist/src/ai/historian.js +1 -5
  51. package/dist/src/ai/navigator.d.ts +8 -6
  52. package/dist/src/ai/navigator.js +34 -65
  53. package/dist/src/ai/pilot.d.ts +13 -8
  54. package/dist/src/ai/pilot.js +53 -59
  55. package/dist/src/ai/planner.d.ts +5 -4
  56. package/dist/src/ai/planner.js +66 -52
  57. package/dist/src/ai/provider.d.ts +4 -1
  58. package/dist/src/ai/provider.js +77 -137
  59. package/dist/src/ai/quartermaster.d.ts +22 -22
  60. package/dist/src/ai/quartermaster.js +8 -14
  61. package/dist/src/ai/rerunner.d.ts +3 -13
  62. package/dist/src/ai/rerunner.js +23 -44
  63. package/dist/src/ai/researcher/cache.js +11 -18
  64. package/dist/src/ai/researcher/coordinates.js +4 -3
  65. package/dist/src/ai/researcher/deep-analysis.d.ts +2 -0
  66. package/dist/src/ai/researcher/deep-analysis.js +8 -11
  67. package/dist/src/ai/researcher/locators.d.ts +9 -2
  68. package/dist/src/ai/researcher/locators.js +65 -8
  69. package/dist/src/ai/researcher/parser.js +1 -1
  70. package/dist/src/ai/researcher/research-result.d.ts +0 -1
  71. package/dist/src/ai/researcher/research-result.js +0 -19
  72. package/dist/src/ai/researcher/sections.d.ts +2 -0
  73. package/dist/src/ai/researcher/sections.js +7 -3
  74. package/dist/src/ai/researcher.js +40 -106
  75. package/dist/src/ai/rules.d.ts +1 -0
  76. package/dist/src/ai/rules.js +22 -15
  77. package/dist/src/ai/session-analyst.js +8 -5
  78. package/dist/src/ai/task-agent.d.ts +19 -6
  79. package/dist/src/ai/task-agent.js +37 -33
  80. package/dist/src/ai/tester.d.ts +6 -14
  81. package/dist/src/ai/tester.js +46 -111
  82. package/dist/src/ai/tools.d.ts +26 -14
  83. package/dist/src/ai/tools.js +138 -132
  84. package/dist/src/command-handler.js +1 -1
  85. package/dist/src/commands/compact-command.d.ts +1 -1
  86. package/dist/src/commands/compact-command.js +6 -6
  87. package/dist/src/commands/context-aria-command.js +1 -1
  88. package/dist/src/commands/context-command.js +8 -6
  89. package/dist/src/commands/context-data-command.js +2 -2
  90. package/dist/src/commands/context-experience-command.js +2 -2
  91. package/dist/src/commands/context-html-command.js +2 -2
  92. package/dist/src/commands/context-knowledge-command.js +2 -2
  93. package/dist/src/commands/drill-command.js +1 -1
  94. package/dist/src/commands/experience-command.js +1 -1
  95. package/dist/src/commands/explore-command.js +4 -3
  96. package/dist/src/commands/freesail-command.js +2 -2
  97. package/dist/src/commands/knows-command.js +1 -1
  98. package/dist/src/commands/learn-command.js +5 -3
  99. package/dist/src/commands/path-command.js +1 -1
  100. package/dist/src/commands/research-command.js +1 -1
  101. package/dist/src/commands/test-command.js +1 -1
  102. package/dist/src/components/AddKnowledge.d.ts +2 -0
  103. package/dist/src/components/AddKnowledge.js +3 -7
  104. package/dist/src/components/App.js +3 -3
  105. package/dist/src/config.d.ts +19 -0
  106. package/dist/src/config.js +156 -10
  107. package/dist/src/experience-tracker.d.ts +11 -6
  108. package/dist/src/experience-tracker.js +82 -155
  109. package/dist/src/explorbot.d.ts +21 -13
  110. package/dist/src/explorbot.js +84 -80
  111. package/dist/src/explorer.d.ts +66 -102
  112. package/dist/src/explorer.js +332 -623
  113. package/dist/src/knowledge-tracker.d.ts +2 -2
  114. package/dist/src/knowledge-tracker.js +38 -45
  115. package/dist/src/state-manager.d.ts +10 -30
  116. package/dist/src/state-manager.js +11 -129
  117. package/dist/src/utils/aria.d.ts +6 -1
  118. package/dist/src/utils/aria.js +4 -1
  119. package/dist/src/utils/cache.d.ts +15 -0
  120. package/dist/src/utils/cache.js +34 -0
  121. package/dist/src/utils/context-formatter.d.ts +6 -0
  122. package/dist/src/utils/context-formatter.js +15 -27
  123. package/dist/src/utils/hooks-runner.js +2 -4
  124. package/dist/src/utils/html.d.ts +5 -0
  125. package/dist/src/utils/html.js +71 -0
  126. package/dist/src/utils/markdown-files.d.ts +10 -0
  127. package/dist/src/utils/markdown-files.js +21 -0
  128. package/dist/src/utils/markdown-query.d.ts +6 -0
  129. package/dist/src/utils/markdown-query.js +14 -1
  130. package/dist/src/utils/page-readiness.d.ts +1 -0
  131. package/dist/src/utils/page-readiness.js +1 -1
  132. package/dist/src/utils/secrets.js +2 -2
  133. package/dist/src/utils/strings.d.ts +2 -0
  134. package/dist/src/utils/strings.js +10 -0
  135. package/dist/src/utils/test-files.js +1 -1
  136. package/dist/src/utils/web-annotate.d.ts +5 -0
  137. package/dist/src/utils/web-annotate.js +58 -0
  138. package/dist/src/utils/web-eidx.d.ts +2 -0
  139. package/dist/src/utils/web-eidx.js +20 -0
  140. package/dist/src/utils/web-element.d.ts +3 -1
  141. package/dist/src/utils/web-element.js +32 -0
  142. package/dist/src/utils/web-sandbox.d.ts +4 -0
  143. package/dist/src/utils/web-sandbox.js +37 -0
  144. package/models.json +30 -0
  145. package/package.json +23 -3
  146. package/src/action-result.ts +70 -173
  147. package/src/action.ts +38 -132
  148. package/src/ai/agent.ts +20 -0
  149. package/src/ai/captain/idle-mode.ts +1 -1
  150. package/src/ai/captain/test-mode.ts +1 -1
  151. package/src/ai/captain/web-mode.ts +18 -30
  152. package/src/ai/captain.ts +17 -42
  153. package/src/ai/conversation.ts +9 -32
  154. package/src/ai/driller.ts +53 -83
  155. package/src/ai/experience-compactor.ts +16 -35
  156. package/src/ai/fisherman-tools.ts +1 -1
  157. package/src/ai/fisherman.ts +0 -4
  158. package/src/ai/historian/codeceptjs.ts +5 -3
  159. package/src/ai/historian/playwright.ts +7 -4
  160. package/src/ai/historian/screencast.ts +4 -3
  161. package/src/ai/historian/utils.ts +0 -1
  162. package/src/ai/historian.ts +6 -8
  163. package/src/ai/navigator.ts +39 -70
  164. package/src/ai/pilot.ts +59 -61
  165. package/src/ai/planner.ts +76 -62
  166. package/src/ai/provider.ts +85 -146
  167. package/src/ai/quartermaster.ts +39 -45
  168. package/src/ai/rerunner.ts +27 -50
  169. package/src/ai/researcher/cache.ts +10 -16
  170. package/src/ai/researcher/coordinates.ts +4 -3
  171. package/src/ai/researcher/deep-analysis.ts +10 -11
  172. package/src/ai/researcher/locators.ts +66 -9
  173. package/src/ai/researcher/parser.ts +1 -1
  174. package/src/ai/researcher/research-result.ts +0 -18
  175. package/src/ai/researcher/sections.ts +9 -3
  176. package/src/ai/researcher.ts +42 -119
  177. package/src/ai/rules.ts +24 -13
  178. package/src/ai/session-analyst.ts +8 -5
  179. package/src/ai/task-agent.ts +45 -38
  180. package/src/ai/tester.ts +51 -129
  181. package/src/ai/tools.ts +145 -153
  182. package/src/command-handler.ts +1 -1
  183. package/src/commands/compact-command.ts +7 -8
  184. package/src/commands/context-aria-command.ts +1 -1
  185. package/src/commands/context-command.ts +8 -6
  186. package/src/commands/context-data-command.ts +2 -2
  187. package/src/commands/context-experience-command.ts +2 -2
  188. package/src/commands/context-html-command.ts +2 -2
  189. package/src/commands/context-knowledge-command.ts +2 -2
  190. package/src/commands/drill-command.ts +1 -1
  191. package/src/commands/experience-command.ts +1 -1
  192. package/src/commands/explore-command.ts +4 -3
  193. package/src/commands/freesail-command.ts +2 -2
  194. package/src/commands/knows-command.ts +1 -1
  195. package/src/commands/learn-command.ts +5 -3
  196. package/src/commands/path-command.ts +1 -1
  197. package/src/commands/research-command.ts +1 -1
  198. package/src/commands/test-command.ts +1 -1
  199. package/src/components/AddKnowledge.tsx +5 -7
  200. package/src/components/App.tsx +3 -3
  201. package/src/config.ts +185 -11
  202. package/src/experience-tracker.ts +89 -152
  203. package/src/explorbot.ts +88 -83
  204. package/src/explorer.ts +402 -693
  205. package/src/knowledge-tracker.ts +42 -49
  206. package/src/state-manager.ts +20 -155
  207. package/src/utils/aria.ts +11 -2
  208. package/src/utils/cache.ts +40 -0
  209. package/src/utils/context-formatter.ts +12 -15
  210. package/src/utils/hooks-runner.ts +2 -4
  211. package/src/utils/html.ts +79 -0
  212. package/src/utils/markdown-files.ts +30 -0
  213. package/src/utils/markdown-query.ts +15 -1
  214. package/src/utils/page-readiness.ts +1 -1
  215. package/src/utils/secrets.ts +2 -3
  216. package/src/utils/strings.ts +12 -0
  217. package/src/utils/test-files.ts +1 -1
  218. package/src/utils/web-annotate.ts +64 -0
  219. package/src/utils/web-eidx.ts +21 -0
  220. package/src/utils/web-element.ts +34 -0
  221. package/src/utils/web-sandbox.ts +43 -0
  222. package/dist/boat/api-tester/example/apibot.config.js +0 -30
@@ -1,5 +1,6 @@
1
1
  import dedent from 'dedent';
2
2
  import type { ActionResult } from '../../action-result.js';
3
+ import type { ExplorbotConfig } from '../../config.ts';
3
4
  import { executionController } from '../../execution-controller.ts';
4
5
  import type Explorer from '../../explorer.ts';
5
6
  import type { StateManager } from '../../state-manager.js';
@@ -20,6 +21,7 @@ export function WithSections<T extends Constructor>(Base: T) {
20
21
  declare explorer: Explorer;
21
22
  declare provider: Provider;
22
23
  declare stateManager: StateManager;
24
+ declare config: ExplorbotConfig;
23
25
  declare actionResult: ActionResult | undefined;
24
26
 
25
27
  async researchBySections(): Promise<string> {
@@ -64,11 +66,11 @@ export function WithSections<T extends Constructor>(Base: T) {
64
66
  }
65
67
 
66
68
  private async _detectFocusCss(): Promise<string | null> {
67
- const focusSections = (this.explorer.getConfig().ai?.agents?.researcher as any)?.focusSections as string[] | undefined;
69
+ const focusSections = (this.config.ai?.agents?.researcher as any)?.focusSections as string[] | undefined;
68
70
  if (!focusSections?.length) return null;
69
71
 
70
72
  for (const css of focusSections) {
71
- const count = await this.explorer.playwrightLocatorCount((page: any) => page.locator(css)).catch(() => 0);
73
+ const count = await this.explorer.withPage((page) => page.locator(css).count()).catch(() => 0);
72
74
  if (count > 0) return css;
73
75
  }
74
76
  return null;
@@ -105,9 +107,13 @@ export function WithSections<T extends Constructor>(Base: T) {
105
107
  </section_format>
106
108
 
107
109
  <rules>
108
- - Every element with eidx MUST appear in the table.
110
+ - List only elements physically inside this section's declared container.
111
+ - Do not copy global toolbar, navigation, list, or detail elements into this section unless they are descendants of this section container.
112
+ - Every element with eidx inside this section's container MUST appear in the table.
109
113
  - Every row needs CSS; ARIA may be "-" for icon-only buttons.
110
114
  - ARIA locator JSON uses keys "role" and "text" (NOT "name").
115
+ - Elements marked data-explorbot-hit="covered" or "offscreen" are not directly actionable; describe the covering or focused UI first.
116
+ - In split-pane pages, entity detail panels are active detail context; include close/back/pin controls in the detail panel section when present.
111
117
  </rules>
112
118
 
113
119
  ${generalLocatorRuleText}
@@ -1,24 +1,22 @@
1
- import { join } from 'node:path';
2
1
  import dedent from 'dedent';
3
2
  import { ActionResult } from '../action-result.js';
4
3
  import { setActivity } from '../activity.ts';
5
- import { ConfigParser, outputPath } from '../config.ts';
4
+ import { ConfigParser, type ExplorbotConfig, outputPath } from '../config.ts';
6
5
  import { executionController } from '../execution-controller.ts';
7
6
  import type { ExperienceTracker } from '../experience-tracker.ts';
8
7
  import type Explorer from '../explorer.ts';
9
- import type { KnowledgeTracker } from '../knowledge-tracker.ts';
10
8
  import { Observability } from '../observability.ts';
11
9
  import type { StateManager } from '../state-manager.js';
12
10
  import { WebPageState } from '../state-manager.js';
13
11
  import { Stats } from '../stats.ts';
14
12
  import { diffAriaSnapshots } from '../utils/aria.ts';
15
13
  import { ErrorPageError, detectPageCondition } from '../utils/error-page.ts';
16
- import { HooksRunner } from '../utils/hooks-runner.ts';
17
14
  import { isBodyEmpty } from '../utils/html.ts';
18
- import { createDebug, pluralize, tag } from '../utils/logger.js';
15
+ import { createDebug, tag } from '../utils/logger.js';
19
16
  import { mdq } from '../utils/markdown-query.ts';
20
17
  import { RulesLoader } from '../utils/rules-loader.ts';
21
- import type { Agent } from './agent.js';
18
+ import { annotatePageElements } from '../utils/web-annotate.ts';
19
+ import type { Agent, AgentDeps } from './agent.js';
22
20
  import type { Navigator } from './navigator.ts';
23
21
  import { ContextLengthError, type Provider } from './provider.js';
24
22
  import { findSimilarResearch, getCachedResearch, saveResearch } from './researcher/cache.ts';
@@ -56,20 +54,16 @@ export class Researcher extends ResearcherBase implements Agent {
56
54
  declare explorer: Explorer;
57
55
  declare provider: Provider;
58
56
  declare stateManager: StateManager;
57
+ declare config: ExplorbotConfig;
59
58
  private experienceTracker!: ExperienceTracker;
60
59
  private hasScreenshotToAnalyze = false;
61
60
  declare actionResult: ActionResult | undefined;
62
- private hooksRunner!: HooksRunner;
63
61
 
64
- constructor(explorer: Explorer, provider: Provider) {
65
- super();
66
- this.explorer = explorer;
67
- this.provider = provider;
68
- this.stateManager = explorer.getStateManager();
69
- this.experienceTracker = this.stateManager.getExperienceTracker();
70
- this.hooksRunner = new HooksRunner(explorer, explorer.getConfig());
62
+ constructor(deps: AgentDeps) {
63
+ super(deps);
64
+ this.experienceTracker = deps.stateManager.getExperienceTracker();
71
65
 
72
- const ai = explorer.getConfig().ai;
66
+ const ai = deps.config.ai;
73
67
  if (ai) {
74
68
  ai.agents ??= {};
75
69
  ai.agents.researcher ??= {};
@@ -81,18 +75,6 @@ export class Researcher extends ResearcherBase implements Agent {
81
75
  throw new Error('not implemented');
82
76
  }
83
77
 
84
- protected getExperienceTracker(): ExperienceTracker {
85
- return this.experienceTracker;
86
- }
87
-
88
- protected getKnowledgeTracker(): KnowledgeTracker {
89
- return this.explorer.getKnowledgeTracker();
90
- }
91
-
92
- protected getProvider(): Provider {
93
- return this.provider;
94
- }
95
-
96
78
  static getCachedResearch(state: WebPageState): string {
97
79
  return getCachedResearch(state.hash || '');
98
80
  }
@@ -111,7 +93,7 @@ export class Researcher extends ResearcherBase implements Agent {
111
93
 
112
94
  async research(state: WebPageState, opts: { screenshot?: boolean; force?: boolean; deep?: boolean; data?: boolean; fix?: boolean; _retriesLeft?: number } = {}): Promise<string> {
113
95
  const { screenshot = false, force = false, deep = false, data = false, fix = true } = opts;
114
- const maxRetries = (this.explorer.getConfig().ai?.agents?.researcher as any)?.retries ?? 2;
96
+ const maxRetries = (this.config.ai?.agents?.researcher as any)?.retries ?? 2;
115
97
  let retriesLeft = opts._retriesLeft ?? maxRetries;
116
98
  this.actionResult = ActionResult.fromState(state);
117
99
  const stateHash = state.hash || this.actionResult.getStateHash();
@@ -135,9 +117,9 @@ export class Researcher extends ResearcherBase implements Agent {
135
117
  await this.ensureNavigated(displayUrl, screenshot && this.provider.hasVision());
136
118
  await this.hooksRunner.runBeforeHook('researcher', state.url);
137
119
 
138
- const annotatedElements = await this.explorer.annotateElements();
120
+ const { elements: annotatedElements } = await this.explorer.withPage(annotatePageElements);
139
121
  debugLog(`Annotated ${annotatedElements.length} interactive elements with eidx`);
140
- this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot && this.provider.hasVision() });
122
+ this.actionResult = await this.explorer.capture({ screenshot: screenshot && this.provider.hasVision() });
141
123
 
142
124
  const condition = detectPageCondition(this.actionResult!);
143
125
  if (condition === 'error') {
@@ -209,11 +191,10 @@ export class Researcher extends ResearcherBase implements Agent {
209
191
  }
210
192
 
211
193
  if (!interrupted()) {
212
- const containerLocs = result.containerLocators;
213
- await this.testLocators(containerLocs);
214
- const brokenContainers = containerLocs.filter((l) => l.valid === false);
215
- if (containerLocs.length > 0 && brokenContainers.length === containerLocs.length && retriesLeft > 0) {
216
- tag('warning').log(`All ${containerLocs.length} containers broken, retrying research (${maxRetries - retriesLeft + 1}/${maxRetries})...`);
194
+ const brokenContainers = await this.resolveContainers(result);
195
+ const containerCount = result.containers.length;
196
+ if (containerCount > 0 && brokenContainers.length === containerCount && retriesLeft > 0) {
197
+ tag('warning').log(`All ${containerCount} containers broken, retrying research (${maxRetries - retriesLeft + 1}/${maxRetries})...`);
217
198
  await new Promise((r) => setTimeout(r, 2000));
218
199
  return this.research(state, { ...opts, force: true, _retriesLeft: retriesLeft - 1 } as any);
219
200
  }
@@ -227,14 +208,6 @@ export class Researcher extends ResearcherBase implements Agent {
227
208
 
228
209
  const toTest = result.locators.filter((l) => l.valid === null);
229
210
  await this.testLocators(toTest);
230
-
231
- const brokenCount = result.locators.filter((l) => l.valid === false).length;
232
- const brokenRatio = result.locators.length > 0 ? brokenCount / result.locators.length : 0;
233
- if (brokenRatio > 0.8 && retriesLeft > 0) {
234
- tag('warning').log(`${Math.round(brokenRatio * 100)}% locators broken, waiting 3s and retrying research (${maxRetries - retriesLeft + 1}/${maxRetries})...`);
235
- await new Promise((r) => setTimeout(r, 3000));
236
- return this.research(state, { ...opts, force: true, _retriesLeft: retriesLeft - 1 } as any);
237
- }
238
211
  }
239
212
 
240
213
  // Stage 3: Fix broken sections via AI conversation continuation
@@ -246,7 +219,7 @@ export class Researcher extends ResearcherBase implements Agent {
246
219
  // Must run BEFORE visuallyAnnotateContainers — annotation overlays inject z-index 99998+ which would pollute the scoring.
247
220
  if (!interrupted() && this.hasScreenshotToAnalyze) {
248
221
  const sections = parseResearchSections(result.text);
249
- const focused = await this.explorer.runWithBrowserRecovery('detectFocusedSection', () => detectFocusedSection(this.explorer.playwrightHelper.page, sections));
222
+ const focused = await this.explorer.withPage((page) => detectFocusedSection(page, sections));
250
223
  if (focused) markSectionAsFocused(result, focused);
251
224
  }
252
225
 
@@ -255,11 +228,11 @@ export class Researcher extends ResearcherBase implements Agent {
255
228
  const validContainers = extractValidContainers(result.text);
256
229
  result.parseLocators();
257
230
  const freshContainerLocs = result.containerLocators;
258
- await this.testLocators(freshContainerLocs);
231
+ await this.testLocators(freshContainerLocs, { scope: true });
259
232
  const freshBroken = freshContainerLocs.filter((l) => l.valid === false).map((l) => l.locator);
260
233
  const containers = validContainers.filter((c) => !freshBroken.includes(c.css));
261
234
  await this.visuallyAnnotateElements({ containers });
262
- this.actionResult = await this.explorer.capturePageWithScreenshot();
235
+ this.actionResult = await this.explorer.capture({ screenshot: true });
263
236
  const visualResult = await this.analyzeScreenshotForVisualProps();
264
237
  if (visualResult.elements.size > 0) {
265
238
  await this.mergeVisualData(result, visualResult.elements);
@@ -315,11 +288,9 @@ export class Researcher extends ResearcherBase implements Agent {
315
288
  researchFile = saveResearch(stateHash, result.text, combinedHtml);
316
289
  }
317
290
 
318
- const summaryMatch = result.text.match(/## Summary\s*\n+([\s\S]*?)(?=\n##|$)/i);
319
- if (summaryMatch) {
320
- const summaryLine = summaryMatch[1].trim().split('\n')[0].trim().slice(0, 200);
321
- if (summaryLine) this.experienceTracker.updateSummary(this.actionResult!, summaryLine);
322
- }
291
+ const summaryText = mdq(result.text).query('section2(/^summary/)').query('paragraph[0]').text().trim();
292
+ const summaryLine = summaryText.split('\n')[0]?.trim().slice(0, 200);
293
+ if (summaryLine) this.experienceTracker.updateSummary(this.actionResult!, summaryLine);
323
294
 
324
295
  tag('multiline').log(formatResearchSummary(result.text, { visionUsed: this.hasScreenshotToAnalyze }));
325
296
  tag('success').log('Research complete');
@@ -337,8 +308,7 @@ export class Researcher extends ResearcherBase implements Agent {
337
308
  private async ensureNavigated(url: string, screenshot?: boolean): Promise<void> {
338
309
  if (!this.actionResult) {
339
310
  debugLog('No action result, navigating to URL');
340
- await this.explorer.visit(url);
341
- this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot });
311
+ this.actionResult = await this.explorer.visit(url, { screenshot });
342
312
  return;
343
313
  }
344
314
 
@@ -348,7 +318,7 @@ export class Researcher extends ResearcherBase implements Agent {
348
318
 
349
319
  if (!isEmpty && isOnCurrentState) {
350
320
  if ((!this.actionResult.screenshot && screenshot) || !this.actionResult.ariaSnapshot) {
351
- this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot });
321
+ this.actionResult = await this.explorer.capture({ screenshot });
352
322
  }
353
323
  return;
354
324
  }
@@ -356,8 +326,7 @@ export class Researcher extends ResearcherBase implements Agent {
356
326
  if (isEmpty && isOnCurrentState) {
357
327
  debugLog('HTML body empty on current URL, waiting for content');
358
328
  tag('step').log('Page body is empty, waiting for content...');
359
- await this.explorer.visit(url);
360
- this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot ?? false });
329
+ this.actionResult = await this.explorer.visit(url, { screenshot: screenshot ?? false });
361
330
  await this.waitUntilSettled(screenshot ?? false);
362
331
  return;
363
332
  }
@@ -365,22 +334,21 @@ export class Researcher extends ResearcherBase implements Agent {
365
334
  debugLog('Not on current state, navigating to URL');
366
335
  tag('step').log('Navigating to URL...');
367
336
 
368
- await this.explorer.visit(url);
369
- this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot ?? false });
337
+ this.actionResult = await this.explorer.visit(url, { screenshot: screenshot ?? false });
370
338
  }
371
339
 
372
340
  private async waitUntilSettled(screenshot: boolean): Promise<boolean> {
373
- const errorPageTimeout = (this.explorer.getConfig().ai?.agents?.researcher as any)?.errorPageTimeout ?? 10;
341
+ const errorPageTimeout = (this.config.ai?.agents?.researcher as any)?.errorPageTimeout ?? 10;
374
342
  if (errorPageTimeout <= 0) return false;
375
343
 
376
344
  const includeScreenshot = screenshot && this.provider.hasVision();
377
345
 
378
346
  try {
379
- await this.explorer.runWithBrowserRecovery('waitUntilSettled', () => this.explorer.playwrightHelper.page?.waitForLoadState('networkidle', { timeout: errorPageTimeout * 1000 }));
347
+ await this.explorer.withPage((page) => page.waitForLoadState('networkidle', { timeout: errorPageTimeout * 1000 }));
380
348
  } catch {}
381
349
 
382
- await this.explorer.annotateElements();
383
- this.actionResult = await this.explorer.capturePageState({ includeScreenshot });
350
+ await this.explorer.withPage(annotatePageElements);
351
+ this.actionResult = await this.explorer.capture({ screenshot: includeScreenshot });
384
352
 
385
353
  let condition = detectPageCondition(this.actionResult!);
386
354
  if (condition === 'error') {
@@ -390,8 +358,8 @@ export class Researcher extends ResearcherBase implements Agent {
390
358
 
391
359
  for (let i = 0; i < 3; i++) {
392
360
  await new Promise((r) => setTimeout(r, 1000));
393
- await this.explorer.annotateElements();
394
- this.actionResult = await this.explorer.capturePageState({ includeScreenshot });
361
+ await this.explorer.withPage(annotatePageElements);
362
+ this.actionResult = await this.explorer.capture({ screenshot: includeScreenshot });
395
363
  condition = detectPageCondition(this.actionResult!);
396
364
  if (condition === 'error') {
397
365
  throw new ErrorPageError(this.actionResult!.url, this.actionResult!.title, this.actionResult!.httpStatus);
@@ -403,7 +371,7 @@ export class Researcher extends ResearcherBase implements Agent {
403
371
  }
404
372
 
405
373
  private getConfiguredSections(): Record<string, string> {
406
- const configSections = (this.explorer.getConfig().ai?.agents?.researcher as any)?.sections as string[] | undefined;
374
+ const configSections = (this.config.ai?.agents?.researcher as any)?.sections as string[] | undefined;
407
375
  if (!configSections?.length) return POSSIBLE_SECTIONS;
408
376
  const filtered: Record<string, string> = {};
409
377
  for (const key of configSections) {
@@ -428,7 +396,10 @@ export class Researcher extends ResearcherBase implements Agent {
428
396
  - Research all menus and navigational areas.
429
397
  - Ignore decorative sidebars, footer-only links, and external links.
430
398
  - Detect layout patterns: list/detail split, 2-pane, or 3-pane layouts.
431
- - Every element with an eidx attribute MUST appear in the UI map describe icon-only buttons by their visual role.
399
+ - In split-pane layouts, keep global toolbar, list pane, and detail pane as separate sections. Do not copy toolbar/list elements into the detail section unless they are physically inside that detail container.
400
+ - If the URL or page state opens an entity detail panel, describe that panel as the active detail context and include its close/back/pin controls when present.
401
+ - If an element has data-explorbot-hit="covered" or "offscreen", do not present it as directly actionable. Prefer the overlay, drawer, dialog, or focused section covering it, and mention what must be dismissed or revealed first.
402
+ - Every element with an eidx attribute MUST appear in exactly one matching UI map section — describe icon-only buttons by their visual role.
432
403
  - Every UI map row needs a CSS selector; ARIA may be "-" for icon-only buttons, CSS must never be "-".
433
404
  - ARIA locator JSON uses keys "role" and "text" (NOT "name").
434
405
  - Mark elements with likely hover interactions (title, aria-describedby, menu items with submenus) as "(hover)".
@@ -445,6 +416,7 @@ export class Researcher extends ResearcherBase implements Agent {
445
416
  .join('\n')}
446
417
 
447
418
  - Sections can overlap; prefer more detailed sections over broader ones.
419
+ - Section tables must list only elements physically inside the declared container. If a control belongs to a global toolbar, list it only in the toolbar section.
448
420
  - Never name a section "Focus" or "Focused" — use what it contains (Detail, Modal, Form, Content, List).
449
421
  - Omit sections that are not present or not relevant.
450
422
  - Each section needs a container CSS locator; UI map CSS locators are relative to it.
@@ -466,23 +438,7 @@ export class Researcher extends ResearcherBase implements Agent {
466
438
  if (!this.actionResult) throw new Error('actionResult is not set');
467
439
 
468
440
  const html = await this.actionResult.combinedHtml();
469
- const knowledgeFiles = this.stateManager.getRelevantKnowledge();
470
-
471
- let knowledge = '';
472
- if (knowledgeFiles.length > 0) {
473
- const knowledgeContent = knowledgeFiles
474
- .map((k) => k.content)
475
- .filter((k) => !!k)
476
- .join('\n\n');
477
-
478
- tag('operation').log(`Found ${knowledgeFiles.length} relevant knowledge ${pluralize(knowledgeFiles.length, 'file')} for: ${this.actionResult.url}`);
479
- knowledge = `
480
- <hint>
481
- Here is relevant knowledge for this page:
482
-
483
- ${knowledgeContent}
484
- </hint>`;
485
- }
441
+ const knowledge = this.knowledgeTracker.renderRelevantKnowledge(this.actionResult);
486
442
 
487
443
  const ariaSnapshot = this.actionResult.getCompactARIA();
488
444
 
@@ -540,39 +496,6 @@ export class Researcher extends ResearcherBase implements Agent {
540
496
  `;
541
497
  }
542
498
 
543
- async textContent(state: WebPageState): Promise<string> {
544
- const actionResult = ActionResult.fromState(state);
545
- const html = await actionResult.combinedHtml();
546
-
547
- const prompt = dedent`
548
- Transform into markdown.
549
- Identify headers, footers, asides, special application parts and main contant.
550
- Content should be in markdown format. If it is content: tables must be tables, lists must be lists.
551
- Navigation elements should be represented as standalone blocks after the content.
552
- Do not summarize content, just transform it into markdown.
553
- It is important to list all the content text
554
- If it is link it must be linked
555
- You can summarize footers/navigation/aside elements.
556
- But main conteint should be kept as text and formatted as markdown based on its current markup.
557
- Links to external web sites should be avoided in output.
558
-
559
- Break down into sections:
560
-
561
- ## Content Area
562
-
563
- ## Navigation Area
564
-
565
- <page_html>
566
- ${html}
567
- </page_html>
568
- `;
569
-
570
- const model = this.provider.getModelForAgent('researcher');
571
- const r = await this.provider.chat([{ role: 'user', content: prompt }], model, { agentName: 'researcher', telemetryFunctionId: 'researcher.textContent' });
572
-
573
- return r.text;
574
- }
575
-
576
499
  private getScreenshotFromState(state: WebPageState): { actionResult: ActionResult; image: Buffer } | null {
577
500
  const actionResult = ActionResult.fromState(state);
578
501
  const image = actionResult.screenshot;
@@ -776,9 +699,9 @@ export class Researcher extends ResearcherBase implements Agent {
776
699
  async cancelInUi() {
777
700
  const beforeAria = this.stateManager.getCurrentState()?.ariaSnapshot || null;
778
701
 
779
- await this.explorer.executeAction('I.clickXY(0, 0)');
780
- if (diffAriaSnapshots(beforeAria, this.stateManager.getCurrentState()?.ariaSnapshot || null)) return;
702
+ await this.explorer.action().execute('I.clickXY(0, 0)');
703
+ if (diffAriaSnapshots(beforeAria, this.stateManager.getCurrentState()?.ariaSnapshot || null).text) return;
781
704
 
782
- await this.explorer.executeAction(`I.pressKey('Escape')`);
705
+ await this.explorer.action().execute(`I.pressKey('Escape')`);
783
706
  }
784
707
  }
package/src/ai/rules.ts CHANGED
@@ -2,7 +2,7 @@ import dedent from 'dedent';
2
2
 
3
3
  export const recommendedCodeceptCommands = ['I.click', 'I.type', 'I.fillField', 'I.see', 'I.seeElement'] as const;
4
4
 
5
- export const locatorRule = dedent`
5
+ const locatorPriorityRule = dedent`
6
6
  <locator_priority>
7
7
  Use the following priority when selecting locators:
8
8
 
@@ -19,10 +19,12 @@ export const locatorRule = dedent`
19
19
  Example: '#login-btn', '[data-testid="submit"]', 'form#login input[name="email"]'
20
20
 
21
21
  4. XPath (last resort) - for complex hierarchy or when CSS can't express the path
22
- Always start with //, never use positional indices like [1], [2]
22
+ Always start with //. Avoid positional indices like [1], [2] except as a last-resort disambiguator
23
23
  Example: '//form[@id="login"]//input[@name="email"]'
24
24
  </locator_priority>
25
+ `;
25
26
 
27
+ const contextSimplificationRule = dedent`
26
28
  <context_simplification>
27
29
  When container is available from UI map sections:
28
30
  - Text + container is simplest and PREFERRED: I.click('Save', '.modal')
@@ -30,7 +32,9 @@ export const locatorRule = dedent`
30
32
  - ALWAYS use context parameter unless locator is XPath or unique ID
31
33
  - No need for complex ARIA when container narrows scope sufficiently
32
34
  </context_simplification>
35
+ `;
33
36
 
37
+ const locatorStrategyRule = dedent`
34
38
  <disambiguation>
35
39
  When multiple elements could match the request, select based on intent:
36
40
  1. Match the context of recent actions - if filling a form, use elements in that same form
@@ -48,8 +52,9 @@ export const locatorRule = dedent`
48
52
  - Use aria-label value if present: { "role": "button", "text": "Close" } (from aria-label="Close")
49
53
  - Use title attribute if present: { "role": "button", "text": "Settings" } (from title="Settings")
50
54
  - If no accessible name exists, mark ARIA as "-" and use CSS/XPath:
51
- * CSS: use partial href a[href*="settings"] or SVG icon class a:has(svg.md-icon-cog)
52
- * XPath: use contains(@href,"settings") or SVG class //a[.//svg[contains(@class,"md-icon-cog")]]
55
+ * CSS: use partial href a[href*="settings"] or SVG icon class a:has(svg.icon-settings)
56
+ * XPath: use contains(@href,"settings") or SVG class //a[.//svg[contains(@class,"icon-settings")]]
57
+ - In inline create/edit rows, confirmation can be an icon-only control near the edited field instead of a text Save button. Anchor the locator to the same row/form as the field and target the adjacent confirm icon/control.
53
58
  - NEVER use empty text: { "role": "button", "text": "" } is INVALID and useless
54
59
 
55
60
  <good_aria_locator_example>
@@ -80,9 +85,9 @@ export const locatorRule = dedent`
80
85
  - Vue: data-v-* attributes
81
86
  Avoid locators that seem to have generated ids or class names (long random numbers, uuids, hashes, etc)
82
87
  Prefer text or ARIA locators over href-based ones. But for icon-only links with no accessible name, use:
83
- - Partial href match: a[href*="settings"], a[href*="requirements"] (use path segments, not full URLs)
84
- - SVG icon class: a:has(svg.md-icon-cog), button:has(svg.md-icon-plus) (target the SVG class inside the link/button)
85
- Avoid full absolute href like a[href="/projects/imr_manual12/settings"] — use generic path segments instead
88
+ - Partial href match: a[href*="settings"], a[href*="reports"] (use path segments, not full URLs)
89
+ - SVG icon class: a:has(svg.icon-settings), button:has(svg.icon-add) (target the SVG class inside the link/button)
90
+ Avoid full absolute href like a[href="/items/12345/settings"] — use generic path segments instead
86
91
  Avoid CSS framework utility classes as containers (Tailwind: flex, grid, space-x-*, justify-*, items-*, w-*, h-*, p-*, m-*, etc; Bootstrap: col-*, row, d-flex, etc)
87
92
  Prefer semantic class names, roles, data attributes, or element hierarchy for containers
88
93
 
@@ -94,7 +99,7 @@ export const locatorRule = dedent`
94
99
 
95
100
  <xpath_rules>
96
101
  XPath locators must start with //.
97
- XPath should use positional indices [1], [2], [3] and contains(., "text") for disambiguation.
102
+ XPath may use positional indices [1], [2] and contains(., "text") as a last-resort disambiguator when attribute/text strategies are exhausted.
98
103
  XPath should rely less on class names — prefer element hierarchy, position, and text content.
99
104
  XPath and CSS MUST provide different strategies for finding the same element.
100
105
  </xpath_rules>
@@ -106,22 +111,26 @@ export const locatorRule = dedent`
106
111
  '#content-top #user_name'
107
112
  '#content-top form input[name="name"]'
108
113
  'a.nav-item[href*="settings"]' // icon-only link matched by partial href
109
- 'a.nav-item:has(svg.md-icon-cog)' // icon-only link matched by SVG icon class
114
+ 'a.nav-item:has(svg.icon-settings)' // icon-only link matched by SVG icon class
110
115
  '//nav//a[contains(@href,"settings")]' // XPath for icon-only nav link
111
116
  </good locator example>
112
117
 
113
118
  <bad locator example>
114
- 'a.filter-tab:nth-of-type(1)' // WRONG: positional in CSS, use :has-text("Manual") instead
115
- '//a[contains(@class,"filter-tab") and contains(@class,"active")]' // WRONG: XPath repeats CSS approach, use positional //a[contains(@class,"filter-tab")][1]
119
+ 'a.filter-tab:nth-of-type(1)' // WRONG: positional in CSS, use :has-text("Active") instead
120
+ '//a[contains(@class,"filter-tab") and contains(@class,"active")]' // WRONG: XPath repeats CSS approach, provide a different strategy e.g. //a[contains(@class,"filter-tab")][1]
116
121
  '//table//tbody/tr[1]//button[contains(@onclick,'fn()')]' // onclick is not semantic attribute
117
122
  '//html/body/vue-button-123' // vue-framework specific locator
118
- 'link "New Template"' // WRONG: malformed string, use {"role":"link","text":"New Template"}
119
- 'a[href="/projects/imr_manual12/settings"]' // WRONG: full absolute href, use a[href*="settings"] instead
123
+ 'link "New Item"' // WRONG: malformed string, use {"role":"link","text":"New Item"}
124
+ 'a[href="/items/12345/settings"]' // WRONG: full absolute href, use a[href*="settings"] instead
120
125
  </bad locator example>
121
126
 
122
127
  HTML locators must be valid JS strings
123
128
  `;
124
129
 
130
+ export const locatorRule = [locatorPriorityRule, contextSimplificationRule, locatorStrategyRule].join('\n\n');
131
+
132
+ export const drillLocatorRule = [locatorPriorityRule, locatorStrategyRule].join('\n\n');
133
+
125
134
  export const fileUploadRule = dedent`
126
135
  <file_upload>
127
136
  Explorbot CAN upload files using I.attachFile() via form() tool.
@@ -307,6 +316,7 @@ export const actionRule = dedent`
307
316
  Use context parameter (second argument) to narrow click area when:
308
317
  - The same text/button appears multiple times on page
309
318
  - You need to click inside a specific form, modal, or section
319
+ - You need an icon-only confirm/save control next to a field in an inline create/edit row
310
320
  Context should be a CSS selector pointing to a unique container.
311
321
 
312
322
  <example>
@@ -319,6 +329,7 @@ export const actionRule = dedent`
319
329
  </example>
320
330
 
321
331
  Prefer text/ARIA locators with context over complex CSS/XPath selectors.
332
+ For inline create/edit flows, after filling a field verify it contains the value, then confirm using the nearest explicit button/link, an adjacent icon-only confirm control in the same row/form, or Enter if the field remains focused.
322
333
  If locator doesn't work, try CSS or XPath locators.
323
334
  If nothing works, use I.clickXY(x, y) as last resort.
324
335
 
@@ -49,6 +49,9 @@ export class SessionAnalyst implements Agent {
49
49
  ## Severity (defects only)
50
50
  [High] blocks a core flow · [Medium] degrades a flow but workaround exists · [Low] cosmetic / edge case
51
51
 
52
+ ## Naming tests
53
+ Reference every test by its full label wrapped in backticks: \`\`\`ET-<number> <test title>\`\`\` — the \`ref\` and \`scenario\` from its \`<test>\` block. Backticks only, never bold. Never write a bare \`#2\`.
54
+
52
55
  ## Format
53
56
 
54
57
  # Session Analysis
@@ -60,22 +63,22 @@ export class SessionAnalyst implements Agent {
60
63
  - Features: <capabilities>
61
64
 
62
65
  ## What works
63
- - **<feature>** — #2, #7, #8
66
+ - **<feature>** — \`ET-2 <title>\`, \`ET-7 <title>\`
64
67
 
65
68
  ## Defects
66
69
 
67
70
  ### [Medium] <plain-English bug title>
68
- Affects: #3, #5
71
+ Affects: \`ET-3 <title>\`, \`ET-5 <title>\`
69
72
  Reproduce:
70
73
  1. <concrete UI step>
71
74
  2. <next>
72
75
  Evidence: <one short observation>
73
76
 
74
77
  ## UX issues
75
- - **<feature>** — <what's confusing> (#7)
78
+ - **<feature>** — <what's confusing> (\`ET-7 <title>\`)
76
79
 
77
80
  ## Execution Issues
78
- - **#2 <scenario>** — <≤10 words, what was unreliable>
81
+ - \`ET-2 <title>\` — <≤10 words, what was unreliable>
79
82
 
80
83
  ## Brevity rules
81
84
 
@@ -143,7 +146,7 @@ export class SessionAnalyst implements Agent {
143
146
  : 'verification_status: none';
144
147
 
145
148
  return dedent`
146
- <test ref="#${ref}">
149
+ <test ref="ET-${ref}">
147
150
  url: ${test.startUrl || '/'}
148
151
  scenario: ${test.scenario}
149
152
  result: ${test.result || 'unknown'}