explorbot 0.1.4 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/dist/rules/planner/styles/curious.md +18 -5
  2. package/dist/rules/planner/styles/normal.md +4 -4
  3. package/dist/rules/planner/styles/psycho.md +14 -11
  4. package/dist/rules/researcher/container-rules.md +15 -0
  5. package/dist/rules/researcher/section-example.md +12 -0
  6. package/dist/src/ai/captain/web-mode.js +9 -1
  7. package/dist/src/ai/historian.js +6 -0
  8. package/dist/src/ai/pilot.js +23 -2
  9. package/dist/src/ai/researcher/cache.js +6 -0
  10. package/dist/src/ai/researcher/deep-analysis.js +65 -9
  11. package/dist/src/ai/researcher/sections.js +103 -0
  12. package/dist/src/ai/researcher.js +30 -103
  13. package/dist/src/ai/task-agent.js +7 -27
  14. package/dist/src/ai/tester.js +19 -4
  15. package/dist/src/ai/tools.js +27 -2
  16. package/dist/src/commands/explore-command.js +4 -9
  17. package/dist/src/experience-tracker.js +126 -1
  18. package/dist/src/explorbot.js +9 -2
  19. package/dist/src/explorer.js +12 -1
  20. package/dist/src/utils/aria.js +39 -2
  21. package/package.json +1 -1
  22. package/rules/planner/styles/curious.md +18 -5
  23. package/rules/planner/styles/normal.md +4 -4
  24. package/rules/planner/styles/psycho.md +14 -11
  25. package/rules/researcher/container-rules.md +15 -0
  26. package/rules/researcher/section-example.md +12 -0
  27. package/src/ai/captain/web-mode.ts +9 -1
  28. package/src/ai/historian.ts +7 -0
  29. package/src/ai/pilot.ts +23 -3
  30. package/src/ai/researcher/cache.ts +5 -0
  31. package/src/ai/researcher/deep-analysis.ts +74 -9
  32. package/src/ai/researcher/sections.ts +122 -0
  33. package/src/ai/researcher.ts +31 -105
  34. package/src/ai/task-agent.ts +7 -31
  35. package/src/ai/tester.ts +20 -4
  36. package/src/ai/tools.ts +33 -1
  37. package/src/commands/explore-command.ts +4 -9
  38. package/src/config.ts +1 -0
  39. package/src/experience-tracker.ts +136 -1
  40. package/src/explorbot.ts +9 -2
  41. package/src/explorer.ts +10 -1
  42. package/src/utils/aria.ts +40 -2
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "explorbot",
3
- "version": "0.1.4",
3
+ "version": "0.1.6",
4
4
  "description": "CLI app built with React Ink, CodeceptJS, and Playwright",
5
5
  "license": "Elastic-2.0",
6
6
  "type": "module",
@@ -1,5 +1,12 @@
1
1
  Detect new valid paths that previous tests missed. Prioritize mining experience and research together before inventing abstract scenarios.
2
2
 
3
+ Rank every scenario you build by the **strength of its outcome**, from strongest to weakest:
4
+ 1. **Data change** — the backend, storage, or persisted state registers a difference (a record is created, edited, or deleted; a setting is persisted; a message is sent; a job is triggered; an item is shared or exported).
5
+ 2. **State change** — the application moves to a different addressable or remembered state (route or URL change, a filter or sort actually applied to real data, a mode or auth change that the application remembers, the page showing a different underlying dataset).
6
+ 3. **UI change only** — a control opens, closes, is cancelled, is dismissed, is hovered, is toggled for display only, or the view expands/collapses without the application registering anything new.
7
+
8
+ Prefer scenarios whose ending falls into category 1. Propose a category 2 scenario when no category 1 outcome is reachable for the control under test. Propose a category 3 scenario last, and only when the UI-only behaviour itself has a verifiable side effect worth checking (a warning prompt, a persisted draft, a state rollback, a badge appearing). A page may expose several paths that reach a data or state change — different buttons, different menus, different keyboard shortcuts, different confirmation flows. Pick whichever path reaches category 1 or 2; do not assume a single "primary action" exists.
9
+
3
10
  When <previously_tested_flows> is present, treat it as the ground truth for what already worked:
4
11
  - List items under Successful Flow describe the path that was executed
5
12
  - Lines in blockquotes (lines starting with >) are discoveries: extra fields, side panels, conditional UI, inputs called out during that run
@@ -11,7 +18,7 @@ When <previously_tested_flows> is NOT present, use <tested_scenarios> as the gro
11
18
  Read the step lines for each test to understand which controls were actually interacted with.
12
19
  Identify elements from <page_research> that appear in NO test steps — these are coverage gaps.
13
20
 
14
- Cross-read with <page_research>: for each form and Extended Research subsection, compare against those flows. Which text inputs, selects, checkboxes, toggles, and side controls were skipped or touched once with a single value? Prefer filling those gaps over repeating the same path.
21
+ Cross-read with <page_research>: for each section and Extended Research subsection, compare against those flows. Which text inputs, selects, checkboxes, toggles, and side controls were skipped or touched once with a single value? Prefer filling those gaps over repeating the same path.
15
22
 
16
23
  The Type column in <page_research> tables shows the ARIA role of each element.
17
24
  Cross-reference these types with the steps listed in <tested_scenarios> or <previously_tested_flows>:
@@ -24,16 +31,22 @@ Coverage gaps to look for:
24
31
  - Action buttons that were never clicked as part of a complete workflow
25
32
  - Dependent UI: controls that appear or change based on another control's value
26
33
 
27
- When proposing tests for forms, prefer filling ALL visible fields — not just required ones.
34
+ A coverage gap for an untested control is only **closed** when the scenario built around it reaches a data change or state change. A scenario that exercises the untested control but ends in a UI-only outcome does not close the gap the application never registered the variation, so nothing distinguishes that scenario from not running it at all.
35
+
36
+ Exercising an untested control and testing a UI-only dismissal (cancel, close, navigate away, discard) are **two different categories of scenario**. Do not merge them by appending a dismissal ending to a variation scenario — the variation loses its value because the system never receives it. A dismissal or UI-only ending deserves its own dedicated scenario only when that dismissal itself has a verifiable side effect.
37
+
38
+ When multiple inputs or configurable controls contribute to the same outcome, prefer scenarios that configure **several of them together** before triggering the data or state change, rather than touching one control in isolation and ending there.
28
39
  Vary input strategies: try short values, multi-word values, edge-of-valid values.
29
- When a form has sections, tabs, or conditional panels, propose tests that exercise each section.
30
- If a control has downstream effects (e.g., selecting a type reveals extra fields), build a test around that interaction chain.
40
+ When sections, tabs, or conditional panels exist, exercise each section.
41
+ When a control has downstream effects (selecting one option reveals extra fields, toggling one setting enables another), build the scenario around that interaction chain — and still end it in a data or state change.
31
42
 
32
43
  Combinatorial coverage (valid data only):
33
44
  - For each select or equivalent, ensure each option is exercised in at least one scenario, or one scenario whose steps walk through distinct options in sequence if that fits the task constraints better
34
45
  - Exercise each checkbox or binary control in both states when behavior can differ
35
46
  - Combine checkboxes and related toggles in small sets (pairs or triples) when they plausibly change validation, visible sections, or outcomes — avoid exploding into huge Cartesian products
36
47
 
37
- When heavy forms are not the focus, still pursue: unvisited state transitions, follow-ups after creates (share, export, duplicate), alternative routes to the same goal, preconditions that unlock UI, and visible controls never clicked.
48
+ Each proposed combination must be exercised in a scenario that reaches a data change or state change. Combinations that only change the UI and never reach a registerable outcome do not count as coverage — the system never distinguishes them from each other.
49
+
50
+ When the page is not heavy on inputs, still pursue: unvisited state transitions, follow-ups after data-changing operations (share, export, duplicate, re-open), alternative paths to the same data change, preconditions that unlock new data-changing actions, and visible controls never clicked. Again, prioritise scenarios whose ending falls into category 1 or 2.
38
51
 
39
52
  Skip the Menu/Navigation section — we are testing THIS page.
@@ -2,18 +2,18 @@ Study the page and figure out its business purpose. What is this page FOR? What
2
2
 
3
3
  Based on the page type, propose tests for COMPLETE user workflows:
4
4
  - If this is a data page (lists, tables): test CRUD operations end-to-end (create item → verify in list, edit item → verify changes saved, delete item → verify removed)
5
- - If this is a form page: test full submission flow, not just "form appears"
5
+ - If the page has inputs to fill in: test the full commit flow, not just that the controls render
6
6
  - If this has filters and search: test filtering AND verify results change, not just "filter tab clicked"
7
7
  - If this has modals/dropdowns: test the ACTION inside them, not just opening/closing them
8
8
 
9
- Each test should end with the application in a different state than it started.
9
+ Each test should end in a **data change** (record created/edited/deleted, setting persisted, message sent) or a **state change** (route change, filter applied to real data, mode change the app remembers). Tests ending in UI-only outcomes (open, close, hover, expand) are the weakest and should be rare.
10
10
 
11
11
  IMPORTANT: Distribute tests across DIFFERENT feature areas from the research.
12
12
  Do not propose more than 2 tests for the same feature area.
13
13
  Every Extended Research section (modal, dropdown, panel) with actionable features deserves at least one test.
14
- Tests that change application data MUST come first — create, update, delete records before testing filters, search, or pagination. You are aiming to change application state.
14
+ Tests that change application data MUST come first — create, update, delete records before testing filters, search, or pagination.
15
15
  If the research shows multiple ways to create or modify data (different types, forms, or options), propose a separate test for each.
16
- View only tests (tab switching, pagination, view toggles) should be proposed only after data-changing interactions are covered.
16
+ UI-only tests (tab switching, pagination, view toggles) should be proposed only after data-changing and state-changing interactions are covered.
17
17
 
18
18
  Skip the Menu/Navigation section — we are testing THIS page.
19
19
 
@@ -1,14 +1,17 @@
1
- Stress-test the page by filling invalid, empty, and extreme values into every input.
1
+ Stress-test the page by feeding invalid, empty, or extreme values to its controls and committing.
2
2
 
3
- Focus on:
4
- - Empty states: submit forms with no data, clear required fields, remove default values
5
- - Long values: paste 10000 characters into inputs, use extremely long names and descriptions
6
- - Boundary values: zero, negative numbers, special characters, unicode, HTML tags in text fields
7
- - Invalid formats: wrong email formats, letters in number fields, SQL injection strings, script tags
8
- - Invalid combinations: select incompatible options, mix conflicting settings
9
- - Combining states: apply multiple filters at once, use conflicting form values together
10
- - Out-of-range values: dates in the past/future, quantities beyond limits, prices with too many decimals
3
+ **Match attack breadth to controls reachable.** If only ONE control is reachable, attack it alone. If several are reachable, attack **all of them in the same scenario** — each with a different strange value. Never stress one while leaving the rest untouched: attacking one-at-a-time hides interaction bugs and wastes plan budget.
11
4
 
12
- Push every input to its limits. Find what breaks when the form receives unexpected data.
5
+ Do not produce multiple scenarios that each isolate one control of the same section. Fold those attacks into fewer scenarios that push every reachable control strangely at once. Vary the **mix** between scenarios — which control receives SQL, which receives 10000 chars, which receives unicode, which receives a conflicting combination — not the single control under attack.
13
6
 
14
- Skip the Menu/Navigation section we are testing THIS page.
7
+ **Attack categories** (combine across controls, not one-per-scenario):
8
+ empty • very long (10000+ chars) • boundary (zero, negative, unicode, HTML, special chars) • invalid formats (malformed email/url/number, SQL, script tags) • invalid combinations (mutually exclusive toggles together, conflicting modes) • out-of-range (far dates, quantities beyond limits, excess decimals) • dependent-UI stress (flip a control that reveals more, attack those too).
9
+
10
+ **Prefer scenarios that:**
11
+ - Push every reachable control to a different bad-data category, then commit
12
+ - Trigger a conditional section, attack revealed and base controls together, then commit
13
+ - Combine mutually exclusive control states with invalid values, then commit
14
+
15
+ End each scenario with the state **committed** (saved, applied, sent, triggered). A scenario that enters bad data then cancels or navigates away reveals nothing — the application never received the payload.
16
+
17
+ Skip the Menu/Navigation section — we are testing THIS page.
@@ -0,0 +1,15 @@
1
+ <container_rules>
2
+ Container CSS must be a SINGLE semantic selector — one class, one id, or one attribute. No spaces, no combinators, no descendant paths.
3
+
4
+ - INVALID: bare tags (`div`, `section`, `nav`), combinators (`div > .content`, `.a .b`), layout/utility classes (flex-, col-, mt-, p-, bg-, text-, items-, rounded-)
5
+ - VALID: semantic class names that describe what the section IS (`.product-list`, `.sidebar-menu`, `.user-profile`, `.search-results`), semantic roles (`[role="dialog"]`), semantic ids (`#main-content`)
6
+
7
+ The container must uniquely identify a semantic wrapper, not a path through the DOM.
8
+ </container_rules>
9
+
10
+ <css_selector_rules>
11
+ CSS selectors inside the UI map must point to the actual interactive element (input, button, a, select), not to wrapper divs.
12
+
13
+ - Prefer distinguishing attributes on the interactive element (`type`, `value`, `name`, `href`, `aria-label`) over wrapper ids.
14
+ - For buttons with similar text, include `type` or `value` or form context to stay unique.
15
+ </css_selector_rules>
@@ -0,0 +1,12 @@
1
+ <section_example>
2
+ ## List
3
+
4
+ Product catalog showing available items with sorting and filtering.
5
+
6
+ > Container: '.product-list'
7
+
8
+ | Element | ARIA | CSS | eidx |
9
+ | 'Sort by price' | { role: 'button', text: 'Sort by price' } | '.sort-btn' | 3 |
10
+ | 'Add to cart' | { role: 'button', text: 'Add to cart' } | '.add-btn' | 4 |
11
+ | 'Product name' | { role: 'link', text: 'Widget Pro' } | 'a.product-link' | 5 |
12
+ </section_example>
@@ -1,6 +1,7 @@
1
1
  import { tool } from 'ai';
2
2
  import dedent from 'dedent';
3
3
  import { z } from 'zod';
4
+ import { ActionResult } from '../../action-result.ts';
4
5
  import { actionRule, locatorRule, sectionContextRule } from '../rules.ts';
5
6
  import { createAgentTools, createCodeceptJSTools } from '../tools.ts';
6
7
  import { type Constructor, type ModeContext, debugLog } from './mixin.ts';
@@ -9,13 +10,19 @@ export function WithWebMode<T extends Constructor>(Base: T) {
9
10
  return class extends Base {
10
11
  webModeTools(ctx: ModeContext): Record<string, any> {
11
12
  const explorer = ctx.explorBot.getExplorer();
13
+ const stateManager = explorer.getStateManager();
12
14
  const codeceptTools = createCodeceptJSTools(explorer, ctx.task);
13
15
  const agentTools = createAgentTools({
14
16
  explorer,
15
17
  researcher: ctx.explorBot.agentResearcher(),
16
18
  navigator: ctx.explorBot.agentNavigator(),
19
+ experienceTracker: stateManager.getExperienceTracker(),
20
+ getState: () => {
21
+ const state = stateManager.getCurrentState();
22
+ return state ? ActionResult.fromState(state) : null;
23
+ },
17
24
  });
18
- const { see, context, visualClick } = agentTools;
25
+ const { see, context, visualClick, learn_experience } = agentTools;
19
26
 
20
27
  return {
21
28
  navigate: tool({
@@ -96,6 +103,7 @@ export function WithWebMode<T extends Constructor>(Base: T) {
96
103
  see,
97
104
  context,
98
105
  visualClick,
106
+ learn_experience,
99
107
  };
100
108
  }
101
109
 
@@ -22,6 +22,7 @@ export class Historian {
22
22
  private experienceTracker: ExperienceTracker;
23
23
  private reporter?: Reporter;
24
24
  private stateManager?: StateManager;
25
+ private savedFiles = new Set<string>();
25
26
 
26
27
  constructor(provider: Provider, experienceTracker?: ExperienceTracker, reporter?: Reporter, stateManager?: StateManager) {
27
28
  this.provider = provider;
@@ -30,6 +31,10 @@ export class Historian {
30
31
  this.stateManager = stateManager;
31
32
  }
32
33
 
34
+ getSavedFiles(): string[] {
35
+ return [...this.savedFiles];
36
+ }
37
+
33
38
  async saveSession(task: Task, initialState: ActionResult, conversation: Conversation): Promise<void> {
34
39
  debugLog('Saving session experience');
35
40
 
@@ -433,6 +438,7 @@ export class Historian {
433
438
  const filename = plan.title.replace(/[^a-zA-Z0-9]/g, '_').toLowerCase();
434
439
  const filePath = join(testsDir, `${filename}.js`);
435
440
  writeFileSync(filePath, lines.join('\n'));
441
+ this.savedFiles.add(filePath);
436
442
 
437
443
  tag('substep').log(`Saved plan tests to: ${filePath}`);
438
444
  return filePath;
@@ -447,6 +453,7 @@ export class Historian {
447
453
  }
448
454
 
449
455
  writeFileSync(filePath, content);
456
+ this.savedFiles.add(filePath);
450
457
  tag('substep').log(`Updated test file with healed steps: ${filePath}`);
451
458
  }
452
459
 
package/src/ai/pilot.ts CHANGED
@@ -1,8 +1,9 @@
1
1
  import { tool } from 'ai';
2
2
  import dedent from 'dedent';
3
3
  import { z } from 'zod';
4
- import type { ActionResult } from '../action-result.ts';
4
+ import { ActionResult } from '../action-result.ts';
5
5
  import { ConfigParser } from '../config.ts';
6
+ import { type ExperienceTracker, renderExperienceToc } from '../experience-tracker.ts';
6
7
  import type Explorer from '../explorer.ts';
7
8
  import { type Test, TestResult } from '../test-plan.ts';
8
9
  import { collectInteractiveNodes, detectFocusArea, extractFocusedElement } from '../utils/aria.ts';
@@ -28,12 +29,14 @@ export class Pilot implements Agent {
28
29
  private researcher: Researcher;
29
30
  private explorer: Explorer;
30
31
  private fisherman: Fisherman | null = null;
32
+ private experienceTracker: ExperienceTracker | null;
31
33
 
32
- constructor(provider: Provider, agentTools: any, researcher: Researcher, explorer: Explorer) {
34
+ constructor(provider: Provider, agentTools: any, researcher: Researcher, explorer: Explorer, experienceTracker?: ExperienceTracker) {
33
35
  this.provider = provider;
34
36
  this.agentTools = agentTools;
35
37
  this.researcher = researcher;
36
38
  this.explorer = explorer;
39
+ this.experienceTracker = experienceTracker || null;
37
40
  }
38
41
 
39
42
  setFisherman(fisherman: Fisherman): void {
@@ -376,7 +379,15 @@ export class Pilot implements Agent {
376
379
 
377
380
  private async sendToPilot(userText: string, functionId: string, opts: { tools?: boolean; maxToolRoundtrips?: number; task?: Test } = {}): Promise<string> {
378
381
  debugLog(`sendToPilot: ${functionId}, tools: ${!!opts.tools}, roundtrips: ${opts.maxToolRoundtrips ?? 0}`);
379
- this.conversation!.addUserText(userText);
382
+
383
+ let finalUserText = userText;
384
+ if (opts.tools) {
385
+ const tocBlock = this.getExperienceToc();
386
+ if (tocBlock) {
387
+ finalUserText = `${tocBlock}\n\n${userText}`;
388
+ }
389
+ }
390
+ this.conversation!.addUserText(finalUserText);
380
391
  let tools = opts.tools ? this.agentTools : undefined;
381
392
 
382
393
  if (opts.tools && opts.task) {
@@ -391,6 +402,15 @@ export class Pilot implements Agent {
391
402
  return result?.response?.text || '';
392
403
  }
393
404
 
405
+ private getExperienceToc(): string {
406
+ if (!this.experienceTracker) return '';
407
+ const state = this.explorer.getStateManager().getCurrentState();
408
+ if (!state) return '';
409
+ const actionResult = ActionResult.fromState(state);
410
+ const toc = this.experienceTracker.getExperienceTableOfContents(actionResult);
411
+ return renderExperienceToc(toc);
412
+ }
413
+
394
414
  private buildPreconditionTool(task: Test) {
395
415
  return {
396
416
  precondition: tool({
@@ -27,6 +27,11 @@ function getFingerprintWorker(): Worker {
27
27
  return fingerprintWorker;
28
28
  }
29
29
 
30
+ export function clearResearchCache(): void {
31
+ for (const key of Object.keys(memoryCache)) delete memoryCache[key];
32
+ for (const key of Object.keys(memoryCacheTimestamps)) delete memoryCacheTimestamps[key];
33
+ }
34
+
30
35
  export function getCachedResearch(hash: string): string {
31
36
  if (!hash) return '';
32
37
  const now = Date.now();
@@ -3,10 +3,12 @@ import { ActionResult, type Diff } from '../../action-result.js';
3
3
  import type Explorer from '../../explorer.ts';
4
4
  import type { StateManager } from '../../state-manager.js';
5
5
  import { WebPageState } from '../../state-manager.js';
6
- import { diffAriaSnapshots } from '../../utils/aria.ts';
6
+ import { detectFocusArea, diffAriaSnapshots } from '../../utils/aria.ts';
7
7
  import { executionController } from '../../execution-controller.ts';
8
8
  import { tag } from '../../utils/logger.js';
9
+ import { mdq } from '../../utils/markdown-query.ts';
9
10
  import type { Provider } from '../provider.js';
11
+ import { getCachedResearch, saveResearch } from './cache.ts';
10
12
  import { type Constructor, debugLog } from './mixin.ts';
11
13
  import { type ResearchElement, parseResearchSections } from './parser.ts';
12
14
  import type { ResearchResult } from './research-result.ts';
@@ -76,6 +78,55 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
76
78
  }
77
79
  }
78
80
 
81
+ async researchOverlay(current: ActionResult, previous: ActionResult, pageStateHash: string): Promise<string | null> {
82
+ const focusArea = detectFocusArea(current.ariaSnapshot);
83
+ if (!focusArea.detected || !focusArea.name) return null;
84
+ if (focusArea.type !== 'dialog' && focusArea.type !== 'modal') return null;
85
+
86
+ const cached = getCachedResearch(pageStateHash);
87
+ if (!cached) return null;
88
+
89
+ const escaped = focusArea.name.replace(/"/g, '\\"');
90
+ if (mdq(cached).query(`section3(~"${escaped}")`).count() > 0) {
91
+ debugLog(`Overlay "${focusArea.name}" already in cached research, skipping`);
92
+ return null;
93
+ }
94
+
95
+ const diff = await current.diff(previous);
96
+ await diff.calculate();
97
+
98
+ if (!diff.ariaChanged && diff.htmlParts.length === 0) {
99
+ debugLog(`No diff between current and previous state for overlay "${focusArea.name}"`);
100
+ return null;
101
+ }
102
+
103
+ const alreadyExpanded = this._summarizeExpanded(
104
+ parseResearchSections(cached)
105
+ .filter((s) => s.elements.length > 0)
106
+ .map((s) => s.rawMarkdown)
107
+ );
108
+
109
+ tag('substep').log(`Researching overlay: ${focusArea.name}`);
110
+ const sectionMarkdown = await this._analyzeExpandedAction('', focusArea.name, diff, alreadyExpanded);
111
+ if (!sectionMarkdown) {
112
+ debugLog(`Overlay "${focusArea.name}" produced no meaningful expansion`);
113
+ return null;
114
+ }
115
+
116
+ const extQuery = mdq(cached).query('section1(~"Extended Research")');
117
+ let updated: string;
118
+ if (extQuery.count() > 0) {
119
+ const existing = extQuery.text().trimEnd();
120
+ updated = extQuery.replace(`${existing}\n\n${sectionMarkdown}\n`);
121
+ } else {
122
+ updated = `${cached.trimEnd()}\n\n# Extended Research\n\n${sectionMarkdown}\n`;
123
+ }
124
+
125
+ saveResearch(pageStateHash, updated);
126
+ tag('substep').log(`Overlay research appended: ${focusArea.name}`);
127
+ return sectionMarkdown;
128
+ }
129
+
79
130
  private async _discoverExpandables(researchText: string): Promise<ExpandableElement[]> {
80
131
  const allElements = new Map<string, ExpandableElement>();
81
132
  for (const section of parseResearchSections(researchText)) {
@@ -314,8 +365,27 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
314
365
  private async _analyzeExpandedAction(code: string, description: string, diff: Diff, alreadyExpanded: string[]): Promise<string | null> {
315
366
  const alreadyHint = alreadyExpanded.length > 0 ? `\nAlready expanded sections:\n${alreadyExpanded.join('\n')}` : '';
316
367
 
368
+ let intro: string;
369
+ if (code) {
370
+ intro = `An action on "${description}" (\`${code}\`) revealed new UI content.`;
371
+ } else {
372
+ intro = `An overlay "${description}" appeared on the page.`;
373
+ }
374
+
375
+ let actionBlock = '';
376
+ if (code) {
377
+ actionBlock = dedent`
378
+ Action:
379
+
380
+ \`\`\`js
381
+ ${code}
382
+ \`\`\`
383
+
384
+ `;
385
+ }
386
+
317
387
  const prompt = dedent`
318
- An action on "${description}" (\`${code}\`) revealed new UI content.
388
+ ${intro}
319
389
  Analyze the changes and produce a UI map section.
320
390
 
321
391
  ARIA changes:
@@ -329,13 +399,7 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
329
399
 
330
400
  ### <Short descriptive name>
331
401
 
332
- Action:
333
-
334
- \`\`\`js
335
- ${code}
336
- \`\`\`
337
-
338
- <One sentence: what appeared — dropdown menu, modal, tab content, expanded panel, etc.>
402
+ ${actionBlock}<One sentence: what appeared — dropdown menu, modal, tab content, expanded panel, etc.>
339
403
 
340
404
  | Element | ARIA | CSS |
341
405
  |---------|------|-----|
@@ -409,4 +473,5 @@ interface ExpandableElement extends ResearchElement {
409
473
 
410
474
  export interface DeepAnalysisMethods {
411
475
  performDeepAnalysis(state: WebPageState, result: ResearchResult): Promise<void>;
476
+ researchOverlay(current: ActionResult, previous: ActionResult, pageStateHash: string): Promise<string | null>;
412
477
  }
@@ -0,0 +1,122 @@
1
+ import dedent from 'dedent';
2
+ import type { ActionResult } from '../../action-result.js';
3
+ import { executionController } from '../../execution-controller.ts';
4
+ import type Explorer from '../../explorer.ts';
5
+ import type { StateManager } from '../../state-manager.js';
6
+ import { tag } from '../../utils/logger.js';
7
+ import { RulesLoader } from '../../utils/rules-loader.ts';
8
+ import type { Provider } from '../provider.js';
9
+ import { locatorRule as generalLocatorRuleText } from '../rules.js';
10
+ import type { Constructor } from './mixin.ts';
11
+
12
+ export interface SectionMethods {
13
+ researchBySections(): Promise<string>;
14
+ }
15
+
16
+ export function WithSections<T extends Constructor>(Base: T) {
17
+ return class extends Base {
18
+ declare explorer: Explorer;
19
+ declare provider: Provider;
20
+ declare stateManager: StateManager;
21
+ declare actionResult: ActionResult | undefined;
22
+
23
+ async researchBySections(): Promise<string> {
24
+ const ariaSnapshot = this.actionResult?.getCompactARIA() || '';
25
+ const configured = (this as any).getConfiguredSections() as Record<string, string>;
26
+ const focusCss = await this._detectFocusCss();
27
+
28
+ let targets: Array<[string, string]>;
29
+ if (focusCss) {
30
+ targets = [['Focus', `element bounded by CSS container '${focusCss}'`]];
31
+ tag('info').log(`Focus element detected via selector '${focusCss}', researching focused area only`);
32
+ } else {
33
+ targets = Object.entries(configured);
34
+ tag('info').log(`Splitting research into ${targets.length} per-section requests`);
35
+ }
36
+
37
+ const parts: string[] = [];
38
+ for (const [name, description] of targets) {
39
+ if (executionController.isInterrupted()) break;
40
+ const text = await this._researchSingleSection(name, description, ariaSnapshot, focusCss);
41
+ if (!text) continue;
42
+ const trimmed = text.trim();
43
+ if (trimmed === 'NOT_PRESENT' || trimmed.startsWith('NOT_PRESENT')) continue;
44
+ parts.push(trimmed);
45
+ }
46
+
47
+ if (parts.length === 0) {
48
+ throw new Error('Per-section research produced no sections — AI responses all empty or NOT_PRESENT');
49
+ }
50
+
51
+ let merged = parts.join('\n\n');
52
+ if (focusCss) merged += '\n\n> Focused: Focus';
53
+ return merged;
54
+ }
55
+
56
+ private async _detectFocusCss(): Promise<string | null> {
57
+ const focusSections = (this.explorer.getConfig().ai?.agents?.researcher as any)?.focusSections as string[] | undefined;
58
+ if (!focusSections?.length) return null;
59
+
60
+ for (const css of focusSections) {
61
+ const count = await this.explorer.playwrightLocatorCount((page: any) => page.locator(css)).catch(() => 0);
62
+ if (count > 0) return css;
63
+ }
64
+ return null;
65
+ }
66
+
67
+ private async _researchSingleSection(name: string, description: string, ariaSnapshot: string, focusCss: string | null): Promise<string> {
68
+ const currentUrl = this.stateManager.getCurrentState()?.url || '';
69
+ const rules = RulesLoader.loadRules('researcher', ['ui-map-table', 'list-element', 'container-rules'], currentUrl);
70
+ const url = this.actionResult?.url || 'Unknown';
71
+ const title = this.actionResult?.title || 'Unknown';
72
+
73
+ let focusHint = '';
74
+ if (focusCss) {
75
+ focusHint = dedent`
76
+ The user's focus is the element matching CSS '${focusCss}'.
77
+ Use that CSS as the Container for this section.
78
+ `;
79
+ }
80
+
81
+ const prompt = dedent`
82
+ <task>
83
+ Identify the "${name}" section on this page: ${description}
84
+ If this section is NOT present on the page, respond with ONLY: NOT_PRESENT
85
+ Otherwise output only this single section in the format below.
86
+ ${focusHint}
87
+ </task>
88
+
89
+ <section_format>
90
+ ## ${name}
91
+
92
+ > Container: '.semantic-container-selector'
93
+
94
+ | Element | ARIA | CSS | eidx |
95
+ </section_format>
96
+
97
+ <rules>
98
+ - Every element with eidx MUST appear in the table.
99
+ - Every row needs CSS; ARIA may be "-" for icon-only buttons.
100
+ - ARIA locator JSON uses keys "role" and "text" (NOT "name").
101
+ </rules>
102
+
103
+ ${generalLocatorRuleText}
104
+
105
+ ${rules}
106
+
107
+ URL: ${url}
108
+ Title: ${title}
109
+
110
+ <aria>
111
+ ${ariaSnapshot}
112
+ </aria>
113
+ `;
114
+
115
+ const conversation = this.provider.startConversation((this as any).getSystemMessage(), 'researcher');
116
+ conversation.addUserText(prompt);
117
+
118
+ const result = await this.provider.invokeConversation(conversation, undefined, { agentName: 'researcher' });
119
+ return result?.response.text || '';
120
+ }
121
+ };
122
+ }