explorbot 0.4.4 → 0.4.5

package/src/ai/tools.ts

@@ -12,6 +12,7 @@ import { cleanHtmlSnippet } from '../utils/html.ts';
 import { createDebug, tag } from '../utils/logger.js';
 import { compactErrorMessage, normalizeInlineText, truncate } from '../utils/strings.ts';
 import { pause } from '../utils/loop.js';
+import { ariaRefSelector, describeRef, refIsGone } from '../utils/aria-ref.ts';
 import { WebElement } from '../utils/web-element.ts';
 import type { ToolDeps } from './agent.ts';
 import { Navigator } from './navigator.ts';
@@ -66,7 +67,6 @@ export function createCodeceptJSTools({ explorer, stateManager }: ToolDeps, task
           2. I.click(ARIA, container) - e.g. I.click({"role":"button","text":"Save"}, ".modal")
           3. I.click(CSS, container) - e.g. I.click("#btn", ".modal")
           4. I.click(CSS) or I.click(XPath) - when locator already includes context (ID, XPath)
-          5. I.clickXY(x, y) - coordinates fallback
           After a result reporting multiple matches, reuse that locator with step.opts({ elementIndex: N }) as the last argument.
         `),
         explanation: z.string().describe('Why you are clicking this element'),
@@ -79,11 +79,21 @@ export function createCodeceptJSTools({ explorer, stateManager }: ToolDeps, task
           return failedToolResult('click', 'No commands provided');
         }
 
-        const invalidCommands = rawCommands.map((cmd) => cmd.trim()).filter((cmd) => cmd.startsWith('I.') && !cmd.startsWith('I.click'));
+        const trimmedCommands = rawCommands.map((cmd) => cmd.trim());
+        const coordinateCommands = trimmedCommands.filter((cmd) => cmd.startsWith('I.clickXY'));
+
+        if (coordinateCommands.length > 0) {
+          activeNote.commit(TestResult.FAILED);
+          return failedToolResult('click', `Coordinate commands are not locators: ${coordinateCommands.join(', ')}. A coordinate click always runs, so it cannot tell you whether the element was there.`, {
+            suggestion: 'Name the element instead. Use visualClick() when you can see the target but cannot address it, or form() for a deliberate coordinate click such as dismissing a layer.',
+          });
+        }
+
+        const invalidCommands = trimmedCommands.filter((cmd) => cmd.startsWith('I.') && !cmd.startsWith('I.click'));
 
         if (invalidCommands.length > 0) {
           activeNote.commit(TestResult.FAILED);
-          return failedToolResult('click', `Invalid commands: ${invalidCommands.join(', ')}. Click tool only accepts I.click() or I.clickXY() commands.`, {
+          return failedToolResult('click', `Invalid commands: ${invalidCommands.join(', ')}. Click tool only accepts I.click() commands.`, {
             suggestion: 'Use form() tool for typing text or multiple actions, or exitIframe() to leave iframe context.',
           });
         }
@@ -111,6 +121,17 @@ export function createCodeceptJSTools({ explorer, stateManager }: ToolDeps, task
 
           if (success) {
             const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, command);
+
+            if (!hasObservablePageChange(toolResult)) {
+              activeNote.commit(TestResult.FAILED);
+              return failedToolResult('click', 'Click executed, but no observable page change was captured.', {
+                ...toolResult,
+                attempts,
+                code: command,
+                suggestion: 'Treat the element as not clicked. It may be covered by another layer, disabled, or the locator may have matched a non-interactive ancestor. Re-locate via xpathCheck(), which reports whether the element is covered or offscreen, before retrying.',
+              });
+            }
+
             await commitNote(activeNote, TestResult.PASSED, toolResult, action);
             return successToolResult('click', { ...toolResult, attempts, code: command }, action);
           }
@@ -461,10 +482,18 @@ export function createRefTools({ explorer, stateManager }: ToolDeps, task: Task)
       }),
       execute: async ({ ref, element }) => {
         const activeNote = task.startNote(`Click ${element}`);
+
+        if (await refIsGone(explorer, ref)) {
+          activeNote.commit(TestResult.FAILED);
+          return failedToolResult('clickRef', `Ref ${ref} names no element on the page any more.`, {
+            suggestion: 'The page has been rebuilt since you were given that ref. Call context() and use the ref it gives, or fall back to click() with a locator.',
+          });
+        }
+
         const previousState = ActionResult.fromState(stateManager.getCurrentState()!);
         const action = explorer.action();
         const named = await describeRef(explorer, ref);
-        const run = `I.usePlaywrightTo(${JSON.stringify(`click ${element}`)}, async ({ page }) => page.locator(${JSON.stringify(`aria-ref=${ref}`)}).click())`;
+        const run = `I.usePlaywrightTo(${JSON.stringify(`click ${element}`)}, async ({ page }) => page.locator(${JSON.stringify(ariaRefSelector(ref))}).click())`;
 
         if (!(await action.attempt(run, `Click ${element}`))) {
           activeNote.commit(TestResult.FAILED);
@@ -1169,21 +1198,6 @@ export async function commitNote(activeNote: any, result: TestResult, toolResult
   activeNote.commit(result);
 }
 
-async function describeRef(explorer: any, ref: string): Promise<{ role: string; text: string } | null> {
-  return Promise.resolve(
-    explorer?.withPage?.((page: any) =>
-      page.locator(`aria-ref=${ref}`).evaluate((el: any) => {
-        const tag = el.tagName.toLowerCase();
-        const roles: Record<string, string> = { a: 'link', button: 'button', select: 'combobox', textarea: 'textbox' };
-        const role = el.getAttribute('role') || roles[tag] || tag;
-        const text = (el.getAttribute('aria-label') || el.innerText || el.value || '').trim().split('\n')[0];
-        if (!text) return null;
-        return { role, text };
-      })
-    )
-  ).catch(() => null);
-}
-
 async function hasFocusedElement(explorer: any): Promise<boolean> {
   return explorer.withPage((page: any) => page.evaluate(() => !!document.activeElement && document.activeElement !== document.body)).catch(() => true);
 }
@@ -1238,6 +1252,7 @@ function hasObservablePageChange(data?: Record<string, any>): boolean {
   if (data.pageDiff.urlChanged === true) return true;
   if (data.pageDiff.ariaChanges) return true;
   if (data.pageDiff.messages?.length) return true;
+  if (data.pageDiff.requests?.length) return true;
   return Array.isArray(data.pageDiff.htmlParts) && data.pageDiff.htmlParts.length > 0;
 }
 
@@ -1248,12 +1263,11 @@ export async function failedToolResult(action: string, message: string, data?: R
   }
 
   const errorTexts = [message, ...(data?.attempts?.map((a: any) => a.error || '') || [])];
-  const hasMultipleElements = errorTexts.some((t: string) => t.toLowerCase().includes(MULTIPLE_ELEMENTS_PATTERN));
-  const multipleElementsSuggestion = hasMultipleElements ? getMultipleElementsSuggestion() : null;
-  if (multipleElementsSuggestion) {
-    result.suggestion = multipleElementsSuggestion;
+  if (errorTexts.some((t: string) => t.toLowerCase().includes(MULTIPLE_ELEMENTS_PATTERN))) {
+    const matched = await extractWebElements(error);
+    result.suggestion = getMultipleElementsSuggestion(matched);
     result.multipleElementsDetected = true;
-    result.elements = await formatMatchedElements(error);
+    result.elements = formatElementList(matched);
     return result;
   }
 
@@ -1266,11 +1280,16 @@ export async function failedToolResult(action: string, message: string, data?: R
   return result;
 }
 
-function getMultipleElementsSuggestion(): string {
+function getMultipleElementsSuggestion(matched: MatchedElement[] | null): string {
+  const visible = (matched || []).filter((element) => element.visible !== false);
+  let onlyVisible = '';
+  if (matched && visible.length === 1) onlyVisible = `\nOnly element ${matched.indexOf(visible[0]) + 1} is on screen, so that is the one to act on.`;
+
   return dedent`
     Multiple elements matched your locator, so that command did nothing — it selected no element and acted on none.
-    Read the numbered elements list and click the one you meant by its number:
+    Read the numbered elements list and act on the one you meant by its number:
     reuse the same locator with step.opts({ elementIndex: N }) as the last argument.
+    A match reported as not visible can never be acted on — pick one that is.${onlyVisible}
     If none of them is the element you want, narrow the locator with a container or its full unique text.
     If the list is missing, call xpathCheck() to see what the locator matches.
   `;
@@ -1294,7 +1313,7 @@ export function clickFailureSuggestion(attempts: Array<{ error?: string }>): str
   }
 
   if (errors.some((e) => e.includes('is not visible'))) {
-    return 'Element is in the DOM but not visible. Reveal it first — scroll to it, expand its section, or open the panel holding it.';
+    return 'Element is in the DOM but not visible. Reveal it — scroll to it, expand its section, open the panel holding it — or, when the page carries several copies of the same control, target the one that is on screen.';
   }
 
   if (errors.some((e) => e.includes('SyntaxError'))) {
@@ -1319,19 +1338,20 @@ const MAX_DISAMBIGUATE_TEXT = 80;
 const MAX_DISAMBIGUATE_HTML = 300;
 const MULTIPLE_ELEMENTS_PATTERN = 'multiple elements';
 
-async function extractWebElements(error: Error | null | undefined): Promise<Array<{ xpath: string; html: string; text: string }> | null> {
+async function extractWebElements(error: Error | null | undefined): Promise<MatchedElement[] | null> {
   if (!error || error.name !== 'MultipleElementsFound') return null;
 
-  const elements = (error as any).webElements as Array<{ toAbsoluteXPath: () => Promise<string>; toOuterHTML: () => Promise<string>; getText: () => Promise<string | null> }> | undefined;
+  const elements = (error as any).webElements as Array<{ toAbsoluteXPath: () => Promise<string>; toOuterHTML: () => Promise<string>; getText: () => Promise<string | null>; isVisible?: () => Promise<boolean> }> | undefined;
   if (!elements?.length) return null;
 
-  const result: Array<{ xpath: string; html: string; text: string }> = [];
+  const result: MatchedElement[] = [];
   for (let i = 0; i < Math.min(elements.length, MAX_DISAMBIGUATE_ELEMENTS); i++) {
     try {
       const xpath = await elements[i].toAbsoluteXPath();
       const html = truncate(cleanHtmlSnippet(await elements[i].toOuterHTML()), MAX_DISAMBIGUATE_HTML);
       const text = truncate(normalizeInlineText((await elements[i].getText()) || ''), MAX_DISAMBIGUATE_TEXT);
-      result.push({ xpath, html, text });
+      const visible = await Promise.resolve(elements[i].isVisible?.()).catch(() => undefined);
+      result.push({ xpath, html, text, visible });
     } catch (e) {
       debugLog('Failed to get details for element %d: %s', i, e);
     }
@@ -1339,14 +1359,20 @@ async function extractWebElements(error: Error | null | undefined): Promise<Arra
   return result.length > 0 ? result : null;
 }
 
-function formatElementList(details: Array<{ xpath: string; html: string; text: string }>): string {
-  return details.map((el, i) => `Element ${i + 1}:\nText: "${el.text}"\nXPath: ${el.xpath}\nHTML: ${el.html}`).join('\n\n');
+function formatElementList(matched: MatchedElement[] | null): string {
+  if (!matched) return 'Could not fetch element details. Repeat the action to get better info.';
+  return matched
+    .map((el, i) => {
+      const lines = [`Element ${i + 1}:`, `Text: "${el.text}"`];
+      if (el.visible !== undefined) lines.push(`Visible: ${el.visible}`);
+      lines.push(`XPath: ${el.xpath}`, `HTML: ${el.html}`);
+      return lines.join('\n');
+    })
+    .join('\n\n');
 }
 
 export async function formatMatchedElements(error: Error | null | undefined): Promise<string | null> {
-  const details = await extractWebElements(error);
-  if (!details) return 'Could not fetch element details. Repeat the action to get better info.';
-  return formatElementList(details);
+  return formatElementList(await extractWebElements(error));
 }
 
 function getNotFoundSuggestion(errorMessage: string): string | null {
@@ -1362,3 +1388,10 @@ function getNotFoundSuggestion(errorMessage: string): string | null {
     4. Prefer ARIA locators: { "role": "button", "text": "visible text" }
   `;
 }
+
+interface MatchedElement {
+  xpath: string;
+  html: string;
+  text: string;
+  visible?: boolean;
+}

package/src/application-spec.ts

@@ -17,7 +17,7 @@ export class ApplicationSpec {
   }
 
   renderFor(state: ActionResult): string {
-    const relevant = this.pages.filter((page) => state.isMatchedBy({ url: page.url }));
+    const relevant = this.relevantPages(state);
     if (relevant.length === 0) return '';
 
     tag('operation').log(`Found application specification for ${state.url}`);
@@ -30,6 +30,10 @@ export class ApplicationSpec {
     `;
   }
 
+  matchedUrls(state: ActionResult): string[] {
+    return this.relevantPages(state).map((page) => page.url);
+  }
+
   get pageCount(): number {
     return this.pages.length;
   }
@@ -75,12 +79,26 @@ export class ApplicationSpec {
   }
 
   private resolveSourcePath(sourcePath: string): string {
-    if (path.isAbsolute(sourcePath)) return path.resolve(sourcePath);
-    const configParser = ConfigParser.getInstance();
-    return path.resolve(configParser.resolveProjectDir(sourcePath));
+    return resolveSpecSource(sourcePath);
+  }
+
+  private relevantPages(state: ActionResult): ApplicationSpecPage[] {
+    return this.pages.filter((page) => state.isMatchedBy({ url: page.url }));
   }
 }
 
+export function resolveSpecBundlePath(sourcePath: string): string | null {
+  const resolved = resolveSpecSource(sourcePath);
+  if (!existsSync(resolved)) return null;
+  if (statSync(resolved).isDirectory()) return resolved;
+  return path.dirname(resolved);
+}
+
+function resolveSpecSource(sourcePath: string): string {
+  if (path.isAbsolute(sourcePath)) return path.resolve(sourcePath);
+  return path.resolve(ConfigParser.getInstance().resolveProjectDir(sourcePath));
+}
+
 interface ApplicationSpecPage {
   url: string;
   content: string;

package/src/config.ts

@@ -130,6 +130,11 @@ interface RerunnerAgentConfig extends AgentConfig {
 interface PlannerAgentConfig extends AgentConfig {
   styles?: string[];
   stylesDir?: string;
+  docsWeight?: number;
+}
+
+interface ScoutAgentConfig extends AgentConfig {
+  dirs?: string[];
 }
 
 interface ScreencastConfig {
@@ -154,6 +159,7 @@ interface AgentsConfig {
   quartermaster?: AgentConfig;
   historian?: HistorianAgentConfig;
   fisherman?: AgentConfig;
+  scout?: ScoutAgentConfig;
   chief?: AgentConfig;
   curler?: AgentConfig;
   rerunner?: RerunnerAgentConfig;
@@ -293,6 +299,7 @@ export type {
   ResearcherAgentConfig,
   NavigatorAgentConfig,
   PlannerAgentConfig,
+  ScoutAgentConfig,
   RerunnerAgentConfig,
   HealRecipe,
   Hook,

package/src/explorbot.ts

@@ -14,12 +14,15 @@ import { AIProvider } from './ai/provider.ts';
 import { Quartermaster } from './ai/quartermaster.ts';
 import { Rerunner } from './ai/rerunner.ts';
 import { Researcher } from './ai/researcher.ts';
+import { Scout } from './ai/scout.ts';
+import { loadScoutCorpus } from './ai/scout/tools.ts';
 import { SessionAnalyst } from './ai/session-analyst.ts';
 import { Tester } from './ai/tester.ts';
 import { createAgentTools } from './ai/tools.ts';
 import { ApiClient } from './api/api-client.ts';
 import { RequestStore } from './api/request-store.ts';
 import { loadSpec } from './api/spec-reader.ts';
+import { resolveSpecBundlePath } from './application-spec.ts';
 import type { ExplorbotConfig, ReporterConfig } from './config.js';
 import { ConfigParser } from './config.ts';
 import { ExperienceTracker } from './experience-tracker.ts';
@@ -231,6 +234,8 @@ export class ExplorBot {
       this.agents.planner = this.createAgent((deps) => new Planner(deps, this.agentResearcher()));
       const fisherman = this.agentFisherman();
       if (fisherman) this.agents.planner.setFisherman(fisherman);
+      const scout = this.agentScout();
+      if (scout) this.agents.planner.setScout(scout);
     }
     return this.agents.planner;
   }
@@ -367,6 +372,37 @@ export class ExplorBot {
     return this.agents.fisherman;
   }
 
+  agentScout(): Scout | null {
+    const scoutConfig = this.config.ai?.agents?.scout;
+    if (scoutConfig?.enabled !== true) return null;
+
+    const dirs = this.scoutCorpusDirs();
+    if (dirs.length === 0) {
+      tag('warning').log('Scout enabled but no documentation found — set --spec or ai.agents.scout.dirs');
+      return null;
+    }
+
+    return (this.agents.scout ||= this.createAgent(({ ai }) => new Scout(ai, loadScoutCorpus(dirs))));
+  }
+
+  private scoutCorpusDirs(): string[] {
+    const dirs: string[] = [];
+
+    const specPath = this.options.applicationSpec || this.config.dirs?.spec;
+    if (specPath) {
+      const bundle = resolveSpecBundlePath(specPath);
+      const pagesDir = bundle && path.join(bundle, 'pages');
+      if (pagesDir && existsSync(pagesDir)) dirs.push(pagesDir);
+    }
+
+    for (const dir of this.config.ai?.agents?.scout?.dirs || []) {
+      const resolved = this.configParser.resolveProjectDir(dir);
+      if (existsSync(resolved)) dirs.push(resolved);
+    }
+
+    return [...new Set(dirs)];
+  }
+
   getCurrentPlan(): Plan | undefined {
     return this.currentPlan;
   }

package/src/knowledge-tracker.ts

@@ -102,6 +102,10 @@ export class KnowledgeTracker {
     return this.applicationSpec?.renderFor(state) || '';
   }
 
+  applicationSpecUrls(state: ActionResult): string[] {
+    return this.applicationSpec?.matchedUrls(state) || [];
+  }
+
   addKnowledge(urlPattern: string, description: string, opts?: { replace?: boolean }): { filename: string; filePath: string; isNewFile: boolean } {
     const configParser = ConfigParser.getInstance();
     const configPath = configParser.getConfigPath();

package/src/utils/aria-ref.ts

@@ -0,0 +1,61 @@
+import { WebElement } from './web-element.ts';
+
+const REF_LINE_PATTERN = /^(\s*)-\s+(\w+)\s*(?:"([^"]*)")?.*?\[ref=(e\d+)\]/;
+const ARIA_REF_PATTERN = /^(f\d+)?e\d+$/i;
+
+const REF_ROLES: Record<string, string> = { a: 'link', button: 'button', select: 'combobox', textarea: 'textbox' };
+
+export function ariaRefSelector(ref: string): string {
+  return `aria-ref=${ref}`;
+}
+
+export function isAriaRef(ref: string): boolean {
+  return ARIA_REF_PATTERN.test(ref);
+}
+
+export function ariaRefSnapshot(page: any): Promise<string> {
+  return page.locator('body').ariaSnapshot({ mode: 'ai' });
+}
+
+export function parseAriaRefs(ariaSnapshot: string): AriaRefEntry[] {
+  const entries: AriaRefEntry[] = [];
+  for (const line of ariaSnapshot.split('\n')) {
+    const match = line.match(REF_LINE_PATTERN);
+    if (!match) continue;
+    entries.push({ role: match[2], name: match[3] || '', ref: match[4] });
+  }
+  return entries;
+}
+
+export async function elementFromAriaRef(page: any, ref: string): Promise<WebElement | null> {
+  if (!isAriaRef(ref)) return null;
+  return WebElement.fromPlaywrightLocator(page.locator(ariaRefSelector(ref)));
+}
+
+export async function refIsGone(explorer: any, ref: string): Promise<boolean> {
+  const count = () => Promise.resolve(explorer?.withPage?.((page: any) => page.locator(ariaRefSelector(ref)).count())).catch(() => undefined);
+  if ((await count()) !== 0) return false;
+  await Promise.resolve(explorer?.withPage?.(ariaRefSnapshot)).catch(() => null);
+  return (await count()) === 0;
+}
+
+export async function describeRef(explorer: any, ref: string): Promise<{ role: string; text: string } | null> {
+  return Promise.resolve(
+    explorer?.withPage?.((page: any) =>
+      page.locator(ariaRefSelector(ref)).evaluate((el: any, roles: Record<string, string>) => {
+        const tag = el.tagName.toLowerCase();
+        const role = el.getAttribute('role') || roles[tag];
+        if (!role) return null;
+        const text = (el.getAttribute('aria-label') || el.innerText || el.value || '').trim().split('\n')[0];
+        if (!text) return null;
+        return { role, text };
+      }, REF_ROLES)
+    )
+  ).catch(() => null);
+}
+
+export interface AriaRefEntry {
+  role: string;
+  name: string;
+  ref: string;
+}

package/src/utils/aria.ts

@@ -431,9 +431,9 @@ const formatDiffSection = (label: string, items: string[]): string[] => {
   const summary = countBy(items);
   if (summary.size === 0) return [`  ${label}: []`];
 
-  const sorted = Array.from(summary.entries()).sort(([aItem, aCount], [bItem, bCount]) => bCount - aCount || aItem.localeCompare(bItem));
-  const top = sorted.slice(0, TOP_DIFF_ITEMS);
-  const rest = sorted.slice(TOP_DIFF_ITEMS);
+  const ordered = Array.from(summary.entries());
+  const top = ordered.slice(0, TOP_DIFF_ITEMS);
+  const rest = ordered.slice(TOP_DIFF_ITEMS);
 
   const lines = [`  ${label}:`];
   for (const [item, count] of top) {

package/src/utils/web-annotate.ts

@@ -1,27 +1,15 @@
+import { ariaRefSnapshot, parseAriaRefs } from './aria-ref.ts';
 import { ELEMENT_EXTRACTION_CONFIG, getElementDataExtractorSource } from './html.ts';
 import { createDebug } from './logger.js';
 import { WebElement } from './web-element.ts';
 
 const debugLog = createDebug('explorbot:web-annotate');
 
-const REF_LINE_PATTERN = /^(\s*)-\s+(\w+)\s*(?:"([^"]*)")?.*?\[ref=(e\d+)\]/;
-
 const ANNOTATABLE_ROLES = new Set(['button', 'link', 'textbox', 'searchbox', 'checkbox', 'radio', 'switch', 'combobox', 'tab', 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'option', 'slider', 'spinbutton', 'treeitem']);
 
-function parseAriaRefs(ariaSnapshot: string): Array<{ role: string; name: string; ref: string }> {
-  const entries: Array<{ role: string; name: string; ref: string }> = [];
-  for (const line of ariaSnapshot.split('\n')) {
-    const match = line.match(REF_LINE_PATTERN);
-    if (!match) continue;
-    if (!ANNOTATABLE_ROLES.has(match[2])) continue;
-    entries.push({ role: match[2], name: match[3] || '', ref: match[4] });
-  }
-  return entries;
-}
-
 export async function annotatePageElements(page: any): Promise<{ ariaSnapshot: string; elements: WebElement[] }> {
-  const ariaSnapshot: string = await page.locator('body').ariaSnapshot({ mode: 'ai' });
-  const refEntries = parseAriaRefs(ariaSnapshot);
+  const ariaSnapshot: string = await ariaRefSnapshot(page);
+  const refEntries = parseAriaRefs(ariaSnapshot).filter((entry) => ANNOTATABLE_ROLES.has(entry.role));
 
   const byRole = new Map<string, Array<{ name: string; ref: string }>>();
   for (const { role, name, ref } of refEntries) {

package/src/utils/web-element.ts

@@ -125,15 +125,6 @@ export class WebElement {
     return WebElement.fromPlaywrightLocator(page.locator(`[${EXPLORBOT_ATTRS.eidx}="${eidx}"]`));
   }
 
-  static isAriaRef(ref: string): boolean {
-    return /^(f\d+)?e\d+$/i.test(ref);
-  }
-
-  static async fromAriaRef(page: any, ref: string): Promise<WebElement | null> {
-    if (!WebElement.isAriaRef(ref)) return null;
-    return WebElement.fromPlaywrightLocator(page.locator(`aria-ref=${ref}`));
-  }
-
   static async fromEidxList(page: any, eidxList: string[]): Promise<WebElement[]> {
     const validEidxList = eidxList.filter((eidx) => /^e\d+$/i.test(eidx));
     if (validEidxList.length === 0) return [];