explorbot 0.4.1 → 0.4.2

package/src/action.ts

@@ -14,6 +14,7 @@ import { browserErrorMessage, isFatalBrowserError, isNavigationTransitionError }
 import { captureHtmlForSnapshot, htmlCombinedSnapshot, minifyHtml } from './utils/html.js';
 import { createDebug, setStepSpanParent, tag } from './utils/logger.js';
 import { Overlay, OverlayPage } from './utils/overlay.js';
+import type { Region } from './utils/region.js';
 import { sleep, waitForPageReadiness } from './utils/page-readiness.ts';
 import { safeFilename } from './utils/strings.ts';
 import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCodeBlock } from './utils/web-sandbox.ts';
@@ -181,7 +182,7 @@ class Action {
         focusedElement,
         iframeURL: frame ? frame.url?.() || 'iframe' : undefined,
       });
-      if (!frame) await this.detectRegionOfInterest(result).catch((err: Error) => debugLog('Region detection failed:', err.message));
+      if (!frame) await this.detectRegion(result).catch((err: Error) => debugLog('Region detection failed:', err.message));
       this.stateManager.updateState(result, codeBlock);
       return result;
     } catch (err) {
@@ -193,36 +194,36 @@ class Action {
     }
   }
 
-  private async detectRegionOfInterest(result: ActionResult): Promise<void> {
+  private async detectRegion(result: ActionResult): Promise<void> {
     const previousState = this.stateManager.getCurrentState();
     if (!previousState) return;
     const previous = ActionResult.fromState(previousState);
-    const previousOverlay = previous.overlay;
+    const previousRegion = previous.overlay;
     const sameUrl = !!previous.url && result.isSameUrl({ url: previous.url });
     const overlayPage = new OverlayPage(this.playwrightHelper.page);
 
-    if (result.overlay.detected && previousOverlay.detected && previousOverlay.root && previousOverlay.type === result.overlay.type && previousOverlay.name === result.overlay.name) {
-      result.overlay = previousOverlay;
+    if (result.overlay.isModal && previousRegion.isModal && previousRegion.root && previousRegion.type === result.overlay.type && previousRegion.name === result.overlay.name) {
+      result.overlay = previousRegion;
       return;
     }
 
     if (!previous.html) return;
 
     if (previous.html === result.html) {
-      if (sameUrl && previousOverlay.present && previousOverlay.xpath && !result.overlay.detected) result.overlay = previousOverlay;
+      if (sameUrl && previousRegion.isOpen && previousRegion.xpath && !result.overlay.isModal) result.overlay = previousRegion;
       return;
     }
 
-    let carried: Overlay | null = null;
-    if (sameUrl && previousOverlay.present && previousOverlay.xpath) {
-      if (await overlayPage.isStillOpen(previousOverlay)) {
-        carried = previousOverlay;
+    let carried: Region | null = null;
+    if (sameUrl && previousRegion.isOpen && previousRegion.xpath) {
+      if (await overlayPage.isStillOpen(previousRegion)) {
+        carried = previousRegion;
       } else {
-        debugLog(`Region closed: ${previousOverlay.name || previousOverlay.type}`);
-        if (!result.overlay.detected) {
-          const parent = previousOverlay.parent;
+        debugLog(`Region closed: ${previousRegion.name || previousRegion.type}`);
+        if (!result.overlay.isModal) {
+          const parent = previousRegion.parent;
           if (parent?.xpath) {
-            const restored = new Overlay(parent);
+            const restored = Overlay.resolve({ overlay: parent });
             if (await overlayPage.isStillOpen(restored)) result.overlay = restored;
           }
           return;
@@ -239,7 +240,7 @@ class Action {
       previousHtml: previous.html,
     });
 
-    if (result.overlay.detected) {
+    if (result.overlay instanceof Overlay) {
       if (detected) result.overlay = result.overlay.withGeometry(detected);
       return;
     }

package/src/ai/fisherman/request-haul.ts

@@ -0,0 +1,32 @@
+import type { RequestResult } from '../../api/request-result.ts';
+import { type RequestStore, isFailedRequest } from '../../api/request-store.ts';
+
+export class RequestHaul {
+  private start: number;
+
+  constructor(private store: RequestStore) {
+    this.start = store.getMadeRequests().length;
+  }
+
+  requests(): RequestResult[] {
+    return this.store.getMadeRequests().slice(this.start);
+  }
+
+  failed(): RequestResult[] {
+    return this.requests().filter(isFailedRequest);
+  }
+
+  successfulWrites(): RequestResult[] {
+    return this.requests().filter((r) => r.isWrite && !r.error && r.status >= 200 && r.status < 400);
+  }
+
+  byId(): Map<string, RequestResult> {
+    const map = new Map<string, RequestResult>();
+    for (const request of this.successfulWrites()) {
+      const { id } = request.extractIdAndTitle();
+      if (id === undefined) continue;
+      map.set(String(id), request);
+    }
+    return map;
+  }
+}

package/src/ai/fisherman-tools.ts

@@ -6,21 +6,18 @@ import type { RequestResult } from '../api/request-result.ts';
 import type { RequestStore } from '../api/request-store.ts';
 import { extractEndpointDefinition } from '../api/spec-reader.ts';
 import { tag } from '../utils/logger.ts';
-import { RequestMap } from '../utils/request-map.ts';
 import { isDynamicSegment } from '../utils/url-matcher.ts';
+import type { RequestHaul } from './fisherman/request-haul.ts';
 
-export function createFishermanTools(apiClient: ApiClient, requestStore: RequestStore, opts: { spec?: any; baseEndpoint?: string }) {
+export function createFishermanTools(apiClient: ApiClient, requestStore: RequestStore, haul: RequestHaul, opts: { spec?: any; baseEndpoint?: string }) {
   let finished = false;
   let result: FishermanResult | null = null;
-  const ledgerStart = requestStore.getMadeRequests().length;
 
-  const runRequests = () => requestStore.getMadeRequests().slice(ledgerStart);
-  const successfulWrites = () => runRequests().filter((r) => r.isWrite && !r.error && r.status >= 200 && r.status < 400);
-  const getResult = () => result ?? synthesizeResult(runRequests(), successfulWrites(), false);
+  const getResult = () => result ?? synthesizeResult(haul, false);
   const isFinished = () => finished;
   const finishFromText = (text?: string) => {
     finished = true;
-    const synthesized = synthesizeResult(runRequests(), successfulWrites(), true);
+    const synthesized = synthesizeResult(haul, true);
     if (text && synthesized.success) synthesized.summary = text;
     result = synthesized;
   };
@@ -160,32 +157,12 @@ export function createFishermanTools(apiClient: ApiClient, requestStore: Request
           .describe('List of items that could not be created'),
       }),
       execute: async ({ summary, created, failed }) => {
-        const writes = successfulWrites();
-        if (writes.length === 0) {
-          tag('warning').log('Fisherman: finish rejected — no successful write request in this run');
-          return { finished: false, error: 'No successful write request was made in this run, so nothing was created. Keep working, or call stop if the data cannot be prepared.' };
-        }
-
-        const createdRequests = new RequestMap(writes);
-
-        const verified: FishermanResult['created'] = [];
-        for (const item of created) {
-          if (item.id === undefined) {
-            verified.push(item);
-            continue;
-          }
-          const request = createdRequests.get(item.id);
-          if (!request) {
-            tag('warning').log(`Fisherman: dropped unverified created item ${item.type} (id: ${item.id})`);
-            continue;
-          }
-          verified.push({ ...item, request: request.toEndpoint() });
-        }
-        if (verified.length === 0) verified.push(...writes.map(toCreatedItem));
+        const { result: verified, error } = verifyFinish(haul, { summary, created, failed });
+        if (!verified) return { finished: false, error };
 
         tag('success').log(`Fisherman done: ${summary}`);
         finished = true;
-        result = { success: true, summary, created: verified, failed: failed || [] };
+        result = verified;
         return { finished: true };
       },
     }),
@@ -207,8 +184,37 @@ export function createFishermanTools(apiClient: ApiClient, requestStore: Request
   return { tools, getResult, isFinished, finishFromText };
 }
 
-function synthesizeResult(made: RequestResult[], writes: RequestResult[], declaredDone: boolean): FishermanResult {
-  const failures = made.filter((r) => r.status >= 400 || r.error);
+export function verifyFinish(haul: RequestHaul, input: { summary: string; created: FishermanResult['created']; failed?: FishermanResult['failed'] }): { result: FishermanResult | null; error?: string } {
+  const writes = haul.successfulWrites();
+  if (writes.length === 0) {
+    tag('warning').log('Fisherman: finish rejected — no successful write request in this run');
+    return { result: null, error: 'No successful write request was made in this run, so nothing was created. Keep working, or call stop if the data cannot be prepared.' };
+  }
+
+  const createdRequests = haul.byId();
+
+  const verified: FishermanResult['created'] = [];
+  for (const item of input.created) {
+    if (item.id === undefined) {
+      verified.push(item);
+      continue;
+    }
+    const request = createdRequests.get(String(item.id));
+    if (!request) {
+      tag('warning').log(`Fisherman: dropped unverified created item ${item.type} (id: ${item.id})`);
+      continue;
+    }
+    verified.push({ ...item, request: request.toEndpoint() });
+  }
+  if (verified.length === 0) verified.push(...writes.map(toCreatedItem));
+
+  return { result: { success: true, summary: input.summary, created: verified, failed: input.failed || [] } };
+}
+
+function synthesizeResult(haul: RequestHaul, declaredDone: boolean): FishermanResult {
+  const made = haul.requests();
+  const writes = haul.successfulWrites();
+  const failures = haul.failed();
   let summary = `Stopped before finishing: ${made.length} requests, ${writes.length} successful writes, ${failures.length} failed`;
   const lastFailure = failures[failures.length - 1];
   if (lastFailure) summary += `; last failure: ${lastFailure.toSummary()}`;

package/src/ai/fisherman.ts

@@ -1,6 +1,6 @@
 import dedent from 'dedent';
 import type { ApiClient } from '../api/api-client.ts';
-import type { RequestStore } from '../api/request-store.ts';
+import { type RequestStore, isFailedRequest } from '../api/request-store.ts';
 import { listAllEndpoints } from '../api/spec-reader.ts';
 import { createDebug, tag } from '../utils/logger.ts';
 
@@ -8,6 +8,7 @@ const debugLog = createDebug('explorbot:fisherman');
 import { loop } from '../utils/loop.ts';
 import type { Agent } from './agent.ts';
 import { type FishermanResult, createFishermanTools } from './fisherman-tools.ts';
+import { RequestHaul } from './fisherman/request-haul.ts';
 import type { Provider } from './provider.ts';
 import { dataProtectionRules } from './rules.ts';
 
@@ -79,11 +80,11 @@ export class Fisherman implements Agent {
     await this.refreshAuth();
     debugLog(`auth headers: ${Object.keys(this.apiClient.getHeaders()).join(', ')}`);
 
-    const { tools, getResult, isFinished, finishFromText } = createFishermanTools(this.apiClient, this.requestStore, {
+    const haul = new RequestHaul(this.requestStore);
+    const { tools, getResult, isFinished, finishFromText } = createFishermanTools(this.apiClient, this.requestStore, haul, {
       spec: this.spec,
       baseEndpoint: this.baseEndpoint,
     });
-    const ledgerStart = this.requestStore.getMadeRequests().length;
 
     const conversation = this.provider.startConversation(this.buildSystemPrompt(endpointList, Object.keys(tools), scopeUrl), 'fisherman');
     conversation.addUserText(this.buildTaskPrompt(instructions));
@@ -109,7 +110,7 @@ export class Fisherman implements Agent {
           return;
         }
 
-        if (this.isStuckOnEndpoint(ledgerStart)) {
+        if (this.isStuckOnEndpoint(haul)) {
           tag('warning').log('Fisherman: repeated failures on the same endpoint — stopping');
           stop();
           return;
@@ -228,12 +229,12 @@ export class Fisherman implements Agent {
     `;
   }
 
-  private isStuckOnEndpoint(ledgerStart: number): boolean {
-    const made = this.requestStore.getMadeRequests().slice(ledgerStart);
+  private isStuckOnEndpoint(haul: RequestHaul): boolean {
+    const made = haul.requests();
     if (made.length < REPEATED_FAILURE_LIMIT) return false;
     const recent = made.slice(-REPEATED_FAILURE_LIMIT);
     const first = recent[0];
-    return recent.every((r) => (r.status >= 400 || r.error) && r.method === first.method && r.path === first.path);
+    return recent.every((r) => isFailedRequest(r) && r.method === first.method && r.path === first.path);
   }
 
   private buildTaskPrompt(instructions: string): string {

package/src/ai/pilot.ts

@@ -31,6 +31,7 @@ const META_TOOLS = ['record', 'reset', 'stop', 'finish'];
 const PILOT_REASONING_LIMIT = 500;
 const PILOT_MESSAGE_LIMIT = 2;
 const PILOT_MESSAGE_MAX_LENGTH = 160;
+const PILOT_REQUEST_LIMIT = 5;
 
 export class Pilot implements Agent {
   emoji = '🧭';
@@ -826,17 +827,17 @@ export class Pilot implements Agent {
     lines.push(`h3: ${state.h3 || ''}`);
     lines.push(`h4: ${state.h4 || ''}`);
 
-    const focusArea = state.overlay;
-    if (focusArea.detected) {
-      let line = `modal: ${focusArea.name || focusArea.type}`;
-      if (focusArea.root) line += ` (root: ${focusArea.root})`;
+    const region = state.overlay;
+    if (region.isModal) {
+      let line = `overlay: ${region.name || region.type}`;
+      if (region.root) line += ` (root: ${region.root})`;
       lines.push(line);
-    } else if (focusArea.present) {
-      let line = `region: ${focusArea.name || 'unnamed'} (inline`;
-      if (focusArea.root) line += `, root: ${focusArea.root}`;
+    } else if (region.isOpen) {
+      let line = `region: ${region.name || 'unnamed'} (inline`;
+      if (region.root) line += `, root: ${region.root}`;
       lines.push(`${line})`);
     } else {
-      lines.push('modal: none');
+      lines.push('overlay: none');
     }
 
     const tabs = this.stateManager.otherTabs;
@@ -1071,9 +1072,13 @@ export class Pilot implements Agent {
 
         if (t.output?.pageDiff?.urlChanged) line += `\n   moved: ${t.output.pageDiff.previousUrl} → ${t.output.pageDiff.currentUrl}`;
 
-        const failedRequests = (t.output?.pageDiff?.requests ?? []).filter((r: any) => r.status >= 400);
-        if (failedRequests.length > 0) {
-          line += `\n   requests: ${failedRequests.map((r: any) => `${r.method} ${r.path} → ${r.status}`).join(', ')}`;
+        const pageRequests = t.output?.pageDiff?.requests ?? [];
+        const requests = pageRequests
+          .filter((r: any) => r.status >= 400)
+          .concat(pageRequests.filter((r: any) => r.status < 400))
+          .slice(0, PILOT_REQUEST_LIMIT);
+        if (requests.length > 0) {
+          line += `\n   requests: ${requests.map((r: any) => `${r.method} ${r.path} → ${r.status}`).join(', ')}`;
         }
 
         const messages = (t.output?.pageDiff?.messages ?? []).slice(0, PILOT_MESSAGE_LIMIT);
@@ -1139,15 +1144,15 @@ export class Pilot implements Agent {
 
       Diagnostic patterns (use <state>, executed/element/skipped fields, ariaDiff):
       - Click failed + button in "disabled buttons" → required field missing. Instruct fill first.
-      - "modal: none" but Tester targets a modal → modal closed; re-trigger.
+      - "overlay: none" but Tester targets an overlay → overlay closed; re-trigger.
       - "region:" in <state> → a large area appeared in place without navigation (subview, wizard step, panel). Direct Tester to act inside it; the rest of the page is still usable.
       - Action SUCCESS but ariaDiff empty → may have worked without visible DOM change; check result message.
-      - MultipleElementsFound → xpathCheck() to identify the right one, then precise locator or visualClick().
+      - MultipleElementsFound → nothing was clicked. Tell Tester to reuse the same locator with step.opts({ elementIndex: N }) from the numbered elements list.
       - Wrong page (settings vs feature) → getVisitedStates() then back() or reset(). Don't try breadcrumbs (SPA back-nav is unreliable).
       - Click SUCCESS but executed locator ≠ explanation intent, or "skipped" attempts present → wrong element clicked.
       - form(I.type()) SUCCESS but "element" shows a button/link → keys went to wrong element; click the input first.
       - ariaDiff shows 5+ added/removed → page entered new mode (editor/modal); call context() before guessing selectors.
-      - Empty dropdown/list when items expected → missing data; call precondition() to create it.
+      - Empty dropdown/list when items expected → wait explicitly, then check the state changed: ariaDiff and any GET that loaded data. If still nothing loaded, confirm the empty state with verify().
       - Search-and-select needs SEQUENCE: focus trigger → type to filter → click option. Tell Tester to split into separate tool calls.
       - Multi-action explanation in one tool call → instruct Tester to split.
 

package/src/ai/provider.ts

@@ -11,6 +11,7 @@ import { type AIConfig, configuredModels, modelName as getModelName } from '../c
 import { executionController } from '../execution-controller.ts';
 import { Observability } from '../observability.ts';
 import { Stats } from '../stats.ts';
+import { getCliName } from '../utils/cli-name.js';
 import { createDebug, tag } from '../utils/logger.js';
 import { type RetryOptions, withRetry } from '../utils/retry.js';
 import { RulesLoader } from '../utils/rules-loader.ts';
@@ -51,12 +52,7 @@ export async function flushTelemetry(): Promise<void> {
 const CONTEXT_LENGTH_PATTERNS = ['reduce the length', 'context length', 'maximum context', 'token limit', 'too many tokens', 'max_tokens', 'context_length_exceeded', 'output truncated at maxtokens'];
 
 function extractCachedTokens(usage: any): number {
-  if (!usage) return 0;
-  const direct = usage.inputTokenDetails?.cacheReadTokens ?? usage.cachedInputTokens;
-  if (typeof direct === 'number') return direct;
-  const raw = usage.raw;
-  const fromRaw = raw?.prompt_tokens_details?.cached_tokens ?? raw?.promptTokensDetails?.cachedTokens;
-  return typeof fromRaw === 'number' ? fromRaw : 0;
+  return usage?.inputTokenDetails?.cacheReadTokens ?? 0;
 }
 
 function abortAfterIdle(ms: number, cancel: { cancelled: boolean }, controller: AbortController): Promise<never> {
@@ -110,7 +106,7 @@ export class Provider {
 
   constructor(config: AIConfig) {
     if (!config?.model) {
-      throw new AiError('AI model is not configured. Set ai.model in your config file.');
+      throw new AiError(`AI model is not configured. Set ai.model in your config file, or pick one with ${getCliName()} recommended-models`);
     }
     this.config = config;
     this.initLangfuse();
@@ -243,8 +239,8 @@ export class Provider {
   private recordUsage(agentName: string, modelName: string, usage: any): void {
     if (!usage) return;
     Stats.recordTokens(agentName, modelName, {
-      input: usage.inputTokens ?? usage.promptTokens ?? 0,
-      output: usage.outputTokens ?? usage.completionTokens ?? 0,
+      input: usage.inputTokens ?? 0,
+      output: usage.outputTokens ?? 0,
       total: usage.totalTokens ?? 0,
       cached: extractCachedTokens(usage),
     });
@@ -428,7 +424,7 @@ export class Provider {
     let invalidRequestFeedbackAdded = false;
     const executedStepMessages: ModelMessage[] = [];
     try {
-      const response = await this.withModelRequestSlot(() =>
+      let response = await this.withModelRequestSlot(() =>
         withRetry(async () => {
           const stepMessages: ModelMessage[] = [];
           const onStepEnd = (step: any) => {
@@ -458,7 +454,7 @@ export class Provider {
 
       clearActivity();
 
-      withExecutedSteps(response, executedStepMessages);
+      response = withExecutedSteps(response, executedStepMessages);
 
       // Log tool usage summary
       if (response.toolCalls && response.toolCalls.length > 0) {
@@ -651,7 +647,7 @@ export class Provider {
 
   async processImage(prompt: string, image: string): Promise<any> {
     if (!this.config.visionModel) {
-      throw new Error('Vision model not configured. Please set ai.visionModel in your config.');
+      throw new Error(`Vision model not configured. Set ai.visionModel in your config; ${getCliName()} recommended-models lists the providers that serve one`);
     }
 
     setActivity(`🤖 Processing image with ${this.config.visionModel}`, 'ai');

package/src/ai/researcher/coordinates.ts

@@ -71,6 +71,14 @@ export async function visuallyAnnotateContainers(page: Page, containers: Array<{
   }, containers);
 }
 
+export async function removeVisualAnnotations(page: Page): Promise<void> {
+  try {
+    await page.locator('[data-explorbot-annotation]').evaluateAll((elements) => {
+      for (const element of elements) element.remove();
+    });
+  } catch {}
+}
+
 export function WithCoordinates<T extends Constructor>(Base: T) {
   return class extends Base {
     declare explorer: Explorer;
@@ -85,6 +93,10 @@ export function WithCoordinates<T extends Constructor>(Base: T) {
       return this.explorer.withPage((page) => visuallyAnnotateContainers(page, opts?.containers || []));
     }
 
+    async removeVisualAnnotations(): Promise<void> {
+      await this.explorer.withPage((page) => removeVisualAnnotations(page));
+    }
+
     private async _analyzeScreenshotForVisualProps(): Promise<VisualAnalysisResult> {
       const elements = new Map<string, { coordinates: string | null; color: string | null; icon: string | null }>();
       const emptyResult: VisualAnalysisResult = { elements, pagePurpose: null, primaryActions: null, focusedSection: null };
@@ -96,7 +108,7 @@ export function WithCoordinates<T extends Constructor>(Base: T) {
 
       const prompt = dedent`
         This screenshot has two types of annotations:
-        - **Section containers**: dashed bordered boxes (no labels on them). A legend at the bottom-left maps dashed line colors to section names. Ignore containers for this task.
+        - **Section containers**: dashed bordered boxes (no labels on them). A legend at the bottom-right maps dashed line colors to section names. Ignore containers for this task.
         - **Interactive elements**: solid bordered boxes with eidx numbers in the top-right corner above the box. Adjacent elements use different colors.
 
         For each interactive element (solid border, eidx number), report:

package/src/ai/researcher/deep-analysis.ts

@@ -88,22 +88,22 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
     }
 
     async researchOverlay(current: ActionResult, previous: ActionResult, pageStateHash: string): Promise<string | null> {
-      const focusArea = current.overlay;
-      if (!focusArea.present || !focusArea.name) return null;
+      const region = current.overlay;
+      if (!region.isOpen || !region.name) return null;
 
       const cached = getCachedResearch(pageStateHash);
       if (!cached) return null;
 
-      const escaped = focusArea.name.replace(/"/g, '\\"');
+      const escaped = region.name.replace(/"/g, '\\"');
       if (mdq(cached).query(`section3(~"${escaped}")`).count() > 0) {
-        debugLog(`Overlay "${focusArea.name}" already in cached research, skipping`);
+        debugLog(`Overlay "${region.name}" already in cached research, skipping`);
         return null;
       }
 
       const diff = await current.diff(previous);
 
       if (!diff.ariaChanged && diff.htmlParts.length === 0) {
-        debugLog(`No diff between current and previous state for overlay "${focusArea.name}"`);
+        debugLog(`No diff between current and previous state for overlay "${region.name}"`);
         return null;
       }
 
@@ -113,10 +113,10 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
           .map((s) => s.rawMarkdown)
       );
 
-      tag('substep').log(`Researching overlay: ${focusArea.name}`);
-      const sectionMarkdown = await this._analyzeExpandedAction('', focusArea.name, diff, alreadyExpanded);
+      tag('substep').log(`Researching overlay: ${region.name}`);
+      const sectionMarkdown = await this._analyzeExpandedAction('', region.name, diff, alreadyExpanded);
       if (!sectionMarkdown) {
-        debugLog(`Overlay "${focusArea.name}" produced no meaningful expansion`);
+        debugLog(`Overlay "${region.name}" produced no meaningful expansion`);
         return null;
       }
 
@@ -130,7 +130,7 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
       }
 
       saveResearch({ hash: pageStateHash }, updated);
-      tag('substep').log(`Overlay research appended: ${focusArea.name}`);
+      tag('substep').log(`Overlay research appended: ${region.name}`);
       return sectionMarkdown;
     }
 

package/src/ai/researcher/fingerprint-worker.ts

@@ -3,16 +3,7 @@ import { join } from 'node:path';
 import { parentPort } from 'node:worker_threads';
 import { computeHtmlFingerprint } from '../../utils/html-diff.ts';
 import { isSamePageFamily } from '../../utils/url-matcher.ts';
-
-function diceSimilarity(a: Set<string>, b: Set<string>): number {
-  let intersection = 0;
-  for (const item of a) {
-    if (b.has(item)) intersection++;
-  }
-  const total = a.size + b.size;
-  if (total === 0) return 100;
-  return Math.round(((2 * intersection) / total) * 100);
-}
+import { diceSimilarity } from '../../utils/similarity.ts';
 
 parentPort!.on('message', (data: FingerprintRequest) => {
   const { html, statesDir, maxAgeMs, threshold, url } = data;

package/src/ai/researcher.ts

@@ -236,7 +236,7 @@ export class Researcher extends ResearcherBase implements Agent {
         const containers = validContainers.filter((c) => !freshBroken.includes(c.css));
         await this.visuallyAnnotateElements({ containers });
         this.actionResult = await this.explorer.capture({ screenshot: true });
-        const visualResult = await this.analyzeScreenshotForVisualProps();
+        const visualResult = await this.analyzeScreenshotForVisualProps().finally(() => this.removeVisualAnnotations());
         if (visualResult.elements.size > 0) {
           await this.mergeVisualData(result, visualResult.elements);
           result.parseLocators();

package/src/ai/rules.ts

@@ -335,6 +335,14 @@ export const actionRule = dedent`
   If locator doesn't work, try CSS or XPath locators.
   If nothing works, use I.clickXY(x, y) as last resort.
 
+  When a click result reports several matches, pick one from its numbered list by position rather than guessing a new locator.
+  Reuse the same locator with step.opts({ elementIndex: N }) as the LAST argument. N is the "Element N" number.
+
+  <example>
+    I.click('Remove', step.opts({ elementIndex: 2 }));
+    I.click({ role: 'link', text: 'Details' }, '.panel', step.opts({ elementIndex: 1 }));
+  </example>
+
   For checkboxes, prefer I.checkOption/I.uncheckOption over I.click.
 
 

package/src/ai/tester.ts

@@ -548,17 +548,17 @@ export class Tester extends TaskAgent implements Agent {
     const isNewUrl = this.previousUrl !== currentUrl;
     const isNewState = !isNewUrl && this.previousStateHash !== null && this.previousStateHash !== currentStateHash;
 
-    if (this.previousRegionPresent !== null && this.previousRegionPresent !== currentState.overlay.present) {
+    if (this.previousRegionPresent !== null && this.previousRegionPresent !== currentState.overlay.isOpen) {
       this.regionTransitioned = true;
     }
-    this.previousRegionPresent = currentState.overlay.present;
+    this.previousRegionPresent = currentState.overlay.isOpen;
 
     this.previousUrl = currentUrl;
     this.previousStateHash = currentStateHash;
 
     let context = '';
 
-    const focusArea = currentState.overlay;
+    const region = currentState.overlay;
 
     const focusedElement = currentState.focusedElement;
     if (focusedElement) {
@@ -577,29 +577,29 @@ export class Tester extends TaskAgent implements Agent {
       `;
     }
 
-    if (focusArea.detected) {
-      const areaName = focusArea.name ? ` "${focusArea.name}"` : '';
+    if (region.isModal) {
+      const areaName = region.name ? ` "${region.name}"` : '';
       let rootHint = '';
-      if (focusArea.root) rootHint = `\nIts content lives inside \`${focusArea.root}\` — scope locators to it.`;
+      if (region.root) rootHint = `\nIts content lives inside \`${region.root}\` — scope locators to it.`;
       context += dedent`
-        <focus_scope>
-        A ${focusArea.type}${areaName} is currently open above the page.${rootHint}
-        Scope all interactions to elements inside this ${focusArea.type}.
-        Page navigation, filters, and tabs that exist outside it are not actionable while it is open and may share names or roles with elements inside it — prefer the locator inside the ${focusArea.type}.
-        Use <page_aria> to confirm the element you target is actually inside the ${focusArea.type}.
-        </focus_scope>
+        <overlay>
+        An overlay${areaName} is currently open above the page.${rootHint}
+        Scope all interactions to elements inside this overlay.
+        Page navigation, filters, and tabs that exist outside it are not actionable while it is open and may share names or roles with elements inside it — prefer the locator inside the overlay.
+        Use <page_aria> to confirm the element you target is actually inside the overlay.
+        </overlay>
       `;
     }
 
-    if (!focusArea.detected && focusArea.present && isNewState) {
+    if (!region.isModal && region.isOpen && isNewState) {
       let rootHint = '';
-      if (focusArea.root) rootHint = `\nIt lives inside \`${focusArea.root}\`.`;
+      if (region.root) rootHint = `\nIt lives inside \`${region.root}\`.`;
       context += dedent`
-        <area_of_interest>
-        A large new area "${focusArea.name || 'unnamed area'}" appeared on this page without navigation.${rootHint}
+        <region>
+        A large new region "${region.name || 'unnamed region'}" appeared on this page without navigation.${rootHint}
         The scenario most likely continues inside this area — prefer its elements for your next actions.
         The rest of the page (navigation, menus, filters) is still interactive and remains available.
-        </area_of_interest>
+        </region>
       `;
     }
 
@@ -663,7 +663,7 @@ export class Tester extends TaskAgent implements Agent {
       return context;
     }
 
-    if (focusArea.present && focusArea.name && this.pageStateHash && this.pageActionResult) {
+    if (region.isOpen && region.name && this.pageStateHash && this.pageActionResult) {
       const overlaySection = await this.researcher.researchOverlay(currentState, this.pageActionResult, this.pageStateHash).catch(this.skipResearch);
       if (overlaySection) {
         context += dedent`