explorbot 0.5.0 → 0.5.2

package/docs/basics/providers.md

@@ -22,6 +22,8 @@ Explorbot uses three roles:
 
 Pick a fast, cheap model for the first two and a stronger one for the third. When a provider has no recommended model for one of these roles, combine it with another provider for that role.
 
+An optional [decision model](#decision-model) can answer narrow yes/no and pick-one questions so the other models are called less often.
+
 ### OpenRouter
 
 Start with OpenRouter. One key reaches [many providers and models](https://openrouter.ai/models).
@@ -308,6 +310,41 @@ Laguna XS is an agentic coding model — fast, cheap, and reliable at tool calli
 
 Keep `agenticModel` on another provider. Poolside's endpoint accepts `response_format: json_schema` but does not enforce it, so structured-output calls depend on the model volunteering valid JSON. Laguna XS usually does; Laguna S answers in prose instead, which makes it unusable for the Planner, Pilot, and Captain. Laguna S is also slow enough under page-sized prompts to hit Explorbot's request timeouts, so it is not a substitute for Laguna XS in the `model` role either.
 
+## Decision Model
+
+A decision model is an optional fourth role. It does not generate text or call tools. It receives the current state and one closed question, and returns an answer with a probability. Explorbot uses it for narrow choices that would otherwise take a call to a larger model or a guess.
+
+Explorbot supports TypeSafe's Jev, reached through OpenRouter or through TypeSafe's API directly:
+
+```javascript
+export default {
+  ai: {
+    // ...your model, visionModel, agenticModel...
+    decisionModel: { provider: 'openrouter', model: 'typesafe/jev-1.13' },
+    // or: decisionModel: { provider: 'typesafe', model: 'jev-latest' },
+  },
+};
+```
+
+| Option | Type | Description |
+|--------|------|-------------|
+| `provider` | `'openrouter' \| 'typesafe'` | Authenticates with `OPENROUTER_API_KEY` or `TYPESAFE_API_KEY` |
+| `model` | `string` | Model ID at that provider |
+| `tool` | `boolean` | Give Tester and Pilot a `judge` tool to ask their own questions. Default: `true` |
+| `direct` | `boolean` | Consult the model at Explorbot's built-in decision points. Default: `true` |
+| `threshold` | `number` | How sure the model must be before Explorbot acts on an answer, between 0 and 1. Default: `0.7` |
+
+Without a config file, or to turn it on for one run, set `EXPLORBOT_DECISION_MODEL` or pass `--decision-model` as `provider/model-id`. It overrides `provider` and `model` in the config and keeps `tool` and `direct`:
+
+```bash
+EXPLORBOT_DECISION_MODEL=openrouter/typesafe/jev-1.13 npx explorbot explore
+npx prima-cli go "Settings" --decision-model typesafe/jev-latest
+```
+
+Leave `decisionModel` unset and Explorbot never calls it. An unknown provider, a missing API key, or a `threshold` outside 0 and 1 stops Explorbot at startup.
+
+Explorbot acts on an answer only when it is more sure than `threshold` (70% by default), and otherwise does what it would have done without the decision model. See [Decisions](../web-testing/decisions.md) for where it is consulted, the `judge` tool, and what happens when an answer is rejected.
+
 ## Multi-Provider Configuration
 
 Mix clients the same way you assign `model`, `visionModel`, and `agenticModel`. Each field can use a different provider instance — a fast provider does the token-heavy reading while a stronger one makes the decisions:
@@ -359,4 +396,5 @@ export OPENAI_API_KEY=your-key-here
 export ANTHROPIC_API_KEY=your-key-here
 export GOOGLE_API_KEY=your-key-here
 export MISTRAL_API_KEY=your-key-here
+export TYPESAFE_API_KEY=your-key-here
 ```

package/docs/index.json

@@ -22,6 +22,7 @@
         { "title": "Planner", "file": "web-testing/planner.md", "description": "Tune what gets tested" },
         { "title": "Researcher", "file": "web-testing/researcher.md", "description": "How pages are analyzed, and how to tune it" },
         { "title": "Page interaction", "file": "web-testing/page-interaction.md", "description": "How agents read and act on a page" },
+        { "title": "Decisions", "file": "web-testing/decisions.md", "description": "The optional decision model: built-in decision points, the judge tool, fallbacks" },
         { "title": "Automated tests", "file": "web-testing/automated-tests.md", "description": "The runnable test files you keep" },
         { "title": "Rerun", "file": "web-testing/rerun.md", "description": "Re-run generated tests with AI healing" },
         { "title": "Agents", "file": "web-testing/agents.md", "description": "What each agent does under the hood" },

package/docs/reference/commands.md

@@ -112,6 +112,7 @@ EXPLORBOT_AI_PROVIDER=openrouter \
 | `EXPLORBOT_URL` | Base URL to test; the API boat reads it as the base endpoint |
 | `EXPLORBOT_VISION_MODEL` | Screenshot analysis; overrides the provider recommendation |
 | `EXPLORBOT_AGENTIC_MODEL` | Captain and Pilot decisions; overrides the provider recommendation |
+| `EXPLORBOT_DECISION_MODEL` | Turns on the decision model, as provider/model-id (openrouter/typesafe/jev-1.13 or typesafe/jev-latest); overrides ai.decisionModel |
 | `EXPLORBOT_OUTPUT` | Output root for states, plans, research, and reports. Defaults to the site dir under ~/.explorbot/sites |
 | `EXPLORBOT_EPHEMERAL` | Keep no state between runs — output goes to a fresh temp directory instead of the site dir |
 | `EXPLORBOT_KNOWLEDGE` | Inline knowledge text, applied to every page |

package/docs/reference/configuration.md

@@ -537,6 +537,7 @@ export default {
   ai: {
     model: groq('openai/gpt-oss-20b'),   // Default model instance (required)
     visionModel: groq('meta-llama/llama-4-scout-17b-16e-instruct'),  // Model for screenshot analysis; setting it enables vision features
+    decisionModel: { provider: 'openrouter', model: 'typesafe/jev-1.13' }, // Optional; see providers docs
     config: {},                    // Additional provider config
     langfuse: {                    // Observability settings
       enabled: true,
@@ -624,6 +625,7 @@ export default {
 
 - [API testing](../api-testing/basics.md) — API testing setup and commands
 - [AI providers](../basics/providers.md) — provider setup examples
+- [Decisions](../web-testing/decisions.md) — the optional decision model and its fallbacks
 - [Agents](../web-testing/agents.md) — agent descriptions and workflows
 - [Agent hooks](../web-testing/hooks.md) — custom code before and after an agent runs
 - [Researcher agent](../web-testing/researcher.md) — Researcher configuration and usage

package/docs/web-testing/decisions.md

@@ -0,0 +1,119 @@
+# Decisions
+
+A decision model answers one closed question about the current state: a yes/no statement, or a pick from a list of options. It returns the answer together with its probability. Explorbot uses it to settle narrow choices quickly and cheaply that would otherwise need a call to a larger model or a guess.
+
+The decision model is optional. Without it Explorbot behaves exactly as described elsewhere in these docs. Setup is covered in [AI providers](../basics/providers.md#decision-model).
+
+## Decision rules
+
+Every question has one of two shapes:
+
+| Shape | Asked as | Answer |
+|---|---|---|
+| **Statement** | "The page shows that this claim is true." | Probability that the statement holds |
+| **Choice** | "Which listed element does the intent name?" plus numbered options | The chosen option and its probability |
+
+A choice always includes an extra *undecided* option, so the model can say that none of the options fits. The question tells the model to choose it when it is not sure.
+
+Explorbot acts on an answer only when it is **approved**:
+
+- a statement's probability is above the threshold, or
+- a choice picked a real option, not *undecided*, with a probability above the threshold.
+
+The threshold is **70%** unless `decisionModel.threshold` sets another value.
+
+Anything else counts as **rejected**, including:
+
+- a confident "no", or any probability at or below the threshold
+- *undecided*
+- a choice offered with fewer than two options
+- a request timeout (15 seconds) or a failed request
+- `direct: false` in the config, for built-in decision points
+
+A rejected answer never means "confidently no". It means Explorbot has no usable answer, so it takes its usual path. Every question is worded so that approving it lets Explorbot take a shortcut and rejecting it keeps the usual path. A wrong or unsure answer can therefore cost a shortcut, but never a check.
+
+## Built-in decision points
+
+These are fixed places in the code where Explorbot asks the decision model before doing the usual, more expensive work. Setting `direct: false` turns them off.
+
+### Tester: locator matches several elements
+
+When a click or fill fails because the locator matched more than one element, Explorbot lists the matches and asks which one the step meant.
+
+- **Approved:** a click is repeated right away on the chosen element with `elementIndex`, before the Tester's remaining fallback locators. For other actions, the Tester is told which element to use.
+- **Identical options:** when two matches look the same, the question is not asked. The Tester is told to click by appearance with `visualClick()`.
+- **Fallback:** the Tester receives the numbered list of matches and picks one itself, narrows the locator, or uses a visual click.
+
+### Pilot: progress review
+
+Every few steps, Pilot reviews the Tester's recent actions with the `agenticModel`. On a routine review it first asks whether the recent actions advance the scenario toward its remaining expected outcomes. The decision model sees the scenario, its planned steps, which expected outcomes are checked and which remain, the run log, the pages visited, the current page and the recent actions.
+
+A review forced by a run of failed or empty steps skips the question and goes straight to Pilot.
+
+- **Approved:** the review is skipped and the Tester continues. This saves one `agenticModel` call.
+- **Fallback:** Pilot runs its full review as usual.
+
+### Navigator: claim already verified
+
+Before Navigator writes assertions for a claim, it asks whether a claim it has already verified on this page means the same thing, even if it is worded differently.
+
+- **Approved:** the claim counts as verified and no assertions are generated or run.
+- **Fallback:** Navigator verifies the claim with assertions as usual.
+
+The new claim is not added to the page's verified claims. Only a claim backed by assertions that actually ran is recorded as verified.
+
+### Navigator: claim no assertion can express
+
+Some claims cannot be turned into an assertion. When that happens, Navigator asks whether the page shows the claim is true.
+
+- **Approved:** `verify` reports that the page appears to confirm the claim. The result is marked as a judgement, kept apart from assertions that ran, and the Tester is asked to restate the claim so it can be asserted.
+- **Fallback:** `verify` reports that no assertion could express the claim, and suggests restating it or checking it with `see`.
+
+In both cases `verify` returns a failure, because nothing ran in the browser.
+
+### Prima: `go` to a described page
+
+When `prima go` gets a description of a page instead of a URL, it asks which control on the current page leads there.
+
+- **Approved:** Prima clicks that control. It then asks a second question: whether the new page shows the target. Only when that is approved too does Prima report success.
+- **Fallback:** if the choice is rejected, or the click fails, or the page doesn't change, or arrival is not confirmed, Prima hands off to the Navigator as usual.
+
+### Prima: `check` expected outcomes
+
+When `prima check` settles the expected outcomes of a finished run without a screenshot, it asks what the run established about each outcome: that it happened, or that it did not.
+
+- **Approved:** that outcome is settled as passed or failed without calling the `agenticModel`.
+- **Fallback:** the remaining outcomes are settled by the `agenticModel` as usual. When a screenshot is available, all outcomes go to the vision model and the decision model is not asked.
+
+## The `judge` tool
+
+With `tool: true`, Tester and Pilot get a `judge` tool. They use it when a step depends on reading the page rather than running a command. The tool takes:
+
+| Argument | Description |
+|---|---|
+| `question` | A statement to confirm, or the question a list of options answers |
+| `options` | Possible answers. Leave it out to confirm a statement |
+| `context` | Anything the page itself doesn't show |
+
+The model supplies only the question. The tool gathers the rest of the state itself: the current scenario, the compact ARIA snapshot of the page (up to 12,000 characters), and the last eight test steps.
+
+- **Approved:** the tool returns the answer and its confidence.
+- **Rejected:** the tool returns "Not confirmed" and tells the agent to gather more context or try another route. "Not confirmed" means the page doesn't settle the question. It does not mean the statement is false.
+
+`tool` and `direct` are independent. For example, `tool: false` keeps the built-in decision points but stops the agents from asking questions of their own.
+
+## Where it is not used
+
+- **Pilot's verdict on a test.** Whether a scenario passed, failed or was impossible is always decided by the `agenticModel`.
+- **Filtering experience.** One question per stored experience block, on every prompt, would cost more than it saves.
+- **Generation, screenshots and tool calling.** The decision model reads text and answers questions. It does not write, look at images or act on the page.
+
+## Tracing
+
+Each question is recorded as a `judge.decide` span:
+
+- **Input:** the exact request body sent to the provider: model, state and question. Post it again to the endpoint to replay the decision.
+- **Output:** the raw answer and its probability, before the threshold is applied.
+- **Metadata:** `judgeQuestion` for filtering by question, `judgeDecision` with the approved value, confidence, threshold and model, and `judgeEndpoint`.
+
+A failed request marks the span as an error and records the reason. The TUI shows `⚖️ Asking judge...` while a question is in flight. See [Observability](../contributing/observability.md) for setting up tracing.

package/docs/workflow/agentic-usage.md

@@ -52,6 +52,7 @@ No `init`, no config file, no project directory, no model IDs to look up. These
 | `EXPLORBOT_URL` | yes | Base URL to test; the API boat reads it as the base endpoint |
 | `EXPLORBOT_VISION_MODEL` | no | Screenshot analysis; overrides the provider recommendation |
 | `EXPLORBOT_AGENTIC_MODEL` | no | Captain and Pilot decisions; overrides the provider recommendation |
+| `EXPLORBOT_DECISION_MODEL` | no | Turns on the decision model, as provider/model-id (openrouter/typesafe/jev-1.13 or typesafe/jev-latest); overrides ai.decisionModel |
 | `EXPLORBOT_OUTPUT` | no | Output root for states, plans, research, and reports. Defaults to the site dir under ~/.explorbot/sites |
 | `EXPLORBOT_EPHEMERAL` | no | Keep no state between runs — output goes to a fresh temp directory instead of the site dir |
 | `EXPLORBOT_KNOWLEDGE` | no | Inline knowledge text, applied to every page |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "explorbot",
-  "version": "0.5.0",
+  "version": "0.5.2",
   "description": "CLI app built with React Ink, CodeceptJS, and Playwright",
   "license": "Elastic-2.0",
   "type": "module",
@@ -102,7 +102,7 @@
     "bash-tool": "^1.3.15",
     "chalk": "^5.6.2",
     "cli-highlight": "^2.1.11",
-    "codeceptjs": "^4.2.0-beta.3",
+    "codeceptjs": "^4.2.0",
     "commander": "^14.0.1",
     "debug": "^4.4.3",
     "dedent": "^1.6.0",

package/src/action.ts

@@ -491,6 +491,14 @@ class Action {
     }
   }
 
+  public async attemptExactElementIndex(codeBlock: string, elementIndex: number, originalMessage?: string): Promise<boolean> {
+    const applyElementIndex = (step: any) => {
+      step.opts = { ...step.opts, elementIndex };
+    };
+    codeceptjs.event.dispatcher.once(codeceptjs.event.step.started, applyElementIndex);
+    return this.attempt(codeBlock, originalMessage).finally(() => codeceptjs.event.dispatcher.off(codeceptjs.event.step.started, applyElementIndex));
+  }
+
   async exitIframe(): Promise<void> {
     if (!this.playwrightHelper.frame) return;
     debugLog('Switching to main frame');

package/src/ai/judge-provider.ts

@@ -1,3 +1,6 @@
+import { Observability } from '../observability.ts';
+import { redactSecrets } from '../utils/secrets.ts';
+
 const REQUEST_TIMEOUT_MS = 15000;
 
 export class JudgeProvider {
@@ -21,6 +24,9 @@ export class JudgeProvider {
     let q: Record<string, unknown> = { type: 'noul', instructions: question };
     if (options) q = { type: 'choice', instructions: question, criteria: Object.fromEntries(options.map((option, index) => [String(index), option])) };
     const body = JSON.stringify({ model: this.model, state, questions: { q } });
+    const span = Observability.getSpan();
+    span?.setAttribute('langfuse.observation.input', redactSecrets(body));
+    span?.setAttribute('ai.telemetry.metadata.judgeEndpoint', this.url);
     const controller = new AbortController();
     const timer = setTimeout(() => controller.abort(), this.requestTimeoutMs);
     try {

package/src/ai/judge.ts

@@ -9,6 +9,7 @@ const debugLog = createDebug('explorbot:judge');
 const APPROVAL_THRESHOLD = 0.7;
 
 export const UNDECIDED = 'undecided';
+const UNDECIDED_HINT = `Choose "${UNDECIDED}" if you are not sure or no option fits.`;
 export const JUDGE_PAGE_CAP = 12000;
 
 export class Decision {
@@ -29,12 +30,15 @@ export class Decision {
 export class Judge {
   constructor(
     private provider: JudgeProvider,
-    private enabled: { tool: boolean; direct: boolean }
+    private enabled: { tool: boolean; direct: boolean },
+    private threshold = APPROVAL_THRESHOLD
   ) {}
 
   static fromConfig(config: AIConfig['decisionModel']): Judge | null {
     if (!config) return null;
-    return new Judge(new JudgeProvider(config.provider, config.model), { tool: config.tool !== false, direct: config.direct !== false });
+    const threshold = config.threshold ?? APPROVAL_THRESHOLD;
+    if (!(threshold > 0 && threshold < 1)) throw new Error(`decisionModel.threshold must be between 0 and 1, got ${config.threshold}`);
+    return new Judge(new JudgeProvider(config.provider, config.model), { tool: config.tool !== false, direct: config.direct !== false }, threshold);
   }
 
   get toolEnabled(): boolean {
@@ -46,12 +50,21 @@ export class Judge {
     return this.consult(question, options, state);
   }
 
+  async pick(question: string, options: string[], state: unknown): Promise<number | null> {
+    if (new Set(options).size < options.length) return null;
+    const decision = await this.decide(question, [...options, UNDECIDED], state);
+    if (!decision.value) return null;
+    return options.indexOf(decision.value) + 1;
+  }
+
   async consult(question: string, options: string[] | boolean | null, state: unknown): Promise<Decision> {
     if (Array.isArray(options) && options.length < 2) return new Decision(null, 0);
     return Observability.run('judge.decide', { tags: ['judge'] }, async () => {
       setActivity('⚖️ Asking judge...', 'ai');
       const decision = await this.request(question, options, state).finally(() => clearActivity());
-      Observability.getSpan()?.setAttribute('ai.telemetry.metadata.judgeDecision', JSON.stringify({ question, value: decision.value, confidence: decision.confidence }));
+      const span = Observability.getSpan();
+      span?.setAttribute('ai.telemetry.metadata.judgeQuestion', question);
+      span?.setAttribute('ai.telemetry.metadata.judgeDecision', JSON.stringify({ question, value: decision.value, confidence: decision.confidence, threshold: this.threshold, model: this.provider.model }));
       return decision;
     });
   }
@@ -59,17 +72,23 @@ export class Judge {
   private async request(question: string, options: string[] | boolean | null, state: unknown): Promise<Decision> {
     let list: string[] | undefined;
     if (Array.isArray(options)) list = options;
+    let instructions = question;
+    if (list?.includes(UNDECIDED)) instructions = `${question} ${UNDECIDED_HINT}`;
 
-    const answer = await this.provider.decide(state, question, list).catch((error: unknown) => this.recordFailure(error));
+    const answer = await this.provider.decide(state, instructions, list).catch((error: unknown) => this.recordFailure(error));
     if (!answer) return new Decision(null, 0);
-    if (answer.probability <= APPROVAL_THRESHOLD) return new Decision(null, answer.probability);
+    Observability.getSpan()?.setAttribute('langfuse.observation.output', JSON.stringify(answer));
+    if (answer.probability <= this.threshold) return new Decision(null, answer.probability);
     if (answer.value === UNDECIDED) return new Decision(null, answer.probability);
     return new Decision(answer.value, answer.probability);
   }
 
   private recordFailure(error: unknown): null {
     debugLog('judge declined: %s', error);
-    Observability.getSpan()?.setAttribute('ai.telemetry.metadata.judgeError', String(error));
+    const span = Observability.getSpan();
+    span?.setAttribute('ai.telemetry.metadata.judgeError', String(error));
+    span?.setAttribute('langfuse.observation.level', 'ERROR');
+    span?.setAttribute('langfuse.observation.status_message', String(error));
     return null;
   }
 }

package/src/ai/pilot.ts

@@ -19,7 +19,7 @@ import type { Agent, AgentDeps } from './agent.ts';
 import type { Conversation } from './conversation.ts';
 import type { Fisherman } from './fisherman.ts';
 import { createAskApiTool } from './fisherman/tools.ts';
-import { type Judge, UNDECIDED } from './judge.ts';
+import { JUDGE_PAGE_CAP, type Judge, UNDECIDED } from './judge.ts';
 import type { Navigator } from './navigator.ts';
 import type { Provider } from './provider.ts';
 import type { Researcher } from './researcher.ts';
@@ -542,6 +542,22 @@ export class Pilot implements Agent {
     );
   }
 
+  async periodicAnalyzeProgress(task: Test, currentState: ActionResult, testerConversation: Conversation): Promise<string> {
+    const toolCalls = testerConversation.getToolExecutions().slice(-this.stepsToReview);
+    const healthy = await this.judge?.decide('The recent actions advance the scenario toward its remaining expected outcomes.', null, {
+      scenario: task.scenario,
+      plannedSteps: task.plannedSteps,
+      expectations: this.formatExpectations(task),
+      runLog: task.notesToString() || 'No steps recorded.',
+      visitedUrls: task.getVisitedUrls({ localOnly: true }),
+      state: this.buildStateContext(currentState),
+      page: currentState.getCompactARIA().slice(0, JUDGE_PAGE_CAP),
+      recentActions: this.formatActions(toolCalls),
+    });
+    if (healthy?.approved) return '';
+    return this.analyzeProgress(task, currentState, testerConversation);
+  }
+
   async analyzeProgress(task: Test, currentState: ActionResult, testerConversation: Conversation): Promise<string> {
     tag('substep').log('Pilot analyzing progress...');
 
@@ -555,9 +571,6 @@ export class Pilot implements Agent {
     const actionsContext = this.formatActions(toolCalls);
     const stateContext = this.buildStateContext(currentState);
 
-    const healthy = await this.judge?.decide('The run is moving toward the goal and can continue without a supervisor reviewing it now.', null, { scenario: task.scenario, state: stateContext, recentActions: actionsContext });
-    if (healthy?.approved) return '';
-
     const hasFailures = toolCalls.length === 0 || toolCalls.some((t) => !t.wasSuccessful);
 
     const text = await this.sendToPilot(

package/src/ai/provider.ts

@@ -1,10 +1,10 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
 import { OpenTelemetry } from '@ai-sdk/otel';
 import { LangfuseSpanProcessor } from '@langfuse/otel';
 import { NodeSDK } from '@opentelemetry/sdk-node';
-import { AsyncLocalStorage } from 'node:async_hooks';
-import dedent from 'dedent';
 import { APICallError, NoObjectGeneratedError, asSchema, extractJsonMiddleware, generateObject, generateText, isStepCount, parsePartialJson, registerTelemetry, tool, wrapLanguageModel } from 'ai';
 import type { ModelMessage } from 'ai';
+import dedent from 'dedent';
 import { z } from 'zod';
 import { clearActivity, setActivity } from '../activity.ts';
 import { type AIConfig, configuredModels, modelName as getModelName } from '../config.js';
@@ -60,14 +60,15 @@ function extractCachedTokens(usage: any): number {
   return usage?.inputTokenDetails?.cacheReadTokens ?? 0;
 }
 
-function abortAfterIdle(ms: number, cancel: { cancelled: boolean }, controller: AbortController, busy: { tools: number }): Promise<never> {
+function abortAfterIdle(ms: number, state: { cancelled: boolean; timedOut: boolean }, controller: AbortController, busy: { tools: number }): Promise<never> {
   return new Promise((_, reject) => {
     const tick = () => {
-      if (cancel.cancelled) return;
+      if (state.cancelled) return;
       if (executionController.isAwaitingInput() || busy.tools > 0) {
         setTimeout(tick, ms);
         return;
       }
+      state.timedOut = true;
       controller.abort();
       reject(new Error('AI request timeout'));
     };
@@ -252,13 +253,18 @@ export class Provider {
   }
 
   private async raceWithIdleTimeout<T>(fn: (signal: AbortSignal) => Promise<T>, timeoutMs: number, busy: { tools: number } = { tools: 0 }): Promise<T> {
-    const cancel = { cancelled: false };
+    const state = { cancelled: false, timedOut: false };
     const controller = new AbortController();
     const combinedSignal = combinedAbortSignal(controller);
     try {
-      return await Promise.race([fn(combinedSignal), abortAfterIdle(timeoutMs, cancel, controller, busy)]);
+      return await Promise.race([fn(combinedSignal), abortAfterIdle(timeoutMs, state, controller, busy)]);
+    } catch (error) {
+      if (state.timedOut && !executionController.getAbortSignal()?.aborted) {
+        throw new Error('AI request timeout', { cause: error });
+      }
+      throw error;
     } finally {
-      cancel.cancelled = true;
+      state.cancelled = true;
     }
   }
 
@@ -683,7 +689,7 @@ export class Provider {
       throw new Error(`Vision model not configured. Set ai.visionModel in your config; ${getCliName()} recommended-models lists the providers that serve one`);
     }
 
-    setActivity(`🤖 Processing image with ${this.config.visionModel}`, 'ai');
+    setActivity(`🤖 Processing image with ${getModelName(this.config.visionModel)}`, 'ai');
 
     const messages: ModelMessage[] = [
       {

package/src/ai/rules.ts

@@ -497,6 +497,19 @@ export const actionRule = dedent`
 
   After hovering, use see() or context() to check what appeared.
 
+  ### I.seeInClipboard / I.seeClipboardEquals / I.clearClipboard
+
+  Check what a copy control put into the system clipboard. Run via form(), not click().
+  I.seeInClipboard asserts the clipboard contains the text, I.seeClipboardEquals asserts an exact match,
+  I.clearClipboard empties it — call it before copying so a stale value can't pass the check.
+
+  <example>
+    I.clearClipboard();
+    I.click({ role: 'button', text: 'Copy link' });
+    I.seeInClipboard('/invite/');
+    I.seeClipboardEquals('ABC-123');
+  </example>
+
   [DO NEVER USE OTHER CODECEPTJS COMMANDS THAN PROPOSED HERE]
   [INTERACT ONLY WITH ELEMENTS THAT ARE ON THE PAGE HTML]
   [DO NOT USE WAIT FUNCTIONS]

package/src/ai/tester.ts

@@ -324,7 +324,12 @@ export class Tester extends TaskAgent implements Agent {
               const guidance = await this.pilot.reviewNewPage(task, currentState, conversation);
               if (guidance) nextStep += `\n\n${guidance}`;
             } else if (this.shouldAnalyzeProgress(iteration, currentState) && this.pilot) {
-              const guidance = await this.pilot.analyzeProgress(task, currentState, conversation);
+              let guidance: string;
+              if (this.isStruggling) {
+                guidance = await this.pilot.analyzeProgress(task, currentState, conversation);
+              } else {
+                guidance = await this.pilot.periodicAnalyzeProgress(task, currentState, conversation);
+              }
               if (guidance) nextStep += `\n\n${guidance}`;
               this.consecutiveFailures = 0;
               this.lastAnalyzedStateHash = currentState.hash;
@@ -486,13 +491,16 @@ export class Tester extends TaskAgent implements Agent {
       this.regionTransitioned = false;
       return true;
     }
-    if (this.consecutiveFailures >= 3) return true;
-    if (this.consecutiveEmptyResults >= 2) return true;
+    if (this.isStruggling) return true;
     if (iteration % this.progressCheckInterval !== 0) return false;
     if (this.lastAnalyzedStateHash === currentState.hash) return false;
     return true;
   }
 
+  private get isStruggling(): boolean {
+    return this.consecutiveFailures >= 3 || this.consecutiveEmptyResults >= 2;
+  }
+
   private shouldStopForStalledExecution(task: Test, previousState: ActionResult, toolExecutions: any[]): boolean {
     if (task.hasFinished) return false;
 

package/src/ai/tools.ts

@@ -16,7 +16,7 @@ import { compactErrorMessage, normalizeInlineText, truncate } from '../utils/str
 import { WebElement } from '../utils/web-element.ts';
 import type { ToolDeps } from './agent.ts';
 import { createJudgeTool } from './judge-tool.ts';
-import { JUDGE_PAGE_CAP, type Judge, UNDECIDED } from './judge.ts';
+import { JUDGE_PAGE_CAP, type Judge } from './judge.ts';
 import { Navigator } from './navigator.ts';
 import { Researcher } from './researcher.ts';
 import { sectionContextRule } from './rules.ts';
@@ -113,13 +113,20 @@ export function createCodeceptJSTools({ explorer, stateManager, judge }: ToolDep
 
         for (let i = 0; i < commands.length; i++) {
           const command = transformContainsCommand(commands[i]);
-          const success = await action.attempt(command, explanation);
+          let success = await action.attempt(command, explanation);
 
           const attempt: { command: string; success: boolean; error?: string } = { command, success };
           if (action.lastError) attempt.error = errorText(action.lastError);
           attempts.push(attempt);
 
-          if (!ambiguityError && action.lastError?.name === 'MultipleElementsFound') ambiguityError = action.lastError;
+          if (!ambiguityError && action.lastError?.name === 'MultipleElementsFound') {
+            ambiguityError = action.lastError;
+            if (judge) {
+              const labels = (await extractWebElements(ambiguityError))?.map((el) => el.label) || [];
+              const index = await judge.pick(PICK_ELEMENT_QUESTION, labels, { intent: explanation, task: task.description });
+              if (index) success = await action.attemptExactElementIndex(command, index, explanation);
+            }
+          }
 
           if (success) {
             const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, command);
@@ -158,9 +165,7 @@ export function createCodeceptJSTools({ explorer, stateManager, judge }: ToolDep
             attempts,
             suggestion,
           },
-          ambiguityError || action.lastError,
-          judge,
-          explanation
+          ambiguityError || action.lastError
         );
       },
     }),
@@ -1309,10 +1314,13 @@ export async function failedToolResult(action: string, message: string, data?: R
     result.suggestion = getMultipleElementsSuggestion();
     result.multipleElementsDetected = true;
     result.elements = formatElementList(matched);
-    const labels = (matched || []).map((element) => `${element.text || 'no text'} — ${element.html}`);
-    const pick = await judge?.decide('Which listed element does the intent name?', [...labels, UNDECIDED], { intent });
-    if (!pick?.value) return result;
-    const index = labels.indexOf(pick.value) + 1;
+    if (!judge || !matched) return result;
+    const index = await judge.pick(
+      PICK_ELEMENT_QUESTION,
+      matched.map((el) => el.label),
+      { intent }
+    );
+    if (!index) return result;
     result.suggestion = `Element ${index} is the one meant. Repeat the action with step.opts({ elementIndex: ${index} }) as the last argument.`;
     return result;
   }
@@ -1333,6 +1341,7 @@ function getMultipleElementsSuggestion(): string {
     reuse the same locator with step.opts({ elementIndex: N }) as the last argument.
     A match reported as not visible can never be acted on — pick one that is.
     If none of them is the element you want, narrow the locator with a container or its full unique text.
+    If the matches cannot be told apart, click the one you mean by appearance with visualClick().
     If the list is missing, call xpathCheck() to see what the locator matches.
   `;
 }
@@ -1379,6 +1388,7 @@ const MAX_DISAMBIGUATE_ELEMENTS = 10;
 const MAX_DISAMBIGUATE_TEXT = 80;
 const MAX_DISAMBIGUATE_HTML = 300;
 const MULTIPLE_ELEMENTS_PATTERN = 'multiple elements';
+const PICK_ELEMENT_QUESTION = 'Which listed element does the intent name?';
 
 async function extractWebElements(error: Error | null | undefined): Promise<MatchedElement[] | null> {
   if (!error || error.name !== 'MultipleElementsFound') return null;
@@ -1393,7 +1403,7 @@ async function extractWebElements(error: Error | null | undefined): Promise<Matc
       const html = truncate(cleanHtmlSnippet(await elements[i].toOuterHTML()), MAX_DISAMBIGUATE_HTML);
       const text = truncate(normalizeInlineText((await elements[i].getText()) || ''), MAX_DISAMBIGUATE_TEXT);
       const visible = await Promise.resolve(elements[i].isVisible?.()).catch(() => undefined);
-      result.push({ xpath, html, text, visible });
+      result.push({ xpath, html, text, visible, label: `${text || 'no text'} — ${html}` });
     } catch (e) {
       debugLog('Failed to get details for element %d: %s', i, e);
     }
@@ -1443,5 +1453,6 @@ interface MatchedElement {
   xpath: string;
   html: string;
   text: string;
+  label: string;
   visible?: boolean;
 }

package/src/commands/options/decision-model-option.ts

@@ -0,0 +1,11 @@
+import { BaseOption } from './base-option.js';
+
+export class DecisionModelOption extends BaseOption {
+  flags = '--decision-model <model>';
+  description = 'Turn on the decision model for this run, as provider/model-id (e.g. openrouter/typesafe/jev-1.13)';
+
+  protected apply(options: Record<string, any>): void {
+    if (!options.decisionModel) return;
+    process.env.EXPLORBOT_DECISION_MODEL = options.decisionModel;
+  }
+}

package/src/commands/options/index.ts

@@ -1,7 +1,9 @@
+import { DecisionModelOption } from './decision-model-option.js';
 import { KnowledgeOption } from './knowledge-option.js';
 import { WsOption } from './ws-option.js';
 
 export { BaseOption } from './base-option.js';
 
+export const decisionModelOption = new DecisionModelOption();
 export const knowledgeOption = new KnowledgeOption();
 export const wsOption = new WsOption();