explorbot 0.5.1 → 0.5.2

package/dist/package.json

@@ -1,6 +1,6 @@
 {
   "name": "explorbot",
-  "version": "0.5.1",
+  "version": "0.5.2",
   "description": "CLI app built with React Ink, CodeceptJS, and Playwright",
   "license": "Elastic-2.0",
   "type": "module",

package/dist/src/action.d.ts

@@ -48,6 +48,7 @@ declare class Action {
     captureBrowserLogs(): Promise<any[]>;
     executeOnce(code: string, verbatim?: boolean): Promise<Action>;
     attempt(codeBlock: string, originalMessage?: string): Promise<boolean>;
+    attemptExactElementIndex(codeBlock: string, elementIndex: number, originalMessage?: string): Promise<boolean>;
     exitIframe(): Promise<void>;
     getActor(): CodeceptJS.I;
     getActionResult(): ActionResult | null;

package/dist/src/action.js

@@ -461,6 +461,13 @@ class Action {
             return false;
         }
     }
+    async attemptExactElementIndex(codeBlock, elementIndex, originalMessage) {
+        const applyElementIndex = (step) => {
+            step.opts = { ...step.opts, elementIndex };
+        };
+        codeceptjs.event.dispatcher.once(codeceptjs.event.step.started, applyElementIndex);
+        return this.attempt(codeBlock, originalMessage).finally(() => codeceptjs.event.dispatcher.off(codeceptjs.event.step.started, applyElementIndex));
+    }
     async exitIframe() {
         if (!this.playwrightHelper.frame)
             return;

package/dist/src/ai/judge-provider.js

@@ -1,3 +1,5 @@
+import { Observability } from "../observability.js";
+import { redactSecrets } from "../utils/secrets.js";
 const REQUEST_TIMEOUT_MS = 15000;
 export class JudgeProvider {
     model;
@@ -19,6 +21,9 @@ export class JudgeProvider {
         if (options)
             q = { type: 'choice', instructions: question, criteria: Object.fromEntries(options.map((option, index) => [String(index), option])) };
         const body = JSON.stringify({ model: this.model, state, questions: { q } });
+        const span = Observability.getSpan();
+        span?.setAttribute('langfuse.observation.input', redactSecrets(body));
+        span?.setAttribute('ai.telemetry.metadata.judgeEndpoint', this.url);
         const controller = new AbortController();
         const timer = setTimeout(() => controller.abort(), this.requestTimeoutMs);
         try {

package/dist/src/ai/judge.d.ts

@@ -15,13 +15,15 @@ export declare class Judge {
         tool: boolean;
         direct: boolean;
     };
+    threshold: number;
     constructor(provider: JudgeProvider, enabled: {
         tool: boolean;
         direct: boolean;
-    });
+    }, threshold?: number);
     static fromConfig(config: AIConfig['decisionModel']): Judge | null;
     get toolEnabled(): boolean;
     decide(question: string, options: string[] | boolean | null, state: unknown): Promise<Decision>;
+    pick(question: string, options: string[], state: unknown): Promise<number | null>;
     consult(question: string, options: string[] | boolean | null, state: unknown): Promise<Decision>;
     request(question: string, options: string[] | boolean | null, state: unknown): Promise<Decision>;
     recordFailure(error: unknown): null;

package/dist/src/ai/judge.js

@@ -5,6 +5,7 @@ import { JudgeProvider } from "./judge-provider.js";
 const debugLog = createDebug('explorbot:judge');
 const APPROVAL_THRESHOLD = 0.7;
 export const UNDECIDED = 'undecided';
+const UNDECIDED_HINT = `Choose "${UNDECIDED}" if you are not sure or no option fits.`;
 export const JUDGE_PAGE_CAP = 12000;
 export class Decision {
     value;
@@ -23,14 +24,19 @@ export class Decision {
 export class Judge {
     provider;
     enabled;
-    constructor(provider, enabled) {
+    threshold;
+    constructor(provider, enabled, threshold = APPROVAL_THRESHOLD) {
         this.provider = provider;
         this.enabled = enabled;
+        this.threshold = threshold;
     }
     static fromConfig(config) {
         if (!config)
             return null;
-        return new Judge(new JudgeProvider(config.provider, config.model), { tool: config.tool !== false, direct: config.direct !== false });
+        const threshold = config.threshold ?? APPROVAL_THRESHOLD;
+        if (!(threshold > 0 && threshold < 1))
+            throw new Error(`decisionModel.threshold must be between 0 and 1, got ${config.threshold}`);
+        return new Judge(new JudgeProvider(config.provider, config.model), { tool: config.tool !== false, direct: config.direct !== false }, threshold);
     }
     get toolEnabled() {
         return this.enabled.tool;
@@ -40,13 +46,23 @@ export class Judge {
             return new Decision(null, 0);
         return this.consult(question, options, state);
     }
+    async pick(question, options, state) {
+        if (new Set(options).size < options.length)
+            return null;
+        const decision = await this.decide(question, [...options, UNDECIDED], state);
+        if (!decision.value)
+            return null;
+        return options.indexOf(decision.value) + 1;
+    }
     async consult(question, options, state) {
         if (Array.isArray(options) && options.length < 2)
             return new Decision(null, 0);
         return Observability.run('judge.decide', { tags: ['judge'] }, async () => {
             setActivity('⚖️ Asking judge...', 'ai');
             const decision = await this.request(question, options, state).finally(() => clearActivity());
-            Observability.getSpan()?.setAttribute('ai.telemetry.metadata.judgeDecision', JSON.stringify({ question, value: decision.value, confidence: decision.confidence }));
+            const span = Observability.getSpan();
+            span?.setAttribute('ai.telemetry.metadata.judgeQuestion', question);
+            span?.setAttribute('ai.telemetry.metadata.judgeDecision', JSON.stringify({ question, value: decision.value, confidence: decision.confidence, threshold: this.threshold, model: this.provider.model }));
             return decision;
         });
     }
@@ -54,10 +70,14 @@ export class Judge {
         let list;
         if (Array.isArray(options))
             list = options;
-        const answer = await this.provider.decide(state, question, list).catch((error) => this.recordFailure(error));
+        let instructions = question;
+        if (list?.includes(UNDECIDED))
+            instructions = `${question} ${UNDECIDED_HINT}`;
+        const answer = await this.provider.decide(state, instructions, list).catch((error) => this.recordFailure(error));
         if (!answer)
             return new Decision(null, 0);
-        if (answer.probability <= APPROVAL_THRESHOLD)
+        Observability.getSpan()?.setAttribute('langfuse.observation.output', JSON.stringify(answer));
+        if (answer.probability <= this.threshold)
             return new Decision(null, answer.probability);
         if (answer.value === UNDECIDED)
             return new Decision(null, answer.probability);
@@ -65,7 +85,10 @@ export class Judge {
     }
     recordFailure(error) {
         debugLog('judge declined: %s', error);
-        Observability.getSpan()?.setAttribute('ai.telemetry.metadata.judgeError', String(error));
+        const span = Observability.getSpan();
+        span?.setAttribute('ai.telemetry.metadata.judgeError', String(error));
+        span?.setAttribute('langfuse.observation.level', 'ERROR');
+        span?.setAttribute('langfuse.observation.status_message', String(error));
         return null;
     }
 }

package/dist/src/ai/pilot.d.ts

@@ -41,6 +41,7 @@ export declare class Pilot implements Agent {
     buildVerdictSystemPrompt(task: Test): string;
     planTest(task: Test, currentState: ActionResult): Promise<string>;
     reviewNewPage(task: Test, currentState: ActionResult, testerConversation: Conversation): Promise<string>;
+    periodicAnalyzeProgress(task: Test, currentState: ActionResult, testerConversation: Conversation): Promise<string>;
     analyzeProgress(task: Test, currentState: ActionResult, testerConversation: Conversation): Promise<string>;
     settleExpectations(task: Test, finalState?: ActionResult): Promise<SettledExpectation[]>;
     settleByJudge(task: Test, expectations: string[]): Promise<Map<string, SettledStatus>>;

package/dist/src/ai/pilot.js

@@ -11,7 +11,7 @@ import { createDebug, tag } from "../utils/logger.js";
 const debugLog = createDebug('explorbot:pilot');
 import { truncateJson } from "../utils/strings.js";
 import { createAskApiTool } from "./fisherman/tools.js";
-import { UNDECIDED } from "./judge.js";
+import { JUDGE_PAGE_CAP, UNDECIDED } from "./judge.js";
 import { capabilityGroundingRule, dataProtectionRules } from "./rules.js";
 import { isInteractive } from "./task-agent.js";
 import { withdrawVisionTools } from "./tools.js";
@@ -477,6 +477,22 @@ export class Pilot {
         Tester holds no recipe for this page until you load one — open the <experience> entries whose titles fit a step you are about to instruct.
       `, 'pilot.reviewNewPage', { tools: true, maxToolRoundtrips: 2, task });
     }
+    async periodicAnalyzeProgress(task, currentState, testerConversation) {
+        const toolCalls = testerConversation.getToolExecutions().slice(-this.stepsToReview);
+        const healthy = await this.judge?.decide('The recent actions advance the scenario toward its remaining expected outcomes.', null, {
+            scenario: task.scenario,
+            plannedSteps: task.plannedSteps,
+            expectations: this.formatExpectations(task),
+            runLog: task.notesToString() || 'No steps recorded.',
+            visitedUrls: task.getVisitedUrls({ localOnly: true }),
+            state: this.buildStateContext(currentState),
+            page: currentState.getCompactARIA().slice(0, JUDGE_PAGE_CAP),
+            recentActions: this.formatActions(toolCalls),
+        });
+        if (healthy?.approved)
+            return '';
+        return this.analyzeProgress(task, currentState, testerConversation);
+    }
     async analyzeProgress(task, currentState, testerConversation) {
         tag('substep').log('Pilot analyzing progress...');
         if (!this.conversation) {
@@ -487,9 +503,6 @@ export class Pilot {
         const toolCalls = testerConversation.getToolExecutions().slice(-this.stepsToReview);
         const actionsContext = this.formatActions(toolCalls);
         const stateContext = this.buildStateContext(currentState);
-        const healthy = await this.judge?.decide('The run is moving toward the goal and can continue without a supervisor reviewing it now.', null, { scenario: task.scenario, state: stateContext, recentActions: actionsContext });
-        if (healthy?.approved)
-            return '';
         const hasFailures = toolCalls.length === 0 || toolCalls.some((t) => !t.wasSuccessful);
         const text = await this.sendToPilot(dedent `
         START URL: ${task.startUrl}

package/dist/src/ai/provider.js

@@ -1,9 +1,9 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
 import { OpenTelemetry } from '@ai-sdk/otel';
 import { LangfuseSpanProcessor } from '@langfuse/otel';
 import { NodeSDK } from '@opentelemetry/sdk-node';
-import { AsyncLocalStorage } from 'node:async_hooks';
-import dedent from 'dedent';
 import { APICallError, NoObjectGeneratedError, asSchema, extractJsonMiddleware, generateObject, generateText, isStepCount, parsePartialJson, registerTelemetry, tool, wrapLanguageModel } from 'ai';
+import dedent from 'dedent';
 import { z } from 'zod';
 import { clearActivity, setActivity } from "../activity.js";
 import { configuredModels, modelName as getModelName } from '../config.js';
@@ -51,15 +51,16 @@ const CONTEXT_LENGTH_PATTERNS = ['reduce the length', 'context length', 'maximum
 function extractCachedTokens(usage) {
     return usage?.inputTokenDetails?.cacheReadTokens ?? 0;
 }
-function abortAfterIdle(ms, cancel, controller, busy) {
+function abortAfterIdle(ms, state, controller, busy) {
     return new Promise((_, reject) => {
         const tick = () => {
-            if (cancel.cancelled)
+            if (state.cancelled)
                 return;
             if (executionController.isAwaitingInput() || busy.tools > 0) {
                 setTimeout(tick, ms);
                 return;
             }
+            state.timedOut = true;
             controller.abort();
             reject(new Error('AI request timeout'));
         };
@@ -232,14 +233,20 @@ export class Provider {
         });
     }
     async raceWithIdleTimeout(fn, timeoutMs, busy = { tools: 0 }) {
-        const cancel = { cancelled: false };
+        const state = { cancelled: false, timedOut: false };
         const controller = new AbortController();
         const combinedSignal = combinedAbortSignal(controller);
         try {
-            return await Promise.race([fn(combinedSignal), abortAfterIdle(timeoutMs, cancel, controller, busy)]);
+            return await Promise.race([fn(combinedSignal), abortAfterIdle(timeoutMs, state, controller, busy)]);
+        }
+        catch (error) {
+            if (state.timedOut && !executionController.getAbortSignal()?.aborted) {
+                throw new Error('AI request timeout', { cause: error });
+            }
+            throw error;
         }
         finally {
-            cancel.cancelled = true;
+            state.cancelled = true;
         }
     }
     async recoverFromContextLength(error, messages, options, retry) {

package/dist/src/ai/tester.d.ts

@@ -51,6 +51,7 @@ export declare class Tester extends TaskAgent implements Agent {
         success: boolean;
     }>;
     shouldAnalyzeProgress(iteration: number, currentState: ActionResult): boolean;
+    get isStruggling(): boolean;
     shouldStopForStalledExecution(task: Test, previousState: ActionResult, toolExecutions: any[]): boolean;
     prepareInstructionsForNextStep(task: Test): Promise<string>;
     reinjectContextIfNeeded(iteration: number, currentState: ActionResult): Promise<string>;

package/dist/src/ai/tester.js

@@ -285,7 +285,13 @@ export class Tester extends TaskAgent {
                             nextStep += `\n\n${guidance}`;
                     }
                     else if (this.shouldAnalyzeProgress(iteration, currentState) && this.pilot) {
-                        const guidance = await this.pilot.analyzeProgress(task, currentState, conversation);
+                        let guidance;
+                        if (this.isStruggling) {
+                            guidance = await this.pilot.analyzeProgress(task, currentState, conversation);
+                        }
+                        else {
+                            guidance = await this.pilot.periodicAnalyzeProgress(task, currentState, conversation);
+                        }
                         if (guidance)
                             nextStep += `\n\n${guidance}`;
                         this.consecutiveFailures = 0;
@@ -432,9 +438,7 @@ export class Tester extends TaskAgent {
             this.regionTransitioned = false;
             return true;
         }
-        if (this.consecutiveFailures >= 3)
-            return true;
-        if (this.consecutiveEmptyResults >= 2)
+        if (this.isStruggling)
             return true;
         if (iteration % this.progressCheckInterval !== 0)
             return false;
@@ -442,6 +446,9 @@ export class Tester extends TaskAgent {
             return false;
         return true;
     }
+    get isStruggling() {
+        return this.consecutiveFailures >= 3 || this.consecutiveEmptyResults >= 2;
+    }
     shouldStopForStalledExecution(task, previousState, toolExecutions) {
         if (task.hasFinished)
             return false;

package/dist/src/ai/tools.js

@@ -14,7 +14,7 @@ import { pause } from '../utils/loop.js';
 import { compactErrorMessage, normalizeInlineText, truncate } from "../utils/strings.js";
 import { WebElement } from "../utils/web-element.js";
 import { createJudgeTool } from "./judge-tool.js";
-import { JUDGE_PAGE_CAP, UNDECIDED } from "./judge.js";
+import { JUDGE_PAGE_CAP } from "./judge.js";
 import { sectionContextRule } from "./rules.js";
 import { isInteractive } from "./task-agent.js";
 const debugLog = createDebug('explorbot:tools');
@@ -92,13 +92,20 @@ export function createCodeceptJSTools({ explorer, stateManager, judge }, task) {
                 let ambiguityError = null;
                 for (let i = 0; i < commands.length; i++) {
                     const command = transformContainsCommand(commands[i]);
-                    const success = await action.attempt(command, explanation);
+                    let success = await action.attempt(command, explanation);
                     const attempt = { command, success };
                     if (action.lastError)
                         attempt.error = errorText(action.lastError);
                     attempts.push(attempt);
-                    if (!ambiguityError && action.lastError?.name === 'MultipleElementsFound')
+                    if (!ambiguityError && action.lastError?.name === 'MultipleElementsFound') {
                         ambiguityError = action.lastError;
+                        if (judge) {
+                            const labels = (await extractWebElements(ambiguityError))?.map((el) => el.label) || [];
+                            const index = await judge.pick(PICK_ELEMENT_QUESTION, labels, { intent: explanation, task: task.description });
+                            if (index)
+                                success = await action.attemptExactElementIndex(command, index, explanation);
+                        }
+                    }
                     if (success) {
                         const toolResult = await ActionResult.fromState(stateManager.getCurrentState()).toToolResult(previousState, command);
                         if (!hasObservablePageChange(toolResult)) {
@@ -127,7 +134,7 @@ export function createCodeceptJSTools({ explorer, stateManager, judge }, task) {
                     ...toolResult,
                     attempts,
                     suggestion,
-                }, ambiguityError || action.lastError, judge, explanation);
+                }, ambiguityError || action.lastError);
             },
         }),
         hover: tool({
@@ -1148,11 +1155,11 @@ export async function failedToolResult(action, message, data, error, judge, inte
         result.suggestion = getMultipleElementsSuggestion();
         result.multipleElementsDetected = true;
         result.elements = formatElementList(matched);
-        const labels = (matched || []).map((element) => `${element.text || 'no text'} — ${element.html}`);
-        const pick = await judge?.decide('Which listed element does the intent name?', [...labels, UNDECIDED], { intent });
-        if (!pick?.value)
+        if (!judge || !matched)
+            return result;
+        const index = await judge.pick(PICK_ELEMENT_QUESTION, matched.map((el) => el.label), { intent });
+        if (!index)
             return result;
-        const index = labels.indexOf(pick.value) + 1;
         result.suggestion = `Element ${index} is the one meant. Repeat the action with step.opts({ elementIndex: ${index} }) as the last argument.`;
         return result;
     }
@@ -1170,6 +1177,7 @@ function getMultipleElementsSuggestion() {
     reuse the same locator with step.opts({ elementIndex: N }) as the last argument.
     A match reported as not visible can never be acted on — pick one that is.
     If none of them is the element you want, narrow the locator with a container or its full unique text.
+    If the matches cannot be told apart, click the one you mean by appearance with visualClick().
     If the list is missing, call xpathCheck() to see what the locator matches.
   `;
 }
@@ -1206,6 +1214,7 @@ const MAX_DISAMBIGUATE_ELEMENTS = 10;
 const MAX_DISAMBIGUATE_TEXT = 80;
 const MAX_DISAMBIGUATE_HTML = 300;
 const MULTIPLE_ELEMENTS_PATTERN = 'multiple elements';
+const PICK_ELEMENT_QUESTION = 'Which listed element does the intent name?';
 async function extractWebElements(error) {
     if (!error || error.name !== 'MultipleElementsFound')
         return null;
@@ -1219,7 +1228,7 @@ async function extractWebElements(error) {
             const html = truncate(cleanHtmlSnippet(await elements[i].toOuterHTML()), MAX_DISAMBIGUATE_HTML);
             const text = truncate(normalizeInlineText((await elements[i].getText()) || ''), MAX_DISAMBIGUATE_TEXT);
             const visible = await Promise.resolve(elements[i].isVisible?.()).catch(() => undefined);
-            result.push({ xpath, html, text, visible });
+            result.push({ xpath, html, text, visible, label: `${text || 'no text'} — ${html}` });
         }
         catch (e) {
             debugLog('Failed to get details for element %d: %s', i, e);

package/dist/src/config.d.ts

@@ -162,6 +162,7 @@ interface AIConfig {
         model: string;
         tool?: boolean;
         direct?: boolean;
+        threshold?: number;
     };
     retryAttempts?: number;
     retryDelay?: number;

package/docs/basics/providers.md

@@ -332,6 +332,7 @@ export default {
 | `model` | `string` | Model ID at that provider |
 | `tool` | `boolean` | Give Tester and Pilot a `judge` tool to ask their own questions. Default: `true` |
 | `direct` | `boolean` | Consult the model at Explorbot's built-in decision points. Default: `true` |
+| `threshold` | `number` | How sure the model must be before Explorbot acts on an answer, between 0 and 1. Default: `0.7` |
 
 Without a config file, or to turn it on for one run, set `EXPLORBOT_DECISION_MODEL` or pass `--decision-model` as `provider/model-id`. It overrides `provider` and `model` in the config and keeps `tool` and `direct`:
 
@@ -340,9 +341,9 @@ EXPLORBOT_DECISION_MODEL=openrouter/typesafe/jev-1.13 npx explorbot explore
 npx prima-cli go "Settings" --decision-model typesafe/jev-latest
 ```
 
-Leave `decisionModel` unset and Explorbot never calls it. An unknown provider or a missing API key stops Explorbot at startup.
+Leave `decisionModel` unset and Explorbot never calls it. An unknown provider, a missing API key, or a `threshold` outside 0 and 1 stops Explorbot at startup.
 
-Explorbot acts on an answer only when it is more than 70% sure, and otherwise does what it would have done without the decision model. See [Decisions](../web-testing/decisions.md) for where it is consulted, the `judge` tool, and what happens when an answer is rejected.
+Explorbot acts on an answer only when it is more sure than `threshold` (70% by default), and otherwise does what it would have done without the decision model. See [Decisions](../web-testing/decisions.md) for where it is consulted, the `judge` tool, and what happens when an answer is rejected.
 
 ## Multi-Provider Configuration
 

package/docs/web-testing/decisions.md

@@ -13,16 +13,18 @@ Every question has one of two shapes:
 | **Statement** | "The page shows that this claim is true." | Probability that the statement holds |
 | **Choice** | "Which listed element does the intent name?" plus numbered options | The chosen option and its probability |
 
-A choice always includes an extra *undecided* option, so the model can say that none of the options fits.
+A choice always includes an extra *undecided* option, so the model can say that none of the options fits. The question tells the model to choose it when it is not sure.
 
 Explorbot acts on an answer only when it is **approved**:
 
-- a statement's probability is above **70%**, or
-- a choice picked a real option, not *undecided*, with a probability above **70%**.
+- a statement's probability is above the threshold, or
+- a choice picked a real option, not *undecided*, with a probability above the threshold.
+
+The threshold is **70%** unless `decisionModel.threshold` sets another value.
 
 Anything else counts as **rejected**, including:
 
-- a confident "no", or any probability of 70% or lower
+- a confident "no", or any probability at or below the threshold
 - *undecided*
 - a choice offered with fewer than two options
 - a request timeout (15 seconds) or a failed request
@@ -38,12 +40,15 @@ These are fixed places in the code where Explorbot asks the decision model befor
 
 When a click or fill fails because the locator matched more than one element, Explorbot lists the matches and asks which one the step meant.
 
-- **Approved:** the Tester is told which element to use and how to target it with `elementIndex`.
+- **Approved:** a click is repeated right away on the chosen element with `elementIndex`, before the Tester's remaining fallback locators. For other actions, the Tester is told which element to use.
+- **Identical options:** when two matches look the same, the question is not asked. The Tester is told to click by appearance with `visualClick()`.
 - **Fallback:** the Tester receives the numbered list of matches and picks one itself, narrows the locator, or uses a visual click.
 
 ### Pilot: progress review
 
-Every few steps, Pilot reviews the Tester's recent actions with the `agenticModel`. First it asks whether the run is moving toward its goal and can continue unsupervised for now.
+Every few steps, Pilot reviews the Tester's recent actions with the `agenticModel`. On a routine review it first asks whether the recent actions advance the scenario toward its remaining expected outcomes. The decision model sees the scenario, its planned steps, which expected outcomes are checked and which remain, the run log, the pages visited, the current page and the recent actions.
+
+A review forced by a run of failed or empty steps skips the question and goes straight to Pilot.
 
 - **Approved:** the review is skipped and the Tester continues. This saves one `agenticModel` call.
 - **Fallback:** Pilot runs its full review as usual.
@@ -105,4 +110,10 @@ The model supplies only the question. The tool gathers the rest of the state its
 
 ## Tracing
 
-Each question is recorded as a `judge.decide` span with the question, the answer and its confidence. A failed request also records the reason. The TUI shows `⚖️ Asking judge...` while a question is in flight. See [Observability](../contributing/observability.md) for setting up tracing.
+Each question is recorded as a `judge.decide` span:
+
+- **Input:** the exact request body sent to the provider: model, state and question. Post it again to the endpoint to replay the decision.
+- **Output:** the raw answer and its probability, before the threshold is applied.
+- **Metadata:** `judgeQuestion` for filtering by question, `judgeDecision` with the approved value, confidence, threshold and model, and `judgeEndpoint`.
+
+A failed request marks the span as an error and records the reason. The TUI shows `⚖️ Asking judge...` while a question is in flight. See [Observability](../contributing/observability.md) for setting up tracing.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "explorbot",
-  "version": "0.5.1",
+  "version": "0.5.2",
   "description": "CLI app built with React Ink, CodeceptJS, and Playwright",
   "license": "Elastic-2.0",
   "type": "module",

package/src/action.ts

@@ -491,6 +491,14 @@ class Action {
     }
   }
 
+  public async attemptExactElementIndex(codeBlock: string, elementIndex: number, originalMessage?: string): Promise<boolean> {
+    const applyElementIndex = (step: any) => {
+      step.opts = { ...step.opts, elementIndex };
+    };
+    codeceptjs.event.dispatcher.once(codeceptjs.event.step.started, applyElementIndex);
+    return this.attempt(codeBlock, originalMessage).finally(() => codeceptjs.event.dispatcher.off(codeceptjs.event.step.started, applyElementIndex));
+  }
+
   async exitIframe(): Promise<void> {
     if (!this.playwrightHelper.frame) return;
     debugLog('Switching to main frame');

package/src/ai/judge-provider.ts

@@ -1,3 +1,6 @@
+import { Observability } from '../observability.ts';
+import { redactSecrets } from '../utils/secrets.ts';
+
 const REQUEST_TIMEOUT_MS = 15000;
 
 export class JudgeProvider {
@@ -21,6 +24,9 @@ export class JudgeProvider {
     let q: Record<string, unknown> = { type: 'noul', instructions: question };
     if (options) q = { type: 'choice', instructions: question, criteria: Object.fromEntries(options.map((option, index) => [String(index), option])) };
     const body = JSON.stringify({ model: this.model, state, questions: { q } });
+    const span = Observability.getSpan();
+    span?.setAttribute('langfuse.observation.input', redactSecrets(body));
+    span?.setAttribute('ai.telemetry.metadata.judgeEndpoint', this.url);
     const controller = new AbortController();
     const timer = setTimeout(() => controller.abort(), this.requestTimeoutMs);
     try {

package/src/ai/judge.ts

@@ -9,6 +9,7 @@ const debugLog = createDebug('explorbot:judge');
 const APPROVAL_THRESHOLD = 0.7;
 
 export const UNDECIDED = 'undecided';
+const UNDECIDED_HINT = `Choose "${UNDECIDED}" if you are not sure or no option fits.`;
 export const JUDGE_PAGE_CAP = 12000;
 
 export class Decision {
@@ -29,12 +30,15 @@ export class Decision {
 export class Judge {
   constructor(
     private provider: JudgeProvider,
-    private enabled: { tool: boolean; direct: boolean }
+    private enabled: { tool: boolean; direct: boolean },
+    private threshold = APPROVAL_THRESHOLD
   ) {}
 
   static fromConfig(config: AIConfig['decisionModel']): Judge | null {
     if (!config) return null;
-    return new Judge(new JudgeProvider(config.provider, config.model), { tool: config.tool !== false, direct: config.direct !== false });
+    const threshold = config.threshold ?? APPROVAL_THRESHOLD;
+    if (!(threshold > 0 && threshold < 1)) throw new Error(`decisionModel.threshold must be between 0 and 1, got ${config.threshold}`);
+    return new Judge(new JudgeProvider(config.provider, config.model), { tool: config.tool !== false, direct: config.direct !== false }, threshold);
   }
 
   get toolEnabled(): boolean {
@@ -46,12 +50,21 @@ export class Judge {
     return this.consult(question, options, state);
   }
 
+  async pick(question: string, options: string[], state: unknown): Promise<number | null> {
+    if (new Set(options).size < options.length) return null;
+    const decision = await this.decide(question, [...options, UNDECIDED], state);
+    if (!decision.value) return null;
+    return options.indexOf(decision.value) + 1;
+  }
+
   async consult(question: string, options: string[] | boolean | null, state: unknown): Promise<Decision> {
     if (Array.isArray(options) && options.length < 2) return new Decision(null, 0);
     return Observability.run('judge.decide', { tags: ['judge'] }, async () => {
       setActivity('⚖️ Asking judge...', 'ai');
       const decision = await this.request(question, options, state).finally(() => clearActivity());
-      Observability.getSpan()?.setAttribute('ai.telemetry.metadata.judgeDecision', JSON.stringify({ question, value: decision.value, confidence: decision.confidence }));
+      const span = Observability.getSpan();
+      span?.setAttribute('ai.telemetry.metadata.judgeQuestion', question);
+      span?.setAttribute('ai.telemetry.metadata.judgeDecision', JSON.stringify({ question, value: decision.value, confidence: decision.confidence, threshold: this.threshold, model: this.provider.model }));
       return decision;
     });
   }
@@ -59,17 +72,23 @@ export class Judge {
   private async request(question: string, options: string[] | boolean | null, state: unknown): Promise<Decision> {
     let list: string[] | undefined;
     if (Array.isArray(options)) list = options;
+    let instructions = question;
+    if (list?.includes(UNDECIDED)) instructions = `${question} ${UNDECIDED_HINT}`;
 
-    const answer = await this.provider.decide(state, question, list).catch((error: unknown) => this.recordFailure(error));
+    const answer = await this.provider.decide(state, instructions, list).catch((error: unknown) => this.recordFailure(error));
     if (!answer) return new Decision(null, 0);
-    if (answer.probability <= APPROVAL_THRESHOLD) return new Decision(null, answer.probability);
+    Observability.getSpan()?.setAttribute('langfuse.observation.output', JSON.stringify(answer));
+    if (answer.probability <= this.threshold) return new Decision(null, answer.probability);
     if (answer.value === UNDECIDED) return new Decision(null, answer.probability);
     return new Decision(answer.value, answer.probability);
   }
 
   private recordFailure(error: unknown): null {
     debugLog('judge declined: %s', error);
-    Observability.getSpan()?.setAttribute('ai.telemetry.metadata.judgeError', String(error));
+    const span = Observability.getSpan();
+    span?.setAttribute('ai.telemetry.metadata.judgeError', String(error));
+    span?.setAttribute('langfuse.observation.level', 'ERROR');
+    span?.setAttribute('langfuse.observation.status_message', String(error));
     return null;
   }
 }

package/src/ai/pilot.ts

@@ -19,7 +19,7 @@ import type { Agent, AgentDeps } from './agent.ts';
 import type { Conversation } from './conversation.ts';
 import type { Fisherman } from './fisherman.ts';
 import { createAskApiTool } from './fisherman/tools.ts';
-import { type Judge, UNDECIDED } from './judge.ts';
+import { JUDGE_PAGE_CAP, type Judge, UNDECIDED } from './judge.ts';
 import type { Navigator } from './navigator.ts';
 import type { Provider } from './provider.ts';
 import type { Researcher } from './researcher.ts';
@@ -542,6 +542,22 @@ export class Pilot implements Agent {
     );
   }
 
+  async periodicAnalyzeProgress(task: Test, currentState: ActionResult, testerConversation: Conversation): Promise<string> {
+    const toolCalls = testerConversation.getToolExecutions().slice(-this.stepsToReview);
+    const healthy = await this.judge?.decide('The recent actions advance the scenario toward its remaining expected outcomes.', null, {
+      scenario: task.scenario,
+      plannedSteps: task.plannedSteps,
+      expectations: this.formatExpectations(task),
+      runLog: task.notesToString() || 'No steps recorded.',
+      visitedUrls: task.getVisitedUrls({ localOnly: true }),
+      state: this.buildStateContext(currentState),
+      page: currentState.getCompactARIA().slice(0, JUDGE_PAGE_CAP),
+      recentActions: this.formatActions(toolCalls),
+    });
+    if (healthy?.approved) return '';
+    return this.analyzeProgress(task, currentState, testerConversation);
+  }
+
   async analyzeProgress(task: Test, currentState: ActionResult, testerConversation: Conversation): Promise<string> {
     tag('substep').log('Pilot analyzing progress...');
 
@@ -555,9 +571,6 @@ export class Pilot implements Agent {
     const actionsContext = this.formatActions(toolCalls);
     const stateContext = this.buildStateContext(currentState);
 
-    const healthy = await this.judge?.decide('The run is moving toward the goal and can continue without a supervisor reviewing it now.', null, { scenario: task.scenario, state: stateContext, recentActions: actionsContext });
-    if (healthy?.approved) return '';
-
     const hasFailures = toolCalls.length === 0 || toolCalls.some((t) => !t.wasSuccessful);
 
     const text = await this.sendToPilot(

package/src/ai/provider.ts

@@ -1,10 +1,10 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
 import { OpenTelemetry } from '@ai-sdk/otel';
 import { LangfuseSpanProcessor } from '@langfuse/otel';
 import { NodeSDK } from '@opentelemetry/sdk-node';
-import { AsyncLocalStorage } from 'node:async_hooks';
-import dedent from 'dedent';
 import { APICallError, NoObjectGeneratedError, asSchema, extractJsonMiddleware, generateObject, generateText, isStepCount, parsePartialJson, registerTelemetry, tool, wrapLanguageModel } from 'ai';
 import type { ModelMessage } from 'ai';
+import dedent from 'dedent';
 import { z } from 'zod';
 import { clearActivity, setActivity } from '../activity.ts';
 import { type AIConfig, configuredModels, modelName as getModelName } from '../config.js';
@@ -60,14 +60,15 @@ function extractCachedTokens(usage: any): number {
   return usage?.inputTokenDetails?.cacheReadTokens ?? 0;
 }
 
-function abortAfterIdle(ms: number, cancel: { cancelled: boolean }, controller: AbortController, busy: { tools: number }): Promise<never> {
+function abortAfterIdle(ms: number, state: { cancelled: boolean; timedOut: boolean }, controller: AbortController, busy: { tools: number }): Promise<never> {
   return new Promise((_, reject) => {
     const tick = () => {
-      if (cancel.cancelled) return;
+      if (state.cancelled) return;
       if (executionController.isAwaitingInput() || busy.tools > 0) {
         setTimeout(tick, ms);
         return;
       }
+      state.timedOut = true;
       controller.abort();
       reject(new Error('AI request timeout'));
     };
@@ -252,13 +253,18 @@ export class Provider {
   }
 
   private async raceWithIdleTimeout<T>(fn: (signal: AbortSignal) => Promise<T>, timeoutMs: number, busy: { tools: number } = { tools: 0 }): Promise<T> {
-    const cancel = { cancelled: false };
+    const state = { cancelled: false, timedOut: false };
     const controller = new AbortController();
     const combinedSignal = combinedAbortSignal(controller);
     try {
-      return await Promise.race([fn(combinedSignal), abortAfterIdle(timeoutMs, cancel, controller, busy)]);
+      return await Promise.race([fn(combinedSignal), abortAfterIdle(timeoutMs, state, controller, busy)]);
+    } catch (error) {
+      if (state.timedOut && !executionController.getAbortSignal()?.aborted) {
+        throw new Error('AI request timeout', { cause: error });
+      }
+      throw error;
     } finally {
-      cancel.cancelled = true;
+      state.cancelled = true;
     }
   }
 

package/src/ai/tester.ts

@@ -324,7 +324,12 @@ export class Tester extends TaskAgent implements Agent {
               const guidance = await this.pilot.reviewNewPage(task, currentState, conversation);
               if (guidance) nextStep += `\n\n${guidance}`;
             } else if (this.shouldAnalyzeProgress(iteration, currentState) && this.pilot) {
-              const guidance = await this.pilot.analyzeProgress(task, currentState, conversation);
+              let guidance: string;
+              if (this.isStruggling) {
+                guidance = await this.pilot.analyzeProgress(task, currentState, conversation);
+              } else {
+                guidance = await this.pilot.periodicAnalyzeProgress(task, currentState, conversation);
+              }
               if (guidance) nextStep += `\n\n${guidance}`;
               this.consecutiveFailures = 0;
               this.lastAnalyzedStateHash = currentState.hash;
@@ -486,13 +491,16 @@ export class Tester extends TaskAgent implements Agent {
       this.regionTransitioned = false;
       return true;
     }
-    if (this.consecutiveFailures >= 3) return true;
-    if (this.consecutiveEmptyResults >= 2) return true;
+    if (this.isStruggling) return true;
     if (iteration % this.progressCheckInterval !== 0) return false;
     if (this.lastAnalyzedStateHash === currentState.hash) return false;
     return true;
   }
 
+  private get isStruggling(): boolean {
+    return this.consecutiveFailures >= 3 || this.consecutiveEmptyResults >= 2;
+  }
+
   private shouldStopForStalledExecution(task: Test, previousState: ActionResult, toolExecutions: any[]): boolean {
     if (task.hasFinished) return false;
 

package/src/ai/tools.ts

@@ -16,7 +16,7 @@ import { compactErrorMessage, normalizeInlineText, truncate } from '../utils/str
 import { WebElement } from '../utils/web-element.ts';
 import type { ToolDeps } from './agent.ts';
 import { createJudgeTool } from './judge-tool.ts';
-import { JUDGE_PAGE_CAP, type Judge, UNDECIDED } from './judge.ts';
+import { JUDGE_PAGE_CAP, type Judge } from './judge.ts';
 import { Navigator } from './navigator.ts';
 import { Researcher } from './researcher.ts';
 import { sectionContextRule } from './rules.ts';
@@ -113,13 +113,20 @@ export function createCodeceptJSTools({ explorer, stateManager, judge }: ToolDep
 
         for (let i = 0; i < commands.length; i++) {
           const command = transformContainsCommand(commands[i]);
-          const success = await action.attempt(command, explanation);
+          let success = await action.attempt(command, explanation);
 
           const attempt: { command: string; success: boolean; error?: string } = { command, success };
           if (action.lastError) attempt.error = errorText(action.lastError);
           attempts.push(attempt);
 
-          if (!ambiguityError && action.lastError?.name === 'MultipleElementsFound') ambiguityError = action.lastError;
+          if (!ambiguityError && action.lastError?.name === 'MultipleElementsFound') {
+            ambiguityError = action.lastError;
+            if (judge) {
+              const labels = (await extractWebElements(ambiguityError))?.map((el) => el.label) || [];
+              const index = await judge.pick(PICK_ELEMENT_QUESTION, labels, { intent: explanation, task: task.description });
+              if (index) success = await action.attemptExactElementIndex(command, index, explanation);
+            }
+          }
 
           if (success) {
             const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, command);
@@ -158,9 +165,7 @@ export function createCodeceptJSTools({ explorer, stateManager, judge }: ToolDep
             attempts,
             suggestion,
           },
-          ambiguityError || action.lastError,
-          judge,
-          explanation
+          ambiguityError || action.lastError
         );
       },
     }),
@@ -1309,10 +1314,13 @@ export async function failedToolResult(action: string, message: string, data?: R
     result.suggestion = getMultipleElementsSuggestion();
     result.multipleElementsDetected = true;
     result.elements = formatElementList(matched);
-    const labels = (matched || []).map((element) => `${element.text || 'no text'} — ${element.html}`);
-    const pick = await judge?.decide('Which listed element does the intent name?', [...labels, UNDECIDED], { intent });
-    if (!pick?.value) return result;
-    const index = labels.indexOf(pick.value) + 1;
+    if (!judge || !matched) return result;
+    const index = await judge.pick(
+      PICK_ELEMENT_QUESTION,
+      matched.map((el) => el.label),
+      { intent }
+    );
+    if (!index) return result;
     result.suggestion = `Element ${index} is the one meant. Repeat the action with step.opts({ elementIndex: ${index} }) as the last argument.`;
     return result;
   }
@@ -1333,6 +1341,7 @@ function getMultipleElementsSuggestion(): string {
     reuse the same locator with step.opts({ elementIndex: N }) as the last argument.
     A match reported as not visible can never be acted on — pick one that is.
     If none of them is the element you want, narrow the locator with a container or its full unique text.
+    If the matches cannot be told apart, click the one you mean by appearance with visualClick().
     If the list is missing, call xpathCheck() to see what the locator matches.
   `;
 }
@@ -1379,6 +1388,7 @@ const MAX_DISAMBIGUATE_ELEMENTS = 10;
 const MAX_DISAMBIGUATE_TEXT = 80;
 const MAX_DISAMBIGUATE_HTML = 300;
 const MULTIPLE_ELEMENTS_PATTERN = 'multiple elements';
+const PICK_ELEMENT_QUESTION = 'Which listed element does the intent name?';
 
 async function extractWebElements(error: Error | null | undefined): Promise<MatchedElement[] | null> {
   if (!error || error.name !== 'MultipleElementsFound') return null;
@@ -1393,7 +1403,7 @@ async function extractWebElements(error: Error | null | undefined): Promise<Matc
       const html = truncate(cleanHtmlSnippet(await elements[i].toOuterHTML()), MAX_DISAMBIGUATE_HTML);
       const text = truncate(normalizeInlineText((await elements[i].getText()) || ''), MAX_DISAMBIGUATE_TEXT);
       const visible = await Promise.resolve(elements[i].isVisible?.()).catch(() => undefined);
-      result.push({ xpath, html, text, visible });
+      result.push({ xpath, html, text, visible, label: `${text || 'no text'} — ${html}` });
     } catch (e) {
       debugLog('Failed to get details for element %d: %s', i, e);
     }
@@ -1443,5 +1453,6 @@ interface MatchedElement {
   xpath: string;
   html: string;
   text: string;
+  label: string;
   visible?: boolean;
 }

package/src/config.ts

@@ -185,7 +185,7 @@ interface AIConfig {
   vision?: boolean;
   visionModel?: any;
   agenticModel?: any;
-  decisionModel?: { provider: string; model: string; tool?: boolean; direct?: boolean };
+  decisionModel?: { provider: string; model: string; tool?: boolean; direct?: boolean; threshold?: number };
   retryAttempts?: number;
   retryDelay?: number;
   maxParallelRequests?: number;