explorbot 0.5.1 → 0.5.3

package/dist/src/config.js

@@ -65,6 +65,7 @@ export class ConfigParser {
     site = null;
     siteStartPath = '/';
     siteConfigPath = null;
+    runBase = null;
     constructor() { }
     static loadEnv(filePath, keepExisting = false) {
         const resolved = resolve(filePath);
@@ -96,6 +97,7 @@ export class ConfigParser {
         if (this.config && !options?.config && !options?.path && this.runtimeTarget === target) {
             return this.config;
         }
+        this.runBase = parseRunBase(options?.baseUrl);
         // Store the initial working directory for reference
         if (!process.env.INITIAL_CWD) {
             process.env.INITIAL_CWD = process.cwd();
@@ -123,12 +125,19 @@ export class ConfigParser {
                 let envUrl = options?.baseUrl;
                 if (!envUrl && target?.startsWith('http'))
                     envUrl = target;
-                const outputRoot = resolveOutputRoot(process.env.EXPLORBOT_URL || envUrl);
+                if (options?.baseUrl) {
+                    const envOrigin = URL.parse(process.env.EXPLORBOT_URL || '')?.origin;
+                    const baseOrigin = URL.parse(options.baseUrl)?.origin;
+                    if (envOrigin && envOrigin !== baseOrigin) {
+                        tag('warning').log(`--base-url ${baseOrigin} selects the site for this run — EXPLORBOT_URL ${envOrigin} is ignored`);
+                    }
+                }
+                const outputRoot = resolveOutputRoot(options?.baseUrl || process.env.EXPLORBOT_URL || envUrl);
                 loadedConfig = await this.buildEnvConfig(envUrl, outputRoot);
                 sourcePath = join(outputRoot, 'explorbot.config.js');
                 log(`Configuration built from EXPLORBOT_* environment variables. Output: ${outputRoot}`);
             }
-            let config = this.resolveConfig(loadedConfig, options);
+            let config = this.resolveConfig(loadedConfig);
             await resolveConfigModels(config.ai);
             this.site = null;
             this.siteConfigPath = null;
@@ -195,6 +204,22 @@ export class ConfigParser {
     getSiteConfigPath() {
         return this.siteConfigPath;
     }
+    applyBasePath(target) {
+        const base = this.runBase?.path;
+        if (!base)
+            return target;
+        if (!target.startsWith('/'))
+            return target;
+        if (target === base || target.startsWith(`${base}/`) || target.startsWith(`${base}?`))
+            return target;
+        return `${base}${target}`;
+    }
+    getBaseQuery() {
+        return this.runBase?.query || '';
+    }
+    getBasePath() {
+        return this.runBase?.path || '';
+    }
     resolveTargetPath(target) {
         if (!this.site) {
             const configured = this.config?.playwright?.url || this.config?.web?.url;
@@ -203,13 +228,15 @@ export class ConfigParser {
             if (targetOrigin && baseOrigin && targetOrigin !== baseOrigin) {
                 tag('warning').log(`Exploring ${targetOrigin} but base URL is ${baseOrigin}. Relative navigation resolves against the base URL — set web.url to ${targetOrigin} to avoid it.`);
             }
-            return target || '/';
+            return this.applyBasePath(target || '/');
         }
         if (!target)
             return this.siteStartPath;
         const resolved = resolveSiteTarget(target, this.site.url);
         if (resolved.baseUrl !== this.site.url)
             return target;
+        if (target.startsWith('/'))
+            return this.applyBasePath(resolved.path);
         return resolved.path;
     }
     getStatesDir() {
@@ -232,6 +259,7 @@ export class ConfigParser {
             ConfigParser.instance.site = null;
             ConfigParser.instance.siteStartPath = '/';
             ConfigParser.instance.siteConfigPath = null;
+            ConfigParser.instance.runBase = null;
         }
     }
     // For testing purposes only - sets up minimal default config
@@ -393,14 +421,14 @@ export class ConfigParser {
             return JSON.parse(content);
         }
     }
-    resolveConfig(config, options) {
+    resolveConfig(config) {
         if (config.web?.url && !config.playwright?.url) {
             config.playwright = config.playwright || { browser: 'chromium', url: '' };
             config.playwright.url = config.web.url;
         }
-        if (options?.baseUrl) {
+        if (this.runBase) {
             config.playwright = config.playwright || { browser: 'chromium', url: '' };
-            config.playwright.url = options.baseUrl;
+            config.playwright.url = this.runBase.origin;
         }
         resolveLangfuse(config.ai);
         return config;
@@ -609,3 +637,11 @@ export async function createModel(provider, modelId) {
     }
     return (await info.load())(modelId);
 }
+function parseRunBase(baseUrl) {
+    if (!baseUrl)
+        return null;
+    const url = URL.parse(baseUrl);
+    if (!url)
+        throw new Error(`Base URL must be a full URL like https://app.example.com/team/acme, got "${baseUrl}"`);
+    return { origin: url.origin, path: url.pathname.replace(/\/+$/, ''), query: url.search };
+}

package/dist/src/explorbot.js

@@ -449,6 +449,10 @@ export class ExplorBot {
         this.currentPlan = plan;
         if (!plan)
             return;
+        if (plan.url)
+            plan.url = this.configParser.applyBasePath(plan.url);
+        for (const test of plan.tests)
+            test.startUrl = this.configParser.applyBasePath(test.startUrl);
         if (!this.sessionPlans.includes(plan)) {
             this.sessionPlans.push(plan);
         }

package/dist/src/explorer.d.ts

@@ -57,6 +57,8 @@ declare class Explorer {
     convertToCodeceptConfig(config: ExplorbotConfig): any;
     connectOrLaunchBrowser(): Promise<void>;
     openContextPage(): Promise<void>;
+    applyBasePathToHelper(): void;
+    pinBaseQuery(): Promise<void>;
     createBrowserContextOptions(): BrowserContextOptions;
     attachXhrCapture(): void;
     runWithRecovery<T>(label: string, operation: () => Promise<T>): Promise<T>;

package/dist/src/explorer.js

@@ -73,6 +73,7 @@ class Explorer {
         if (!this.playwrightHelper) {
             throw new Error('Playwright helper not available');
         }
+        this.applyBasePathToHelper();
         await this.connectOrLaunchBrowser();
         const hasSession = !this.options?.attachedBrowser && this.options?.session && existsSync(this.options.session);
         await this.openContextPage();
@@ -124,7 +125,7 @@ class Explorer {
     }
     async visit(url, opts = {}) {
         return this.runWithRecovery('visit', async () => {
-            const action = await this.visitOnce(url);
+            const action = await this.visitOnce(ConfigParser.getInstance().applyBasePath(url));
             if (opts.screenshot)
                 return action.capturePageState({ includeScreenshot: true });
             return action.getActionResult() ?? action.capturePageState();
@@ -279,6 +280,7 @@ class Explorer {
         const attached = this.options?.attachedBrowser;
         if (!attached) {
             await this.playwrightHelper._createContextPage(this.createBrowserContextOptions());
+            await this.pinBaseQuery();
             return;
         }
         const context = attached.contexts()[0] || (await attached.newContext());
@@ -287,6 +289,36 @@ class Explorer {
         this.playwrightHelper.browserContext = context;
         await this.playwrightHelper._setPage(page);
         debugLog(`Adopted attached browser page: ${page.url()}`);
+        await this.pinBaseQuery();
+    }
+    applyBasePathToHelper() {
+        if (!ConfigParser.getInstance().getBasePath())
+            return;
+        const helper = this.playwrightHelper;
+        const original = helper.amOnPage.bind(helper);
+        helper.amOnPage = (url) => original(ConfigParser.getInstance().applyBasePath(url));
+    }
+    async pinBaseQuery() {
+        const query = ConfigParser.getInstance().getBaseQuery();
+        if (!query)
+            return;
+        const pinned = new URLSearchParams(query);
+        const origin = URL.parse(this.config.playwright.url)?.origin;
+        await this.playwrightHelper.browserContext.route('**/*', async (route) => {
+            const request = route.request();
+            if (request.resourceType() !== 'document')
+                return route.continue();
+            const url = URL.parse(request.url());
+            if (!url || url.origin !== origin)
+                return route.continue();
+            const missing = [...pinned].filter(([key]) => !url.searchParams.has(key));
+            if (!missing.length)
+                return route.continue();
+            for (const [key, value] of missing)
+                url.searchParams.set(key, value);
+            return route.continue({ url: url.toString() });
+        });
+        tag('info').log(`Every page load of ${origin} carries ${query}`);
     }
     createBrowserContextOptions() {
         const helperOptions = this.playwrightHelper.options || {};

package/dist/src/global-config.js

@@ -116,7 +116,7 @@ export function resolveSiteTarget(target, defaultBaseUrl) {
         return { baseUrl: url.origin, path: `${url.pathname}${url.search}${url.hash}` };
     }
     if (raw.startsWith('/')) {
-        const base = defaultBaseUrl || process.env.EXPLORBOT_URL;
+        const base = process.env.EXPLORBOT_URL || defaultBaseUrl;
         if (!base) {
             throw new Error(withSites(`Cannot resolve path "${raw}" without a site.`));
         }

package/dist/src/utils/test-files.d.ts

@@ -1,4 +1,9 @@
 export declare function listTestFiles(testsDir: string): string[];
+/**
+ * Generated tests import `codeceptjs/steps`, but in global mode they lie in ~/.explorbot,
+ * far from any node_modules. Link the bot's own codeceptjs next to them so Node resolves it.
+ */
+export declare function ensureCodeceptResolvable(dir: string): void;
 export declare function loadTestSuites(testsDir: string): any[];
 export declare function printTestList(suites: any[]): void;
 export declare function dryRunTestFile(filePath: string): Promise<void>;

package/dist/src/utils/test-files.js

@@ -1,4 +1,5 @@
-import { existsSync, readdirSync } from 'node:fs';
+import { existsSync, mkdirSync, readdirSync, symlinkSync } from 'node:fs';
+import { createRequire } from 'node:module';
 import path from 'node:path';
 import chalk from 'chalk';
 import { highlight } from 'cli-highlight';
@@ -8,6 +9,7 @@ import storeListener from 'codeceptjs/lib/listener/store';
 import store from 'codeceptjs/lib/store';
 import figureSet from 'figures';
 import { ConfigParser } from "../config.js";
+const nodeRequire = createRequire(import.meta.url);
 export function listTestFiles(testsDir) {
     if (!existsSync(testsDir))
         return [];
@@ -15,10 +17,34 @@ export function listTestFiles(testsDir) {
         .filter((f) => f.endsWith('.js'))
         .map((f) => path.resolve(testsDir, f));
 }
+/**
+ * Generated tests import `codeceptjs/steps`, but in global mode they lie in ~/.explorbot,
+ * far from any node_modules. Link the bot's own codeceptjs next to them so Node resolves it.
+ */
+export function ensureCodeceptResolvable(dir) {
+    const link = path.join(dir, 'node_modules', 'codeceptjs');
+    if (existsSync(link))
+        return;
+    let target;
+    try {
+        target = path.dirname(nodeRequire.resolve('codeceptjs/package.json'));
+    }
+    catch {
+        return;
+    }
+    try {
+        mkdirSync(path.dirname(link), { recursive: true });
+        symlinkSync(target, link, process.platform === 'win32' ? 'junction' : 'dir');
+    }
+    catch {
+        // Best effort: a project install already resolves codeceptjs without the link.
+    }
+}
 export function loadTestSuites(testsDir) {
     const jsFiles = listTestFiles(testsDir);
     if (jsFiles.length === 0)
         return [];
+    ensureCodeceptResolvable(testsDir);
     codeceptjs.container.createMocha();
     const mocha = codeceptjs.container.mocha();
     mocha.files = jsFiles;
@@ -79,6 +105,7 @@ export async function dryRunTestFile(filePath) {
     const mocha = codeceptjs.container.mocha();
     mocha.reporter(class {
     });
+    ensureCodeceptResolvable(path.dirname(absPath));
     mocha.files = [absPath];
     mocha.loadFiles();
     let currentSuite = '';

package/docs/basics/providers.md

@@ -332,6 +332,7 @@ export default {
 | `model` | `string` | Model ID at that provider |
 | `tool` | `boolean` | Give Tester and Pilot a `judge` tool to ask their own questions. Default: `true` |
 | `direct` | `boolean` | Consult the model at Explorbot's built-in decision points. Default: `true` |
+| `threshold` | `number` | How sure the model must be before Explorbot acts on an answer, between 0 and 1. Default: `0.7` |
 
 Without a config file, or to turn it on for one run, set `EXPLORBOT_DECISION_MODEL` or pass `--decision-model` as `provider/model-id`. It overrides `provider` and `model` in the config and keeps `tool` and `direct`:
 
@@ -340,9 +341,9 @@ EXPLORBOT_DECISION_MODEL=openrouter/typesafe/jev-1.13 npx explorbot explore
 npx prima-cli go "Settings" --decision-model typesafe/jev-latest
 ```
 
-Leave `decisionModel` unset and Explorbot never calls it. An unknown provider or a missing API key stops Explorbot at startup.
+Leave `decisionModel` unset and Explorbot never calls it. An unknown provider, a missing API key, or a `threshold` outside 0 and 1 stops Explorbot at startup.
 
-Explorbot acts on an answer only when it is more than 70% sure, and otherwise does what it would have done without the decision model. See [Decisions](../web-testing/decisions.md) for where it is consulted, the `judge` tool, and what happens when an answer is rejected.
+Explorbot acts on an answer only when it is more sure than `threshold` (70% by default), and otherwise does what it would have done without the decision model. See [Decisions](../web-testing/decisions.md) for where it is consulted, the `judge` tool, and what happens when an answer is rejected.
 
 ## Multi-Provider Configuration
 

package/docs/reference/commands.md

@@ -66,11 +66,23 @@ Every CLI command that drives a browser accepts these options (`start`, `explore
 | `--debug` | Enable debug logging (same as `--verbose`) |
 | `-c, --config <path>` | Path to configuration file |
 | `-p, --path <path>` | Working directory path |
+| `--base-url <url>` | Run against this URL: its path scopes relative paths, its query params ride along with every page load |
 | `-s, --show` | Show browser window |
 | `--headless` | Run browser in headless mode |
 | `--incognito` | Run without recording experiences |
 | `--session [file]` | Save/restore browser session (cookies, localStorage) from file |
 
+### `--base-url`
+
+Points one run at one area of a site, leaving every config file alone. The origin selects the site, so the knowledge and experience already stored for that host still apply. The path prefixes every relative path a plan or a command gives, and the query params are added to each page load.
+
+```bash
+npx explorbot test tests/plans/runs.md '*' \
+  --base-url 'https://app.example.com/teams/acme/?preview=pr-42'
+```
+
+A plan whose tests start at `/runs` runs them at `/teams/acme/runs`, and every page load carries `?preview=pr-42` — enough to keep a preview deployment selected for a whole session. The param travels on the request rather than in the address bar, so Explorbot navigates to clean URLs, though an app that redirects can still echo the param back. Absolute URLs are never prefixed: `https://app.example.com/users/sign_in` still points outside the scoped area, as a login page usually must.
+
 ### `--knowledge`
 
 Passes facts to the run without creating a file in `knowledge/`. Plain text applies everywhere; add frontmatter to scope it to a page or an API endpoint. Repeat the flag for several facts. See [Knowledge](../workflow/knowledge.md#per-session-knowledge).

package/docs/web-testing/decisions.md

@@ -13,16 +13,18 @@ Every question has one of two shapes:
 | **Statement** | "The page shows that this claim is true." | Probability that the statement holds |
 | **Choice** | "Which listed element does the intent name?" plus numbered options | The chosen option and its probability |
 
-A choice always includes an extra *undecided* option, so the model can say that none of the options fits.
+A choice always includes an extra *undecided* option, so the model can say that none of the options fits. The question tells the model to choose it when it is not sure.
 
 Explorbot acts on an answer only when it is **approved**:
 
-- a statement's probability is above **70%**, or
-- a choice picked a real option, not *undecided*, with a probability above **70%**.
+- a statement's probability is above the threshold, or
+- a choice picked a real option, not *undecided*, with a probability above the threshold.
+
+The threshold is **70%** unless `decisionModel.threshold` sets another value.
 
 Anything else counts as **rejected**, including:
 
-- a confident "no", or any probability of 70% or lower
+- a confident "no", or any probability at or below the threshold
 - *undecided*
 - a choice offered with fewer than two options
 - a request timeout (15 seconds) or a failed request
@@ -38,12 +40,15 @@ These are fixed places in the code where Explorbot asks the decision model befor
 
 When a click or fill fails because the locator matched more than one element, Explorbot lists the matches and asks which one the step meant.
 
-- **Approved:** the Tester is told which element to use and how to target it with `elementIndex`.
+- **Approved:** a click is repeated right away on the chosen element with `elementIndex`, before the Tester's remaining fallback locators. For other actions, the Tester is told which element to use.
+- **Identical options:** when two matches look the same, the question is not asked. The Tester is told to click by appearance with `visualClick()`.
 - **Fallback:** the Tester receives the numbered list of matches and picks one itself, narrows the locator, or uses a visual click.
 
 ### Pilot: progress review
 
-Every few steps, Pilot reviews the Tester's recent actions with the `agenticModel`. First it asks whether the run is moving toward its goal and can continue unsupervised for now.
+Every few steps, Pilot reviews the Tester's recent actions with the `agenticModel`. On a routine review it first asks whether the recent actions advance the scenario toward its remaining expected outcomes. The decision model sees the scenario, its planned steps, which expected outcomes are checked and which remain, the run log, the pages visited, the current page and the recent actions.
+
+A review forced by a run of failed or empty steps skips the question and goes straight to Pilot.
 
 - **Approved:** the review is skipped and the Tester continues. This saves one `agenticModel` call.
 - **Fallback:** Pilot runs its full review as usual.
@@ -105,4 +110,10 @@ The model supplies only the question. The tool gathers the rest of the state its
 
 ## Tracing
 
-Each question is recorded as a `judge.decide` span with the question, the answer and its confidence. A failed request also records the reason. The TUI shows `⚖️ Asking judge...` while a question is in flight. See [Observability](../contributing/observability.md) for setting up tracing.
+Each question is recorded as a `judge.decide` span:
+
+- **Input:** the exact request body sent to the provider: model, state and question. Post it again to the endpoint to replay the decision.
+- **Output:** the raw answer and its probability, before the threshold is applied.
+- **Metadata:** `judgeQuestion` for filtering by question, `judgeDecision` with the approved value, confidence, threshold and model, and `judgeEndpoint`.
+
+A failed request marks the span as an error and records the reason. The TUI shows `⚖️ Asking judge...` while a question is in flight. See [Observability](../contributing/observability.md) for setting up tracing.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "explorbot",
-  "version": "0.5.1",
+  "version": "0.5.3",
   "description": "CLI app built with React Ink, CodeceptJS, and Playwright",
   "license": "Elastic-2.0",
   "type": "module",

package/src/action.ts

@@ -114,6 +114,7 @@ class Action {
       if (includeScreenshot) {
         const filename = safeFilename(`${stateHash}_${timestamp}`, '.png');
         const screenshotPath = join(statesDir, filename);
+        await page?.screencast?.hideOverlays().catch(() => {});
         screenshotFile = await page
           ?.screenshot({ path: screenshotPath, fullPage: true })
           .then(() => {
@@ -124,6 +125,7 @@ class Action {
             debugLog('Screenshot failed, continuing without it:', err);
             return undefined;
           });
+        await page?.screencast?.showOverlays().catch(() => {});
       }
 
       // Save HTML to file
@@ -491,6 +493,14 @@ class Action {
     }
   }
 
+  public async attemptExactElementIndex(codeBlock: string, elementIndex: number, originalMessage?: string): Promise<boolean> {
+    const applyElementIndex = (step: any) => {
+      step.opts = { ...step.opts, elementIndex };
+    };
+    codeceptjs.event.dispatcher.once(codeceptjs.event.step.started, applyElementIndex);
+    return this.attempt(codeBlock, originalMessage).finally(() => codeceptjs.event.dispatcher.off(codeceptjs.event.step.started, applyElementIndex));
+  }
+
   async exitIframe(): Promise<void> {
     if (!this.playwrightHelper.frame) return;
     debugLog('Switching to main frame');

package/src/ai/historian/codeceptjs.ts

@@ -8,6 +8,7 @@ import { tag } from '../../utils/logger.ts';
 import { relativeToCwd } from '../../utils/next-steps.ts';
 import { CODECEPT_TOOLS, isNonReusableCode, stripComments } from '../../utils/step-analyzer.ts';
 import { safeFilename } from '../../utils/strings.ts';
+import { ensureCodeceptResolvable } from '../../utils/test-files.ts';
 import type { Conversation } from '../conversation.ts';
 import { ASSERTION_TOOLS } from '../tools.ts';
 import type { Constructor } from './mixin.ts';
@@ -99,6 +100,7 @@ export function WithCodeceptJS<T extends Constructor>(Base: T) {
 
       const testsDir = ConfigParser.getInstance().getTestsDir();
       mkdirSync(testsDir, { recursive: true });
+      ensureCodeceptResolvable(testsDir);
 
       const filePath = join(testsDir, safeFilename(plan.title, '.js'));
       writeFileSync(filePath, lines.join('\n'));

package/src/ai/judge-provider.ts

@@ -1,3 +1,6 @@
+import { Observability } from '../observability.ts';
+import { redactSecrets } from '../utils/secrets.ts';
+
 const REQUEST_TIMEOUT_MS = 15000;
 
 export class JudgeProvider {
@@ -21,6 +24,9 @@ export class JudgeProvider {
     let q: Record<string, unknown> = { type: 'noul', instructions: question };
     if (options) q = { type: 'choice', instructions: question, criteria: Object.fromEntries(options.map((option, index) => [String(index), option])) };
     const body = JSON.stringify({ model: this.model, state, questions: { q } });
+    const span = Observability.getSpan();
+    span?.setAttribute('langfuse.observation.input', redactSecrets(body));
+    span?.setAttribute('ai.telemetry.metadata.judgeEndpoint', this.url);
     const controller = new AbortController();
     const timer = setTimeout(() => controller.abort(), this.requestTimeoutMs);
     try {

package/src/ai/judge-tool.ts

@@ -11,7 +11,11 @@ export function createJudgeTool(deps: ToolDeps, buildState: () => Promise<Record
   return {
     judge: tool({
       description: dedent`
-        Settle one judgement about the current page instead of guessing. Phrase it literally and concretely.
+        Fast yes/no or pick-one decision about the current page, answered from <page_aria> and your recent steps.
+        No screenshot, no HTML, produces no test code.
+        Use for in-flow judgements you would otherwise guess: did the last action already take effect, which of several similar elements the scenario means, is this the expected page or dialog.
+        Not for final assertions (use verify/finish) and not for anything page_aria does not show, such as styling or visual state (use see).
+        Phrase the question literally and concretely.
       `,
       inputSchema: z.object({
         question: z.string().describe('The statement to confirm, or the question the options answer'),

package/src/ai/judge.ts

@@ -9,6 +9,7 @@ const debugLog = createDebug('explorbot:judge');
 const APPROVAL_THRESHOLD = 0.7;
 
 export const UNDECIDED = 'undecided';
+const UNDECIDED_HINT = `Choose "${UNDECIDED}" if you are not sure or no option fits.`;
 export const JUDGE_PAGE_CAP = 12000;
 
 export class Decision {
@@ -29,12 +30,15 @@ export class Decision {
 export class Judge {
   constructor(
     private provider: JudgeProvider,
-    private enabled: { tool: boolean; direct: boolean }
+    private enabled: { tool: boolean; direct: boolean },
+    private threshold = APPROVAL_THRESHOLD
   ) {}
 
   static fromConfig(config: AIConfig['decisionModel']): Judge | null {
     if (!config) return null;
-    return new Judge(new JudgeProvider(config.provider, config.model), { tool: config.tool !== false, direct: config.direct !== false });
+    const threshold = config.threshold ?? APPROVAL_THRESHOLD;
+    if (!(threshold > 0 && threshold < 1)) throw new Error(`decisionModel.threshold must be between 0 and 1, got ${config.threshold}`);
+    return new Judge(new JudgeProvider(config.provider, config.model), { tool: config.tool !== false, direct: config.direct !== false }, threshold);
   }
 
   get toolEnabled(): boolean {
@@ -46,12 +50,21 @@ export class Judge {
     return this.consult(question, options, state);
   }
 
+  async pick(question: string, options: string[], state: unknown): Promise<number | null> {
+    if (new Set(options).size < options.length) return null;
+    const decision = await this.decide(question, [...options, UNDECIDED], state);
+    if (!decision.value) return null;
+    return options.indexOf(decision.value) + 1;
+  }
+
   async consult(question: string, options: string[] | boolean | null, state: unknown): Promise<Decision> {
     if (Array.isArray(options) && options.length < 2) return new Decision(null, 0);
     return Observability.run('judge.decide', { tags: ['judge'] }, async () => {
       setActivity('⚖️ Asking judge...', 'ai');
       const decision = await this.request(question, options, state).finally(() => clearActivity());
-      Observability.getSpan()?.setAttribute('ai.telemetry.metadata.judgeDecision', JSON.stringify({ question, value: decision.value, confidence: decision.confidence }));
+      const span = Observability.getSpan();
+      span?.setAttribute('ai.telemetry.metadata.judgeQuestion', question);
+      span?.setAttribute('ai.telemetry.metadata.judgeDecision', JSON.stringify({ question, value: decision.value, confidence: decision.confidence, threshold: this.threshold, model: this.provider.model }));
       return decision;
     });
   }
@@ -59,17 +72,23 @@ export class Judge {
   private async request(question: string, options: string[] | boolean | null, state: unknown): Promise<Decision> {
     let list: string[] | undefined;
     if (Array.isArray(options)) list = options;
+    let instructions = question;
+    if (list?.includes(UNDECIDED)) instructions = `${question} ${UNDECIDED_HINT}`;
 
-    const answer = await this.provider.decide(state, question, list).catch((error: unknown) => this.recordFailure(error));
+    const answer = await this.provider.decide(state, instructions, list).catch((error: unknown) => this.recordFailure(error));
     if (!answer) return new Decision(null, 0);
-    if (answer.probability <= APPROVAL_THRESHOLD) return new Decision(null, answer.probability);
+    Observability.getSpan()?.setAttribute('langfuse.observation.output', JSON.stringify(answer));
+    if (answer.probability <= this.threshold) return new Decision(null, answer.probability);
     if (answer.value === UNDECIDED) return new Decision(null, answer.probability);
     return new Decision(answer.value, answer.probability);
   }
 
   private recordFailure(error: unknown): null {
     debugLog('judge declined: %s', error);
-    Observability.getSpan()?.setAttribute('ai.telemetry.metadata.judgeError', String(error));
+    const span = Observability.getSpan();
+    span?.setAttribute('ai.telemetry.metadata.judgeError', String(error));
+    span?.setAttribute('langfuse.observation.level', 'ERROR');
+    span?.setAttribute('langfuse.observation.status_message', String(error));
     return null;
   }
 }

package/src/ai/pilot.ts

@@ -19,7 +19,7 @@ import type { Agent, AgentDeps } from './agent.ts';
 import type { Conversation } from './conversation.ts';
 import type { Fisherman } from './fisherman.ts';
 import { createAskApiTool } from './fisherman/tools.ts';
-import { type Judge, UNDECIDED } from './judge.ts';
+import { JUDGE_PAGE_CAP, type Judge, UNDECIDED } from './judge.ts';
 import type { Navigator } from './navigator.ts';
 import type { Provider } from './provider.ts';
 import type { Researcher } from './researcher.ts';
@@ -542,6 +542,22 @@ export class Pilot implements Agent {
     );
   }
 
+  async periodicAnalyzeProgress(task: Test, currentState: ActionResult, testerConversation: Conversation): Promise<string> {
+    const toolCalls = testerConversation.getToolExecutions().slice(-this.stepsToReview);
+    const healthy = await this.judge?.decide('The recent actions advance the scenario toward its remaining expected outcomes.', null, {
+      scenario: task.scenario,
+      plannedSteps: task.plannedSteps,
+      expectations: this.formatExpectations(task),
+      runLog: task.notesToString() || 'No steps recorded.',
+      visitedUrls: task.getVisitedUrls({ localOnly: true }),
+      state: this.buildStateContext(currentState),
+      page: currentState.getCompactARIA().slice(0, JUDGE_PAGE_CAP),
+      recentActions: this.formatActions(toolCalls),
+    });
+    if (healthy?.approved) return '';
+    return this.analyzeProgress(task, currentState, testerConversation);
+  }
+
   async analyzeProgress(task: Test, currentState: ActionResult, testerConversation: Conversation): Promise<string> {
     tag('substep').log('Pilot analyzing progress...');
 
@@ -555,9 +571,6 @@ export class Pilot implements Agent {
     const actionsContext = this.formatActions(toolCalls);
     const stateContext = this.buildStateContext(currentState);
 
-    const healthy = await this.judge?.decide('The run is moving toward the goal and can continue without a supervisor reviewing it now.', null, { scenario: task.scenario, state: stateContext, recentActions: actionsContext });
-    if (healthy?.approved) return '';
-
     const hasFailures = toolCalls.length === 0 || toolCalls.some((t) => !t.wasSuccessful);
 
     const text = await this.sendToPilot(
@@ -1149,6 +1162,9 @@ export class Pilot implements Agent {
         const consoleError = t.output?.pageDiff?.consoleErrors?.[0];
         if (consoleError) line += `\n   console: ${consoleError.slice(0, PILOT_MESSAGE_MAX_LENGTH)}`;
 
+        const disabledReason = t.output?.disabledReason?.[0];
+        if (disabledReason) line += `\n   shown on hover of disabled target: ${disabledReason.slice(0, PILOT_MESSAGE_MAX_LENGTH)}`;
+
         return line;
       })
       .join('\n\n');

package/src/ai/provider.ts

@@ -1,10 +1,10 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
 import { OpenTelemetry } from '@ai-sdk/otel';
 import { LangfuseSpanProcessor } from '@langfuse/otel';
 import { NodeSDK } from '@opentelemetry/sdk-node';
-import { AsyncLocalStorage } from 'node:async_hooks';
-import dedent from 'dedent';
 import { APICallError, NoObjectGeneratedError, asSchema, extractJsonMiddleware, generateObject, generateText, isStepCount, parsePartialJson, registerTelemetry, tool, wrapLanguageModel } from 'ai';
 import type { ModelMessage } from 'ai';
+import dedent from 'dedent';
 import { z } from 'zod';
 import { clearActivity, setActivity } from '../activity.ts';
 import { type AIConfig, configuredModels, modelName as getModelName } from '../config.js';
@@ -60,14 +60,15 @@ function extractCachedTokens(usage: any): number {
   return usage?.inputTokenDetails?.cacheReadTokens ?? 0;
 }
 
-function abortAfterIdle(ms: number, cancel: { cancelled: boolean }, controller: AbortController, busy: { tools: number }): Promise<never> {
+function abortAfterIdle(ms: number, state: { cancelled: boolean; timedOut: boolean }, controller: AbortController, busy: { tools: number }): Promise<never> {
   return new Promise((_, reject) => {
     const tick = () => {
-      if (cancel.cancelled) return;
+      if (state.cancelled) return;
       if (executionController.isAwaitingInput() || busy.tools > 0) {
         setTimeout(tick, ms);
         return;
       }
+      state.timedOut = true;
       controller.abort();
       reject(new Error('AI request timeout'));
     };
@@ -252,13 +253,18 @@ export class Provider {
   }
 
   private async raceWithIdleTimeout<T>(fn: (signal: AbortSignal) => Promise<T>, timeoutMs: number, busy: { tools: number } = { tools: 0 }): Promise<T> {
-    const cancel = { cancelled: false };
+    const state = { cancelled: false, timedOut: false };
     const controller = new AbortController();
     const combinedSignal = combinedAbortSignal(controller);
     try {
-      return await Promise.race([fn(combinedSignal), abortAfterIdle(timeoutMs, cancel, controller, busy)]);
+      return await Promise.race([fn(combinedSignal), abortAfterIdle(timeoutMs, state, controller, busy)]);
+    } catch (error) {
+      if (state.timedOut && !executionController.getAbortSignal()?.aborted) {
+        throw new Error('AI request timeout', { cause: error });
+      }
+      throw error;
     } finally {
-      cancel.cancelled = true;
+      state.cancelled = true;
     }
   }