explorbot 0.2.4 → 0.2.5

package/dist/src/ai/navigator.js

@@ -4,11 +4,14 @@ import { z } from 'zod';
 import { ActionResult } from '../action-result.js';
 import { normalizeUrl } from '../state-manager.js';
 import { renderAssertion } from "../playwright-recorder.js";
+import { isFatalBrowserError } from "../utils/browser-errors.js";
+import { getCliName } from "../utils/cli-name.js";
 import { extractCodeBlocks } from '../utils/code-extractor.js';
 import { HooksRunner } from "../utils/hooks-runner.js";
 import { createDebug, pluralize, tag } from '../utils/logger.js';
 import { loop, pause } from '../utils/loop.js';
 import { RulesLoader } from "../utils/rules-loader.js";
+import { normalizeInlineText } from "../utils/strings.js";
 import { extractStatePath, matchesNavigationUrl } from '../utils/url-matcher.js';
 import { Researcher } from "./researcher.js";
 import { actionRule, locatorRule, unexpectedPopupRule } from './rules.js';
@@ -22,6 +25,7 @@ class Navigator {
     experienceTracker;
     hooksRunner;
     MAX_ATTEMPTS = Number.parseInt(process.env.MAX_ATTEMPTS || '5');
+    lastFailureReason = null;
     systemPrompt = dedent `
   <role>
     You are senior test automation engineer with master QA skills.
@@ -90,8 +94,12 @@ class Navigator {
         const currentState = stateManager.getCurrentState();
         if (!currentState)
             return '';
-        const current = /^https?:\/\//i.test(expectedUrl) ? currentState.fullUrl || currentState.url || '' : currentState.url || '';
-        return current;
+        return this.comparableUrl(currentState, expectedUrl);
+    }
+    comparableUrl(state, expectedUrl) {
+        if (/^https?:\/\//i.test(expectedUrl))
+            return state.fullUrl || state.url || '';
+        return state.url || '';
     }
     isSameExpectedOrigin(expectedUrl, stateManager) {
         const currentState = stateManager.getCurrentState();
@@ -139,7 +147,7 @@ class Navigator {
                 const originalMessage = `Navigate to: ${url}. Current page: ${actualPath}`;
                 const resolved = await this.resolveState(originalMessage, actionResult, { action, expectedUrl: url });
                 if (!resolved) {
-                    throw new Error(`Navigation to ${url} failed: redirected to ${actualPath} and could not resolve`);
+                    throw this.navigationError(url, `redirected to ${actualPath} and could not resolve`);
                 }
             }
             else if (action.lastError) {
@@ -151,7 +159,7 @@ class Navigator {
         `.trim();
                 const resolved = await this.resolveState(originalMessage, actionResult, { action, expectedUrl: url });
                 if (!resolved) {
-                    throw new Error(`Navigation to ${url} failed: ${action.lastError?.message}`);
+                    throw this.navigationError(url, action.lastError?.message || 'Navigation failed');
                 }
             }
             await this.explorer.capture({ screenshot: true });
@@ -167,57 +175,32 @@ class Navigator {
             throw error;
         }
     }
+    navigationError(url, fallback) {
+        if (this.lastFailureReason)
+            return new Error(`Navigation to ${url} failed: ${this.lastFailureReason}`);
+        return new Error(`Navigation to ${url} failed: ${fallback}`);
+    }
+    failureReason(stopReason, knowledge, url) {
+        const reasons = [];
+        if (stopReason)
+            reasons.push(stopReason);
+        if (!knowledge) {
+            const path = extractStatePath(url).split('?')[0].split('#')[0];
+            reasons.push(`no knowledge is set for ${path} — teach it what this page needs (credentials, hints) with: ${getCliName()} learn "${path}" "<facts>"`);
+        }
+        return reasons.join('; ') || null;
+    }
     async resolveState(message, actionResult, opts) {
         if (!this.provider)
             throw new Error('AI-assisted recovery is unavailable: no AI model is configured.');
+        this.lastFailureReason = null;
         tag('info').log('AI Navigator resolving state at', actionResult.url);
         debugLog('Resolution message:', message);
         const action = opts?.action ?? this.explorer.action();
         const expectedUrl = opts?.expectedUrl;
         const knowledge = this.knowledgeTracker.renderRelevantContext(actionResult);
-        let experience = '';
-        if (!actionResult.isInsideIframe) {
-            const successful = this.experienceTracker.getSuccessfulExperience(actionResult);
-            if (successful.length > 0) {
-                tag('operation').log(`Found ${successful.length} experience ${pluralize(successful.length, 'file')} for: ${actionResult.url}`);
-                experience = `<experience>\nPast successful recipes recorded from prior runs for this page. Prefer these solutions first if they match the goal.\n\n${successful.join('\n\n')}\n</experience>`;
-            }
-        }
-        const prompt = dedent `
-      <message>
-        ${message}
-      </message>
-
-      <page>
-        ${actionResult.toAiContext()}
-
-        <page_html>
-        ${await actionResult.combinedHtml()}
-        </page_html>
-      </page>
-
-      <task>
-        Identify the actual request of the user.
-        Identify what is expected by user.
-        Identify what might have caused the error.
-        Propose different solutions to achieve the result.
-        Solution should be valid CodeceptJS code.
-        Use only data from the <page> context to plan the solution.
-        Try various ways to achieve the result
-      </task>
-
-      ${actionRule}
-
-      ${unexpectedPopupRule}
-
-      ${RulesLoader.loadRules('navigator', ['multiple-locator', 'output'], actionResult.url || '').replace('{{maxAttempts}}', String(this.MAX_ATTEMPTS))}
-
-      ${experience}
-
-      ${knowledge}
-    `;
         const conversation = this.provider.startConversation(this.systemPrompt, 'navigator');
-        conversation.addUserText(prompt);
+        conversation.addUserText(await this.buildResolutionPrompt(message, actionResult));
         let stopReason = null;
         const tools = {
             stop: tool({
@@ -243,6 +226,7 @@ class Navigator {
         let htmlContextAdded = false;
         let codeBlockIndex = 0;
         let totalAttempts = 0;
+        let lastFailure = null;
         const progressBlocks = [];
         const batchFailures = [];
         let resolved = false;
@@ -252,7 +236,6 @@ class Navigator {
                 if (!result)
                     return;
                 if (stopReason) {
-                    tag('error').log(`Navigator stopped: ${stopReason}`);
                     resolved = false;
                     stop();
                     return;
@@ -274,173 +257,242 @@ class Navigator {
                     return;
                 }
                 tag('operation').log('Feeding failures back to AI for a new batch...');
-                let contextMsg = 'Previous solutions did not work. Analyze the failures and try DIFFERENT strategies (not syntactic variants of the same locator).\n\n';
-                if (batchFailures.length > 0) {
-                    const lines = batchFailures
-                        .map((f) => {
-                        const head = `- \`${f.code.split('\n')[0]}\` → ${f.error}`;
-                        if (!f.ariaChanges)
-                            return head;
-                        const trimmed = f.ariaChanges.split('\n').slice(0, 12).join('\n    ');
-                        return `${head}\n  • ARIA changes after the action:\n    ${trimmed}`;
-                    })
-                        .join('\n');
-                    contextMsg += `<previous_failures>\n${lines}\n</previous_failures>\n\n`;
-                }
-                if (!htmlContextAdded) {
-                    htmlContextAdded = true;
-                    contextMsg += `Full HTML context:\n\n<page_html>\n${await actionResult.combinedHtml()}\n</page_html>\n\n`;
-                }
-                const pageReacted = batchFailures.some((f) => f.ariaChanges);
-                if (pageReacted) {
-                    contextMsg += dedent `
-              Some steps in the previous batch did not throw, but the URL did not change to the expected target and the page changed in other ways — the ARIA diff for each such step is listed in <previous_failures> above.
-
-              Read those diffs and judge what each step actually triggered. Different action types produce different reactions; the diff is your only evidence of what happened. A diff might show, for example: a new alert / alertdialog / status / validation message appearing near a field or at page level; a modal, dialog, or wizard step opening; a banner, toast, or notification region appearing; a section expanding or collapsing; a tab or accordion switching content. A diff might also be empty or unrelated to the step — that is also a signal.
-
-              Choose exactly ONE path based on what the diffs actually show — do not assume the previous step submitted any particular kind of data:
-
-              A. The diff indicates the application requires something only the user can supply — for example: an authentication failure you cannot guess past, a captcha, a permission the test cannot satisfy, or knowledge that is not present in the provided context. Call the stop() tool and quote what you saw in the diff and what is needed.
-
-              B. The diff indicates the next step is something you can perform from the existing knowledge / hint context — for example: re-emit a step with a value that exists in the knowledge but was used incorrectly; dismiss an unexpected modal; accept a confirmation; take a follow-up step the page now requires. Emit code blocks for that next step. Do NOT change the locator of a step that already produced a reaction.
-
-              C. The diff is empty or unrelated to your step — the action likely missed its target. Propose a different locator strategy.
-            `;
-                }
-                else {
-                    contextMsg += 'Propose new solutions. If errors mention "intercepts pointer events" or timeouts on visible elements, an overlay is blocking — dismiss it first (Escape, click outside, Close button) before retrying the original action.';
-                }
-                conversation.addUserText(contextMsg);
+                conversation.addUserText(await this.buildRetryFeedback(batchFailures, !htmlContextAdded, actionResult));
+                htmlContextAdded = true;
                 codeBlocks = [];
                 batchFailures.length = 0;
                 return;
             }
             codeBlockIndex++;
             totalAttempts++;
-            await action.exitIframe();
             const prevActionResult = action.actionResult ?? actionResult;
             const prevHash = prevActionResult.getStateHash();
-            debugLog(`Attempting resolution: ${codeBlock}`);
-            const attemptOk = await action.attempt(codeBlock, message);
-            const page = action.playwrightHelper?.page;
-            if (page) {
-                try {
-                    await page.waitForLoadState('load', { timeout: 5000 });
-                }
-                catch {
-                    // Navigation did not reach 'load' state within timeout; continue and verify URL
-                }
-            }
-            if (attemptOk)
-                opts?.onAttempt?.({ code: codeBlock });
-            if (!attemptOk) {
-                const raw = action.lastError?.message || 'attempt failed';
-                const firstMeaningful = raw.split('\n').find((l) => l.trim() && !l.trim().startsWith('at ')) || raw;
-                const shortErr = firstMeaningful.replace(/\s+/g, ' ').trim().slice(0, 220);
-                batchFailures.push({ code: codeBlock, error: shortErr });
-                opts?.onAttempt?.({ code: codeBlock, error: shortErr });
+            const attempt = await this.executeAttempt(action, codeBlock, message);
+            opts?.onAttempt?.({ code: codeBlock, error: attempt.error });
+            if (attempt.error) {
+                batchFailures.push({ code: codeBlock, error: attempt.error });
+                lastFailure = attempt.error;
             }
             if (expectedUrl) {
-                if (page) {
-                    try {
-                        await page.waitForURL((url) => matchesNavigationUrl(expectedUrl, `${url.pathname}${url.search}${url.hash}`), { timeout: 5000 });
-                    }
-                    catch {
-                        // URL did not transition to expectedUrl within timeout
-                    }
-                }
-                const freshState = await this.explorer.capture();
-                const currentUrl = /^https?:\/\//i.test(expectedUrl) ? freshState.fullUrl || freshState.url || '' : freshState.url || '';
-                const urlMatches = this.isSameExpectedOrigin(expectedUrl, action.stateManager) && matchesNavigationUrl(expectedUrl, currentUrl);
-                const stateChanged = freshState.getStateHash() !== actionResult.getStateHash();
-                resolved = urlMatches && stateChanged;
-                if (!resolved && attemptOk) {
-                    let ariaChanges = null;
-                    if (freshState.getStateHash() !== prevHash) {
-                        try {
-                            const diff = await freshState.diff(prevActionResult);
-                            ariaChanges = diff.ariaChanged;
-                        }
-                        catch (err) {
-                            debugLog('Failed to compute pageDiff for failed URL verification:', err);
-                        }
-                    }
+                const check = await this.verifyNavigation(action, expectedUrl);
+                const freshHash = check.freshState.getStateHash();
+                resolved = check.urlMatches && freshHash !== actionResult.getStateHash();
+                if (!resolved && attempt.ok) {
+                    lastFailure = `URL did not change (still ${check.freshState.url})`;
                     batchFailures.push({
                         code: codeBlock,
-                        error: `URL did not change (still ${freshState.url})`,
-                        ariaChanges,
-                        urlAfter: freshState.url,
+                        error: lastFailure,
+                        ariaChanges: await this.ariaDiff(check.freshState, prevActionResult),
+                        urlAfter: check.freshState.url,
                     });
-                    tag('warning').log(`URL verification failed: expected ${expectedUrl}, got ${freshState.url}`);
+                    tag('warning').log(`URL verification failed: expected ${expectedUrl}, got ${check.freshState.url}`);
                 }
-                if (freshState.getStateHash() !== prevHash && (attemptOk || urlMatches)) {
+                if (freshHash !== prevHash && (attempt.ok || check.urlMatches)) {
                     progressBlocks.push(codeBlock);
                 }
             }
             else {
-                resolved = attemptOk;
-                if (attemptOk)
+                resolved = attempt.ok;
+                if (attempt.ok)
                     progressBlocks.push(codeBlock);
             }
-            if (resolved) {
-                tag('success').log('Navigation resolved successfully');
-                let scenario = message.split('\n')[0];
-                if (expectedUrl) {
-                    const fromPath = extractStatePath(actionResult.url || '');
-                    const toPath = extractStatePath(expectedUrl);
-                    scenario = `reach ${toPath} from ${fromPath}`;
-                }
-                const recipe = progressBlocks
-                    .join('\n')
-                    .split('\n')
-                    .filter((line) => !/^\s*I\.amOnPage\s*\(/.test(line))
-                    .join('\n')
-                    .trim();
-                if (recipe) {
-                    const body = `## FLOW: ${scenario}\n\n* ${scenario}\n\n\`\`\`js\n${recipe}\n\`\`\`\n\n---\n`;
-                    this.experienceTracker.writeFlow(actionResult, body);
-                }
-                stop();
+            if (!resolved)
                 return;
-            }
+            tag('success').log('Navigation resolved successfully');
+            this.saveFlow(message, expectedUrl, actionResult, progressBlocks);
+            stop();
         }, {
             maxAttempts: this.MAX_ATTEMPTS * 2,
             observability: {
                 agent: 'navigator',
             },
-            catch: async (error) => {
+            catch: async ({ error }) => {
+                if (isFatalBrowserError(error))
+                    throw error;
                 debugLog(error);
                 resolved = false;
             },
         });
-        if (!resolved && expectedUrl) {
-            await action.getActor().wait(1);
-            if (this.isOnExpectedPage(expectedUrl, action.stateManager)) {
-                resolved = true;
-                tag('success').log('Navigation resolved after delayed redirect');
-            }
-        }
+        if (!resolved && expectedUrl)
+            resolved = await this.rescueDelayedRedirect(action, expectedUrl);
         if (!resolved && stopReason) {
             tag('error').log(`Navigator stopped: ${stopReason}`);
         }
         else if (!resolved && totalAttempts > 0) {
             tag('error').log(`Navigation failed after ${totalAttempts} attempts`);
         }
-        if (!resolved && isInteractive()) {
-            const stopLine = stopReason ? `Navigator stopped: ${stopReason}\n` : '';
-            const userInput = await pause(`${stopLine}Navigator failed to resolve. Current: ${action.stateManager.getCurrentState()?.url}\n` + `Target: ${expectedUrl ?? '(none)'}\nEnter CodeceptJS commands (or press Enter to skip):`);
-            if (userInput?.trim()) {
-                resolved = await action.attempt(userInput, message);
-                if (resolved && expectedUrl) {
-                    await action.getActor().wait(1);
-                    if (!this.isOnExpectedPage(expectedUrl, action.stateManager)) {
-                        resolved = false;
-                    }
-                }
-            }
+        if (!resolved && isInteractive())
+            resolved = await this.askUserToResolve(action, message, expectedUrl, stopReason);
+        if (!resolved) {
+            let cause = 'the AI proposed no working solution for this page';
+            if (lastFailure)
+                cause = `${totalAttempts} ${pluralize(totalAttempts, 'attempt')} failed, last: ${lastFailure}`;
+            if (stopReason)
+                cause = stopReason;
+            this.lastFailureReason = this.failureReason(cause, knowledge, actionResult.url || '');
         }
         return resolved;
     }
+    async buildResolutionPrompt(message, actionResult) {
+        let experience = '';
+        if (!actionResult.isInsideIframe) {
+            const successful = this.experienceTracker.getSuccessfulExperience(actionResult);
+            if (successful.length > 0) {
+                tag('operation').log(`Found ${successful.length} experience ${pluralize(successful.length, 'file')} for: ${actionResult.url}`);
+                experience = `<experience>\nPast successful recipes recorded from prior runs for this page. Prefer these solutions first if they match the goal.\n\n${successful.join('\n\n')}\n</experience>`;
+            }
+        }
+        return dedent `
+      <message>
+        ${message}
+      </message>
+
+      <page>
+        ${actionResult.toAiContext()}
+
+        <page_html>
+        ${await actionResult.combinedHtml()}
+        </page_html>
+      </page>
+
+      <task>
+        Identify the actual request of the user.
+        Identify what is expected by user.
+        Identify what might have caused the error.
+        Propose different solutions to achieve the result.
+        Solution should be valid CodeceptJS code.
+        Use only data from the <page> context to plan the solution.
+        Try various ways to achieve the result
+      </task>
+
+      ${actionRule}
+
+      ${unexpectedPopupRule}
+
+      ${RulesLoader.loadRules('navigator', ['multiple-locator', 'output'], actionResult.url || '').replace('{{maxAttempts}}', String(this.MAX_ATTEMPTS))}
+
+      ${experience}
+
+      ${this.knowledgeTracker.renderRelevantContext(actionResult)}
+    `;
+    }
+    async buildRetryFeedback(failures, includeHtml, actionResult) {
+        let contextMsg = 'Previous solutions did not work. Analyze the failures and try DIFFERENT strategies (not syntactic variants of the same locator).\n\n';
+        if (failures.length > 0) {
+            const lines = failures
+                .map((f) => {
+                const head = `- \`${f.code.split('\n')[0]}\` → ${f.error}`;
+                if (!f.ariaChanges)
+                    return head;
+                const trimmed = f.ariaChanges.split('\n').slice(0, 12).join('\n    ');
+                return `${head}\n  • ARIA changes after the action:\n    ${trimmed}`;
+            })
+                .join('\n');
+            contextMsg += `<previous_failures>\n${lines}\n</previous_failures>\n\n`;
+        }
+        if (includeHtml) {
+            contextMsg += `Full HTML context:\n\n<page_html>\n${await actionResult.combinedHtml()}\n</page_html>\n\n`;
+        }
+        if (!failures.some((f) => f.ariaChanges)) {
+            return `${contextMsg}Propose new solutions. If errors mention "intercepts pointer events" or timeouts on visible elements, an overlay is blocking — dismiss it first (Escape, click outside, Close button) before retrying the original action.`;
+        }
+        return (contextMsg +
+            dedent `
+      Some steps in the previous batch did not throw, but the URL did not change to the expected target and the page changed in other ways — the ARIA diff for each such step is listed in <previous_failures> above.
+
+      Read those diffs and judge what each step actually triggered. Different action types produce different reactions; the diff is your only evidence of what happened. A diff might show, for example: a new alert / alertdialog / status / validation message appearing near a field or at page level; a modal, dialog, or wizard step opening; a banner, toast, or notification region appearing; a section expanding or collapsing; a tab or accordion switching content. A diff might also be empty or unrelated to the step — that is also a signal.
+
+      Choose exactly ONE path based on what the diffs actually show — do not assume the previous step submitted any particular kind of data:
+
+      A. The diff indicates the application requires something only the user can supply — for example: an authentication failure you cannot guess past, a captcha, a permission the test cannot satisfy, or knowledge that is not present in the provided context. Call the stop() tool and quote what you saw in the diff and what is needed.
+
+      B. The diff indicates the next step is something you can perform from the existing knowledge / hint context — for example: re-emit a step with a value that exists in the knowledge but was used incorrectly; dismiss an unexpected modal; accept a confirmation; take a follow-up step the page now requires. Emit code blocks for that next step. Do NOT change the locator of a step that already produced a reaction.
+
+      C. The diff is empty or unrelated to your step — the action likely missed its target. Propose a different locator strategy.
+    `);
+    }
+    async executeAttempt(action, codeBlock, message) {
+        await action.exitIframe();
+        debugLog(`Attempting resolution: ${codeBlock}`);
+        const ok = await action.attempt(codeBlock, message);
+        const page = action.playwrightHelper?.page;
+        if (page) {
+            try {
+                await page.waitForLoadState('load', { timeout: 5000 });
+            }
+            catch {
+                // Navigation did not reach 'load' state within timeout; continue and verify URL
+            }
+        }
+        if (ok)
+            return { ok };
+        const raw = action.lastError?.message || 'attempt failed';
+        const firstMeaningful = raw.split('\n').find((l) => l.trim() && !l.trim().startsWith('at ')) || raw;
+        return { ok, error: normalizeInlineText(firstMeaningful).slice(0, 220) || 'attempt failed' };
+    }
+    async verifyNavigation(action, expectedUrl) {
+        const page = action.playwrightHelper?.page;
+        if (page) {
+            try {
+                await page.waitForURL((url) => matchesNavigationUrl(expectedUrl, `${url.pathname}${url.search}${url.hash}`), { timeout: 5000 });
+            }
+            catch {
+                // URL did not transition to expectedUrl within timeout
+            }
+        }
+        const freshState = await this.explorer.capture();
+        const urlMatches = this.isSameExpectedOrigin(expectedUrl, action.stateManager) && matchesNavigationUrl(expectedUrl, this.comparableUrl(freshState, expectedUrl));
+        return { freshState, urlMatches };
+    }
+    async ariaDiff(freshState, previous) {
+        if (freshState.getStateHash() === previous.getStateHash())
+            return null;
+        try {
+            const diff = await freshState.diff(previous);
+            return diff.ariaChanged;
+        }
+        catch (err) {
+            debugLog('Failed to compute pageDiff for failed URL verification:', err);
+            return null;
+        }
+    }
+    saveFlow(message, expectedUrl, actionResult, progressBlocks) {
+        let scenario = message.split('\n')[0];
+        if (expectedUrl) {
+            scenario = `reach ${extractStatePath(expectedUrl)} from ${extractStatePath(actionResult.url || '')}`;
+        }
+        const recipe = progressBlocks
+            .join('\n')
+            .split('\n')
+            .filter((line) => !/^\s*I\.amOnPage\s*\(/.test(line))
+            .join('\n')
+            .trim();
+        if (!recipe)
+            return;
+        this.experienceTracker.writeFlow(actionResult, `## FLOW: ${scenario}\n\n* ${scenario}\n\n\`\`\`js\n${recipe}\n\`\`\`\n\n---\n`);
+    }
+    async rescueDelayedRedirect(action, expectedUrl) {
+        await action.getActor().wait(1);
+        if (!this.isOnExpectedPage(expectedUrl, action.stateManager))
+            return false;
+        tag('success').log('Navigation resolved after delayed redirect');
+        return true;
+    }
+    async askUserToResolve(action, message, expectedUrl, stopReason) {
+        let stopLine = '';
+        if (stopReason)
+            stopLine = `Navigator stopped: ${stopReason}\n`;
+        const userInput = await pause(`${stopLine}Navigator failed to resolve. Current: ${action.stateManager.getCurrentState()?.url}\nTarget: ${expectedUrl ?? '(none)'}\nEnter CodeceptJS commands (or press Enter to skip):`);
+        if (!userInput?.trim())
+            return false;
+        const resolved = await action.attempt(userInput, message);
+        if (!resolved)
+            return false;
+        if (!expectedUrl)
+            return true;
+        await action.getActor().wait(1);
+        return this.isOnExpectedPage(expectedUrl, action.stateManager);
+    }
     buildExperienceTools() {
         if (!this.experienceTracker)
             return undefined;

package/dist/src/ai/pilot.d.ts

@@ -40,10 +40,7 @@ export declare class Pilot implements Agent {
     planTest(task: Test, currentState: ActionResult): Promise<string>;
     reviewNewPage(task: Test, currentState: ActionResult, testerConversation: Conversation): Promise<string>;
     analyzeProgress(task: Test, currentState: ActionResult, testerConversation: Conversation): Promise<string>;
-    settleExpectations(task: Test): Promise<Array<{
-        text: string;
-        status: 'passed' | 'failed' | 'unverified';
-    }>>;
+    settleExpectations(task: Test, finalState?: ActionResult): Promise<SettledExpectation[]>;
     formatExpectations(task: Test): string;
     sendToPilot(userText: string, functionId: string, opts: {
         tools?: boolean;
@@ -90,3 +87,9 @@ export declare class Pilot implements Agent {
     buildDeletionScope(task: Test): string;
     getSystemPrompt(task: Test, initialState: ActionResult): string;
 }
+export type SettledStatus = 'passed' | 'failed' | 'unverified' | 'contradiction';
+export interface SettledExpectation {
+    text: string;
+    status: SettledStatus;
+    evidence?: string;
+}

package/dist/src/ai/pilot.js

@@ -3,6 +3,7 @@ import dedent from 'dedent';
 import { z } from 'zod';
 import { ActionResult } from "../action-result.js";
 import { ConfigParser } from "../config.js";
+import { Stats } from "../stats.js";
 import { TestResult } from "../test-plan.js";
 import { collectInteractiveNodes, detectFocusArea } from "../utils/aria.js";
 import { ErrorPageError } from "../utils/error-page.js";
@@ -485,21 +486,43 @@ export class Pilot {
         }
         return text;
     }
-    async settleExpectations(task) {
-        const undecided = task.expected.filter((text) => !task.getCheckedExpectations().includes(text));
+    async settleExpectations(task, finalState) {
+        let image = null;
+        if (finalState?.screenshot && this.provider.hasVision())
+            image = `data:image/png;base64,${finalState.screenshot.toString('base64')}`;
         const decided = (text) => {
             if (task.hasAchievedAny() && !task.getRemainingExpectations().includes(text))
                 return 'passed';
             return 'failed';
         };
+        let undecided = task.expected.filter((text) => !task.getCheckedExpectations().includes(text));
+        if (image)
+            undecided = task.expected;
         if (!undecided.length)
             return task.expected.map((text) => ({ text, status: decided(text) }));
         const schema = z.object({
             outcomes: z.array(z.object({
                 expectation: z.string().describe('The expected outcome, repeated exactly as it was given'),
-                status: z.enum(['passed', 'failed', 'unverified']).describe('passed = the log shows it happened, failed = the log shows it did not, unverified = the run never established either way'),
+                status: z.enum(['passed', 'failed', 'unverified', 'contradiction']).describe('passed = the evidence shows it happened, failed = the evidence shows it did not, unverified = the run never established either way, contradiction = the picture and the run disagree'),
+                evidence: z.string().nullable().describe('What settled it. For a contradiction, what each side shows. Null when there is nothing to add'),
             })),
         });
+        let pageEvidence = '';
+        if (image) {
+            pageEvidence = dedent `
+        A screenshot of the whole page as the run left it is attached. It is the proof: an outcome is satisfied
+        when the page shows it to somebody looking at it. The log only says what the run did.
+
+        Not finding something in the picture is not by itself a disagreement. Report "contradiction" only when
+        the picture shows something incompatible with what the run claims — a list visibly empty, an error where
+        a result was expected, the old value still displayed, a control visibly disabled. When you simply cannot
+        make it out, say "unverified" and name what you could not find.
+
+        The picture covers the full page, but not the inside of a region that scrolls on its own, and not the
+        state of the page before the run ended. An outcome established earlier stays established even when the
+        page has moved past it, and that is not a contradiction.
+      `;
+        }
         const userContent = dedent `
       A test run has finished. Decide, for each expected outcome, what the run established about it.
 
@@ -511,22 +534,41 @@ export class Pilot {
       ${task.notesToString() || 'No steps recorded.'}
       </run_log>
 
+      ${pageEvidence}
+
       The log is written in the tester's own words, so an outcome can be satisfied by a step that describes it
       differently. Judge by what the steps show happened, not by whether the wording matches.
-      Choose "unverified" only when the log neither shows the outcome happening nor shows it failing —
+      Choose "unverified" only when the evidence neither shows the outcome happening nor shows it failing —
       that is a statement about the run, not about the application.
     `;
-        const response = await this.provider
-            .generateObject([{ role: 'user', content: userContent }], schema, this.provider.getAgenticModel('pilot'), {
+        const settle = (content, model) => this.provider
+            .generateObject([{ role: 'user', content }], schema, model, {
             agentName: 'pilot',
             telemetry: { functionId: 'pilot.settleExpectations' },
         })
             .catch(() => null);
-        const judged = new Map((response?.object?.outcomes || []).map((outcome) => [outcome.expectation, outcome.status]));
+        let response = null;
+        if (image) {
+            const seen = [
+                { type: 'text', text: userContent },
+                { type: 'file', mediaType: 'image/png', data: image },
+            ];
+            response = await settle(seen, this.provider.getVisionModel());
+            if (!response) {
+                Stats.visionDisabled = true;
+                tag('warning').log('⚠️ Vision model could not judge the outcomes. Settling them from the run log instead.');
+            }
+        }
+        if (!response)
+            response = await settle(userContent, this.provider.getAgenticModel('pilot'));
+        const judged = new Map((response?.object?.outcomes || []).map((outcome) => [outcome.expectation, outcome]));
         return task.expected.map((text) => {
             if (!undecided.includes(text))
                 return { text, status: decided(text) };
-            return { text, status: judged.get(text) || 'unverified' };
+            const outcome = judged.get(text);
+            if (!outcome)
+                return { text, status: 'unverified' };
+            return { text, status: outcome.status || 'unverified', evidence: outcome.evidence };
         });
     }
     formatExpectations(task) {

package/dist/src/ai/provider.d.ts

@@ -1,6 +1,6 @@
 import { NodeSDK } from '@opentelemetry/sdk-node';
 import type { ModelMessage } from 'ai';
-import type { AIConfig } from '../config.js';
+import { type AIConfig } from '../config.js';
 import { type RetryOptions } from '../utils/retry.js';
 import { Conversation } from './conversation.js';
 declare class AiError extends Error {
@@ -14,10 +14,10 @@ export declare class Provider {
     defaultRetryOptions: RetryOptions;
     lastConversation: Conversation | null;
     constructor(config: AIConfig);
-    getModelName(model: any): string;
     validateConnection(): Promise<void>;
     getModelForAgent(agentName?: string): any;
     getAgenticModel(agentName?: string): any;
+    getVisionModel(): any;
     getConfiguredModels(): Record<string, string>;
     getSystemPromptForAgent(agentName: string, currentUrl?: string): string | undefined;
     getProviderOptionsForAgent(agentName: string): Record<string, any> | undefined;