explorbot 0.2.4 → 0.2.5

package/dist/boat/prima/src/cli.js

@@ -20,16 +20,26 @@ const helpContract = dedent `
     prima pw "({ page }) => page.click('[data-test=submit]')"
 `;
 const checkHelp = dedent `
-  check takes an outcome rather than a click path, and works out how to reach it.
+  check takes an outcome rather than a click path, and works out how to reach it. It runs
+  on the page you are already on and never reloads it, so an open dialog survives the check.
   --expected  one outcome the run must reach, repeatable for several. Without it the
               scenario text is the single expected outcome. Each comes back under
-              ### Expected outcomes as PASSED, FAILED or not verified - "not verified"
-              means the run never checked it, which is not the same as false.
+              ### Expected outcomes as PASSED, FAILED, CONTRADICTION or not verified.
+              "not verified" means the run never checked it, which is not the same
+              as false.
+  Outcomes are settled against a screenshot of the whole page: what a user can see is
+  the proof, and the run log only says what was done. CONTRADICTION means the two
+  disagree - reported with both sides rather than settled one way, and ### Artifacts
+  then names the html, aria and screenshot on disk so you can judge it yourself. Not
+  finding something in the picture is not enough on its own; that is "not verified".
+  ok: follows those outcomes - false when one FAILED or CONTRADICTED, or when the run
+  could not complete, which is reported as such rather than as an app failure.
   Page problems seen on the way appear under ### Answer, not as step failures.
 `;
 const doHelp = dedent `
-  Each instruction is numbered and accounted for: ### Steps reports each as ok or FAIL
-  with what proved it. One that could not be carried out fails the command and says why.
+  Each instruction is numbered and accounted for: ### Steps reports each as ok, FAIL or ??.
+  ?? means the action ran but the run ended without confirming that instruction - read the
+  steps above it. Only FAIL and an instruction the page could not carry out fail the command.
   Nothing runs past the last instruction given. A whole remaining sequence in one call is
   what makes this tier cheap.
 `;
@@ -165,10 +175,12 @@ export function createPrimaCommands(name = 'prima') {
             options.baseUrl = target;
         await runPrima(options, `go ${target}`, (prima) => prima.go(target));
     });
-    addCommonOptions(cmd.command('config').description('Show the AI models prima runs on and the config file they come from')).action(async (options) => {
+    addCommonOptions(cmd.command('config').description('Show models, config file and paths used by this run'))
+        .option('--json', 'Print the resolved config as JSON')
+        .action(async (options) => {
         setQuietMode(!isVerboseMode());
         const prima = primaFor(options);
-        console.log(await prima.config().catch((error) => browserErrorMessage(error)));
+        console.log(await prima.config(options.json).catch((error) => browserErrorMessage(error)));
         await prima.stop().catch(() => { });
         process.exit(0);
     });

package/dist/boat/prima/src/envelope.js

@@ -1,12 +1,13 @@
 import { mkdirSync, writeFileSync } from 'node:fs';
 import path from 'node:path';
 const EXPECTATION_LABELS = {
-    passed: 'PASSED      ',
-    failed: 'FAILED      ',
-    unverified: 'not verified',
+    passed: 'PASSED       ',
+    failed: 'FAILED       ',
+    unverified: 'not verified ',
+    contradiction: 'CONTRADICTION',
 };
 export function renderEnvelope(data) {
-    const sections = [renderResult(data), renderPage(data), renderValue(data), renderChanges(data), renderSteps(data), renderExpectations(data), renderOutcome(data), ...renderFailure(data), renderInstance(data), renderArtifacts(data)];
+    const sections = [renderResult(data), renderPage(data), renderValue(data), renderChanges(data), renderSteps(data), renderExpectations(data), renderWarning(data), renderOutcome(data), ...renderFailure(data), renderInstance(data), renderArtifacts(data)];
     return sections.filter((section) => section).join('\n\n');
 }
 export function writeArtifacts(dir, snapshot) {
@@ -17,6 +18,10 @@ export function writeArtifacts(dir, snapshot) {
     };
     writeFileSync(paths.aria, snapshot.aria ?? '', 'utf-8');
     writeFileSync(paths.html, snapshot.html ?? '', 'utf-8');
+    if (snapshot.screenshot) {
+        paths.screenshot = path.resolve(dir, 'page.png');
+        writeFileSync(paths.screenshot, snapshot.screenshot);
+    }
     if (!snapshot.requests.length)
         return paths;
     paths.network = path.resolve(dir, 'network.jsonl');
@@ -55,7 +60,12 @@ function renderSteps(data) {
         return null;
     const lines = [];
     data.steps.forEach((step, index) => {
-        lines.push(`${index + 1}. ${step.ok ? 'ok  ' : 'FAIL'} ${step.label}`);
+        let mark = 'FAIL';
+        if (step.ok)
+            mark = 'ok  ';
+        if (step.unconfirmed)
+            mark = '??  ';
+        lines.push(`${index + 1}. ${mark} ${step.label}`);
         for (const line of (step.proof || '').split('\n').filter(Boolean))
             lines.push(`      ${line}`);
     });
@@ -66,9 +76,21 @@ function renderSteps(data) {
 function renderExpectations(data) {
     if (!data.expectations?.length)
         return null;
-    const lines = data.expectations.map((expectation, index) => `${index + 1}. ${EXPECTATION_LABELS[expectation.status]} ${expectation.text}`);
+    const lines = [];
+    data.expectations.forEach((expectation, index) => {
+        lines.push(`${index + 1}. ${EXPECTATION_LABELS[expectation.status]} ${expectation.text}`);
+        if (expectation.status !== 'contradiction' && expectation.status !== 'failed')
+            return;
+        for (const line of (expectation.evidence || '').split('\n').filter(Boolean))
+            lines.push(`      ${line}`);
+    });
     return section('Expected outcomes', lines.join('\n'));
 }
+function renderWarning(data) {
+    if (!data.warning)
+        return null;
+    return section('Warning', data.warning);
+}
 function renderOutcome(data) {
     if (data.answer)
         return section('Answer', data.answer);
@@ -127,9 +149,11 @@ function tabsLabel(tabs) {
 export function renderArtifacts(data) {
     if (!data.artifacts)
         return null;
-    const lines = [`aria: ${data.artifacts.aria}`, `html: ${data.artifacts.html}`];
+    const lines = [`aria:       ${data.artifacts.aria}`, `html:       ${data.artifacts.html}`];
+    if (data.artifacts.screenshot)
+        lines.push(`screenshot: ${data.artifacts.screenshot}`);
     if (data.artifacts.network)
-        lines.push(`network: ${data.artifacts.network}`);
+        lines.push(`network:    ${data.artifacts.network}`);
     return section('Artifacts', lines.join('\n'));
 }
 function align(label, marker, width) {

package/dist/boat/prima/src/prima.js

@@ -4,28 +4,29 @@ import { createRequire } from 'node:module';
 import path from 'node:path';
 import { tool } from 'ai';
 import dedent from 'dedent';
-import { z } from 'zod';
 import * as playwright from 'playwright';
+import { z } from 'zod';
 import { ActionResult } from "../../../src/action-result.js";
+import { getPreviousResearch } from "../../../src/ai/researcher/cache.js";
 import { actionRule, locatorRule } from "../../../src/ai/rules.js";
 import { createAgentTools, createCodeceptJSTools } from "../../../src/ai/tools.js";
 import { getAliveEndpoint, launchServer, listInstances, stopServer } from "../../../src/browser-server.js";
-import { listSites } from "../../../src/global-config.js";
+import { ConfigCommand } from "../../../src/commands/config-command.js";
 import { ConfigMissingError, ConfigParser, outputPath } from "../../../src/config.js";
 import { ExplorBot } from "../../../src/explorbot.js";
+import { listSites } from "../../../src/global-config.js";
 import { Reporter } from "../../../src/reporter.js";
 import { Stats } from "../../../src/stats.js";
 import { Task, Test, TestResult } from "../../../src/test-plan.js";
-import { getPreviousResearch } from "../../../src/ai/researcher/cache.js";
 import { compactAriaSnapshot } from "../../../src/utils/aria.js";
-import { mdq } from "../../../src/utils/markdown-query.js";
 import { browserErrorMessage } from "../../../src/utils/browser-errors.js";
 import { pluralize } from "../../../src/utils/logger.js";
+import { mdq } from "../../../src/utils/markdown-query.js";
 import { safeFilename } from "../../../src/utils/strings.js";
 import { writeArtifacts } from "./envelope.js";
 import { isFunctionExpression, takePwValue, toCodeceptWrapper } from "./pw-parser.js";
-import { latestSessionFile, readSession, recordCommand, sessionFile, sessionsDir } from "./session-log.js";
 import { readDescriptors, selectDescriptor } from "./pw-registry.js";
+import { latestSessionFile, readSession, recordCommand, sessionFile, sessionsDir } from "./session-log.js";
 const TESTER_ONLY_TOOLS = ['learnExperience', 'askUser'];
 const ITERATIONS_PER_INSTRUCTION = 2;
 const MAX_INSTRUCTION_ITERATIONS = 24;
@@ -36,7 +37,7 @@ const AI_AGENT_NAME = 'prima';
 const CONNECT_TIMEOUT = 3000;
 const requireLib = createRequire(import.meta.url);
 const VOLATILE_COLUMNS = ['CSS', 'XPath', 'Coordinates', 'eidx'];
-const UNACCOUNTED = { open: 'never reported — the run ended with this instruction still open' };
+const UNACCOUNTED = { open: 'the run ended without confirming this one — the actions above are everything that ran' };
 function dropVolatileColumns(markdown) {
     return mdq(markdown)
         .query('table')
@@ -67,6 +68,7 @@ export class Prima {
     server = null;
     attached = null;
     session = null;
+    artifacts;
     constructor(options = {}) {
         this.options = options;
         this.bot = new ExplorBot({
@@ -208,18 +210,18 @@ export class Prima {
         }
         if (aiError)
             return this.failureEnvelope(command, aiError, previousState);
-        if (used.length && ledger.some((entry) => entry.status === 'open')) {
+        if (trace.length && ledger.some((entry) => entry.status === 'open')) {
             await this.settleLedger(conversation, provider, ledger, trace);
         }
         const unfinished = ledger.filter((entry) => entry.status !== 'done');
-        const steps = [...trace, ...ledger.filter((entry) => entry.status === 'open').map((entry) => ({ label: `unreported: ${entry.text}`, ok: false, proof: UNACCOUNTED.open }))];
+        const steps = [...trace, ...ledger.filter((entry) => entry.status === 'open').map((entry) => ({ label: entry.text, ok: false, unconfirmed: true, proof: UNACCOUNTED.open }))];
         if (failure && unfinished.length) {
             const envelope = await this.failureEnvelope(command, failure.message, previousState);
             envelope.steps = steps;
             envelope.stepFiles = this.statusDir();
             return envelope;
         }
-        if (!used.length && unfinished.length === ledger.length) {
+        if (!trace.length && unfinished.length === ledger.length) {
             const reason = ['No action was performed for these instructions on the current page.', narration].filter(Boolean).join(' ');
             return this.failureEnvelope(command, reason, previousState);
         }
@@ -230,10 +232,10 @@ export class Prima {
         // the step log already reports every action and what it changed
         envelope.used = undefined;
         envelope.changes = undefined;
-        const unmet = unfinished.map((entry) => `${entry.status}: ${entry.text}${entry.proof ? ` — ${entry.proof}` : ''}`);
-        if (unmet.length) {
+        const blocked = ledger.filter((entry) => entry.status === 'blocked');
+        if (blocked.length) {
             envelope.ok = false;
-            envelope.failure = { error: unmet.join('\n') };
+            envelope.failure = { error: blocked.map((entry) => `blocked: ${entry.text}${entry.proof ? ` — ${entry.proof}` : ''}`).join('\n') };
         }
         return envelope;
     }
@@ -281,7 +283,13 @@ export class Prima {
       instructions have moved it on, and something you confirmed earlier stays confirmed even if it is gone.
       completed() for those, blocked() for the ones the page could not do. Report every one — nothing else runs after this.
     `);
-        const invoked = await provider.invokeConversation(conversation, { completed: this.completedTool(), blocked: this.blockedTool() }, { maxToolRoundtrips: 2, toolChoice: 'required', agentName: AI_AGENT_NAME }).catch(() => null);
+        let settleError = null;
+        const invoked = await provider.invokeConversation(conversation, { completed: this.completedTool(), blocked: this.blockedTool() }, { maxToolRoundtrips: 2, toolChoice: 'required', agentName: AI_AGENT_NAME }).catch((error) => {
+            settleError = error;
+            return null;
+        });
+        if (settleError)
+            trace.push({ label: 'settling which instructions were satisfied', ok: false, proof: browserErrorMessage(settleError) });
         for (const execution of invoked?.toolExecutions || []) {
             this.applyLedgerReport(execution, ledger, trace);
         }
@@ -305,17 +313,32 @@ export class Prima {
         const outcomes = expected.length ? expected : [scenario];
         const test = new Test(scenario, 'normal', outcomes, previousState?.url || this.options.url || '');
         const tester = this.bot.agentTester();
-        const outcome = await tester.test(test);
+        await tester.test(test, { startOnCurrentPage: true });
         const notes = Object.values(test.notes || {});
-        const result = await this.capturedResult(this.bot.stateManager().getCurrentState());
-        const envelope = await this.reportEnvelope(command, result, previousState, { ok: outcome.success });
+        const result = await this.capturedResult(this.bot.stateManager().getCurrentState(), { screenshot: this.visionEnabled() });
+        const envelope = await this.reportEnvelope(command, result, previousState, {});
         const recorded = notes.filter((note) => !note.observation && !outcomes.includes(note.message));
         const failed = recorded.filter((note) => note.status === TestResult.FAILED);
         envelope.steps = failed.map((note) => ({ label: note.message, ok: false, proof: note.log || '' }));
         const routine = recorded.length - failed.length;
         if (routine)
             envelope.steps.push({ label: `${routine} further ${pluralize(routine, 'step')} ran without failing — prima status ${envelope.status} for the full log`, ok: true, proof: '' });
-        envelope.expectations = await this.bot.agentPilot().settleExpectations(test);
+        envelope.expectations = await this.bot.agentPilot().settleExpectations(test, result);
+        if (!result.screenshot || !this.visionEnabled()) {
+            envelope.warning = 'These outcomes were settled from the run log alone — no screenshot backed them. Set ai.visionModel, or check anything visual with prima ask.';
+        }
+        const unreached = envelope.expectations.filter((expectation) => expectation.status === 'failed');
+        const contradicted = envelope.expectations.filter((expectation) => expectation.status === 'contradiction');
+        envelope.ok = !unreached.length && !contradicted.length;
+        const problems = [...unreached.map((expectation) => `not reached: ${expectation.text}`), ...contradicted.map((expectation) => `the picture and the run disagree about: ${expectation.text}`)];
+        if (problems.length)
+            envelope.failure = { error: problems.join('\n') };
+        if (contradicted.length)
+            envelope.artifacts = this.artifacts;
+        if (!test.hasFinished || test.isSkipped) {
+            envelope.ok = false;
+            envelope.failure = { error: `the run did not complete, so it established nothing about the app: ${notes.at(-1)?.message || 'no steps were recorded'}` };
+        }
         const observations = notes.filter((note) => note.observation).map((note) => note.message);
         if (observations.length)
             envelope.answer = ['Page problems noticed while running, not step failures:', ...observations.map((line) => `- ${line}`)].join('\n');
@@ -338,7 +361,14 @@ export class Prima {
         const previousState = this.bot.stateManager().getCurrentState();
         const result = await this.capturedResult(previousState);
         const verification = await this.bot.agentNavigator().verifyState(assertion, result);
-        return this.reportEnvelope(command, result, previousState, { assertions: verification.results || [] });
+        const outcome = { assertions: verification.results || [] };
+        if (verification.inexpressible) {
+            const question = `Judging only from the screenshot, is this true of the page: "${assertion}"? Answer true, false or undetermined, and say what settles it.`;
+            const seen = await this.visionAnswer(question, await this.capturedResult(previousState, { screenshot: this.visionEnabled() }));
+            if (seen)
+                outcome.answer = `No assertion could express this claim, so it was judged from a screenshot instead.\n\n${seen}`;
+        }
+        return this.reportEnvelope(command, result, previousState, outcome);
     }
     async research(opts = {}) {
         const flags = [opts.data && '--data', opts.deep && '--deep', opts.fresh && '--fresh'].filter(Boolean);
@@ -398,27 +428,13 @@ export class Prima {
         }
         return stopped;
     }
-    async config() {
+    async config(json) {
         const [site] = listSites();
         if (site && !this.configBaseUrl())
             this.sessionUrl = site.url;
         const config = await this.loadConfig();
-        const named = (model) => {
-            if (typeof model === 'string')
-                return model;
-            return model?.modelId || model?.model || 'unknown';
-        };
-        const ai = config.ai || {};
-        const roles = [
-            ['model', ai.model],
-            ['agenticModel', ai.agenticModel],
-            ['visionModel', ai.visionModel],
-        ];
-        const lines = roles.filter(([, model]) => model).map(([role, model]) => `${role.padEnd(14)} ${named(model)}`);
-        lines.push(`config         ${ConfigParser.getInstance().getConfigPath() || 'built-in defaults'}`);
-        if (ai.langfuse?.enabled)
-            lines.push('telemetry      langfuse');
-        return lines.join('\n');
+        const parser = ConfigParser.getInstance();
+        return ConfigCommand.render(config, { configPath: parser.getConfigPath(), root: parser.getProjectRoot(), json });
     }
     record(envelope, durationMs) {
         if (!this.session)
@@ -706,6 +722,7 @@ export class Prima {
       <proof>
       An instruction is done only when a change on the page shows it. After each action read the reported change and decide which part of it proves the instruction.
       That part is what completed() takes as its proof. Do not restate the action as if it were the outcome.
+      How much of the page moved is not evidence of whether it happened — a change confined to one region proves an instruction as well as one that redraws everything.
       An instruction that only inspects the page is satisfied by what you can see, including seeing that something is absent — those need no action at all.
       </proof>
 
@@ -856,6 +873,8 @@ export class Prima {
     visionEnabled() {
         if (this.options.noVision)
             return false;
+        if (Stats.visionDisabled)
+            return false;
         return this.bot.getProvider().hasVision?.() === true;
     }
     async answer(question, result) {
@@ -976,7 +995,6 @@ export class Prima {
             ok: true,
             command: `status ${hash}`,
             page: saved.page,
-            changes: saved.changes,
             instance: await this.instanceInfo(),
             artifacts: { aria: path.join(dir, 'aria.yml'), html: path.join(dir, 'page.html') },
         };
@@ -984,7 +1002,7 @@ export class Prima {
     async saveStatus(result) {
         const hash = this.statusHash();
         await this.writeSnapshot(result);
-        writeFileSync(path.join(this.statusDir(hash), 'status.json'), JSON.stringify({ page: this.pageBlock(result, null), changes: compactAriaSnapshot(result.ariaSnapshot, true) }), 'utf-8');
+        writeFileSync(path.join(this.statusDir(hash), 'status.json'), JSON.stringify({ page: this.pageBlock(result, null) }), 'utf-8');
         return hash;
     }
     async writeStepFiles(index, label, diff) {
@@ -1001,12 +1019,12 @@ export class Prima {
             writeFileSync(`${stem}.diff.yaml`, diff, 'utf-8');
     }
     async writeSnapshot(result) {
-        writeArtifacts(this.statusDir(), {
+        this.artifacts = writeArtifacts(this.statusDir(), {
             aria: result.ariaSnapshot,
             html: await result.combinedHtml(),
+            screenshot: result.screenshot,
             requests: this.bot.requestStore().getRequests(),
         });
-        return undefined;
     }
     statusHash() {
         this.hash ||= createHash('sha1')

package/dist/package.json

@@ -1,6 +1,6 @@
 {
   "name": "explorbot",
-  "version": "0.2.4",
+  "version": "0.2.5",
   "description": "CLI app built with React Ink, CodeceptJS, and Playwright",
   "license": "Elastic-2.0",
   "type": "module",

package/dist/src/action.js

@@ -50,6 +50,7 @@ class Action {
             await this.actor.saveScreenshot(filename);
             if (currentState)
                 currentState.screenshotFile = filename;
+            tag('data').log('screenshot', { path: outputPath('states', filename) });
             return filename;
         }
         catch (err) {
@@ -97,7 +98,10 @@ class Action {
                 const screenshotPath = join(statesDir, filename);
                 screenshotFile = await page
                     ?.screenshot({ path: screenshotPath, fullPage: true })
-                    .then(() => filename)
+                    .then(() => {
+                    tag('data').log('screenshot', { path: screenshotPath });
+                    return filename;
+                })
                     .catch((err) => {
                     debugLog('Screenshot failed, continuing without it:', err);
                     return undefined;

package/dist/src/ai/navigator.d.ts

@@ -15,6 +15,7 @@ declare class Navigator implements Agent {
     experienceTracker: ExperienceTracker;
     hooksRunner: HooksRunner;
     MAX_ATTEMPTS: number;
+    lastFailureReason: string | null;
     systemPrompt: string;
     freeSailSystemPrompt: string;
     explorer: Explorer;
@@ -25,10 +26,16 @@ declare class Navigator implements Agent {
     get verifyTimeout(): number;
     getBaseOrigin(): string | null;
     getComparableCurrentUrl(stateManager: any, expectedUrl: string): string;
+    comparableUrl(state: {
+        url?: string;
+        fullUrl?: string;
+    }, expectedUrl: string): string;
     isSameExpectedOrigin(expectedUrl: string, stateManager: any): boolean;
     isOnExpectedPage(expectedUrl: string, stateManager: any): boolean;
     visit(url: string): Promise<void>;
     visitOnce(url: string): Promise<void>;
+    navigationError(url: string, fallback: string): Error;
+    failureReason(stopReason: string | null, knowledge: string, url: string): string | null;
     resolveState(message: string, actionResult: ActionResult, opts?: {
         action?: Action;
         expectedUrl?: string;
@@ -37,6 +44,20 @@ declare class Navigator implements Agent {
             error?: string;
         }) => void;
     }): Promise<boolean>;
+    buildResolutionPrompt(message: string, actionResult: ActionResult): Promise<string>;
+    buildRetryFeedback(failures: BatchFailure[], includeHtml: boolean, actionResult: ActionResult): Promise<string>;
+    executeAttempt(action: Action, codeBlock: string, message: string): Promise<{
+        ok: boolean;
+        error?: string;
+    }>;
+    verifyNavigation(action: Action, expectedUrl: string): Promise<{
+        freshState: ActionResult;
+        urlMatches: boolean;
+    }>;
+    ariaDiff(freshState: ActionResult, previous: ActionResult): Promise<string | null>;
+    saveFlow(message: string, expectedUrl: string | undefined, actionResult: ActionResult, progressBlocks: string[]): void;
+    rescueDelayedRedirect(action: Action, expectedUrl: string): Promise<boolean>;
+    askUserToResolve(action: Action, message: string, expectedUrl: string | undefined, stopReason: string | null): Promise<boolean>;
     buildExperienceTools(): {
         learnExperience: unknown;
     } | undefined;
@@ -61,6 +82,12 @@ declare class Navigator implements Agent {
     }>;
     checkAlreadyVerified(aiResponse: string, actionResult: ActionResult): boolean;
 }
+type BatchFailure = {
+    code: string;
+    error: string;
+    ariaChanges?: string | null;
+    urlAfter?: string;
+};
 export type AssertionResult = {
     code: string;
     passed: boolean;