explorbot 0.2.3 → 0.2.5

package/dist/src/ai/pilot.js

@@ -3,14 +3,16 @@ import dedent from 'dedent';
 import { z } from 'zod';
 import { ActionResult } from "../action-result.js";
 import { ConfigParser } from "../config.js";
+import { Stats } from "../stats.js";
 import { TestResult } from "../test-plan.js";
-import { collectInteractiveNodes, detectFocusArea, extractFocusedElement } from "../utils/aria.js";
+import { collectInteractiveNodes, detectFocusArea } from "../utils/aria.js";
 import { ErrorPageError } from "../utils/error-page.js";
 import { createDebug, tag } from "../utils/logger.js";
 const debugLog = createDebug('explorbot:pilot');
 import { truncateJson } from "../utils/strings.js";
 import { capabilityGroundingRule, dataProtectionRules } from "./rules.js";
 import { isInteractive } from "./task-agent.js";
+import { withdrawVisionTools } from "./tools.js";
 const CHECK_TOOLS = ['verify', 'see', 'research', 'context'];
 const META_TOOLS = ['record', 'reset', 'stop', 'finish'];
 export class Pilot {
@@ -484,6 +486,91 @@ export class Pilot {
         }
         return text;
     }
+    async settleExpectations(task, finalState) {
+        let image = null;
+        if (finalState?.screenshot && this.provider.hasVision())
+            image = `data:image/png;base64,${finalState.screenshot.toString('base64')}`;
+        const decided = (text) => {
+            if (task.hasAchievedAny() && !task.getRemainingExpectations().includes(text))
+                return 'passed';
+            return 'failed';
+        };
+        let undecided = task.expected.filter((text) => !task.getCheckedExpectations().includes(text));
+        if (image)
+            undecided = task.expected;
+        if (!undecided.length)
+            return task.expected.map((text) => ({ text, status: decided(text) }));
+        const schema = z.object({
+            outcomes: z.array(z.object({
+                expectation: z.string().describe('The expected outcome, repeated exactly as it was given'),
+                status: z.enum(['passed', 'failed', 'unverified', 'contradiction']).describe('passed = the evidence shows it happened, failed = the evidence shows it did not, unverified = the run never established either way, contradiction = the picture and the run disagree'),
+                evidence: z.string().nullable().describe('What settled it. For a contradiction, what each side shows. Null when there is nothing to add'),
+            })),
+        });
+        let pageEvidence = '';
+        if (image) {
+            pageEvidence = dedent `
+        A screenshot of the whole page as the run left it is attached. It is the proof: an outcome is satisfied
+        when the page shows it to somebody looking at it. The log only says what the run did.
+
+        Not finding something in the picture is not by itself a disagreement. Report "contradiction" only when
+        the picture shows something incompatible with what the run claims — a list visibly empty, an error where
+        a result was expected, the old value still displayed, a control visibly disabled. When you simply cannot
+        make it out, say "unverified" and name what you could not find.
+
+        The picture covers the full page, but not the inside of a region that scrolls on its own, and not the
+        state of the page before the run ended. An outcome established earlier stays established even when the
+        page has moved past it, and that is not a contradiction.
+      `;
+        }
+        const userContent = dedent `
+      A test run has finished. Decide, for each expected outcome, what the run established about it.
+
+      <expected_outcomes>
+      ${undecided.map((text) => `- ${text}`).join('\n')}
+      </expected_outcomes>
+
+      <run_log>
+      ${task.notesToString() || 'No steps recorded.'}
+      </run_log>
+
+      ${pageEvidence}
+
+      The log is written in the tester's own words, so an outcome can be satisfied by a step that describes it
+      differently. Judge by what the steps show happened, not by whether the wording matches.
+      Choose "unverified" only when the evidence neither shows the outcome happening nor shows it failing —
+      that is a statement about the run, not about the application.
+    `;
+        const settle = (content, model) => this.provider
+            .generateObject([{ role: 'user', content }], schema, model, {
+            agentName: 'pilot',
+            telemetry: { functionId: 'pilot.settleExpectations' },
+        })
+            .catch(() => null);
+        let response = null;
+        if (image) {
+            const seen = [
+                { type: 'text', text: userContent },
+                { type: 'file', mediaType: 'image/png', data: image },
+            ];
+            response = await settle(seen, this.provider.getVisionModel());
+            if (!response) {
+                Stats.visionDisabled = true;
+                tag('warning').log('⚠️ Vision model could not judge the outcomes. Settling them from the run log instead.');
+            }
+        }
+        if (!response)
+            response = await settle(userContent, this.provider.getAgenticModel('pilot'));
+        const judged = new Map((response?.object?.outcomes || []).map((outcome) => [outcome.expectation, outcome]));
+        return task.expected.map((text) => {
+            if (!undecided.includes(text))
+                return { text, status: decided(text) };
+            const outcome = judged.get(text);
+            if (!outcome)
+                return { text, status: 'unverified' };
+            return { text, status: outcome.status || 'unverified', evidence: outcome.evidence };
+        });
+    }
     formatExpectations(task) {
         const checked = task.getCheckedExpectations();
         const remaining = task.getRemainingExpectations();
@@ -547,6 +634,7 @@ export class Pilot {
             planning.learnExperience = learnExperience;
         if (askUser)
             planning.askUser = askUser;
+        withdrawVisionTools(planning);
         return planning;
     }
     buildPreconditionTool(task) {
@@ -624,7 +712,7 @@ export class Pilot {
         const lines = [];
         lines.push(`url: ${state.url}`);
         lines.push(`title: ${state.title || 'unknown'}`);
-        const focused = extractFocusedElement(state.ariaSnapshot);
+        const focused = state.focusedElement;
         if (focused) {
             const valuePart = focused.value ? ` (value: "${focused.value}")` : '';
             lines.push(`focused: ${focused.role} "${focused.name}"${valuePart}`);

package/dist/src/ai/provider.d.ts

@@ -1,6 +1,6 @@
 import { NodeSDK } from '@opentelemetry/sdk-node';
 import type { ModelMessage } from 'ai';
-import type { AIConfig } from '../config.js';
+import { type AIConfig } from '../config.js';
 import { type RetryOptions } from '../utils/retry.js';
 import { Conversation } from './conversation.js';
 declare class AiError extends Error {
@@ -14,10 +14,10 @@ export declare class Provider {
     defaultRetryOptions: RetryOptions;
     lastConversation: Conversation | null;
     constructor(config: AIConfig);
-    getModelName(model: any): string;
     validateConnection(): Promise<void>;
     getModelForAgent(agentName?: string): any;
     getAgenticModel(agentName?: string): any;
+    getVisionModel(): any;
     getConfiguredModels(): Record<string, string>;
     getSystemPromptForAgent(agentName: string, currentUrl?: string): string | undefined;
     getProviderOptionsForAgent(agentName: string): Record<string, any> | undefined;

package/dist/src/ai/provider.js

@@ -3,6 +3,7 @@ import { LangfuseSpanProcessor } from '@langfuse/otel';
 import { NodeSDK } from '@opentelemetry/sdk-node';
 import { generateObject, generateText, isStepCount, registerTelemetry } from 'ai';
 import { clearActivity, setActivity } from "../activity.js";
+import { configuredModels, modelName as getModelName } from '../config.js';
 import { executionController } from "../execution-controller.js";
 import { Observability } from "../observability.js";
 import { Stats } from "../stats.js";
@@ -79,9 +80,6 @@ export class Provider {
         this.config = config;
         this.initLangfuse();
     }
-    getModelName(model) {
-        return model?.modelId || model?.model || 'unknown';
-    }
     async validateConnection() {
         try {
             await generateText({
@@ -108,16 +106,13 @@ export class Provider {
         }
         return this.config.agenticModel || this.config.model;
     }
+    getVisionModel() {
+        return this.config.visionModel;
+    }
     getConfiguredModels() {
-        const models = { model: this.getModelName(this.config.model) };
-        if (this.config.agenticModel)
-            models.agenticModel = this.getModelName(this.config.agenticModel);
-        if (this.config.visionModel)
-            models.visionModel = this.getModelName(this.config.visionModel);
-        for (const [agent, agentConfig] of Object.entries(this.config.agents || {})) {
-            if (agentConfig?.model)
-                models[agent] = this.getModelName(agentConfig.model);
-        }
+        const models = {};
+        for (const [role, model] of Object.entries(configuredModels(this.config)))
+            models[role] = model.name;
         return models;
     }
     getSystemPromptForAgent(agentName, currentUrl) {
@@ -207,11 +202,7 @@ export class Provider {
         return retry(reduced.messages, { ...options, _contextRetryLevel: reduced.nextLevel });
     }
     initLangfuse() {
-        const langfuseConfig = this.config.langfuse;
-        const publicKey = langfuseConfig?.publicKey || process.env.LANGFUSE_PUBLIC_KEY;
-        const secretKey = langfuseConfig?.secretKey || process.env.LANGFUSE_SECRET_KEY;
-        const baseUrl = langfuseConfig?.baseUrl || process.env.LANGFUSE_BASE_URL || process.env.LANGFUSE_HOST;
-        const enabled = langfuseConfig?.enabled ?? Boolean(publicKey && secretKey);
+        const { enabled, publicKey, secretKey, baseUrl } = this.config.langfuse || {};
         if (!enabled || !publicKey || !secretKey) {
             return;
         }
@@ -282,7 +273,7 @@ export class Provider {
         return { conversation, response, toolExecutions };
     }
     async chat(messages, model, options = {}) {
-        const modelName = this.getModelName(model);
+        const modelName = getModelName(model);
         setActivity(`🤖 Asking ${modelName}`, 'ai');
         promptLog(`Using model: ${modelName}`);
         const config = this.buildGenerateConfig({ maxOutputTokens: 16384 }, { model, abortSignal: executionController.getAbortSignal() }, options);
@@ -290,6 +281,7 @@ export class Provider {
         try {
             const response = await withRetry(async () => {
                 const result = await generateText({ messages, ...config });
+                this.recordUsage(options.agentName || 'unknown', modelName, result.usage);
                 if (!result.text) {
                     debugLog(result);
                     if (result.finishReason === 'length') {
@@ -304,7 +296,6 @@ export class Provider {
             }, this.getRetryOptions(options));
             clearActivity();
             responseLog(response.text);
-            this.recordUsage(options.agentName || 'unknown', modelName, response.usage);
             return response;
         }
         catch (error) {
@@ -324,7 +315,7 @@ export class Provider {
         }
     }
     async generateWithTools(messages, model, tools, options = {}) {
-        const modelName = this.getModelName(model);
+        const modelName = getModelName(model);
         setActivity(`🤖 Asking ${modelName} with dynamic tools`, 'ai');
         promptLog(`Using model: ${modelName}`);
         const toolNames = Object.keys(tools || {});
@@ -340,6 +331,7 @@ export class Provider {
         try {
             const response = await withRetry(async () => {
                 const result = (await this.raceWithIdleTimeout((signal) => generateText({ messages, ...config, abortSignal: signal }), config.timeout || 30000));
+                this.recordUsage(options.agentName || 'unknown', modelName, result.usage);
                 const hasToolCall = (result.toolCalls?.length || 0) > 0;
                 if (!result.text && !hasToolCall && result.finishReason === 'length') {
                     throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase maxOutputTokens in config or use a model with higher output capacity.');
@@ -355,7 +347,6 @@ export class Provider {
                 });
             }
             responseLog(response.text);
-            this.recordUsage(options.agentName || 'unknown', modelName, response.usage);
             return response;
         }
         catch (error) {
@@ -379,7 +370,7 @@ export class Provider {
     }
     async generateObject(messages, schema, model, options = {}) {
         const modelToUse = model || this.config.model;
-        const modelName = this.getModelName(modelToUse);
+        const modelName = getModelName(modelToUse);
         setActivity(`🤖 Asking ${modelName} for structured output`, 'ai');
         promptLog(`Using model: ${modelName}`);
         const config = this.buildGenerateConfig({ schema }, { model: modelToUse }, options);
@@ -557,7 +548,7 @@ export class Provider {
             }, this.getRetryOptions());
             clearActivity();
             responseLog(response.text);
-            this.recordUsage('vision', this.getModelName(this.config.visionModel), response.usage);
+            this.recordUsage('vision', getModelName(this.config.visionModel), response.usage);
             return response;
         }
         catch (error) {

package/dist/src/ai/rerunner.js

@@ -21,7 +21,7 @@ import { RulesLoader } from "../utils/rules-loader.js";
 import { toolExecutionLabel } from "./conversation.js";
 import { actionRule, locatorRule, sectionContextRule } from "./rules.js";
 import { TaskAgent } from "./task-agent.js";
-import { createCodeceptJSTools } from "./tools.js";
+import { createCodeceptJSTools, withdrawVisionTools } from "./tools.js";
 const debugLog = createDebug('explorbot:rerunner');
 export class Rerunner extends TaskAgent {
     ACTION_TOOLS = ['click', 'pressKey', 'form'];
@@ -322,6 +322,7 @@ export class Rerunner extends TaskAgent {
                     },
                 }),
             };
+            withdrawVisionTools(tools);
             const conversation = this.provider.startConversation(this.getHealSystemPrompt(), 'rerunner');
             conversation.addUserText(this.getHealUserPrompt(failedCode, error));
             await loop(async ({ stop }) => {

package/dist/src/ai/researcher/cache.d.ts

@@ -1,3 +1,5 @@
+export declare function researchPath(hash: string): string;
+export declare function reportResearch(hash: string, text: string): void;
 export declare function clearResearchCache(): void;
 export declare function getCachedResearch(hash: string): string;
 export declare function getPreviousResearch(hash: string): string;

package/dist/src/ai/researcher/cache.js

@@ -4,6 +4,7 @@ import { Worker } from 'node:worker_threads';
 import { outputPath } from "../../config.js";
 import { TTLCache } from "../../utils/cache.js";
 import { computeHtmlFingerprint } from "../../utils/html-diff.js";
+import { tag } from "../../utils/logger.js";
 import { debugLog } from "./mixin.js";
 const CACHE_TTL_MS = 6 * 60 * 60 * 1000; // 6 hours
 const FINGERPRINT_MAX_AGE_MS = 60 * 60 * 1000; // 1 hour
@@ -11,6 +12,12 @@ const FINGERPRINT_WORKER_TIMEOUT_MS = 10_000;
 const SIMILARITY_THRESHOLD = 90;
 const memoryCache = new TTLCache(CACHE_TTL_MS);
 let fingerprintWorker = null;
+export function researchPath(hash) {
+    return outputPath('research', `${hash}.md`);
+}
+export function reportResearch(hash, text) {
+    tag('data').log('research', { path: researchPath(hash), hash, content: text });
+}
 function getStatesDir() {
     return outputPath('states');
 }
@@ -30,7 +37,7 @@ export function getCachedResearch(hash) {
     const cached = memoryCache.get(hash);
     if (cached !== undefined)
         return cached;
-    const researchFile = outputPath('research', `${hash}.md`);
+    const researchFile = researchPath(hash);
     if (!existsSync(researchFile))
         return '';
     const stats = statSync(researchFile);
@@ -43,7 +50,7 @@ export function getCachedResearch(hash) {
 export function getPreviousResearch(hash) {
     if (!hash)
         return '';
-    const researchFile = outputPath('research', `${hash}.md`);
+    const researchFile = researchPath(hash);
     if (!existsSync(researchFile))
         return '';
     return readFileSync(researchFile, 'utf8');
@@ -55,6 +62,7 @@ export function saveResearch(hash, text, combinedHtml) {
         mkdirSync(researchDir, { recursive: true });
     writeFileSync(researchFile, text);
     memoryCache.set(hash, text);
+    reportResearch(hash, text);
     debugLog(`Research saved to ${researchFile}`);
     if (combinedHtml) {
         const statesDir = getStatesDir();

package/dist/src/ai/researcher.js

@@ -13,7 +13,7 @@ import { mdq } from "../utils/markdown-query.js";
 import { RulesLoader } from "../utils/rules-loader.js";
 import { annotatePageElements } from "../utils/web-annotate.js";
 import { ContextLengthError } from './provider.js';
-import { findSimilarResearch, getCachedResearch, saveResearch } from "./researcher/cache.js";
+import { findSimilarResearch, getCachedResearch, reportResearch, saveResearch } from "./researcher/cache.js";
 import { WithCoordinates } from "./researcher/coordinates.js";
 import { WithDeepAnalysis } from "./researcher/deep-analysis.js";
 import { detectFocusedSection, hasFocusedSection, markSectionAsFocused, pickDefaultFocusedSection } from "./researcher/focus.js";
@@ -76,7 +76,8 @@ export class Researcher extends ResearcherBase {
             const cached = getCachedResearch(stateHash);
             if (cached) {
                 debugLog('Previous research result found');
-                return `!! UI MAP IS CACHED AND MAY NOT REPRESENT CURRENT STATE; REFRESH RESEARCH IF YOU NOTICE ISSUES !!\n\n${cached}`;
+                reportResearch(stateHash, cached);
+                return cached;
             }
         }
         Stats.researches++;

package/dist/src/ai/rules.js

@@ -2,13 +2,18 @@ import dedent from 'dedent';
 export const recommendedCodeceptCommands = ['I.click', 'I.type', 'I.fillField', 'I.see', 'I.seeElement'];
 const locatorPriorityRule = dedent `
   <locator_priority>
+  When the page context shows the element a ref, such as [ref=e14], there is no locator to select: click it with clickRef
+  and that ref. A ref names one exact element, so it never matches the wrong one and never has to be narrowed. Everything
+  below is for elements the context gives no ref for.
+
   Use the following priority when selecting locators:
 
   1. ARIA locators (first choice) - target browser's accessibility tree, most reliable
      Use JSON format: { "role": "button", "text": "Login" }
-     Best for: buttons, links, inputs, form controls, dropdowns, checkboxes, radio buttons
+     Copy role and text VERBATIM from the ARIA snapshot or UI map — never guess the pair.
+     If the element is absent from the snapshot, do not invent one; use text or CSS instead.
 
-  2. Text locators (second choice) - use only when text is unique on the page
+  2. Text locators (second choice) - exact visible text, use only when unique on the page
      Example: 'Login', 'Submit', 'Username'
      Skip if the same text appears multiple times on the page
 
@@ -23,10 +28,10 @@ const locatorPriorityRule = dedent `
 `;
 const contextSimplificationRule = dedent `
   <context_simplification>
-  When container is available from UI map sections:
-  - Text + container is simplest and PREFERRED: I.click('Save', '.modal')
-  - ARIA + container for disambiguation: I.click({"role":"button","text":"Save"}, '.modal')
-  - ALWAYS use context parameter unless locator is XPath or unique ID
+  - Add a container when the target may match several elements, or the UI map gives a verified
+    section container: I.click('Save', '.modal')
+  - Skip the container when the locator is already unique (XPath, unique ID, unique text)
+  - A wrong or stale container guarantees failure: always add one fallback command WITHOUT a container
   - No need for complex ARIA when container narrows scope sufficiently
   </context_simplification>
 `;
@@ -55,7 +60,7 @@ const locatorStrategyRule = dedent `
 
   <good_aria_locator_example>
   { "role": "button", "text": "Login" },
-  { "role": "input", "text": "Name" },
+  { "role": "textbox", "text": "Name" },
   { "role": "link", "text": "Forgot your password?" },
   { "role": "link", "text": "Sign Up" },
   { "role": "button", "text": "Sign In" },
@@ -101,7 +106,7 @@ const locatorStrategyRule = dedent `
   </xpath_rules>
 
   <good locator example>
-    'div[role=input][placeholder="Name"]'
+    'input[placeholder="Name"]'
     '[aria-label="Name"]'
     'form#user_form input[name="name"]'
     '#content-top #user_name'
@@ -231,7 +236,8 @@ export const unexpectedPopupRule = dedent `
 `;
 export const sectionContextRule = dedent `
   <section_context_rule>
-  Context parameter is DEFAULT for all interactions. ALWAYS use container from UI map sections unless locator is XPath or unique ID.
+  Use a container from UI map sections when the target may match several elements. A container that is
+  wrong or stale guarantees failure, so prefer a verified one and keep a containerless fallback.
 
   1. Identify which section contains the target element
   2. Get the Context Locator from that section in the UI map
@@ -249,9 +255,10 @@ export const sectionContextRule = dedent `
   For CSS locators - prepend section context:
   - I.click('.main button.submit')  // instead of I.click('button.submit')
 
-  Only omit context when:
+  Omit context when the locator already resolves to one element:
   - Locator is XPath (already includes path context)
   - Locator is a unique ID (#specific-element)
+  - Text or ARIA that appears only once on the page
   </section_context_rule>
 
   ${unexpectedPopupRule}

package/dist/src/ai/session-analyst.js

@@ -3,6 +3,7 @@ import path from 'node:path';
 import dedent from 'dedent';
 import { outputPath } from "../config.js";
 import { Stats } from "../stats.js";
+import { tag } from "../utils/logger.js";
 export class SessionAnalyst {
     emoji = '🧐';
     provider;
@@ -103,6 +104,7 @@ export class SessionAnalyst {
         if (!existsSync(dir))
             mkdirSync(dir, { recursive: true });
         writeFileSync(filePath, markdown);
+        tag('data').log('report', { path: filePath, content: markdown });
         return filePath;
     }
     serializeTest(test, ref) {

package/dist/src/ai/task-agent.js

@@ -1,6 +1,9 @@
+import { executionController } from "../execution-controller.js";
 import { HooksRunner } from "../utils/hooks-runner.js";
 export function isInteractive() {
-    return process.env.INK_RUNNING === 'true';
+    if (process.env.INK_RUNNING === 'true')
+        return true;
+    return executionController.hasInputCallback();
 }
 function createNullProxy() {
     return new Proxy({}, {

package/dist/src/ai/tester.d.ts

@@ -31,7 +31,6 @@ export declare class Tester extends TaskAgent implements Agent {
     seenUiMapUrls: Set<string>;
     lastAnalyzedStateHash: string | null;
     stalledIterations: number;
-    hasSuccessfulAssertion: boolean;
     readonly MAX_STALLED_ITERATIONS = 3;
     constructor(deps: AgentDeps, researcher: Researcher, navigator: Navigator, agentTools?: any);
     getNavigator(): Navigator;
@@ -40,16 +39,17 @@ export declare class Tester extends TaskAgent implements Agent {
     getCurrentState(): ActionResult;
     get progressCheckInterval(): number;
     getConversation(): Conversation | null;
-    test(task: Test): Promise<{
+    test(task: Test, opts?: TestOptions): Promise<{
         success: boolean;
     }>;
-    runTestSession(task: Test, initialState: ActionResult, conversation: Conversation, handlers: TestSessionHandlers): Promise<{
+    runTestSession(task: Test, initialState: ActionResult, conversation: Conversation, handlers: TestSessionHandlers, opts: TestOptions): Promise<{
         success: boolean;
     }>;
     shouldAnalyzeProgress(iteration: number, currentState: ActionResult): boolean;
     shouldStopForStalledExecution(task: Test, previousState: ActionResult, toolExecutions: any[]): boolean;
     prepareInstructionsForNextStep(task: Test): Promise<string>;
     reinjectContextIfNeeded(iteration: number, currentState: ActionResult): Promise<string>;
+    interactiveAriaWithRefs(state: ActionResult): Promise<string>;
     finishTest(task: Test): void;
     abortStartedTestOnErrorPage(task: Test, actionResult: ActionResult): Promise<{
         success: boolean;
@@ -93,4 +93,7 @@ export declare class Tester extends TaskAgent implements Agent {
 interface TestSessionHandlers {
     offFailedRequest?: () => void;
 }
+export interface TestOptions {
+    startOnCurrentPage?: boolean;
+}
 export {};