explorbot 0.2.3 → 0.2.5

package/src/ai/pilot.ts

@@ -7,8 +7,9 @@ import { ConfigParser } from '../config.ts';
 import type Explorer from '../explorer.ts';
 import type { PlaywrightRecorder } from '../playwright-recorder.ts';
 import type { StateManager } from '../state-manager.ts';
+import { Stats } from '../stats.ts';
 import { type Test, TestResult } from '../test-plan.ts';
-import { collectInteractiveNodes, detectFocusArea, extractFocusedElement } from '../utils/aria.ts';
+import { collectInteractiveNodes, detectFocusArea } from '../utils/aria.ts';
 import { ErrorPageError } from '../utils/error-page.ts';
 import { createDebug, tag } from '../utils/logger.ts';
 
@@ -22,6 +23,7 @@ import type { Provider } from './provider.ts';
 import type { Researcher } from './researcher.ts';
 import { capabilityGroundingRule, dataProtectionRules } from './rules.ts';
 import { isInteractive } from './task-agent.ts';
+import { withdrawVisionTools } from './tools.ts';
 
 const CHECK_TOOLS = ['verify', 'see', 'research', 'context'];
 const META_TOOLS = ['record', 'reset', 'stop', 'finish'];
@@ -561,6 +563,97 @@ export class Pilot implements Agent {
     return text;
   }
 
+  async settleExpectations(task: Test, finalState?: ActionResult): Promise<SettledExpectation[]> {
+    let image: string | null = null;
+    if (finalState?.screenshot && this.provider.hasVision()) image = `data:image/png;base64,${finalState.screenshot.toString('base64')}`;
+
+    const decided = (text: string): 'passed' | 'failed' => {
+      if (task.hasAchievedAny() && !task.getRemainingExpectations().includes(text)) return 'passed';
+      return 'failed';
+    };
+
+    let undecided = task.expected.filter((text) => !task.getCheckedExpectations().includes(text));
+    if (image) undecided = task.expected;
+    if (!undecided.length) return task.expected.map((text) => ({ text, status: decided(text) }));
+
+    const schema = z.object({
+      outcomes: z.array(
+        z.object({
+          expectation: z.string().describe('The expected outcome, repeated exactly as it was given'),
+          status: z.enum(['passed', 'failed', 'unverified', 'contradiction']).describe('passed = the evidence shows it happened, failed = the evidence shows it did not, unverified = the run never established either way, contradiction = the picture and the run disagree'),
+          evidence: z.string().nullable().describe('What settled it. For a contradiction, what each side shows. Null when there is nothing to add'),
+        })
+      ),
+    });
+
+    let pageEvidence = '';
+    if (image) {
+      pageEvidence = dedent`
+        A screenshot of the whole page as the run left it is attached. It is the proof: an outcome is satisfied
+        when the page shows it to somebody looking at it. The log only says what the run did.
+
+        Not finding something in the picture is not by itself a disagreement. Report "contradiction" only when
+        the picture shows something incompatible with what the run claims — a list visibly empty, an error where
+        a result was expected, the old value still displayed, a control visibly disabled. When you simply cannot
+        make it out, say "unverified" and name what you could not find.
+
+        The picture covers the full page, but not the inside of a region that scrolls on its own, and not the
+        state of the page before the run ended. An outcome established earlier stays established even when the
+        page has moved past it, and that is not a contradiction.
+      `;
+    }
+
+    const userContent = dedent`
+      A test run has finished. Decide, for each expected outcome, what the run established about it.
+
+      <expected_outcomes>
+      ${undecided.map((text) => `- ${text}`).join('\n')}
+      </expected_outcomes>
+
+      <run_log>
+      ${task.notesToString() || 'No steps recorded.'}
+      </run_log>
+
+      ${pageEvidence}
+
+      The log is written in the tester's own words, so an outcome can be satisfied by a step that describes it
+      differently. Judge by what the steps show happened, not by whether the wording matches.
+      Choose "unverified" only when the evidence neither shows the outcome happening nor shows it failing —
+      that is a statement about the run, not about the application.
+    `;
+
+    const settle = (content: any, model: any) =>
+      this.provider
+        .generateObject([{ role: 'user' as const, content }], schema, model, {
+          agentName: 'pilot',
+          telemetry: { functionId: 'pilot.settleExpectations' },
+        })
+        .catch(() => null);
+
+    let response = null;
+    if (image) {
+      const seen = [
+        { type: 'text', text: userContent },
+        { type: 'file', mediaType: 'image/png', data: image },
+      ];
+      response = await settle(seen, this.provider.getVisionModel());
+      if (!response) {
+        Stats.visionDisabled = true;
+        tag('warning').log('⚠️ Vision model could not judge the outcomes. Settling them from the run log instead.');
+      }
+    }
+
+    if (!response) response = await settle(userContent, this.provider.getAgenticModel('pilot'));
+
+    const judged = new Map((response?.object?.outcomes || []).map((outcome: any) => [outcome.expectation, outcome]));
+    return task.expected.map((text) => {
+      if (!undecided.includes(text)) return { text, status: decided(text) };
+      const outcome = judged.get(text) as { status: SettledStatus; evidence?: string } | undefined;
+      if (!outcome) return { text, status: 'unverified' as SettledStatus };
+      return { text, status: outcome.status || 'unverified', evidence: outcome.evidence };
+    });
+  }
+
   private formatExpectations(task: Test): string {
     const checked = task.getCheckedExpectations();
     const remaining = task.getRemainingExpectations();
@@ -620,6 +713,7 @@ export class Pilot implements Agent {
     if (xpathCheck) planning.xpathCheck = xpathCheck;
     if (learnExperience) planning.learnExperience = learnExperience;
     if (askUser) planning.askUser = askUser;
+    withdrawVisionTools(planning);
     return planning;
   }
 
@@ -703,7 +797,7 @@ export class Pilot implements Agent {
     lines.push(`url: ${state.url}`);
     lines.push(`title: ${state.title || 'unknown'}`);
 
-    const focused = extractFocusedElement(state.ariaSnapshot);
+    const focused = state.focusedElement;
     if (focused) {
       const valuePart = focused.value ? ` (value: "${focused.value}")` : '';
       lines.push(`focused: ${focused.role} "${focused.name}"${valuePart}`);
@@ -1065,3 +1159,11 @@ export class Pilot implements Agent {
     `;
   }
 }
+
+export type SettledStatus = 'passed' | 'failed' | 'unverified' | 'contradiction';
+
+export interface SettledExpectation {
+  text: string;
+  status: SettledStatus;
+  evidence?: string;
+}

package/src/ai/provider.ts

@@ -4,7 +4,7 @@ import { NodeSDK } from '@opentelemetry/sdk-node';
 import { generateObject, generateText, isStepCount, registerTelemetry } from 'ai';
 import type { ModelMessage } from 'ai';
 import { clearActivity, setActivity } from '../activity.ts';
-import type { AIConfig } from '../config.js';
+import { type AIConfig, configuredModels, modelName as getModelName } from '../config.js';
 import { executionController } from '../execution-controller.ts';
 import { Observability } from '../observability.ts';
 import { Stats } from '../stats.ts';
@@ -88,10 +88,6 @@ export class Provider {
     this.initLangfuse();
   }
 
-  private getModelName(model: any): string {
-    return model?.modelId || model?.model || 'unknown';
-  }
-
   async validateConnection(): Promise<void> {
     try {
       await generateText({
@@ -120,13 +116,13 @@ export class Provider {
     return this.config.agenticModel || this.config.model;
   }
 
+  getVisionModel(): any {
+    return this.config.visionModel;
+  }
+
   getConfiguredModels(): Record<string, string> {
-    const models: Record<string, string> = { model: this.getModelName(this.config.model) };
-    if (this.config.agenticModel) models.agenticModel = this.getModelName(this.config.agenticModel);
-    if (this.config.visionModel) models.visionModel = this.getModelName(this.config.visionModel);
-    for (const [agent, agentConfig] of Object.entries(this.config.agents || {})) {
-      if (agentConfig?.model) models[agent] = this.getModelName(agentConfig.model);
-    }
+    const models: Record<string, string> = {};
+    for (const [role, model] of Object.entries(configuredModels(this.config))) models[role] = model.name;
     return models;
   }
 
@@ -223,11 +219,7 @@ export class Provider {
   }
 
   private initLangfuse() {
-    const langfuseConfig = this.config.langfuse;
-    const publicKey = langfuseConfig?.publicKey || process.env.LANGFUSE_PUBLIC_KEY;
-    const secretKey = langfuseConfig?.secretKey || process.env.LANGFUSE_SECRET_KEY;
-    const baseUrl = langfuseConfig?.baseUrl || process.env.LANGFUSE_BASE_URL || process.env.LANGFUSE_HOST;
-    const enabled = langfuseConfig?.enabled ?? Boolean(publicKey && secretKey);
+    const { enabled, publicKey, secretKey, baseUrl } = this.config.langfuse || {};
 
     if (!enabled || !publicKey || !secretKey) {
       return;
@@ -316,7 +308,7 @@ export class Provider {
   }
 
   async chat(messages: ModelMessage[], model: any, options: any = {}): Promise<any> {
-    const modelName = this.getModelName(model);
+    const modelName = getModelName(model);
     setActivity(`🤖 Asking ${modelName}`, 'ai');
     promptLog(`Using model: ${modelName}`);
 
@@ -326,6 +318,7 @@ export class Provider {
     try {
       const response = await withRetry(async () => {
         const result = await generateText({ messages, ...config });
+        this.recordUsage(options.agentName || 'unknown', modelName, result.usage);
         if (!result.text) {
           debugLog(result);
           if (result.finishReason === 'length') {
@@ -342,8 +335,6 @@ export class Provider {
       clearActivity();
       responseLog(response.text);
 
-      this.recordUsage(options.agentName || 'unknown', modelName, response.usage);
-
       return response;
     } catch (error: any) {
       clearActivity();
@@ -361,7 +352,7 @@ export class Provider {
   }
 
   async generateWithTools(messages: ModelMessage[], model: any, tools: any, options: any = {}): Promise<any> {
-    const modelName = this.getModelName(model);
+    const modelName = getModelName(model);
     setActivity(`🤖 Asking ${modelName} with dynamic tools`, 'ai');
     promptLog(`Using model: ${modelName}`);
 
@@ -378,6 +369,7 @@ export class Provider {
     try {
       const response = await withRetry(async () => {
         const result = (await this.raceWithIdleTimeout((signal) => generateText({ messages, ...config, abortSignal: signal }), config.timeout || 30000)) as any;
+        this.recordUsage(options.agentName || 'unknown', modelName, result.usage);
         const hasToolCall = (result.toolCalls?.length || 0) > 0;
         if (!result.text && !hasToolCall && result.finishReason === 'length') {
           throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase maxOutputTokens in config or use a model with higher output capacity.');
@@ -397,8 +389,6 @@ export class Provider {
 
       responseLog(response.text);
 
-      this.recordUsage(options.agentName || 'unknown', modelName, response.usage);
-
       return response;
     } catch (error: any) {
       clearActivity();
@@ -420,7 +410,7 @@ export class Provider {
 
   async generateObject(messages: ModelMessage[], schema: any, model?: any, options: any = {}): Promise<any> {
     const modelToUse = model || this.config.model;
-    const modelName = this.getModelName(modelToUse);
+    const modelName = getModelName(modelToUse);
     setActivity(`🤖 Asking ${modelName} for structured output`, 'ai');
     promptLog(`Using model: ${modelName}`);
 
@@ -622,7 +612,7 @@ export class Provider {
       clearActivity();
       responseLog(response.text);
 
-      this.recordUsage('vision', this.getModelName(this.config.visionModel), response.usage);
+      this.recordUsage('vision', getModelName(this.config.visionModel), response.usage);
 
       return response;
     } catch (error: any) {

package/src/ai/rerunner.ts

@@ -24,7 +24,7 @@ import type { Navigator } from './navigator.ts';
 import { Provider } from './provider.ts';
 import { actionRule, locatorRule, sectionContextRule } from './rules.ts';
 import { TaskAgent } from './task-agent.ts';
-import { createCodeceptJSTools } from './tools.ts';
+import { createCodeceptJSTools, withdrawVisionTools } from './tools.ts';
 
 const debugLog = createDebug('explorbot:rerunner');
 
@@ -361,6 +361,7 @@ export class Rerunner extends TaskAgent implements Agent {
           },
         }),
       };
+      withdrawVisionTools(tools);
 
       const conversation = this.provider.startConversation(this.getHealSystemPrompt(), 'rerunner');
       conversation.addUserText(this.getHealUserPrompt(failedCode, error));

package/src/ai/researcher/cache.ts

@@ -4,6 +4,7 @@ import { Worker } from 'node:worker_threads';
 import { outputPath } from '../../config.ts';
 import { TTLCache } from '../../utils/cache.ts';
 import { computeHtmlFingerprint } from '../../utils/html-diff.ts';
+import { tag } from '../../utils/logger.ts';
 import { debugLog } from './mixin.ts';
 
 const CACHE_TTL_MS = 6 * 60 * 60 * 1000; // 6 hours
@@ -15,6 +16,14 @@ const memoryCache = new TTLCache<string>(CACHE_TTL_MS);
 
 let fingerprintWorker: Worker | null = null;
 
+export function researchPath(hash: string): string {
+  return outputPath('research', `${hash}.md`);
+}
+
+export function reportResearch(hash: string, text: string): void {
+  tag('data').log('research', { path: researchPath(hash), hash, content: text });
+}
+
 function getStatesDir(): string {
   return outputPath('states');
 }
@@ -35,7 +44,7 @@ export function getCachedResearch(hash: string): string {
   if (!hash) return '';
   const cached = memoryCache.get(hash);
   if (cached !== undefined) return cached;
-  const researchFile = outputPath('research', `${hash}.md`);
+  const researchFile = researchPath(hash);
   if (!existsSync(researchFile)) return '';
   const stats = statSync(researchFile);
   if (Date.now() - stats.mtimeMs > CACHE_TTL_MS) return '';
@@ -46,7 +55,7 @@ export function getCachedResearch(hash: string): string {
 
 export function getPreviousResearch(hash: string): string {
   if (!hash) return '';
-  const researchFile = outputPath('research', `${hash}.md`);
+  const researchFile = researchPath(hash);
   if (!existsSync(researchFile)) return '';
   return readFileSync(researchFile, 'utf8');
 }
@@ -57,6 +66,7 @@ export function saveResearch(hash: string, text: string, combinedHtml?: string):
   if (!existsSync(researchDir)) mkdirSync(researchDir, { recursive: true });
   writeFileSync(researchFile, text);
   memoryCache.set(hash, text);
+  reportResearch(hash, text);
   debugLog(`Research saved to ${researchFile}`);
 
   if (combinedHtml) {

package/src/ai/researcher.ts

@@ -19,7 +19,7 @@ import { annotatePageElements } from '../utils/web-annotate.ts';
 import type { Agent, AgentDeps } from './agent.js';
 import type { Navigator } from './navigator.ts';
 import { ContextLengthError, type Provider } from './provider.js';
-import { findSimilarResearch, getCachedResearch, saveResearch } from './researcher/cache.ts';
+import { findSimilarResearch, getCachedResearch, reportResearch, saveResearch } from './researcher/cache.ts';
 import { type CoordinateMethods, WithCoordinates } from './researcher/coordinates.ts';
 import { type DeepAnalysisMethods, WithDeepAnalysis } from './researcher/deep-analysis.ts';
 import { detectFocusedSection, hasFocusedSection, markSectionAsFocused, pickDefaultFocusedSection } from './researcher/focus.ts';
@@ -102,7 +102,8 @@ export class Researcher extends ResearcherBase implements Agent {
       const cached = getCachedResearch(stateHash);
       if (cached) {
         debugLog('Previous research result found');
-        return `!! UI MAP IS CACHED AND MAY NOT REPRESENT CURRENT STATE; REFRESH RESEARCH IF YOU NOTICE ISSUES !!\n\n${cached}`;
+        reportResearch(stateHash, cached);
+        return cached;
       }
     }
 

package/src/ai/rules.ts

@@ -4,13 +4,18 @@ export const recommendedCodeceptCommands = ['I.click', 'I.type', 'I.fillField',
 
 const locatorPriorityRule = dedent`
   <locator_priority>
+  When the page context shows the element a ref, such as [ref=e14], there is no locator to select: click it with clickRef
+  and that ref. A ref names one exact element, so it never matches the wrong one and never has to be narrowed. Everything
+  below is for elements the context gives no ref for.
+
   Use the following priority when selecting locators:
 
   1. ARIA locators (first choice) - target browser's accessibility tree, most reliable
      Use JSON format: { "role": "button", "text": "Login" }
-     Best for: buttons, links, inputs, form controls, dropdowns, checkboxes, radio buttons
+     Copy role and text VERBATIM from the ARIA snapshot or UI map — never guess the pair.
+     If the element is absent from the snapshot, do not invent one; use text or CSS instead.
 
-  2. Text locators (second choice) - use only when text is unique on the page
+  2. Text locators (second choice) - exact visible text, use only when unique on the page
      Example: 'Login', 'Submit', 'Username'
      Skip if the same text appears multiple times on the page
 
@@ -26,10 +31,10 @@ const locatorPriorityRule = dedent`
 
 const contextSimplificationRule = dedent`
   <context_simplification>
-  When container is available from UI map sections:
-  - Text + container is simplest and PREFERRED: I.click('Save', '.modal')
-  - ARIA + container for disambiguation: I.click({"role":"button","text":"Save"}, '.modal')
-  - ALWAYS use context parameter unless locator is XPath or unique ID
+  - Add a container when the target may match several elements, or the UI map gives a verified
+    section container: I.click('Save', '.modal')
+  - Skip the container when the locator is already unique (XPath, unique ID, unique text)
+  - A wrong or stale container guarantees failure: always add one fallback command WITHOUT a container
   - No need for complex ARIA when container narrows scope sufficiently
   </context_simplification>
 `;
@@ -59,7 +64,7 @@ const locatorStrategyRule = dedent`
 
   <good_aria_locator_example>
   { "role": "button", "text": "Login" },
-  { "role": "input", "text": "Name" },
+  { "role": "textbox", "text": "Name" },
   { "role": "link", "text": "Forgot your password?" },
   { "role": "link", "text": "Sign Up" },
   { "role": "button", "text": "Sign In" },
@@ -105,7 +110,7 @@ const locatorStrategyRule = dedent`
   </xpath_rules>
 
   <good locator example>
-    'div[role=input][placeholder="Name"]'
+    'input[placeholder="Name"]'
     '[aria-label="Name"]'
     'form#user_form input[name="name"]'
     '#content-top #user_name'
@@ -245,7 +250,8 @@ export const unexpectedPopupRule = dedent`
 
 export const sectionContextRule = dedent`
   <section_context_rule>
-  Context parameter is DEFAULT for all interactions. ALWAYS use container from UI map sections unless locator is XPath or unique ID.
+  Use a container from UI map sections when the target may match several elements. A container that is
+  wrong or stale guarantees failure, so prefer a verified one and keep a containerless fallback.
 
   1. Identify which section contains the target element
   2. Get the Context Locator from that section in the UI map
@@ -263,9 +269,10 @@ export const sectionContextRule = dedent`
   For CSS locators - prepend section context:
   - I.click('.main button.submit')  // instead of I.click('button.submit')
 
-  Only omit context when:
+  Omit context when the locator already resolves to one element:
   - Locator is XPath (already includes path context)
   - Locator is a unique ID (#specific-element)
+  - Text or ARIA that appears only once on the page
   </section_context_rule>
 
   ${unexpectedPopupRule}

package/src/ai/session-analyst.ts

@@ -4,6 +4,7 @@ import dedent from 'dedent';
 import { outputPath } from '../config.ts';
 import { Stats } from '../stats.ts';
 import type { Test } from '../test-plan.ts';
+import { tag } from '../utils/logger.ts';
 import type { Agent } from './agent.ts';
 import type { Provider } from './provider.ts';
 
@@ -117,6 +118,7 @@ export class SessionAnalyst implements Agent {
     const dir = path.dirname(filePath);
     if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
     writeFileSync(filePath, markdown);
+    tag('data').log('report', { path: filePath, content: markdown });
     return filePath;
   }
 

package/src/ai/task-agent.ts

@@ -1,5 +1,6 @@
 import type { ActionResult } from '../action-result.js';
 import type { ExplorbotConfig } from '../config.ts';
+import { executionController } from '../execution-controller.ts';
 import type { ExperienceTracker } from '../experience-tracker.js';
 import type Explorer from '../explorer.ts';
 import type { KnowledgeTracker } from '../knowledge-tracker.js';
@@ -12,7 +13,8 @@ import type { Provider } from './provider.js';
 import { Quartermaster } from './quartermaster.js';
 
 export function isInteractive(): boolean {
-  return process.env.INK_RUNNING === 'true';
+  if (process.env.INK_RUNNING === 'true') return true;
+  return executionController.hasInputCallback();
 }
 
 function createNullProxy<T extends object>(): T {