explorbot 0.2.3 → 0.2.4

package/src/ai/tools.ts

@@ -3,8 +3,9 @@ import dedent from 'dedent';
 import { z } from 'zod';
 import { ActionResult, type PageDiff, type ToolResultMetadata } from '../action-result.ts';
 import type { ExperienceTracker } from '../experience-tracker.ts';
+import { Stats } from '../stats.ts';
 import { type Task, TestResult } from '../test-plan.js';
-import { LARGE_ARIA_CHANGE_THRESHOLD, extractFocusedElement } from '../utils/aria.ts';
+import { LARGE_ARIA_CHANGE_THRESHOLD } from '../utils/aria.ts';
 import { isFatalBrowserError } from '../utils/browser-errors.ts';
 import { createDebug, tag } from '../utils/logger.js';
 import { pause } from '../utils/loop.js';
@@ -33,6 +34,10 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }: ToolDeps,
       description: dedent`
         Click an element by trying multiple CodeceptJS commands in order until one succeeds.
 
+        Use this only for elements the page context gives you no ref for. When the element shows a ref such as [ref=e14],
+        call clickRef with that ref instead — composing a locator for an element that already has a ref is wasted work,
+        and a locator can match several elements where a ref cannot.
+
         Follow <locator_priority> from system prompt for locator selection.
 
         I.click(locator) - click element matching locator
@@ -51,14 +56,13 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }: ToolDeps,
         commands: z.array(z.string()).describe(dedent`
           FALLBACK LOCATORS for ONE element. All commands must click the SAME element.
           Never mix different elements — use separate click() calls instead.
+          REQUIRED: include at least one command WITHOUT a container — a wrong or stale container always fails.
           Order by reliability:
-          1. I.click(text, container) - PREFERRED when container is known - e.g. I.click("Save", ".modal")
+          1. I.click(text, container) - when the container is verified - e.g. I.click("Save", ".modal")
           2. I.click(ARIA, container) - e.g. I.click({"role":"button","text":"Save"}, ".modal")
           3. I.click(CSS, container) - e.g. I.click("#btn", ".modal")
           4. I.click(CSS) or I.click(XPath) - when locator already includes context (ID, XPath)
           5. I.clickXY(x, y) - coordinates fallback
-          IMPORTANT: Always include at least one command WITHOUT a container as fallback,
-          in case the element moved to a different section (e.g. I.click("Save") without container).
         `),
         explanation: z.string().describe('Why you are clicking this element'),
       }),
@@ -132,13 +136,7 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }: ToolDeps,
         const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, commands[0]);
         await commitNote(activeNote, TestResult.FAILED, toolResult, action);
 
-        let suggestion = "Try xpathCheck() to find the element's actual position, see() for visual analysis, or visualClick() to click by visual appearance.";
-        const lastError = attempts[attempts.length - 1]?.error || '';
-        if (lastError.includes('was not found') || lastError.includes('not found by text')) {
-          suggestion = 'Element was not found in the DOM. Use xpathCheck() to locate it, context() to refresh snapshot, or visualClick() to click by visual appearance.';
-        } else if (lastError.includes('Timeout') || lastError.includes('intercept')) {
-          suggestion = 'Element exists but could not be clicked (possibly covered by overlay or not interactable). Try closing overlapping panels first, or use visualClick().';
-        }
+        const suggestion = clickFailureSuggestion(attempts);
 
         return failedToolResult(
           'click',
@@ -153,6 +151,41 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }: ToolDeps,
       },
     }),
 
+    clickRef: tool({
+      description: dedent`
+        Click an element by the ref the page context gave it, e.g. [ref=e14].
+
+        Prefer this over click() whenever the element you want carries a ref. A ref names one exact element, so it
+        cannot match several by mistake and never needs disambiguating — it is the fastest way to click.
+        Only pass a ref that appears in the page context you were given. Never invent or guess one.
+        If it reports the ref is gone, the page has been rebuilt: get fresh context and use the new ref.
+      `,
+      inputSchema: z.object({
+        ref: z.string().describe('The ref exactly as it appears in the page context, e.g. "e14"'),
+        element: z.string().describe('Role and name of the element you are clicking, for the record'),
+      }),
+      execute: async ({ ref, element }) => {
+        const activeNote = task.startNote(`Click ${element}`);
+        const previousState = ActionResult.fromState(stateManager.getCurrentState()!);
+        const action = explorer.action();
+        const named = await describeRef(explorer, ref);
+        const run = `I.usePlaywrightTo(${JSON.stringify(`click ${element}`)}, async ({ page }) => page.locator(${JSON.stringify(`aria-ref=${ref}`)}).click())`;
+
+        if (!(await action.attempt(run, `Click ${element}`))) {
+          activeNote.commit(TestResult.FAILED);
+          return failedToolResult('clickRef', `Ref ${ref} could not be clicked: ${errorText(action.lastError)}`, {
+            suggestion: 'The ref may belong to an older version of the page. Get fresh context and use the ref it gives, or fall back to click() with a locator.',
+          });
+        }
+
+        // a ref belongs to this session only, so the run is reported as the locator a later test can replay
+        const code = named ? `I.click(${JSON.stringify(named)})` : run;
+        const toolResult = await ActionResult.fromState(stateManager.getCurrentState()!).toToolResult(previousState, code);
+        await commitNote(activeNote, TestResult.PASSED, toolResult, action);
+        return successToolResult('clickRef', { ...toolResult, code }, action);
+      },
+    }),
+
     hover: tool({
       description: dedent`
         Move the mouse cursor to an element to reveal hover-only controls.
@@ -299,15 +332,11 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }: ToolDeps,
           const focusFreeKeys = new Set(['Escape', 'Esc', 'Tab', 'F1', 'F2', 'F3', 'F4', 'F5', 'F6', 'F7', 'F8', 'F9', 'F10', 'F11', 'F12']);
           const needsFocus = !focusFreeKeys.has(keyToUse) && !modifier;
 
-          if (needsFocus) {
-            const currentAriaState = stateManager.getCurrentState()?.ariaSnapshot;
-            const focused = extractFocusedElement(currentAriaState ?? null);
-            if (!focused) {
-              activeNote.commit(TestResult.FAILED);
-              return failedToolResult('pressKey', `No element is focused. Key '${keyToUse}' requires a focused element.`, {
-                suggestion: 'Click the target element first, then press the key.',
-              });
-            }
+          if (needsFocus && !(await hasFocusedElement(explorer))) {
+            activeNote.commit(TestResult.FAILED);
+            return failedToolResult('pressKey', `No element is focused. Key '${keyToUse}' requires a focused element.`, {
+              suggestion: 'Click the target element first, then press the key.',
+            });
           }
 
           const previousState = ActionResult.fromState(stateManager.getCurrentState()!);
@@ -358,6 +387,8 @@ export function createCodeceptJSTools({ explorer, stateManager, ai }: ToolDeps,
         Execute raw CodeceptJS code block with multiple commands.
         USE THIS TOOL for typing text into fields: I.fillField, I.type
 
+        Do not put a click on a ref-bearing element in here — clickRef with its ref is cheaper and cannot mis-target.
+
         Follow <actions> from system prompt for available commands.
         Follow <locator_priority> from system prompt for locator selection.
 
@@ -539,26 +570,25 @@ export function createLearnExperienceTool({ getExperienceTracker, getState }: {
 }
 
 export function createAgentTools({ explorer, stateManager, ai, researcher, navigator, supervisor, withExperience }: AgentToolDeps): any {
-  let visionDisabled = false;
-
   const tools: Record<string, any> = {
     see: tool({
       description: dedent`
-        Check the page contents based on current page state and screenshot.
-        This tool will trigger visual research to check the page contents on request.
-        Use it to verify the actions were performed correctly and the page is in the expected state.
+        Answer a question about the page from a screenshot, for things its structure cannot express:
+        layout and position, what an image or canvas depicts, colour, and whether something is covered or cut off.
+        This runs a second model and is the slowest tool here, so reach for it only when the question is genuinely visual.
+        Do NOT use it to confirm an action landed — every action already reports what changed on the page.
         Input schema has exactly one field: request. Do not pass text, reason, assertion, or other fields.
 
         <example>
-        request: "Check current state of the Login form"
-        result: "Login form is visible with username and password fields, username is filled with 'testuser' and password is empty' 
+        request: "Is the save button covered by anything, and does the chart show any plotted data?"
+        result: "The save button is partly behind a cookie banner at the bottom. The chart area is empty apart from its axes."
         </example>
       `,
       inputSchema: z.object({
         request: z.string().describe('LLM-friendly description of the page contents to look for. 1-3 sentences. No more than 100 words.'),
       }),
       execute: async ({ request }) => {
-        if (visionDisabled) {
+        if (Stats.visionDisabled) {
           return failedToolResult('see', 'Vision tools are disabled for this session. Use context() to get fresh ARIA snapshot and analyze page state from ARIA data.');
         }
 
@@ -583,8 +613,7 @@ export function createAgentTools({ explorer, stateManager, ai, researcher, navig
         } catch (error) {
           throwIfFatalBrowserError(error);
           const errorMessage = errorText(error);
-          visionDisabled = true;
-          tag('warning').log('⚠️ Vision model is not available. Visual checks are disabled for this session.');
+          disableVision();
           return failedToolResult('see', `See tool failed: ${errorMessage}`, {
             suggestion: 'Vision is now disabled. Use context() to get fresh ARIA snapshot and analyze page state from ARIA data.',
           });
@@ -676,6 +705,13 @@ export function createAgentTools({ explorer, stateManager, ai, researcher, navig
             );
           }
 
+          if (result.inexpressible) {
+            return failedToolResult('verify', `No assertion could express this claim: ${assertion}`, {
+              inexpressible: true,
+              suggestion: 'This is not evidence the page is wrong — the claim could not be turned into an assertion. Restate it in terms of what is visible or of a control state, or check it with see().',
+            });
+          }
+
           return failedToolResult('verify', `Verification failed: ${assertion}`, {
             suggestion: 'The assertion could not be verified. Check if the condition is actually present on the page or try a different assertion.',
           });
@@ -804,7 +840,7 @@ export function createAgentTools({ explorer, stateManager, ai, researcher, navig
         context: z.string().describe('What you already tried and why it failed - helps with accurate identification'),
       }),
       execute: async ({ element, context }) => {
-        if (visionDisabled) {
+        if (Stats.visionDisabled) {
           return failedToolResult('visualClick', 'Vision tools are disabled for this session. Use xpathCheck() to find the element, then click() with the discovered locator.');
         }
 
@@ -860,8 +896,7 @@ export function createAgentTools({ explorer, stateManager, ai, researcher, navig
         } catch (error) {
           throwIfFatalBrowserError(error);
           const errorMessage = errorText(error);
-          visionDisabled = true;
-          tag('warning').log('⚠️ Vision model is not available. Visual clicks are disabled for this session.');
+          disableVision();
           return failedToolResult('visualClick', `visualClick tool failed: ${errorMessage}`, {
             suggestion: 'Vision is now disabled. Use xpathCheck() to find the element, then click() with the discovered locator.',
           });
@@ -1011,6 +1046,12 @@ export function createAgentTools({ explorer, stateManager, ai, researcher, navig
     }),
   };
 
+  const disableVision = (): void => {
+    Stats.visionDisabled = true;
+    withdrawVisionTools(tools);
+    tag('warning').log('⚠️ Vision model is not available. Visual tools are disabled for this session.');
+  };
+
   if (withExperience !== false) {
     tools.learnExperience = createLearnExperienceTool({
       getExperienceTracker: () => stateManager.getExperienceTracker(),
@@ -1063,6 +1104,8 @@ export function createAgentTools({ explorer, stateManager, ai, researcher, navig
     });
   }
 
+  withdrawVisionTools(tools);
+
   return tools;
 }
 
@@ -1112,14 +1155,33 @@ function errorText(error: unknown): string {
   return 'Unknown error occurred';
 }
 
-async function commitNote(activeNote: any, result: TestResult, toolResult: any, action: any): Promise<void> {
+export async function commitNote(activeNote: any, result: TestResult, toolResult: any, action: any): Promise<void> {
   if (toolResult?.pageDiff?.ariaChanges || toolResult?.pageDiff?.urlChanged) {
     activeNote.screenshot = await action.saveScreenshot();
   }
   activeNote.commit(result);
 }
 
-function successToolResult(action: string, data?: Record<string, any>, source?: { playwrightGroupId?: string | null; assertionSteps?: any[] }) {
+async function describeRef(explorer: any, ref: string): Promise<{ role: string; text: string } | null> {
+  return Promise.resolve(
+    explorer?.withPage?.((page: any) =>
+      page.locator(`aria-ref=${ref}`).evaluate((el: any) => {
+        const tag = el.tagName.toLowerCase();
+        const roles: Record<string, string> = { a: 'link', button: 'button', select: 'combobox', textarea: 'textbox' };
+        const role = el.getAttribute('role') || roles[tag] || tag;
+        const text = (el.getAttribute('aria-label') || el.innerText || el.value || '').trim().split('\n')[0];
+        if (!text) return null;
+        return { role, text };
+      })
+    )
+  ).catch(() => null);
+}
+
+async function hasFocusedElement(explorer: any): Promise<boolean> {
+  return explorer.withPage((page: any) => page.evaluate(() => !!document.activeElement && document.activeElement !== document.body)).catch(() => true);
+}
+
+export function successToolResult(action: string, data?: Record<string, any>, source?: { playwrightGroupId?: string | null; assertionSteps?: any[] }) {
   const result: Record<string, any> = { success: true, action, ...data };
   if (source?.playwrightGroupId) {
     result.playwrightGroupId = source.playwrightGroupId;
@@ -1155,7 +1217,7 @@ function hasObservablePageChange(data?: Record<string, any>): boolean {
   return Array.isArray(data.pageDiff.htmlParts) && data.pageDiff.htmlParts.length > 0;
 }
 
-async function failedToolResult(action: string, message: string, data?: Record<string, any>, error?: Error | null) {
+export async function failedToolResult(action: string, message: string, data?: Record<string, any>, error?: Error | null) {
   const result: Record<string, any> = { success: false, action, message, ...data };
   if (data?.pageDiff) {
     result.suggestion = data.suggestion ? `${data.suggestion} ${PAGE_DIFF_SUGGESTION}` : PAGE_DIFF_SUGGESTION;
@@ -1192,6 +1254,40 @@ function getMultipleElementsSuggestion(): string {
   `;
 }
 
+export function withdrawVisionTools(tools: Record<string, any>): void {
+  if (!Stats.visionDisabled) return;
+  Reflect.deleteProperty(tools, 'see');
+  Reflect.deleteProperty(tools, 'visualClick');
+}
+
+export function clickFailureSuggestion(attempts: Array<{ error?: string }>): string {
+  const errors = attempts.map((a) => a.error || '');
+
+  if (errors.some((e) => e.includes('not enabled'))) {
+    return 'Element exists but is DISABLED — clicking it again cannot work. A precondition is unmet: a required field is empty, nothing is selected, or a dialog is blocking. Satisfy it, then retry.';
+  }
+
+  if (errors.some((e) => e.includes('intercepts pointer events'))) {
+    return 'Element exists but another element covers it. Close the overlapping panel or dialog, then retry.';
+  }
+
+  if (errors.some((e) => e.includes('is not visible'))) {
+    return 'Element is in the DOM but not visible. Reveal it first — scroll to it, expand its section, or open the panel holding it.';
+  }
+
+  const notFound = errors.filter((e) => e.includes('was not found'));
+
+  if (notFound.length && notFound.every((e) => e.includes('was not found inside element'))) {
+    return 'Element was not found inside that container — the container is wrong or stale, and the element may exist elsewhere on the page. Retry the same locator WITHOUT a container, or verify the container with xpathCheck().';
+  }
+
+  if (notFound.length) {
+    return 'Element was not found in the DOM. Use xpathCheck() to locate it, context() to refresh snapshot, or visualClick() to click by visual appearance.';
+  }
+
+  return "Try xpathCheck() to find the element's actual position, see() for visual analysis, or visualClick() to click by visual appearance.";
+}
+
 const MAX_DISAMBIGUATE_ELEMENTS = 10;
 const MULTIPLE_ELEMENTS_PATTERN = 'multiple elements';
 

package/src/config.ts

@@ -1,12 +1,13 @@
 import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import path, { basename, dirname, join, resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
 import { parseEnv } from 'node:util';
 import dedent from 'dedent';
 import matter from 'gray-matter';
 import { type SiteRecord, findGlobalConfig, globalEnvPath, isGlobalConfigPath, registerSite, resolveSiteTarget } from './global-config.js';
 import { getCliName } from './utils/cli-name.js';
-import { log } from './utils/logger.js';
+import { log, tag } from './utils/logger.js';
 
 export const PROVIDERS: Record<string, ProviderInfo> = {
   openai: { envKey: 'OPENAI_API_KEY', load: async () => (await import('@ai-sdk/openai')).createOpenAI() },
@@ -197,6 +198,7 @@ interface HtmlConfig {
 interface ActionConfig {
   delay?: number;
   retries?: number;
+  timeout?: number;
 }
 
 interface ReporterConfig {
@@ -455,7 +457,15 @@ export class ConfigParser {
   }
 
   public resolveTargetPath(target?: string): string {
-    if (!this.site) return target || '/';
+    if (!this.site) {
+      const configured = this.config?.playwright?.url || this.config?.web?.url;
+      const targetOrigin = target ? URL.parse(target)?.origin : null;
+      const baseOrigin = configured ? URL.parse(configured)?.origin : null;
+      if (targetOrigin && baseOrigin && targetOrigin !== baseOrigin) {
+        tag('warning').log(`Exploring ${targetOrigin} but base URL is ${baseOrigin}. Relative navigation resolves against the base URL — set web.url to ${targetOrigin} to avoid it.`);
+      }
+      return target || '/';
+    }
     if (!target) return this.siteStartPath;
 
     const resolved = resolveSiteTarget(target, this.site.url);
@@ -610,17 +620,18 @@ export class ConfigParser {
 
   private async loadConfigModule(configPath: string): Promise<any> {
     const ext = configPath.split('.').pop();
+    const moduleUrl = pathToFileURL(resolve(configPath)).href;
 
     if (ext === 'ts') {
       try {
-        const module = await import(configPath);
+        const module = await import(moduleUrl);
         return module;
       } catch (error) {
         const require = (await import('node:module')).createRequire(import.meta.url);
         return require(configPath);
       }
     } else if (ext === 'js' || ext === 'mjs') {
-      const module = await import(configPath);
+      const module = await import(moduleUrl);
       return module;
     } else {
       const content = readFileSync(configPath, 'utf8');

package/src/execution-controller.ts

@@ -27,6 +27,14 @@ export class ExecutionController extends EventEmitter {
     this.inputCallback = callback;
   }
 
+  hasInputCallback(): boolean {
+    return !!this.inputCallback;
+  }
+
+  clearInputCallback(): void {
+    this.inputCallback = null;
+  }
+
   startExecution(): void {
     this.interrupted = false;
     this.abortController = new AbortController();

package/src/explorbot.ts

@@ -20,7 +20,7 @@ import { createAgentTools } from './ai/tools.ts';
 import { ApiClient } from './api/api-client.ts';
 import { RequestStore } from './api/request-store.ts';
 import { loadSpec } from './api/spec-reader.ts';
-import type { ExplorbotConfig } from './config.js';
+import type { ExplorbotConfig, ReporterConfig } from './config.js';
 import { ConfigParser } from './config.ts';
 import { ExperienceTracker } from './experience-tracker.ts';
 import Explorer from './explorer.ts';
@@ -49,6 +49,7 @@ export interface ExplorBotOptions {
   session?: string | boolean;
   instance?: string;
   optionalAi?: boolean;
+  reporter?: ReporterConfig;
   attachedBrowser?: Browser;
   applicationSpec?: string;
 }
@@ -128,6 +129,7 @@ export class ExplorBot {
     if (this.provider) return;
     this.config = await this.configParser.loadConfig(this.options);
     if (this.options.session === true) this.options.session = path.join(this.configParser.getOutputDir(), 'session.json');
+    if (typeof this.options.session === 'string') this.options.session = path.resolve(this.options.path || '.', this.options.session);
     if (this.options.optionalAi) return this.bootstrapOptionalProvider();
     this.provider = new AIProvider(this.config.ai);
     await this.provider.validateConnection();
@@ -175,7 +177,7 @@ export class ExplorBot {
   }
 
   reporter(): Reporter {
-    return (this._reporter ||= new Reporter(this.config.reporter, this.stateManager()));
+    return (this._reporter ||= new Reporter(this.options.reporter || this.config.reporter, this.stateManager()));
   }
 
   requestStore(): RequestStore {

package/src/explorer.ts

@@ -721,11 +721,11 @@ class Explorer {
     if (this.observedTestPages.has(page)) return;
 
     this.testPageErrorHandler ||= (err: Error) => {
-      this._activeTest?.addNote(`Console error: ${err.message}`, TestResult.FAILED);
+      this._activeTest?.addObservation(`Console error: ${err.message}`);
     };
     this.testConsoleHandler ||= (msg: any) => {
       if (msg.type() !== 'error') return;
-      this._activeTest?.addNote(`Console error: ${msg.text()}`, TestResult.FAILED);
+      this._activeTest?.addObservation(`Console error: ${msg.text()}`);
     };
     this.testDialogHandler ||= (dialog: any) => {
       const dialogType = dialog.type();

package/src/playwright-recorder.ts

@@ -287,8 +287,31 @@ function formatSelectOption(options: any): string {
   return `[${values.map((v) => quote(v)).join(', ')}]`;
 }
 
+function assertionLocator(target: any): string | null {
+  if (typeof target === 'string') return `page.locator(${JSON.stringify(target)})`;
+  if (!target || typeof target !== 'object') return null;
+
+  const role = target.role || target.aria;
+  const name = target.text ?? target.name ?? target.label;
+  if (role && name) return `page.getByRole(${JSON.stringify(String(role))}, { name: ${JSON.stringify(String(name))} })`;
+  if (role) return `page.getByRole(${JSON.stringify(String(role))})`;
+  if (name) return `page.getByText(${JSON.stringify(String(name))})`;
+  if (target.css) return `page.locator(${JSON.stringify(String(target.css))})`;
+  if (target.xpath) return `page.locator(${JSON.stringify(`xpath=${target.xpath}`)})`;
+  return null;
+}
+
 export function renderAssertion(assertion: { name: string; args: any[] }): string {
   const args = assertion.args;
+  const target = assertionLocator(args[0]);
+
+  if (target) {
+    if (assertion.name === 'seeElement') return `await expect(${target}).toBeVisible();`;
+    if (assertion.name === 'dontSeeElement') return `await expect(${target}).toBeHidden();`;
+    if (assertion.name === 'seeInField' && args[1] !== undefined) return `await expect(${target}).toHaveValue(${JSON.stringify(String(args[1]))});`;
+    if (assertion.name === 'dontSeeInField' && args[1] !== undefined) return `await expect(${target}).not.toHaveValue(${JSON.stringify(String(args[1]))});`;
+  }
+
   if (assertion.name === 'see' && typeof args[0] === 'string') {
     return `await expect(page).toContainText(${JSON.stringify(args[0])});`;
   }