explorbot 0.1.27 → 0.1.29

package/src/ai/provider.ts

@@ -1,6 +1,7 @@
+import { OpenTelemetry } from '@ai-sdk/otel';
 import { LangfuseSpanProcessor } from '@langfuse/otel';
 import { NodeSDK } from '@opentelemetry/sdk-node';
-import { generateObject, generateText, stepCountIs } from 'ai';
+import { generateObject, generateText, isStepCount, registerTelemetry } from 'ai';
 import type { ModelMessage } from 'ai';
 import { clearActivity, setActivity } from '../activity.ts';
 import type { AIConfig } from '../config.js';
@@ -19,9 +20,11 @@ const responseLog = createDebug('explorbot:provider:in');
 class AiError extends Error {}
 export class ContextLengthError extends Error {}
 
+let telemetryRegistered = false;
+
 function extractCachedTokens(usage: any): number {
   if (!usage) return 0;
-  const direct = usage.cachedInputTokens ?? usage.inputTokenDetails?.cacheReadTokens;
+  const direct = usage.inputTokenDetails?.cacheReadTokens ?? usage.cachedInputTokens;
   if (typeof direct === 'number') return direct;
   const raw = usage.raw;
   const fromRaw = raw?.prompt_tokens_details?.cached_tokens ?? raw?.promptTokensDetails?.cachedTokens;
@@ -86,7 +89,7 @@ export class Provider {
       await generateText({
         model: this.config.model,
         prompt: 'hi',
-        maxTokens: 1,
+        maxOutputTokens: 1,
       });
     } catch (error: any) {
       throw new AiError(`AI connection failed: ${error.message}`);
@@ -110,6 +113,16 @@ export class Provider {
     return this.config.agenticModel || this.config.model;
   }
 
+  getConfiguredModels(): Record<string, string> {
+    const models: Record<string, string> = { model: this.getModelName(this.config.model) };
+    if (this.config.agenticModel) models.agenticModel = this.getModelName(this.config.agenticModel);
+    if (this.config.visionModel) models.visionModel = this.getModelName(this.config.visionModel);
+    for (const [agent, agentConfig] of Object.entries(this.config.agents || {})) {
+      if (agentConfig?.model) models[agent] = this.getModelName(agentConfig.model);
+    }
+    return models;
+  }
+
   getSystemPromptForAgent(agentName: string, currentUrl?: string): string | undefined {
     const agentConfig = this.config.agents?.[agentName as keyof typeof this.config.agents];
     const parts: string[] = [];
@@ -129,6 +142,12 @@ export class Provider {
     return agentConfig?.providerOptions;
   }
 
+  getReasoningForAgent(agentName?: string): string | undefined {
+    if (!agentName) return undefined;
+    const agentConfig = this.config.agents?.[agentName as keyof typeof this.config.agents];
+    return agentConfig?.reasoning;
+  }
+
   private getRetryOptions(options: any = {}): RetryOptions {
     return {
       ...this.defaultRetryOptions,
@@ -146,6 +165,12 @@ export class Provider {
     };
   }
 
+  private finalizeConfig(config: Record<string, any>, options: any, telemetry: any): void {
+    if (telemetry) config.telemetry = telemetry;
+    const reasoning = this.getReasoningForAgent(options.agentName);
+    if (reasoning) config.reasoning ??= reasoning;
+  }
+
   private initLangfuse() {
     const langfuseConfig = this.config.langfuse;
     const publicKey = langfuseConfig?.publicKey || process.env.LANGFUSE_PUBLIC_KEY;
@@ -167,6 +192,10 @@ export class Provider {
       instrumentations: [],
     });
     void this.otelSdk.start();
+    if (!telemetryRegistered) {
+      registerTelemetry(new OpenTelemetry());
+      telemetryRegistered = true;
+    }
     this.telemetryEnabled = true;
   }
 
@@ -177,20 +206,20 @@ export class Provider {
 
     const runTelemetry = Observability.getTelemetry();
 
-    if (!options.experimental_telemetry) {
-      return runTelemetry || { isEnabled: true };
+    if (!options.telemetry) {
+      return runTelemetry;
     }
 
     if (!runTelemetry) {
-      return options.experimental_telemetry;
+      return options.telemetry;
     }
 
     return {
       ...runTelemetry,
-      ...options.experimental_telemetry,
+      ...options.telemetry,
       metadata: {
         ...runTelemetry.metadata,
-        ...options.experimental_telemetry.metadata,
+        ...options.telemetry.metadata,
       },
     };
   }
@@ -242,15 +271,16 @@ export class Provider {
     const telemetry = this.getTelemetry(options);
     const config = this.mergeProviderOptions(
       {
-        maxTokens: 16384,
+        maxOutputTokens: 16384,
+        allowSystemInMessages: true,
         ...(this.config.config || {}),
         ...options,
-        ...(telemetry ? { experimental_telemetry: telemetry } : {}),
         model,
         abortSignal: executionController.getAbortSignal(),
       },
       options.agentName
     );
+    this.finalizeConfig(config, options, telemetry);
 
     promptLog(messages[messages.length - 1].content);
     try {
@@ -259,7 +289,7 @@ export class Provider {
         if (!result.text) {
           debugLog(result);
           if (result.finishReason === 'length') {
-            throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase maxTokens in config or use a model with higher output capacity.');
+            throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase maxOutputTokens in config or use a model with higher output capacity.');
           }
           throw new Error('No response text from AI');
         }
@@ -315,23 +345,24 @@ export class Provider {
     const telemetry = this.getTelemetry(options);
     const maxRoundtrips = options.maxToolRoundtrips ?? 5;
     const extraStop = options.stopWhen;
-    const stopConditions: any[] = [stepCountIs(maxRoundtrips)];
+    const stopConditions: any[] = [isStepCount(maxRoundtrips)];
     if (extraStop) stopConditions.push(extraStop);
     const { stopWhen: _ignoredStopWhen, ...optionsWithoutStop } = options;
     const config = this.mergeProviderOptions(
       {
         tools,
-        maxTokens: 16384,
+        maxOutputTokens: 16384,
         toolChoice: 'auto',
+        allowSystemInMessages: true,
         ...(this.config.config || {}),
         ...optionsWithoutStop,
         stopWhen: stopConditions,
-        ...(telemetry ? { experimental_telemetry: telemetry } : {}),
         model,
         abortSignal: executionController.getAbortSignal(),
       },
       options.agentName
     );
+    this.finalizeConfig(config, options, telemetry);
     try {
       const response = await withRetry(async () => {
         const timeout = config.timeout || 30000;
@@ -346,7 +377,7 @@ export class Provider {
           ])) as any;
           const hasToolCall = (result.toolCalls?.length || 0) > 0;
           if (!result.text && !hasToolCall && result.finishReason === 'length') {
-            throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase maxTokens in config or use a model with higher output capacity.');
+            throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase maxOutputTokens in config or use a model with higher output capacity.');
           }
           return result;
         } finally {
@@ -409,14 +440,15 @@ export class Provider {
     const config = this.mergeProviderOptions(
       {
         schema,
+        allowSystemInMessages: true,
         ...(this.config.config || {}),
         ...options,
-        ...(telemetry ? { experimental_telemetry: telemetry } : {}),
         model: modelToUse,
         abortSignal: executionController.getAbortSignal(),
       },
       options.agentName
     );
+    this.finalizeConfig(config, options, telemetry);
 
     try {
       promptLog(messages[messages.length - 1].content);
@@ -617,13 +649,13 @@ export class Provider {
     ];
 
     const telemetry = this.getTelemetry({});
-    const config = {
-      maxTokens: 16384,
+    const config: Record<string, any> = {
+      maxOutputTokens: 16384,
       ...(this.config.config || {}),
-      ...(telemetry ? { experimental_telemetry: telemetry } : {}),
       model: this.config.visionModel,
       abortSignal: executionController.getAbortSignal(),
     };
+    if (telemetry) config.telemetry = telemetry;
 
     try {
       promptLog(`Processing image with prompt: ${prompt}`);

package/src/ai/researcher/coordinates.ts

@@ -81,7 +81,7 @@ export function WithCoordinates<T extends Constructor>(Base: T) {
     }
 
     async visuallyAnnotateElements(opts?: { containers?: Array<{ css: string; label: string }> }): Promise<number> {
-      return visuallyAnnotateContainers(this.explorer.playwrightHelper.page, opts?.containers || []);
+      return this.explorer.visuallyAnnotateElements(opts);
     }
 
     private async _analyzeScreenshotForVisualProps(): Promise<VisualAnalysisResult> {
@@ -193,7 +193,6 @@ export function WithCoordinates<T extends Constructor>(Base: T) {
     }
 
     async backfillCoordinates(result: ResearchResult): Promise<void> {
-      const page = this.explorer.playwrightHelper.page;
       const sections = parseResearchSections(result.text);
       const eidxWithoutCoords: string[] = [];
       for (const section of sections) {
@@ -203,7 +202,7 @@ export function WithCoordinates<T extends Constructor>(Base: T) {
       }
       if (eidxWithoutCoords.length === 0) return;
 
-      const webElements = await WebElement.fromEidxList(page, eidxWithoutCoords);
+      const webElements = await this.explorer.runWithBrowserRecovery('backfillCoordinates', () => WebElement.fromEidxList(this.explorer.playwrightHelper.page, eidxWithoutCoords));
       if (webElements.length === 0) return;
 
       const rectMap = new Map(webElements.map((w) => [w.eidx!, w]));

package/src/ai/researcher/deep-analysis.ts

@@ -359,11 +359,10 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
           const isCoordinateClick = el.commands[0].startsWith('I.clickXY(');
           if (!isCoordinateClick) {
             const hoverCmd = el.commands[0].replace('I.click(', 'I.moveCursorTo(');
-            const hoverAction = this.explorer.createAction();
-            await hoverAction.attempt(hoverCmd, undefined, false);
+            await this.explorer.attemptAction(hoverCmd, undefined, false);
             await new Promise((r) => setTimeout(r, 500));
 
-            await this.explorer.createAction().capturePageState();
+            await this.explorer.capturePageState();
             const hoverAR = ActionResult.fromState(this.stateManager.getCurrentState()!);
             const hoverDiff = await hoverAR.diff(previousState);
             await hoverDiff.calculate();
@@ -452,7 +451,7 @@ export function WithDeepAnalysis<T extends Constructor>(Base: T) {
     private async _restorePageState(url: string, originalAria: string): Promise<void> {
       try {
         await (this as any).cancelInUi();
-        await this.explorer.createAction().capturePageState();
+        await this.explorer.capturePageState();
         const currentAria = this.stateManager.getCurrentState()?.ariaSnapshot || '';
         if (!diffAriaSnapshots(originalAria, currentAria)) return;
       } catch (err) {

package/src/ai/researcher/locators.ts

@@ -194,8 +194,7 @@ export function WithLocators<T extends Constructor>(Base: T) {
       }
 
       if (needsXpath.length > 0) {
-        const page = this.explorer.playwrightHelper.page;
-        const webElements = await WebElement.fromEidxList(page, needsXpath);
+        const webElements = await this.explorer.runWithBrowserRecovery('backfillBrokenLocators', () => WebElement.fromEidxList(this.explorer.playwrightHelper.page, needsXpath));
         const changedSections = new Set<(typeof sections)[0]>();
         for (const w of webElements) {
           const entry = needsXpathEls.get(w.eidx!);

package/src/ai/researcher.ts

@@ -68,6 +68,13 @@ export class Researcher extends ResearcherBase implements Agent {
     this.stateManager = explorer.getStateManager();
     this.experienceTracker = this.stateManager.getExperienceTracker();
     this.hooksRunner = new HooksRunner(explorer, explorer.getConfig());
+
+    const ai = explorer.getConfig().ai;
+    if (ai) {
+      ai.agents ??= {};
+      ai.agents.researcher ??= {};
+      ai.agents.researcher.reasoning ??= 'low';
+    }
   }
 
   protected getNavigator(): Navigator {
@@ -130,12 +137,12 @@ export class Researcher extends ResearcherBase implements Agent {
 
       const annotatedElements = await this.explorer.annotateElements();
       debugLog(`Annotated ${annotatedElements.length} interactive elements with eidx`);
-      this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot: screenshot && this.provider.hasVision() });
+      this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot && this.provider.hasVision() });
 
       const condition = detectPageCondition(this.actionResult!);
       if (condition === 'error') {
         tag('warning').log(`Detected error page at ${state.url}`);
-        throw new ErrorPageError(state.url, this.actionResult!.title);
+        throw new ErrorPageError(state.url, this.actionResult!.title, this.actionResult!.httpStatus);
       }
       if (condition === 'loading') {
         const settled = await this.waitUntilSettled(screenshot);
@@ -177,7 +184,7 @@ export class Researcher extends ResearcherBase implements Agent {
       } catch (error) {
         if (!(error instanceof ContextLengthError) || retriesLeft <= 0) {
           if (error instanceof ContextLengthError) {
-            tag('warning').log('Output truncated. Try lowering reasoning effort or increasing maxTokens in ai.config.');
+            tag('warning').log('Output truncated. Try lowering reasoning effort or increasing maxOutputTokens in ai.config.');
           }
           throw error;
         }
@@ -239,7 +246,7 @@ export class Researcher extends ResearcherBase implements Agent {
       // Must run BEFORE visuallyAnnotateContainers — annotation overlays inject z-index 99998+ which would pollute the scoring.
       if (!interrupted() && this.hasScreenshotToAnalyze) {
         const sections = parseResearchSections(result.text);
-        const focused = await detectFocusedSection(this.explorer.playwrightHelper.page, sections);
+        const focused = await this.explorer.runWithBrowserRecovery('detectFocusedSection', () => detectFocusedSection(this.explorer.playwrightHelper.page, sections));
         if (focused) markSectionAsFocused(result, focused);
       }
 
@@ -252,7 +259,7 @@ export class Researcher extends ResearcherBase implements Agent {
         const freshBroken = freshContainerLocs.filter((l) => l.valid === false).map((l) => l.locator);
         const containers = validContainers.filter((c) => !freshBroken.includes(c.css));
         await this.visuallyAnnotateElements({ containers });
-        this.actionResult = await this.explorer.createAction().caputrePageWithScreenshot();
+        this.actionResult = await this.explorer.capturePageWithScreenshot();
         const visualResult = await this.analyzeScreenshotForVisualProps();
         if (visualResult.elements.size > 0) {
           await this.mergeVisualData(result, visualResult.elements);
@@ -331,7 +338,7 @@ export class Researcher extends ResearcherBase implements Agent {
     if (!this.actionResult) {
       debugLog('No action result, navigating to URL');
       await this.explorer.visit(url);
-      this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot: screenshot });
+      this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot });
       return;
     }
 
@@ -341,7 +348,7 @@ export class Researcher extends ResearcherBase implements Agent {
 
     if (!isEmpty && isOnCurrentState) {
       if ((!this.actionResult.screenshot && screenshot) || !this.actionResult.ariaSnapshot) {
-        this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot: screenshot });
+        this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot });
       }
       return;
     }
@@ -349,6 +356,8 @@ export class Researcher extends ResearcherBase implements Agent {
     if (isEmpty && isOnCurrentState) {
       debugLog('HTML body empty on current URL, waiting for content');
       tag('step').log('Page body is empty, waiting for content...');
+      await this.explorer.visit(url);
+      this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot ?? false });
       await this.waitUntilSettled(screenshot ?? false);
       return;
     }
@@ -357,36 +366,35 @@ export class Researcher extends ResearcherBase implements Agent {
     tag('step').log('Navigating to URL...');
 
     await this.explorer.visit(url);
-    this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot: screenshot ?? false });
+    this.actionResult = await this.explorer.capturePageState({ includeScreenshot: screenshot ?? false });
   }
 
   private async waitUntilSettled(screenshot: boolean): Promise<boolean> {
     const errorPageTimeout = (this.explorer.getConfig().ai?.agents?.researcher as any)?.errorPageTimeout ?? 10;
     if (errorPageTimeout <= 0) return false;
 
-    const page = this.explorer.playwrightHelper.page;
     const includeScreenshot = screenshot && this.provider.hasVision();
 
     try {
-      await page?.waitForLoadState('networkidle', { timeout: errorPageTimeout * 1000 });
+      await this.explorer.runWithBrowserRecovery('waitUntilSettled', () => this.explorer.playwrightHelper.page?.waitForLoadState('networkidle', { timeout: errorPageTimeout * 1000 }));
     } catch {}
 
     await this.explorer.annotateElements();
-    this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot });
+    this.actionResult = await this.explorer.capturePageState({ includeScreenshot });
 
     let condition = detectPageCondition(this.actionResult!);
     if (condition === 'error') {
-      throw new ErrorPageError(this.actionResult!.url, this.actionResult!.title);
+      throw new ErrorPageError(this.actionResult!.url, this.actionResult!.title, this.actionResult!.httpStatus);
     }
     if (condition === 'ok') return true;
 
     for (let i = 0; i < 3; i++) {
       await new Promise((r) => setTimeout(r, 1000));
       await this.explorer.annotateElements();
-      this.actionResult = await this.explorer.createAction().capturePageState({ includeScreenshot });
+      this.actionResult = await this.explorer.capturePageState({ includeScreenshot });
       condition = detectPageCondition(this.actionResult!);
       if (condition === 'error') {
-        throw new ErrorPageError(this.actionResult!.url, this.actionResult!.title);
+        throw new ErrorPageError(this.actionResult!.url, this.actionResult!.title, this.actionResult!.httpStatus);
       }
       if (condition === 'ok') return true;
     }
@@ -762,17 +770,15 @@ export class Researcher extends ResearcherBase implements Agent {
   }
 
   async navigateTo(url: string): Promise<void> {
-    const action = this.explorer.createAction();
-    await action.execute(`I.amOnPage("${url}")`);
+    await this.explorer.visit(url);
   }
 
   async cancelInUi() {
     const beforeAria = this.stateManager.getCurrentState()?.ariaSnapshot || null;
-    const action = this.explorer.createAction();
 
-    await action.execute('I.clickXY(0, 0)');
+    await this.explorer.executeAction('I.clickXY(0, 0)');
     if (diffAriaSnapshots(beforeAria, this.stateManager.getCurrentState()?.ariaSnapshot || null)) return;
 
-    await action.execute(`I.pressKey('Escape')`);
+    await this.explorer.executeAction(`I.pressKey('Escape')`);
   }
 }

package/src/ai/rules.ts

@@ -153,6 +153,52 @@ export const protectionRule = dedent`
   </important>
 `;
 
+export const dataProtectionRules = dedent`
+  <data_protection_rules>
+  ${protectionRule}
+
+  If the user request, scenario, focus, or test instructions explicitly prohibit creating,
+  editing, updating, deleting, removing, or otherwise mutating data, do not perform those
+  actions through the UI, API preconditions, cleanup, fallback steps, or Fisherman.
+
+  Do not use Fisherman or API data preparation to bypass a no-mutation, read-only, search,
+  filter, tab, or list-inspection constraint. Use visible existing data when it is available.
+  If no suitable data exists, report the missing precondition instead of creating data.
+
+  Destructive actions are allowed only against disposable data created by the current scenario
+  or prepared for that scenario by Fisherman/API preconditions. Existing application data must
+  remain unchanged.
+  </data_protection_rules>
+`;
+
+export const capabilityGroundingRule = dedent`
+  <capability_grounding>
+  When a scenario depends on a named action, menu item, status, option, workflow, or feature,
+  that capability must be visible or explicitly confirmed in the current research/page context
+  for the same target entity type.
+
+  Do not transfer capabilities between similar entities, rows, lists, detail pages, or menus.
+  Do not replace a requested action with a synonym or related action unless the UI explicitly
+  shows that action for the target entity.
+
+  When an action is described as applying to an item, row, card, record, node, or entity,
+  the target must be grounded as that kind of data entity in the current context. Do not use
+  navigation links, filter tabs, counters, breadcrumbs, headings, toolbar controls, or other
+  page controls as the subject of row/entity actions.
+
+  When a scenario asks to open, view, inspect, or navigate to an entity detail view, success
+  requires evidence of that entity detail context. An active filter, selected tab, visible count,
+  or filtered list is not enough to prove an entity detail view opened.
+
+  Do not rewrite a scenario goal to match a similar outcome that happened accidentally. If the
+  requested entity detail/action/workflow was not achieved, report that mismatch instead of
+  passing the test for a related filter, tab, navigation, or status view.
+
+  If the required capability is not available for the target entity after reasonable discovery,
+  record the missing capability and stop instead of repeatedly trying unrelated locators.
+  </capability_grounding>
+`;
+
 export const focusedElementRule = dedent`
   <focused_element_actions>
   When a text input element is focused (textbox, combobox, contenteditable):

package/src/ai/task-agent.ts

@@ -24,7 +24,7 @@ export abstract class TaskAgent {
   protected consecutiveFailures = 0;
   protected consecutiveEmptyResults = 0;
   protected recentToolCalls: any[] = [];
-  protected abstract readonly ACTION_TOOLS: string[];
+  protected readonly ACTION_TOOLS: string[] = [];
 
   private _historian: Historian | null = null;
   private _quartermaster: Quartermaster | null = null;