explorbot 0.3.0 → 0.3.2

package/bin/explorbot-cli.ts

@@ -9,6 +9,7 @@ import { render } from 'ink';
 import React from 'react';
 import { App } from '../src/components/App.js';
 import { StatusPane } from '../src/components/StatusPane.js';
+import { flushTelemetry } from '../src/ai/provider.js';
 import { ConfigParser, EXPLORBOT_ENV_VARS, PROVIDERS } from '../src/config.js';
 import { ExplorBot, type ExplorBotOptions } from '../src/explorbot.js';
 import { remote } from '../src/remote.js';
@@ -103,9 +104,7 @@ async function startTUI(explorBot: ExplorBot): Promise<void> {
 async function showStatsAndExit(code: number): Promise<never> {
   if (remote.isAttached()) {
     await remote.close(code);
-    process.exit(code);
-  }
-  if (Stats.hasActivity()) {
+  } else if (Stats.hasActivity()) {
     await new Promise<void>((resolve) => {
       const { unmount } = render(
         React.createElement(StatusPane, {
@@ -121,6 +120,7 @@ async function showStatsAndExit(code: number): Promise<never> {
       );
     });
   }
+  await flushTelemetry();
   process.exit(code);
 }
 

package/dist/bin/explorbot-cli.js

@@ -9,6 +9,7 @@ import { render } from 'ink';
 import React from 'react';
 import { App } from '../src/components/App.js';
 import { StatusPane } from '../src/components/StatusPane.js';
+import { flushTelemetry } from '../src/ai/provider.js';
 import { ConfigParser, EXPLORBOT_ENV_VARS, PROVIDERS } from '../src/config.js';
 import { ExplorBot } from '../src/explorbot.js';
 import { remote } from '../src/remote.js';
@@ -78,9 +79,8 @@ async function startTUI(explorBot) {
 async function showStatsAndExit(code) {
     if (remote.isAttached()) {
         await remote.close(code);
-        process.exit(code);
     }
-    if (Stats.hasActivity()) {
+    else if (Stats.hasActivity()) {
         await new Promise((resolve) => {
             const { unmount } = render(React.createElement(StatusPane, {
                 onComplete: () => {
@@ -93,6 +93,7 @@ async function showStatsAndExit(code) {
             });
         });
     }
+    await flushTelemetry();
     process.exit(code);
 }
 addCommonOptions(program.command('start [path]').description('Start web exploration')).action(async (startPath, options) => {

package/dist/package.json

@@ -1,6 +1,6 @@
 {
   "name": "explorbot",
-  "version": "0.3.0",
+  "version": "0.3.2",
   "description": "CLI app built with React Ink, CodeceptJS, and Playwright",
   "license": "Elastic-2.0",
   "type": "module",

package/dist/src/action-result.d.ts

@@ -2,6 +2,7 @@ import { type HtmlConfig } from './config.js';
 import type { Link, WebPageState } from './state-manager.js';
 import { TTLCache } from './utils/cache.js';
 import { type HtmlDiffPart, type HtmlDiffResult } from './utils/html-diff.js';
+import { Overlay } from './utils/overlay.js';
 interface ActionResultData extends WebPageState {
     html?: string;
     fullUrl?: string | undefined;
@@ -28,6 +29,7 @@ interface ActionResultData extends WebPageState {
     focusedElement?: FocusedElement | null;
     iframeURL?: string;
     links?: Link[];
+    overlayHtml?: string;
 }
 export interface PageDiff {
     urlChanged: boolean;
@@ -81,6 +83,7 @@ export declare class ActionResult implements ActionResultData {
     notes: string[];
     links: Link[];
     verifications?: Record<string, boolean>;
+    overlay: Overlay;
     constructor(data: ActionResultData);
     get hash(): string;
     get html(): string;

package/dist/src/action-result.js

@@ -5,6 +5,7 @@ import { TTLCache } from "./utils/cache.js";
 import { htmlDiff, liveRegionMessages } from "./utils/html-diff.js";
 import { extractHeadings, extractLinks, extractTargetedHtml, htmlCombinedSnapshot, htmlMinimalUISnapshot, htmlTextSnapshot, minifyHtml } from "./utils/html.js";
 import { createDebug } from "./utils/logger.js";
+import { Overlay } from "./utils/overlay.js";
 import { slugify } from "./utils/strings.js";
 import { extractStatePath, matchesUrl } from "./utils/url-matcher.js";
 const debugLog = createDebug('explorbot:state');
@@ -37,6 +38,7 @@ export class ActionResult {
     notes = [];
     links = [];
     verifications;
+    overlay = new Overlay();
     constructor(data) {
         this.id = data.id;
         this.timestamp = data.timestamp ?? new Date();
@@ -77,6 +79,7 @@ export class ActionResult {
         if (data.ariaSnapshot !== undefined) {
             this._ariaSnapshot = data.ariaSnapshot;
         }
+        this.overlay = Overlay.resolve(data);
         if (!this.fullUrl && this.url) {
             this.fullUrl = this.url;
         }

package/dist/src/action.d.ts

@@ -32,6 +32,7 @@ declare class Action {
         includeScreenshot?: boolean;
         codeBlock?: string;
     }): Promise<ActionResult>;
+    captureOverlayHtml(): Promise<string>;
     captureMainDocumentStatus(): Promise<number | undefined>;
     captureResponses(): () => void;
     recordNetworkCall(request: any, status: number): void;

package/dist/src/action.js

@@ -8,8 +8,9 @@ import { clearActivity, setActivity } from "./activity.js";
 import { ConfigParser, outputPath } from './config.js';
 import { Observability } from "./observability.js";
 import { browserErrorMessage, isFatalBrowserError, isNavigationTransitionError } from "./utils/browser-errors.js";
-import { captureHtmlForSnapshot, htmlCombinedSnapshot, minifyHtml } from './utils/html.js';
+import { captureHtmlForSnapshot, getVisibleOverlayHtmlExtractorSource, htmlCombinedSnapshot, minifyHtml } from './utils/html.js';
 import { createDebug, setStepSpanParent, tag } from './utils/logger.js';
+import { Overlay } from './utils/overlay.js';
 import { sleep, waitForPageReadiness } from "./utils/page-readiness.js";
 import { safeFilename } from "./utils/strings.js";
 import { codeceptJSSandbox, hasPlaywrightCommands, playwrightSandbox, sanitizeCodeBlock } from "./utils/web-sandbox.js";
@@ -131,10 +132,13 @@ class Action {
             let ariaSnapshot = null;
             let ariaSnapshotFile = undefined;
             let focusedElement = null;
+            let overlayHtml = '';
             try {
                 const page = this.playwrightHelper.page;
                 ariaSnapshot = await page.locator('body').ariaSnapshot();
                 focusedElement = await page.evaluate(readFocusedElement);
+                if (!frame)
+                    overlayHtml = await this.captureOverlayHtml();
             }
             catch (err) {
                 debugLog('ARIA snapshot failed:', err instanceof Error ? `${err.message}\n${err.stack}` : err);
@@ -161,6 +165,7 @@ class Action {
                 ariaSnapshot,
                 ariaSnapshotFile,
                 focusedElement,
+                overlayHtml: overlayHtml || undefined,
                 iframeURL: frame ? frame.url?.() || 'iframe' : undefined,
             });
             this.stateManager.updateState(result, codeBlock);
@@ -175,6 +180,12 @@ class Action {
             return new ActionResult({ url, error: msg });
         }
     }
+    async captureOverlayHtml() {
+        return this.playwrightHelper.page.evaluate(({ extractorSource, config }) => {
+            const extract = new Function(`return ${extractorSource}`)();
+            return extract(config);
+        }, { extractorSource: getVisibleOverlayHtmlExtractorSource(), config: Overlay.captureConfig() });
+    }
     async captureMainDocumentStatus() {
         if (this.mainDocumentStatus)
             return this.mainDocumentStatus;

package/dist/src/ai/conversation.d.ts

@@ -7,6 +7,7 @@ export interface ToolExecution {
 }
 export declare function toToolExecution(toolName: string, input: any, rawOutput: any): ToolExecution;
 export declare function toolExecutionLabel(input: Record<string, any> | undefined): string;
+export declare const NARRATION_TOOL = "commentary";
 export declare class Conversation {
     messages: ModelMessage[];
     model: any;

package/dist/src/ai/conversation.js

@@ -7,6 +7,7 @@ export function toToolExecution(toolName, input, rawOutput) {
 export function toolExecutionLabel(input) {
     return input?.explanation || input?.assertion || input?.reason || input?.request || '';
 }
+export const NARRATION_TOOL = 'commentary';
 const AUTO_COMPACT_ARIA_CHANGES_CUTOFF = 500;
 const AUTO_COMPACT_TARGETED_HTML_CUTOFF = 500;
 export class Conversation {
@@ -203,6 +204,8 @@ export class Conversation {
             for (const part of message.content) {
                 if (part.type !== 'tool-result')
                     continue;
+                if (part.toolName === NARRATION_TOOL)
+                    continue;
                 executions.push(toToolExecution(part.toolName, toolCalls.get(part.toolCallId) || {}, part.output));
             }
         }

package/dist/src/ai/driller.js

@@ -9,6 +9,7 @@ import { collectInteractiveNodes } from "../utils/aria.js";
 import { EXPLORBOT_ATTRS, HTML_COMPOSITE_AREA_HINTS, HTML_COMPOSITE_TARGET_ROLES, HTML_EXTRACTION_LIMITS, HTML_FORM_CONTROL_ROLES, HTML_FORM_CONTROL_TAGS, HTML_INTERACTIVE_ROLES, HTML_SELECTORS, HTML_VISIBILITY_LIMITS, getComponentScopeHtmlExtractorSource, getVisibleOverlayHtmlExtractorSource, inferHtmlRole, } from "../utils/html.js";
 import { createDebug, tag } from "../utils/logger.js";
 import { loop, pause } from "../utils/loop.js";
+import { OVERLAY_SELECTORS } from "../utils/overlay.js";
 import { annotatePageElements } from "../utils/web-annotate.js";
 import { eidxInContainer } from "../utils/web-eidx.js";
 import { WebElement } from "../utils/web-element.js";
@@ -578,7 +579,8 @@ export class Driller extends TaskAgent {
             config: {
                 interactiveContentSelector: HTML_SELECTORS.interactiveContent,
                 limits: HTML_EXTRACTION_LIMITS,
-                overlaySelectors: HTML_SELECTORS.semanticOverlays,
+                overlaySelectors: OVERLAY_SELECTORS.semanticOverlays,
+                overlaySemanticSelector: OVERLAY_SELECTORS.overlaySemanticSelector,
                 visibilityLimits: HTML_VISIBILITY_LIMITS,
             },
         }));

package/dist/src/ai/fisherman.js

@@ -158,7 +158,7 @@ export class Fisherman {
 
       AVAILABLE TOOLS:
       ${toolNames.join(', ')}.
-      Use tool names exactly as listed. Do not invent aliases, combined names, or names with channel markers such as "commentary".
+      Use tool names exactly as listed. Do not invent aliases or combined names.
       Match each tool input schema exactly. Do not invent parameter names or pass extra fields.
 
       WORKFLOW:

package/dist/src/ai/pilot.js

@@ -5,7 +5,7 @@ import { ActionResult } from "../action-result.js";
 import { ConfigParser } from "../config.js";
 import { Stats } from "../stats.js";
 import { TestResult } from "../test-plan.js";
-import { collectInteractiveNodes, detectFocusArea } from "../utils/aria.js";
+import { collectInteractiveNodes } from "../utils/aria.js";
 import { ErrorPageError } from "../utils/error-page.js";
 import { createDebug, tag } from "../utils/logger.js";
 const debugLog = createDebug('explorbot:pilot');
@@ -13,7 +13,8 @@ import { truncateJson } from "../utils/strings.js";
 import { capabilityGroundingRule, dataProtectionRules } from "./rules.js";
 import { isInteractive } from "./task-agent.js";
 import { withdrawVisionTools } from "./tools.js";
-const CHECK_TOOLS = ['verify', 'see', 'research', 'context'];
+const CHECK_TOOLS = ['verify', 'see', 'research'];
+const EVIDENCE_TOOLS = ['verify', 'see'];
 const META_TOOLS = ['record', 'reset', 'stop', 'finish'];
 const PILOT_MESSAGE_LIMIT = 2;
 const PILOT_MESSAGE_MAX_LENGTH = 160;
@@ -280,6 +281,10 @@ export class Pilot {
       overrides the others — weigh them together. Tester's record() notes are the LEAST reliable; always
       cross-check against actual actions and state. Visual screenshot analysis is strong for UI state
       (active tabs, visible counts, colors).
+      Judge every check by WHAT IT ESTABLISHES, never by the fact that it ran. A check that executed
+      successfully is failure evidence when its content negates the scenario goal — the goal's object
+      absent, the action not performed, the interaction impossible. "The check passed" and "the goal was
+      met" are different claims.
       If the final page clearly shows an equivalent success state in a different UI form, do not fail only
       because one narrow assertion targeted a specific badge, count, toast, or wording that the product
       represents differently.
@@ -499,7 +504,7 @@ export class Pilot {
     async settleExpectations(task, finalState) {
         let image = null;
         if (finalState?.screenshot && this.provider.hasVision())
-            image = `data:image/png;base64,${finalState.screenshot.toString('base64')}`;
+            image = finalState.screenshot;
         const decided = (text) => {
             if (task.hasAchievedAny() && !task.getRemainingExpectations().includes(text))
                 return 'passed';
@@ -733,7 +738,7 @@ export class Pilot {
         lines.push(`h2: ${state.h2 || ''}`);
         lines.push(`h3: ${state.h3 || ''}`);
         lines.push(`h4: ${state.h4 || ''}`);
-        const focusArea = detectFocusArea(state.ariaSnapshot);
+        const focusArea = state.overlay;
         if (focusArea.detected) {
             lines.push(`modal: ${focusArea.name || focusArea.type}`);
         }
@@ -897,20 +902,20 @@ export class Pilot {
     hasSuccessfulCheckEvidence(currentState, testerConversation) {
         if (Object.values(currentState.verifications ?? {}).some(Boolean))
             return true;
-        return testerConversation.getToolExecutions().some((t) => CHECK_TOOLS.includes(t.toolName) && t.wasSuccessful);
+        return testerConversation.getToolExecutions().some((t) => EVIDENCE_TOOLS.includes(t.toolName) && t.wasSuccessful);
     }
     formatSuccessfulAssertions(currentState, testerConversation) {
         const lines = [];
         for (const [assertion, passed] of Object.entries(currentState.verifications ?? {})) {
             if (passed)
-                lines.push(`PASS state verification: ${assertion}`);
+                lines.push(`state verification (passed): ${assertion}`);
         }
         for (const exec of testerConversation.getToolExecutions()) {
-            if (!CHECK_TOOLS.includes(exec.toolName) || !exec.wasSuccessful)
+            if (!EVIDENCE_TOOLS.includes(exec.toolName) || !exec.wasSuccessful)
                 continue;
             const description = exec.input?.assertion || exec.input?.request || truncateJson(exec.input);
             const result = exec.output?.message || exec.output?.analysis || exec.output?.result;
-            lines.push(`PASS ${exec.toolName}: ${description}${result ? ` -> ${result}` : ''}`);
+            lines.push(`CHECK ${exec.toolName} (executed successfully): ${description}${result ? ` -> ${result}` : ''}`);
         }
         return [...new Set(lines)].join('\n');
     }
@@ -1028,7 +1033,7 @@ export class Pilot {
 
       Tester tools: click, pressKey, form, see, verify, interact, context, research, xpathCheck,
       visualClick, back, getVisitedStates, reset, stop, finish, record.
-      Use tool names exactly as listed. Do not invent combined names, aliases, or names with channel markers such as "commentary".
+      Use tool names exactly as listed. Do not invent combined names or aliases.
 
       ${capabilityGroundingRule}
 

package/dist/src/ai/provider.d.ts

@@ -7,14 +7,18 @@ declare class AiError extends Error {
 }
 export declare class ContextLengthError extends Error {
 }
+export declare function flushTelemetry(): Promise<void>;
 export declare class Provider {
     config: AIConfig;
     telemetryEnabled: boolean;
     otelSdk: NodeSDK | null;
     defaultRetryOptions: RetryOptions;
     lastConversation: Conversation | null;
+    activeModelCalls: number;
+    modelCallWaiters: (() => void)[];
     constructor(config: AIConfig);
     validateConnection(): Promise<void>;
+    stop(): Promise<void>;
     getModelForAgent(agentName?: string): any;
     getAgenticModel(agentName?: string): any;
     getVisionModel(): any;
@@ -23,6 +27,7 @@ export declare class Provider {
     getProviderOptionsForAgent(agentName: string): Record<string, any> | undefined;
     getReasoningForAgent(agentName?: string): string | undefined;
     getRetryOptions(options?: any): RetryOptions;
+    withModelRequestSlot<T>(fn: () => Promise<T>): Promise<T>;
     mergeProviderOptions(config: Record<string, any>, agentName?: string): Record<string, any>;
     finalizeConfig(config: Record<string, any>, options: any, telemetry: any): void;
     buildGenerateConfig(defaults: Record<string, any>, overrides: Record<string, any>, options: any): Record<string, any>;

package/dist/src/ai/provider.js

@@ -1,7 +1,9 @@
 import { OpenTelemetry } from '@ai-sdk/otel';
 import { LangfuseSpanProcessor } from '@langfuse/otel';
 import { NodeSDK } from '@opentelemetry/sdk-node';
-import { generateObject, generateText, isStepCount, registerTelemetry } from 'ai';
+import { AsyncLocalStorage } from 'node:async_hooks';
+import { generateObject, generateText, isStepCount, registerTelemetry, tool } from 'ai';
+import { z } from 'zod';
 import { clearActivity, setActivity } from "../activity.js";
 import { configuredModels, modelName as getModelName } from '../config.js';
 import { executionController } from "../execution-controller.js";
@@ -10,7 +12,7 @@ import { Stats } from "../stats.js";
 import { createDebug, tag } from '../utils/logger.js';
 import { withRetry } from '../utils/retry.js';
 import { RulesLoader } from "../utils/rules-loader.js";
-import { Conversation, toToolExecution } from './conversation.js';
+import { Conversation, NARRATION_TOOL, toToolExecution } from './conversation.js';
 const debugLog = createDebug('explorbot:provider');
 const promptLog = createDebug('explorbot:provider:out');
 const responseLog = createDebug('explorbot:provider:in');
@@ -18,7 +20,26 @@ class AiError extends Error {
 }
 export class ContextLengthError extends Error {
 }
+const DEFAULT_PARALLEL_REQUESTS = 4;
+const modelSlotContext = new AsyncLocalStorage();
+const HARMONY_CHANNELS = ['commentary', 'analysis', 'final'];
+function createHarmonyChannelFallbackTool() {
+    return tool({
+        description: 'Internal compatibility fallback for model channel output. Do not call directly.',
+        inputSchema: z.record(z.string(), z.any()),
+        execute: async () => ({ message: 'Noted. Continue with your next action.' }),
+    });
+}
 let telemetryRegistered = false;
+let beforeExitFlushHooked = false;
+let activeOtelSdk = null;
+export async function flushTelemetry() {
+    const sdk = activeOtelSdk;
+    activeOtelSdk = null;
+    if (!sdk)
+        return;
+    await sdk.shutdown().catch((error) => debugLog(`Telemetry flush failed: ${error instanceof Error ? error.message : error}`));
+}
 const CONTEXT_LENGTH_PATTERNS = ['reduce the length', 'context length', 'maximum context', 'token limit', 'too many tokens', 'max_tokens', 'context_length_exceeded', 'output truncated at maxtokens'];
 function extractCachedTokens(usage) {
     if (!usage)
@@ -73,6 +94,8 @@ export class Provider {
         },
     };
     lastConversation = null;
+    activeModelCalls = 0;
+    modelCallWaiters = [];
     constructor(config) {
         if (!config?.model) {
             throw new AiError('AI model is not configured. Set ai.model in your config file.');
@@ -91,6 +114,9 @@ export class Provider {
             throw new AiError(`AI connection failed: ${error.message}`);
         }
     }
+    async stop() {
+        await flushTelemetry();
+    }
     getModelForAgent(agentName) {
         if (!agentName) {
             return this.config.model;
@@ -140,9 +166,31 @@ export class Provider {
     getRetryOptions(options = {}) {
         return {
             ...this.defaultRetryOptions,
-            maxAttempts: options.maxRetries || this.defaultRetryOptions.maxAttempts,
+            maxAttempts: options.maxRetries || this.config.retryAttempts || this.defaultRetryOptions.maxAttempts,
+            baseDelay: this.config.retryDelay || this.defaultRetryOptions.baseDelay,
         };
     }
+    async withModelRequestSlot(fn) {
+        if (modelSlotContext.getStore())
+            return fn();
+        const limit = Math.max(1, this.config.maxParallelRequests ?? DEFAULT_PARALLEL_REQUESTS);
+        if (this.activeModelCalls >= limit || this.modelCallWaiters.length > 0) {
+            await new Promise((resolve) => this.modelCallWaiters.push(resolve));
+        }
+        else {
+            this.activeModelCalls++;
+        }
+        try {
+            return await modelSlotContext.run(true, fn);
+        }
+        finally {
+            const next = this.modelCallWaiters.shift();
+            if (next)
+                next();
+            else
+                this.activeModelCalls--;
+        }
+    }
     mergeProviderOptions(config, agentName) {
         if (!agentName)
             return config;
@@ -215,7 +263,12 @@ export class Provider {
             spanProcessors: [processor],
             instrumentations: [],
         });
+        activeOtelSdk = this.otelSdk;
         void this.otelSdk.start();
+        if (!beforeExitFlushHooked) {
+            process.on('beforeExit', () => void flushTelemetry());
+            beforeExitFlushHooked = true;
+        }
         if (!telemetryRegistered) {
             registerTelemetry(new OpenTelemetry());
             telemetryRegistered = true;
@@ -257,7 +310,7 @@ export class Provider {
     }
     async invokeConversation(conversation, tools, options = {}) {
         const response = tools ? await this.generateWithTools(conversation.messages, conversation.model, tools, options) : await this.chat(conversation.messages, conversation.model, options);
-        const responseMessages = response.response?.messages || [];
+        const responseMessages = response.responseMessages || [];
         if (responseMessages.length > 0) {
             conversation.messages.push(...responseMessages);
             tag('debug').log('Added', responseMessages.length, 'messages from response');
@@ -269,7 +322,7 @@ export class Provider {
         const toolCalls = response.toolCalls || [];
         const toolResults = response.toolResults || [];
         const resultsById = new Map(toolResults.map((r) => [r.toolCallId, r]));
-        const toolExecutions = toolCalls.map((call) => toToolExecution(call.toolName || '', call.input, resultsById.get(call.toolCallId)?.output));
+        const toolExecutions = toolCalls.filter((call) => call.toolName !== NARRATION_TOOL).map((call) => toToolExecution(call.toolName || '', call.input, resultsById.get(call.toolCallId)?.output));
         return { conversation, response, toolExecutions };
     }
     async chat(messages, model, options = {}) {
@@ -279,7 +332,7 @@ export class Provider {
         const config = this.buildGenerateConfig({ maxOutputTokens: 16384 }, { model, abortSignal: executionController.getAbortSignal() }, options);
         promptLog(messages[messages.length - 1].content);
         try {
-            const response = await withRetry(async () => {
+            const response = await this.withModelRequestSlot(() => withRetry(async () => {
                 const result = await generateText({ messages, ...config });
                 this.recordUsage(options.agentName || 'unknown', modelName, result.usage);
                 if (!result.text) {
@@ -293,7 +346,7 @@ export class Provider {
                     debugLog('finishReason=length, response may be truncated');
                 }
                 return result;
-            }, this.getRetryOptions(options));
+            }, this.getRetryOptions(options)));
             clearActivity();
             responseLog(response.text);
             return response;
@@ -318,7 +371,8 @@ export class Provider {
         const modelName = getModelName(model);
         setActivity(`🤖 Asking ${modelName} with dynamic tools`, 'ai');
         promptLog(`Using model: ${modelName}`);
-        const toolNames = Object.keys(tools || {});
+        const toolsWithCommentary = tools?.commentary ? tools : { ...tools, commentary: createHarmonyChannelFallbackTool() };
+        const toolNames = Object.keys(toolsWithCommentary || {});
         tag('debug').log(`Tools enabled: [${toolNames.join(', ')}]`);
         promptLog('Available tools:', toolNames);
         promptLog(messages[messages.length - 1].content);
@@ -327,9 +381,9 @@ export class Provider {
         const stopConditions = [isStepCount(maxRoundtrips)];
         if (extraStop)
             stopConditions.push(extraStop);
-        const config = this.buildGenerateConfig({ tools, maxOutputTokens: 16384, toolChoice: 'auto', experimental_repairToolCall: repairToolCall }, { stopWhen: stopConditions, model }, options);
+        const config = this.buildGenerateConfig({ tools: toolsWithCommentary, maxOutputTokens: 16384, toolChoice: 'auto', experimental_repairToolCall: repairToolCall }, { stopWhen: stopConditions, model }, options);
         try {
-            const response = await withRetry(async () => {
+            const response = await this.withModelRequestSlot(() => withRetry(async () => {
                 const result = (await this.raceWithIdleTimeout((signal) => generateText({ messages, ...config, abortSignal: signal }), config.timeout || 30000));
                 this.recordUsage(options.agentName || 'unknown', modelName, result.usage);
                 const hasToolCall = (result.toolCalls?.length || 0) > 0;
@@ -337,7 +391,7 @@ export class Provider {
                     throw new ContextLengthError('AI response empty: output truncated at maxTokens. Increase maxOutputTokens in config or use a model with higher output capacity.');
                 }
                 return result;
-            }, this.getRetryOptions(options));
+            }, this.getRetryOptions(options)));
             clearActivity();
             // Log tool usage summary
             if (response.toolCalls && response.toolCalls.length > 0) {
@@ -376,9 +430,9 @@ export class Provider {
         const config = this.buildGenerateConfig({ schema }, { model: modelToUse }, options);
         try {
             promptLog(messages[messages.length - 1].content);
-            const response = await withRetry(async () => {
+            const response = await this.withModelRequestSlot(() => withRetry(async () => {
                 return (await this.raceWithIdleTimeout((signal) => generateObject({ messages, ...config, abortSignal: signal }), config.timeout || 30000));
-            }, this.getRetryOptions(options));
+            }, this.getRetryOptions(options)));
             clearActivity();
             responseLog(response.object);
             this.recordUsage(options.agentName || 'unknown', modelName, response.usage);
@@ -512,7 +566,6 @@ export class Provider {
             throw new Error('Vision model not configured. Please set ai.visionModel in your config.');
         }
         setActivity(`🤖 Processing image with ${this.config.visionModel}`, 'ai');
-        const imageData = `data:image/png;base64,${image.toString()}`;
         const messages = [
             {
                 role: 'user',
@@ -524,7 +577,7 @@ export class Provider {
                     {
                         type: 'file',
                         mediaType: 'image/png',
-                        data: imageData,
+                        data: image,
                     },
                 ],
             },
@@ -540,12 +593,12 @@ export class Provider {
             config.telemetry = telemetry;
         try {
             promptLog(`Processing image with prompt: ${prompt}`);
-            const response = await withRetry(async () => {
+            const response = await this.withModelRequestSlot(() => withRetry(async () => {
                 return await generateText({
                     messages,
                     ...config,
                 });
-            }, this.getRetryOptions());
+            }, this.getRetryOptions()));
             clearActivity();
             responseLog(response.text);
             this.recordUsage('vision', getModelName(this.config.visionModel), response.usage);
@@ -559,13 +612,13 @@ export class Provider {
         }
     }
     hasVision() {
-        return this.config.visionModel !== undefined;
+        return this.config.visionModel !== undefined && !Stats.visionDisabled;
     }
 }
 function repairToolCall(options) {
     if (options.toolCall.toolName.includes('<|channel|>'))
         return repairChannelMarker(options);
-    return null;
+    return repairHarmonyChannel(options);
 }
 function repairChannelMarker({ toolCall, tools }) {
     const markerIndex = toolCall.toolName.indexOf('<|channel|>');
@@ -577,4 +630,16 @@ function repairChannelMarker({ toolCall, tools }) {
     tag('warning').log(`Repaired tool name '${toolCall.toolName}' → '${toolName}'`);
     return { ...toolCall, toolName };
 }
+function repairHarmonyChannel({ toolCall, tools }) {
+    if (!HARMONY_CHANNELS.includes(toolCall.toolName))
+        return null;
+    if (!tools.commentary)
+        return null;
+    let input = toolCall.input;
+    if (typeof input !== 'string' || !input.trim().startsWith('{')) {
+        input = JSON.stringify({ content: typeof input === 'string' ? input : JSON.stringify(input ?? null) });
+    }
+    tag('warning').log(`Repaired tool name '${toolCall.toolName}' → 'commentary'`);
+    return { ...toolCall, toolName: NARRATION_TOOL, input };
+}
 export { AiError, Provider as AIProvider };

package/dist/src/ai/researcher/deep-analysis.js

@@ -1,7 +1,7 @@
 import dedent from 'dedent';
 import { ActionResult } from '../../action-result.js';
 import { executionController } from "../../execution-controller.js";
-import { detectFocusArea, diffAriaSnapshots } from "../../utils/aria.js";
+import { diffAriaSnapshots } from "../../utils/aria.js";
 import { extractCodeBlocks } from "../../utils/code-extractor.js";
 import { tag } from '../../utils/logger.js';
 import { mdq } from "../../utils/markdown-query.js";
@@ -60,7 +60,7 @@ export function WithDeepAnalysis(Base) {
             this._appendExtendedResearch(result, expandedSections, navigationLinks);
         }
         async researchOverlay(current, previous, pageStateHash) {
-            const focusArea = detectFocusArea(current.ariaSnapshot);
+            const focusArea = current.overlay;
             if (!focusArea.detected || !focusArea.name)
                 return null;
             if (focusArea.type !== 'dialog' && focusArea.type !== 'modal')