@office-ai/aioncli-core 0.30.0 → 0.30.1

package/dist/src/agents/browser/analyzeScreenshot.js

@@ -0,0 +1,183 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * @fileoverview Tool for visual identification via a single model call.
+ *
+ * The semantic browser agent uses this tool when it needs to identify
+ * elements by visual attributes not present in the accessibility tree
+ * (e.g., color, layout, precise coordinates).
+ *
+ * Unlike the semantic agent which works with the accessibility tree,
+ * this tool sends a screenshot to a computer-use model for visual analysis.
+ * It returns the model's analysis (coordinates, element descriptions) back
+ * to the browser agent, which retains full control of subsequent actions.
+ */
+import { DeclarativeTool, BaseToolInvocation, Kind, } from '../../tools/tools.js';
+import { getVisualAgentModel } from './modelAvailability.js';
+import { debugLogger } from '../../utils/debugLogger.js';
+import { LlmRole } from '../../telemetry/llmRole.js';
+/**
+ * System prompt for the visual analysis model call.
+ */
+const VISUAL_SYSTEM_PROMPT = `You are a Visual Analysis Agent. You receive a screenshot of a browser page and an instruction.
+
+Your job is to ANALYZE the screenshot and provide precise information that a browser automation agent can act on.
+
+COORDINATE SYSTEM:
+- Coordinates are pixel-based relative to the viewport
+- (0,0) is top-left of the visible area
+- Estimate element positions from the screenshot
+
+RESPONSE FORMAT:
+- For coordinate identification: provide exact (x, y) pixel coordinates
+- For element identification: describe the element's visual location and appearance
+- For layout analysis: describe the spatial relationships between elements
+- Be concise and actionable — the browser agent will use your response to decide what action to take
+
+IMPORTANT:
+- You are NOT performing actions — you are only providing visual analysis
+- Include coordinates when possible so the caller can use click_at(x, y)
+- If the element is not visible in the screenshot, say so explicitly`;
+/**
+ * Invocation for the analyze_screenshot tool.
+ * Makes a single generateContent call with a screenshot.
+ */
+class AnalyzeScreenshotInvocation extends BaseToolInvocation {
+    browserManager;
+    config;
+    constructor(browserManager, config, params, messageBus) {
+        super(params, messageBus, 'analyze_screenshot', 'Analyze Screenshot');
+        this.browserManager = browserManager;
+        this.config = config;
+    }
+    getDescription() {
+        const instruction = String(this.params['instruction'] ?? '');
+        return `Visual analysis: "${instruction}"`;
+    }
+    async execute(signal) {
+        try {
+            const instruction = String(this.params['instruction'] ?? '');
+            debugLogger.log(`Visual analysis requested: ${instruction}`);
+            // Capture screenshot via MCP tool
+            const screenshotResult = await this.browserManager.callTool('take_screenshot', {});
+            // Extract base64 image data from MCP response.
+            // Search ALL content items for image type — MCP returns [text, image]
+            // where content[0] is a text description and content[1] is the actual PNG.
+            let screenshotBase64 = '';
+            let mimeType = 'image/png';
+            if (screenshotResult.content && Array.isArray(screenshotResult.content)) {
+                for (const item of screenshotResult.content) {
+                    if (item.type === 'image' && item.data) {
+                        screenshotBase64 = item.data;
+                        mimeType = item.mimeType ?? 'image/png';
+                        break;
+                    }
+                }
+            }
+            if (!screenshotBase64) {
+                return {
+                    llmContent: 'Failed to capture screenshot for visual analysis. Use accessibility tree elements instead.',
+                    returnDisplay: 'Screenshot capture failed',
+                    error: { message: 'Screenshot capture failed' },
+                };
+            }
+            // Make a single generateContent call with the visual model
+            const visualModel = getVisualAgentModel(this.config);
+            const contentGenerator = this.config.getContentGenerator();
+            const response = await contentGenerator.generateContent({
+                model: visualModel,
+                config: {
+                    temperature: 0,
+                    topP: 0.95,
+                    systemInstruction: VISUAL_SYSTEM_PROMPT,
+                    abortSignal: signal,
+                },
+                contents: [
+                    {
+                        role: 'user',
+                        parts: [
+                            {
+                                text: `Analyze this screenshot and respond to the following instruction:\n\n${instruction}`,
+                            },
+                            {
+                                inlineData: {
+                                    mimeType,
+                                    data: screenshotBase64,
+                                },
+                            },
+                        ],
+                    },
+                ],
+            }, 'visual-analysis', LlmRole.UTILITY_TOOL);
+            // Extract text from response
+            const responseText = response.candidates?.[0]?.content?.parts
+                ?.filter((p) => p.text)
+                .map((p) => p.text)
+                .join('\n') ?? '';
+            if (!responseText) {
+                return {
+                    llmContent: 'Visual model returned no analysis. Use accessibility tree elements instead.',
+                    returnDisplay: 'Visual analysis returned empty response',
+                    error: { message: 'Empty visual analysis response' },
+                };
+            }
+            debugLogger.log(`Visual analysis complete: ${responseText}`);
+            return {
+                llmContent: `Visual Analysis Result:\n${responseText}`,
+                returnDisplay: `Visual Analysis Result:\n${responseText}`,
+            };
+        }
+        catch (error) {
+            const errorMsg = error instanceof Error ? error.message : String(error);
+            debugLogger.error(`Visual analysis failed: ${errorMsg}`);
+            // Provide a graceful fallback message for model unavailability
+            const isModelError = errorMsg.includes('404') ||
+                errorMsg.includes('403') ||
+                errorMsg.includes('not found') ||
+                errorMsg.includes('permission');
+            const fallbackMsg = isModelError
+                ? 'Visual analysis model is not available. Use accessibility tree elements (uids from take_snapshot) for all interactions instead.'
+                : `Visual analysis failed: ${errorMsg}. Use accessibility tree elements instead.`;
+            return {
+                llmContent: fallbackMsg,
+                returnDisplay: fallbackMsg,
+                error: { message: errorMsg },
+            };
+        }
+    }
+}
+/**
+ * DeclarativeTool for screenshot-based visual analysis.
+ */
+class AnalyzeScreenshotTool extends DeclarativeTool {
+    browserManager;
+    config;
+    constructor(browserManager, config, messageBus) {
+        super('analyze_screenshot', 'analyze_screenshot', 'Analyze the current page visually using a screenshot. Use when you need to identify elements by visual attributes (color, layout, position) not available in the accessibility tree, or when you need precise pixel coordinates for click_at. Returns visual analysis — you perform the actions yourself.', Kind.Other, {
+            type: 'object',
+            properties: {
+                instruction: {
+                    type: 'string',
+                    description: 'What to identify or analyze visually (e.g., "Find the coordinates of the blue submit button", "What is the layout of the navigation menu?").',
+                },
+            },
+            required: ['instruction'],
+        }, messageBus, true, // isOutputMarkdown
+        false);
+        this.browserManager = browserManager;
+        this.config = config;
+    }
+    build(params) {
+        return new AnalyzeScreenshotInvocation(this.browserManager, this.config, params, this.messageBus);
+    }
+}
+/**
+ * Creates the analyze_screenshot tool for the browser agent.
+ */
+export function createAnalyzeScreenshotTool(browserManager, config, messageBus) {
+    return new AnalyzeScreenshotTool(browserManager, config, messageBus);
+}
+//# sourceMappingURL=analyzeScreenshot.js.map

package/dist/src/agents/browser/analyzeScreenshot.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyzeScreenshot.js","sourceRoot":"","sources":["../../../../src/agents/browser/analyzeScreenshot.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;;;;;;;;;;GAWG;AAEH,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,IAAI,GAGL,MAAM,sBAAsB,CAAC;AAI9B,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAErD;;GAEG;AACH,MAAM,oBAAoB,GAAG;;;;;;;;;;;;;;;;;;qEAkBwC,CAAC;AAEtE;;;GAGG;AACH,MAAM,2BAA4B,SAAQ,kBAGzC;IAEoB;IACA;IAFnB,YACmB,cAA8B,EAC9B,MAAc,EAC/B,MAA+B,EAC/B,UAAsB;QAEtB,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,oBAAoB,EAAE,oBAAoB,CAAC,CAAC;QALrD,mBAAc,GAAd,cAAc,CAAgB;QAC9B,WAAM,GAAN,MAAM,CAAQ;IAKjC,CAAC;IAED,cAAc;QACZ,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7D,OAAO,qBAAqB,WAAW,GAAG,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAmB;QAC/B,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC;YAE7D,WAAW,CAAC,GAAG,CAAC,8BAA8B,WAAW,EAAE,CAAC,CAAC;YAE7D,kCAAkC;YAClC,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,QAAQ,CACzD,iBAAiB,EACjB,EAAE,CACH,CAAC;YAEF,+CAA+C;YAC/C,sEAAsE;YACtE,2EAA2E;YAC3E,IAAI,gBAAgB,GAAG,EAAE,CAAC;YAC1B,IAAI,QAAQ,GAAG,WAAW,CAAC;YAC3B,IAAI,gBAAgB,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAC;gBACxE,KAAK,MAAM,IAAI,IAAI,gBAAgB,CAAC,OAAO,EAAE,CAAC;oBAC5C,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;wBACvC,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC;wBAC7B,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,WAAW,CAAC;wBACxC,MAAM;oBACR,CAAC;gBACH,CAAC;YACH,CAAC;YAED,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACtB,OAAO;oBACL,UAAU,EACR,4FAA4F;oBAC9F,aAAa,EAAE,2BAA2B;oBAC1C,KAAK,EAAE,EAAE,OAAO,EAAE,2BAA2B,EAAE;iBAChD,CAAC;YACJ,CAAC;YAED,2DAA2D;YAC3D,MAAM,WAAW,GAAG,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrD,MAAM,gBAAgB,GAAG,IAAI,CAAC,MAAM,CAAC,mBAAmB,EAAE,CAAC;YAE3D,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,eAAe,CACrD;gBACE,KAAK,EAAE,WAAW;gBAClB,MAAM,EAAE;oBACN,WAAW,EAAE,CAAC;oBACd,IAAI,EAAE,IAAI;oBACV,iBAAiB,EAAE,oBAAoB;oBACvC,WAAW,EAAE,MAAM;iBACpB;gBACD,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,MAAM;wBACZ,KAAK,EAAE;4BACL;gCACE,IAAI,EAAE,wEAAwE,WAAW,EAAE;6BAC5F;4BACD;gCACE,UAAU,EAAE;oCACV,QAAQ;oCACR,IAAI,EAAE,gBAAgB;iCACvB;6BACF;yBACF;qBACF;iBACF;aACF,EACD,iBAAiB,EACjB,OAAO,CAAC,YAAY,CACrB,CAAC;YAEF,6BAA6B;YAC7B,MAAM,YAAY,GAChB,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK;gBACtC,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;iBACtB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;iBAClB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YAEtB,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,OAAO;oBACL,UAAU,EACR,6EAA6E;oBAC/E,aAAa,EAAE,yCAAyC;oBACxD,KAAK,EAAE,EAAE,OAAO,EAAE,gCAAgC,EAAE;iBACrD,CAAC;YACJ,CAAC;YAED,WAAW,CAAC,GAAG,CAAC,6BAA6B,YAAY,EAAE,CAAC,CAAC;YAE7D,OAAO;gBACL,UAAU,EAAE,4BAA4B,YAAY,EAAE;gBACtD,aAAa,EAAE,4BAA4B,YAAY,EAAE;aAC1D,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,QAAQ,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACxE,WAAW,CAAC,KAAK,CAAC,2BAA2B,QAAQ,EAAE,CAAC,CAAC;YAEzD,+DAA+D;YAC/D,MAAM,YAAY,GAChB,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC;gBACxB,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC;gBACxB,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC;gBAC9B,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;YAElC,MAAM,WAAW,GAAG,YAAY;gBAC9B,CAAC,CAAC,iIAAiI;gBACnI,CAAC,CAAC,2BAA2B,QAAQ,4CAA4C,CAAC;YAEpF,OAAO;gBACL,UAAU,EAAE,WAAW;gBACvB,aAAa,EAAE,WAAW;gBAC1B,KAAK,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;aAC7B,CAAC;QACJ,CAAC;IACH,CAAC;CACF;AAED;;GAEG;AACH,MAAM,qBAAsB,SAAQ,eAGnC;IAEoB;IACA;IAFnB,YACmB,cAA8B,EAC9B,MAAc,EAC/B,UAAsB;QAEtB,KAAK,CACH,oBAAoB,EACpB,oBAAoB,EACpB,2SAA2S,EAC3S,IAAI,CAAC,KAAK,EACV;YACE,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,WAAW,EAAE;oBACX,IAAI,EAAE,QAAQ;oBACd,WAAW,EACT,8IAA8I;iBACjJ;aACF;YACD,QAAQ,EAAE,CAAC,aAAa,CAAC;SAC1B,EACD,UAAU,EACV,IAAI,EAAE,mBAAmB;QACzB,KAAK,CACN,CAAC;QAvBe,mBAAc,GAAd,cAAc,CAAgB;QAC9B,WAAM,GAAN,MAAM,CAAQ;IAuBjC,CAAC;IAED,KAAK,CACH,MAA+B;QAE/B,OAAO,IAAI,2BAA2B,CACpC,IAAI,CAAC,cAAc,EACnB,IAAI,CAAC,MAAM,EACX,MAAM,EACN,IAAI,CAAC,UAAU,CAChB,CAAC;IACJ,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,2BAA2B,CACzC,cAA8B,EAC9B,MAAc,EACd,UAAsB;IAEtB,OAAO,IAAI,qBAAqB,CAAC,cAAc,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;AACvE,CAAC"}

package/dist/src/agents/browser/analyzeScreenshot.test.d.ts

@@ -0,0 +1,6 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+export {};

package/dist/src/agents/browser/analyzeScreenshot.test.js

@@ -0,0 +1,161 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { createAnalyzeScreenshotTool } from './analyzeScreenshot.js';
+const mockMessageBus = {
+    waitForConfirmation: vi.fn().mockResolvedValue({ approved: true }),
+};
+function createMockBrowserManager(callToolResult) {
+    return {
+        callTool: vi.fn().mockResolvedValue(callToolResult ?? {
+            content: [
+                { type: 'text', text: 'Screenshot captured' },
+                {
+                    type: 'image',
+                    data: 'base64encodeddata',
+                    mimeType: 'image/png',
+                },
+            ],
+        }),
+    };
+}
+function createMockConfig(generateContentResult, generateContentError) {
+    const generateContent = generateContentError
+        ? vi.fn().mockRejectedValue(generateContentError)
+        : vi.fn().mockResolvedValue(generateContentResult ?? {
+            candidates: [
+                {
+                    content: {
+                        parts: [
+                            {
+                                text: 'The blue submit button is at coordinates (250, 400).',
+                            },
+                        ],
+                    },
+                },
+            ],
+        });
+    return {
+        getBrowserAgentConfig: vi.fn().mockReturnValue({
+            customConfig: { visualModel: 'test-visual-model' },
+        }),
+        getContentGenerator: vi.fn().mockReturnValue({
+            generateContent,
+        }),
+    };
+}
+describe('analyzeScreenshot', () => {
+    beforeEach(() => {
+        vi.clearAllMocks();
+    });
+    describe('createAnalyzeScreenshotTool', () => {
+        it('creates a tool with the correct name and schema', () => {
+            const browserManager = createMockBrowserManager();
+            const config = createMockConfig();
+            const tool = createAnalyzeScreenshotTool(browserManager, config, mockMessageBus);
+            expect(tool.name).toBe('analyze_screenshot');
+        });
+    });
+    describe('AnalyzeScreenshotInvocation', () => {
+        it('captures a screenshot and returns visual analysis', async () => {
+            const browserManager = createMockBrowserManager();
+            const config = createMockConfig();
+            const tool = createAnalyzeScreenshotTool(browserManager, config, mockMessageBus);
+            const invocation = tool.build({
+                instruction: 'Find the blue submit button',
+            });
+            const result = await invocation.execute(new AbortController().signal);
+            // Verify screenshot was captured
+            expect(browserManager.callTool).toHaveBeenCalledWith('take_screenshot', {});
+            // Verify the visual model was called
+            const contentGenerator = config.getContentGenerator();
+            expect(contentGenerator.generateContent).toHaveBeenCalledWith(expect.objectContaining({
+                model: 'test-visual-model',
+                contents: expect.arrayContaining([
+                    expect.objectContaining({
+                        role: 'user',
+                        parts: expect.arrayContaining([
+                            expect.objectContaining({
+                                inlineData: {
+                                    mimeType: 'image/png',
+                                    data: 'base64encodeddata',
+                                },
+                            }),
+                        ]),
+                    }),
+                ]),
+            }), 'visual-analysis', 'utility_tool');
+            // Verify result
+            expect(result.llmContent).toContain('Visual Analysis Result');
+            expect(result.llmContent).toContain('The blue submit button is at coordinates (250, 400).');
+            expect(result.error).toBeUndefined();
+        });
+        it('returns an error when screenshot capture fails (no image)', async () => {
+            const browserManager = createMockBrowserManager({
+                content: [{ type: 'text', text: 'No screenshot available' }],
+            });
+            const config = createMockConfig();
+            const tool = createAnalyzeScreenshotTool(browserManager, config, mockMessageBus);
+            const invocation = tool.build({
+                instruction: 'Find the button',
+            });
+            const result = await invocation.execute(new AbortController().signal);
+            expect(result.error).toBeDefined();
+            expect(result.llmContent).toContain('Failed to capture screenshot');
+            // Should NOT call the visual model
+            const contentGenerator = config.getContentGenerator();
+            expect(contentGenerator.generateContent).not.toHaveBeenCalled();
+        });
+        it('returns an error when visual model returns empty response', async () => {
+            const browserManager = createMockBrowserManager();
+            const config = createMockConfig({
+                candidates: [{ content: { parts: [] } }],
+            });
+            const tool = createAnalyzeScreenshotTool(browserManager, config, mockMessageBus);
+            const invocation = tool.build({
+                instruction: 'Check the layout',
+            });
+            const result = await invocation.execute(new AbortController().signal);
+            expect(result.error).toBeDefined();
+            expect(result.llmContent).toContain('Visual model returned no analysis');
+        });
+        it('returns a model-unavailability fallback for 404 errors', async () => {
+            const browserManager = createMockBrowserManager();
+            const config = createMockConfig(undefined, new Error('Model not found: 404'));
+            const tool = createAnalyzeScreenshotTool(browserManager, config, mockMessageBus);
+            const invocation = tool.build({
+                instruction: 'Find the red error',
+            });
+            const result = await invocation.execute(new AbortController().signal);
+            expect(result.error).toBeDefined();
+            expect(result.llmContent).toContain('Visual analysis model is not available');
+        });
+        it('returns a model-unavailability fallback for 403 errors', async () => {
+            const browserManager = createMockBrowserManager();
+            const config = createMockConfig(undefined, new Error('permission denied: 403'));
+            const tool = createAnalyzeScreenshotTool(browserManager, config, mockMessageBus);
+            const invocation = tool.build({
+                instruction: 'Identify the element',
+            });
+            const result = await invocation.execute(new AbortController().signal);
+            expect(result.error).toBeDefined();
+            expect(result.llmContent).toContain('Visual analysis model is not available');
+        });
+        it('returns a generic error for non-model errors', async () => {
+            const browserManager = createMockBrowserManager();
+            const config = createMockConfig(undefined, new Error('Network timeout'));
+            const tool = createAnalyzeScreenshotTool(browserManager, config, mockMessageBus);
+            const invocation = tool.build({
+                instruction: 'Find something',
+            });
+            const result = await invocation.execute(new AbortController().signal);
+            expect(result.error).toBeDefined();
+            expect(result.llmContent).toContain('Visual analysis failed');
+            expect(result.llmContent).toContain('Network timeout');
+        });
+    });
+});
+//# sourceMappingURL=analyzeScreenshot.test.js.map

package/dist/src/agents/browser/analyzeScreenshot.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyzeScreenshot.test.js","sourceRoot":"","sources":["../../../../src/agents/browser/analyzeScreenshot.test.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC9D,OAAO,EAAE,2BAA2B,EAAE,MAAM,wBAAwB,CAAC;AAKrE,MAAM,cAAc,GAAG;IACrB,mBAAmB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;CAC1C,CAAC;AAE3B,SAAS,wBAAwB,CAC/B,cAAkC;IAElC,OAAO;QACL,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CACjC,cAAc,IAAI;YAChB,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,qBAAqB,EAAE;gBAC7C;oBACE,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,mBAAmB;oBACzB,QAAQ,EAAE,WAAW;iBACtB;aACF;SACF,CACF;KAC2B,CAAC;AACjC,CAAC;AAED,SAAS,gBAAgB,CACvB,qBAA+B,EAC/B,oBAA4B;IAE5B,MAAM,eAAe,GAAG,oBAAoB;QAC1C,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,oBAAoB,CAAC;QACjD,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CACvB,qBAAqB,IAAI;YACvB,UAAU,EAAE;gBACV;oBACE,OAAO,EAAE;wBACP,KAAK,EAAE;4BACL;gCACE,IAAI,EAAE,sDAAsD;6BAC7D;yBACF;qBACF;iBACF;aACF;SACF,CACF,CAAC;IAEN,OAAO;QACL,qBAAqB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC;YAC7C,YAAY,EAAE,EAAE,WAAW,EAAE,mBAAmB,EAAE;SACnD,CAAC;QACF,mBAAmB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC;YAC3C,eAAe;SAChB,CAAC;KACkB,CAAC;AACzB,CAAC;AAED,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;IACjC,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;IACrB,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,6BAA6B,EAAE,GAAG,EAAE;QAC3C,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;YACzD,MAAM,cAAc,GAAG,wBAAwB,EAAE,CAAC;YAClD,MAAM,MAAM,GAAG,gBAAgB,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,2BAA2B,CACtC,cAAc,EACd,MAAM,EACN,cAAc,CACf,CAAC;YAEF,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,6BAA6B,EAAE,GAAG,EAAE;QAC3C,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;YACjE,MAAM,cAAc,GAAG,wBAAwB,EAAE,CAAC;YAClD,MAAM,MAAM,GAAG,gBAAgB,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,2BAA2B,CACtC,cAAc,EACd,MAAM,EACN,cAAc,CACf,CAAC;YAEF,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC;gBAC5B,WAAW,EAAE,6BAA6B;aAC3C,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,IAAI,eAAe,EAAE,CAAC,MAAM,CAAC,CAAC;YAEtE,iCAAiC;YACjC,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,oBAAoB,CAClD,iBAAiB,EACjB,EAAE,CACH,CAAC;YAEF,qCAAqC;YACrC,MAAM,gBAAgB,GAAG,MAAM,CAAC,mBAAmB,EAAE,CAAC;YACtD,MAAM,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC,oBAAoB,CAC3D,MAAM,CAAC,gBAAgB,CAAC;gBACtB,KAAK,EAAE,mBAAmB;gBAC1B,QAAQ,EAAE,MAAM,CAAC,eAAe,CAAC;oBAC/B,MAAM,CAAC,gBAAgB,CAAC;wBACtB,IAAI,EAAE,MAAM;wBACZ,KAAK,EAAE,MAAM,CAAC,eAAe,CAAC;4BAC5B,MAAM,CAAC,gBAAgB,CAAC;gCACtB,UAAU,EAAE;oCACV,QAAQ,EAAE,WAAW;oCACrB,IAAI,EAAE,mBAAmB;iCAC1B;6BACF,CAAC;yBACH,CAAC;qBACH,CAAC;iBACH,CAAC;aACH,CAAC,EACF,iBAAiB,EACjB,cAAc,CACf,CAAC;YAEF,gBAAgB;YAChB,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,wBAAwB,CAAC,CAAC;YAC9D,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,SAAS,CACjC,sDAAsD,CACvD,CAAC;YACF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,aAAa,EAAE,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;YACzE,MAAM,cAAc,GAAG,wBAAwB,CAAC;gBAC9C,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,yBAAyB,EAAE,CAAC;aAC7D,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,gBAAgB,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,2BAA2B,CACtC,cAAc,EACd,MAAM,EACN,cAAc,CACf,CAAC;YAEF,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC;gBAC5B,WAAW,EAAE,iBAAiB;aAC/B,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,IAAI,eAAe,EAAE,CAAC,MAAM,CAAC,CAAC;YAEtE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,8BAA8B,CAAC,CAAC;YACpE,mCAAmC;YACnC,MAAM,gBAAgB,GAAG,MAAM,CAAC,mBAAmB,EAAE,CAAC;YACtD,MAAM,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QAClE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;YACzE,MAAM,cAAc,GAAG,wBAAwB,EAAE,CAAC;YAClD,MAAM,MAAM,GAAG,gBAAgB,CAAC;gBAC9B,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC;aACzC,CAAC,CAAC;YACH,MAAM,IAAI,GAAG,2BAA2B,CACtC,cAAc,EACd,MAAM,EACN,cAAc,CACf,CAAC;YAEF,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC;gBAC5B,WAAW,EAAE,kBAAkB;aAChC,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,IAAI,eAAe,EAAE,CAAC,MAAM,CAAC,CAAC;YAEtE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,mCAAmC,CAAC,CAAC;QAC3E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wDAAwD,EAAE,KAAK,IAAI,EAAE;YACtE,MAAM,cAAc,GAAG,wBAAwB,EAAE,CAAC;YAClD,MAAM,MAAM,GAAG,gBAAgB,CAC7B,SAAS,EACT,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAClC,CAAC;YACF,MAAM,IAAI,GAAG,2BAA2B,CACtC,cAAc,EACd,MAAM,EACN,cAAc,CACf,CAAC;YAEF,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC;gBAC5B,WAAW,EAAE,oBAAoB;aAClC,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,IAAI,eAAe,EAAE,CAAC,MAAM,CAAC,CAAC;YAEtE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,SAAS,CACjC,wCAAwC,CACzC,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wDAAwD,EAAE,KAAK,IAAI,EAAE;YACtE,MAAM,cAAc,GAAG,wBAAwB,EAAE,CAAC;YAClD,MAAM,MAAM,GAAG,gBAAgB,CAC7B,SAAS,EACT,IAAI,KAAK,CAAC,wBAAwB,CAAC,CACpC,CAAC;YACF,MAAM,IAAI,GAAG,2BAA2B,CACtC,cAAc,EACd,MAAM,EACN,cAAc,CACf,CAAC;YAEF,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC;gBAC5B,WAAW,EAAE,sBAAsB;aACpC,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,IAAI,eAAe,EAAE,CAAC,MAAM,CAAC,CAAC;YAEtE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,SAAS,CACjC,wCAAwC,CACzC,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;YAC5D,MAAM,cAAc,GAAG,wBAAwB,EAAE,CAAC;YAClD,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,EAAE,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC;YACzE,MAAM,IAAI,GAAG,2BAA2B,CACtC,cAAc,EACd,MAAM,EACN,cAAc,CACf,CAAC;YAEF,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC;gBAC5B,WAAW,EAAE,gBAAgB;aAC9B,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,IAAI,eAAe,EAAE,CAAC,MAAM,CAAC,CAAC;YAEtE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,wBAAwB,CAAC,CAAC;YAC9D,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/src/agents/browser/automationOverlay.d.ts

@@ -0,0 +1,26 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * @fileoverview Automation overlay utilities for visual indication during browser automation.
+ *
+ * Provides functions to inject and remove a pulsating blue border overlay
+ * that indicates when the browser is under AI agent control.
+ *
+ * Uses the Web Animations API instead of injected <style> tags so the
+ * animation works on sites with strict Content Security Policies (e.g. google.com).
+ *
+ * The script strings are passed to chrome-devtools-mcp's evaluate_script tool
+ * which expects a plain function expression (NOT an IIFE).
+ */
+import type { BrowserManager } from './browserManager.js';
+/**
+ * Injects the automation overlay into the current page.
+ */
+export declare function injectAutomationOverlay(browserManager: BrowserManager, signal?: AbortSignal): Promise<void>;
+/**
+ * Removes the automation overlay from the current page.
+ */
+export declare function removeAutomationOverlay(browserManager: BrowserManager, signal?: AbortSignal): Promise<void>;

package/dist/src/agents/browser/automationOverlay.js

@@ -0,0 +1,100 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { debugLogger } from '../../utils/debugLogger.js';
+const OVERLAY_ELEMENT_ID = '__gemini_automation_overlay';
+/**
+ * Builds the JavaScript function string that injects the automation overlay.
+ *
+ * Returns a plain arrow-function expression (no trailing invocation) because
+ * chrome-devtools-mcp's evaluate_script tool invokes it internally.
+ *
+ * Avoids nested template literals by using string concatenation for cssText.
+ */
+function buildInjectionScript() {
+    return `() => {
+    const id = '${OVERLAY_ELEMENT_ID}';
+    const existing = document.getElementById(id);
+    if (existing) existing.remove();
+
+    const overlay = document.createElement('div');
+    overlay.id = id;
+    overlay.setAttribute('aria-hidden', 'true');
+    overlay.setAttribute('role', 'presentation');
+
+    Object.assign(overlay.style, {
+      position: 'fixed',
+      top: '0',
+      left: '0',
+      right: '0',
+      bottom: '0',
+      zIndex: '2147483647',
+      pointerEvents: 'none',
+      border: '6px solid rgba(66, 133, 244, 1.0)',
+    });
+
+    document.documentElement.appendChild(overlay);
+
+    try {
+      overlay.animate([
+        { borderColor: 'rgba(66,133,244,0.3)', boxShadow: 'inset 0 0 8px rgba(66,133,244,0.15)' },
+        { borderColor: 'rgba(66,133,244,1.0)', boxShadow: 'inset 0 0 16px rgba(66,133,244,0.5)' },
+        { borderColor: 'rgba(66,133,244,0.3)', boxShadow: 'inset 0 0 8px rgba(66,133,244,0.15)' }
+      ], { duration: 2000, iterations: Infinity, easing: 'ease-in-out' });
+    } catch (e) {
+      // Silently ignore animation errors, as they can happen on sites with strict CSP.
+      // The border itself is the most important visual indicator.
+    }
+
+    return 'overlay-injected';
+  }`;
+}
+/**
+ * Builds the JavaScript function string that removes the automation overlay.
+ */
+function buildRemovalScript() {
+    return `() => {
+    const el = document.getElementById('${OVERLAY_ELEMENT_ID}');
+    if (el) el.remove();
+    return 'overlay-removed';
+  }`;
+}
+/**
+ * Injects the automation overlay into the current page.
+ */
+export async function injectAutomationOverlay(browserManager, signal) {
+    try {
+        debugLogger.log('Injecting automation overlay...');
+        const result = await browserManager.callTool('evaluate_script', { function: buildInjectionScript() }, signal);
+        if (result.isError) {
+            debugLogger.warn('Failed to inject automation overlay:', result);
+        }
+        else {
+            debugLogger.log('Automation overlay injected successfully');
+        }
+    }
+    catch (error) {
+        debugLogger.warn('Error injecting automation overlay:', error);
+    }
+}
+/**
+ * Removes the automation overlay from the current page.
+ */
+export async function removeAutomationOverlay(browserManager, signal) {
+    try {
+        debugLogger.log('Removing automation overlay...');
+        const result = await browserManager.callTool('evaluate_script', { function: buildRemovalScript() }, signal);
+        if (result.isError) {
+            debugLogger.warn('Failed to remove automation overlay:', result);
+        }
+        else {
+            debugLogger.log('Automation overlay removed successfully');
+        }
+    }
+    catch (error) {
+        debugLogger.warn('Error removing automation overlay:', error);
+    }
+}
+//# sourceMappingURL=automationOverlay.js.map

package/dist/src/agents/browser/automationOverlay.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"automationOverlay.js","sourceRoot":"","sources":["../../../../src/agents/browser/automationOverlay.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAgBH,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAEzD,MAAM,kBAAkB,GAAG,6BAA6B,CAAC;AAEzD;;;;;;;GAOG;AACH,SAAS,oBAAoB;IAC3B,OAAO;kBACS,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAkChC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB;IACzB,OAAO;0CACiC,kBAAkB;;;IAGxD,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,cAA8B,EAC9B,MAAoB;IAEpB,IAAI,CAAC;QACH,WAAW,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;QAEnD,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,QAAQ,CAC1C,iBAAiB,EACjB,EAAE,QAAQ,EAAE,oBAAoB,EAAE,EAAE,EACpC,MAAM,CACP,CAAC;QAEF,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,WAAW,CAAC,IAAI,CAAC,sCAAsC,EAAE,MAAM,CAAC,CAAC;QACnE,CAAC;aAAM,CAAC;YACN,WAAW,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,WAAW,CAAC,IAAI,CAAC,qCAAqC,EAAE,KAAK,CAAC,CAAC;IACjE,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,cAA8B,EAC9B,MAAoB;IAEpB,IAAI,CAAC;QACH,WAAW,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;QAElD,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,QAAQ,CAC1C,iBAAiB,EACjB,EAAE,QAAQ,EAAE,kBAAkB,EAAE,EAAE,EAClC,MAAM,CACP,CAAC;QAEF,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,WAAW,CAAC,IAAI,CAAC,sCAAsC,EAAE,MAAM,CAAC,CAAC;QACnE,CAAC;aAAM,CAAC;YACN,WAAW,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,WAAW,CAAC,IAAI,CAAC,oCAAoC,EAAE,KAAK,CAAC,CAAC;IAChE,CAAC;AACH,CAAC"}

package/dist/src/agents/browser/browserAgentDefinition.d.ts

@@ -0,0 +1,50 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * @fileoverview Browser Agent definition following the LocalAgentDefinition pattern.
+ *
+ * This agent uses LocalAgentExecutor for its reAct loop, like CodebaseInvestigatorAgent.
+ * It is available ONLY via delegate_to_agent, NOT as a direct tool.
+ *
+ * Tools are configured dynamically at invocation time via browserAgentFactory.
+ */
+import type { LocalAgentDefinition } from '../types.js';
+import type { Config } from '../../config/config.js';
+import { z } from 'zod';
+/** Canonical agent name — used for routing and configuration lookup. */
+export declare const BROWSER_AGENT_NAME = "browser_agent";
+/**
+ * Output schema for browser agent results.
+ */
+export declare const BrowserTaskResultSchema: z.ZodObject<{
+    success: z.ZodBoolean;
+    summary: z.ZodString;
+    data: z.ZodOptional<z.ZodUnknown>;
+}, "strip", z.ZodTypeAny, {
+    success: boolean;
+    summary: string;
+    data?: unknown;
+}, {
+    success: boolean;
+    summary: string;
+    data?: unknown;
+}>;
+/**
+ * System prompt for the semantic browser agent.
+ * Extracted from prototype (computer_use_subagent_cdt branch).
+ *
+ * @param visionEnabled Whether visual tools (analyze_screenshot, click_at) are available.
+ */
+export declare function buildBrowserSystemPrompt(visionEnabled: boolean): string;
+/**
+ * Browser Agent Definition Factory.
+ *
+ * Following the CodebaseInvestigatorAgent pattern:
+ * - Returns a factory function that takes Config for dynamic model selection
+ * - kind: 'local' for LocalAgentExecutor
+ * - toolConfig is set dynamically by browserAgentFactory
+ */
+export declare const BrowserAgentDefinition: (config: Config, visionEnabled?: boolean) => LocalAgentDefinition<typeof BrowserTaskResultSchema>;