@office-ai/aioncli-core 0.26.0 → 0.30.1

package/dist/src/agents/auth-provider/value-resolver.js

@@ -0,0 +1,85 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { debugLogger } from '../../utils/debugLogger.js';
+import { getShellConfiguration, spawnAsync } from '../../utils/shell-utils.js';
+const COMMAND_TIMEOUT_MS = 60_000;
+/**
+ * Resolves a value that may be an environment variable reference,
+ * a shell command, or a literal value.
+ *
+ * Supported formats:
+ * - `$ENV_VAR`: Read from environment variable
+ * - `!command`: Execute shell command and use output (trimmed)
+ * - `$$` or `!!`: Escape prefix, returns rest as literal
+ * - Any other string: Use as literal value
+ *
+ * @param value The value to resolve
+ * @returns The resolved value
+ * @throws Error if environment variable is not set or command fails
+ */
+export async function resolveAuthValue(value) {
+    // Support escaping with double prefix (e.g. $$ or !!).
+    // Strips one prefix char: $$FOO → $FOO, !!cmd → !cmd (literal, not resolved).
+    if (value.startsWith('$$') || value.startsWith('!!')) {
+        return value.slice(1);
+    }
+    // Environment variable: $MY_VAR
+    if (value.startsWith('$')) {
+        const envVar = value.slice(1);
+        const resolved = process.env[envVar];
+        if (resolved === undefined || resolved === '') {
+            throw new Error(`Environment variable '${envVar}' is not set or is empty. ` +
+                `Please set it before using this agent.`);
+        }
+        debugLogger.debug(`[AuthValueResolver] Resolved env var: ${envVar}`);
+        return resolved;
+    }
+    // Shell command: !command arg1 arg2
+    if (value.startsWith('!')) {
+        const command = value.slice(1).trim();
+        if (!command) {
+            throw new Error('Empty command in auth value. Expected format: !command');
+        }
+        debugLogger.debug(`[AuthValueResolver] Executing command for auth value`);
+        const shellConfig = getShellConfiguration();
+        try {
+            const { stdout } = await spawnAsync(shellConfig.executable, [...shellConfig.argsPrefix, command], {
+                signal: AbortSignal.timeout(COMMAND_TIMEOUT_MS),
+                windowsHide: true,
+            });
+            const trimmed = stdout.trim();
+            if (!trimmed) {
+                throw new Error(`Command '${command}' returned empty output`);
+            }
+            return trimmed;
+        }
+        catch (error) {
+            if (error instanceof Error && error.name === 'AbortError') {
+                throw new Error(`Command '${command}' timed out after ${COMMAND_TIMEOUT_MS / 1000} seconds`);
+            }
+            throw error;
+        }
+    }
+    // Literal value - return as-is
+    return value;
+}
+/**
+ * Check if a value needs resolution (is an env var or command reference).
+ */
+export function needsResolution(value) {
+    return value.startsWith('$') || value.startsWith('!');
+}
+/**
+ * Mask a sensitive value for logging purposes.
+ * Shows the first and last 2 characters with asterisks in between.
+ */
+export function maskSensitiveValue(value) {
+    if (value.length <= 12) {
+        return '****';
+    }
+    return `${value.slice(0, 2)}****${value.slice(-2)}`;
+}
+//# sourceMappingURL=value-resolver.js.map

package/dist/src/agents/auth-provider/value-resolver.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"value-resolver.js","sourceRoot":"","sources":["../../../../src/agents/auth-provider/value-resolver.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,qBAAqB,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAE/E,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAElC;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,KAAa;IAClD,uDAAuD;IACvD,8EAA8E;IAC9E,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACrD,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACxB,CAAC;IAED,gCAAgC;IAChC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACrC,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,EAAE,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CACb,yBAAyB,MAAM,4BAA4B;gBACzD,wCAAwC,CAC3C,CAAC;QACJ,CAAC;QACD,WAAW,CAAC,KAAK,CAAC,yCAAyC,MAAM,EAAE,CAAC,CAAC;QACrE,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,oCAAoC;IACpC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC5E,CAAC;QAED,WAAW,CAAC,KAAK,CAAC,sDAAsD,CAAC,CAAC;QAE1E,MAAM,WAAW,GAAG,qBAAqB,EAAE,CAAC;QAC5C,IAAI,CAAC;YACH,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,UAAU,CACjC,WAAW,CAAC,UAAU,EACtB,CAAC,GAAG,WAAW,CAAC,UAAU,EAAE,OAAO,CAAC,EACpC;gBACE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,kBAAkB,CAAC;gBAC/C,WAAW,EAAE,IAAI;aAClB,CACF,CAAC;YAEF,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;YAC9B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,MAAM,IAAI,KAAK,CAAC,YAAY,OAAO,yBAAyB,CAAC,CAAC;YAChE,CAAC;YACD,OAAO,OAAO,CAAC;QACjB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBAC1D,MAAM,IAAI,KAAK,CACb,YAAY,OAAO,qBAAqB,kBAAkB,GAAG,IAAI,UAAU,CAC5E,CAAC;YACJ,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,+BAA+B;IAC/B,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,KAAa;IAC3C,OAAO,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;AACxD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAAa;IAC9C,IAAI,KAAK,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;QACvB,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AACtD,CAAC"}

package/dist/src/agents/auth-provider/value-resolver.test.d.ts

@@ -0,0 +1,6 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+export {};

package/dist/src/agents/auth-provider/value-resolver.test.js

@@ -0,0 +1,100 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { describe, it, expect, afterEach, vi } from 'vitest';
+import { resolveAuthValue, needsResolution, maskSensitiveValue, } from './value-resolver.js';
+describe('value-resolver', () => {
+    describe('resolveAuthValue', () => {
+        describe('environment variables', () => {
+            afterEach(() => {
+                vi.unstubAllEnvs();
+            });
+            it('should resolve environment variable with $ prefix', async () => {
+                vi.stubEnv('TEST_API_KEY', 'secret-key-123');
+                const result = await resolveAuthValue('$TEST_API_KEY');
+                expect(result).toBe('secret-key-123');
+            });
+            it('should throw error for unset environment variable', async () => {
+                await expect(resolveAuthValue('$UNSET_VAR_12345')).rejects.toThrow("Environment variable 'UNSET_VAR_12345' is not set or is empty");
+            });
+            it('should throw error for empty environment variable', async () => {
+                vi.stubEnv('EMPTY_VAR', '');
+                await expect(resolveAuthValue('$EMPTY_VAR')).rejects.toThrow("Environment variable 'EMPTY_VAR' is not set or is empty");
+            });
+        });
+        describe('shell commands', () => {
+            it('should execute shell command with ! prefix', async () => {
+                const result = await resolveAuthValue('!echo hello');
+                expect(result).toBe('hello');
+            });
+            it('should trim whitespace from command output', async () => {
+                const result = await resolveAuthValue('!echo "  hello  "');
+                expect(result).toBe('hello');
+            });
+            it('should throw error for empty command', async () => {
+                await expect(resolveAuthValue('!')).rejects.toThrow('Empty command in auth value');
+            });
+            it('should throw error for command that returns empty output', async () => {
+                await expect(resolveAuthValue('!echo -n ""')).rejects.toThrow('returned empty output');
+            });
+            it('should throw error for failed command', async () => {
+                await expect(resolveAuthValue('!nonexistent-command-12345')).rejects.toThrow(/Command.*failed/);
+            });
+        });
+        describe('literal values', () => {
+            it('should return literal value as-is', async () => {
+                const result = await resolveAuthValue('literal-api-key');
+                expect(result).toBe('literal-api-key');
+            });
+            it('should return empty string as-is', async () => {
+                const result = await resolveAuthValue('');
+                expect(result).toBe('');
+            });
+            it('should not treat values starting with other characters as special', async () => {
+                const result = await resolveAuthValue('api-key-123');
+                expect(result).toBe('api-key-123');
+            });
+        });
+        describe('escaped literals', () => {
+            it('should return $ literal when value starts with $$', async () => {
+                const result = await resolveAuthValue('$$LITERAL');
+                expect(result).toBe('$LITERAL');
+            });
+            it('should return ! literal when value starts with !!', async () => {
+                const result = await resolveAuthValue('!!not-a-command');
+                expect(result).toBe('!not-a-command');
+            });
+        });
+    });
+    describe('needsResolution', () => {
+        it('should return true for environment variable reference', () => {
+            expect(needsResolution('$ENV_VAR')).toBe(true);
+        });
+        it('should return true for command reference', () => {
+            expect(needsResolution('!command')).toBe(true);
+        });
+        it('should return false for literal value', () => {
+            expect(needsResolution('literal')).toBe(false);
+        });
+        it('should return false for empty string', () => {
+            expect(needsResolution('')).toBe(false);
+        });
+    });
+    describe('maskSensitiveValue', () => {
+        it('should mask value longer than 12 characters', () => {
+            expect(maskSensitiveValue('1234567890abcd')).toBe('12****cd');
+        });
+        it('should return **** for short values', () => {
+            expect(maskSensitiveValue('short')).toBe('****');
+        });
+        it('should return **** for exactly 12 characters', () => {
+            expect(maskSensitiveValue('123456789012')).toBe('****');
+        });
+        it('should return **** for empty string', () => {
+            expect(maskSensitiveValue('')).toBe('****');
+        });
+    });
+});
+//# sourceMappingURL=value-resolver.test.js.map

package/dist/src/agents/auth-provider/value-resolver.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"value-resolver.test.js","sourceRoot":"","sources":["../../../../src/agents/auth-provider/value-resolver.test.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC7D,OAAO,EACL,gBAAgB,EAChB,eAAe,EACf,kBAAkB,GACnB,MAAM,qBAAqB,CAAC;AAE7B,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QAChC,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;YACrC,SAAS,CAAC,GAAG,EAAE;gBACb,EAAE,CAAC,aAAa,EAAE,CAAC;YACrB,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;gBACjE,EAAE,CAAC,OAAO,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;gBAC7C,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,eAAe,CAAC,CAAC;gBACvD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YACxC,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;gBACjE,MAAM,MAAM,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAChE,+DAA+D,CAChE,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;gBACjE,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;gBAC5B,MAAM,MAAM,CAAC,gBAAgB,CAAC,YAAY,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAC1D,yDAAyD,CAC1D,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;YAC9B,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;gBAC1D,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,aAAa,CAAC,CAAC;gBACrD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC/B,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;gBAC1D,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,mBAAmB,CAAC,CAAC;gBAC3D,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC/B,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;gBACpD,MAAM,MAAM,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CACjD,6BAA6B,CAC9B,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,0DAA0D,EAAE,KAAK,IAAI,EAAE;gBACxE,MAAM,MAAM,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAC3D,uBAAuB,CACxB,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;gBACrD,MAAM,MAAM,CACV,gBAAgB,CAAC,4BAA4B,CAAC,CAC/C,CAAC,OAAO,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;YACvC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;YAC9B,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;gBACjD,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,iBAAiB,CAAC,CAAC;gBACzD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YACzC,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;gBAChD,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,EAAE,CAAC,CAAC;gBAC1C,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC1B,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,mEAAmE,EAAE,KAAK,IAAI,EAAE;gBACjF,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,aAAa,CAAC,CAAC;gBACrD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YACrC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;YAChC,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;gBACjE,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,WAAW,CAAC,CAAC;gBACnD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAClC,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;gBACjE,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,iBAAiB,CAAC,CAAC;gBACzD,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YACxC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;YAC/D,MAAM,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;YAC/C,MAAM,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;YAC9C,MAAM,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;YACrD,MAAM,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAChE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACnD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;YACtD,MAAM,CAAC,kBAAkB,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;YAC7C,MAAM,CAAC,kBAAkB,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/src/agents/browser/analyzeScreenshot.d.ts

@@ -0,0 +1,35 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * @fileoverview Tool for visual identification via a single model call.
+ *
+ * The semantic browser agent uses this tool when it needs to identify
+ * elements by visual attributes not present in the accessibility tree
+ * (e.g., color, layout, precise coordinates).
+ *
+ * Unlike the semantic agent which works with the accessibility tree,
+ * this tool sends a screenshot to a computer-use model for visual analysis.
+ * It returns the model's analysis (coordinates, element descriptions) back
+ * to the browser agent, which retains full control of subsequent actions.
+ */
+import { DeclarativeTool, type ToolResult, type ToolInvocation } from '../../tools/tools.js';
+import type { MessageBus } from '../../confirmation-bus/message-bus.js';
+import type { BrowserManager } from './browserManager.js';
+import type { Config } from '../../config/config.js';
+/**
+ * DeclarativeTool for screenshot-based visual analysis.
+ */
+declare class AnalyzeScreenshotTool extends DeclarativeTool<Record<string, unknown>, ToolResult> {
+    private readonly browserManager;
+    private readonly config;
+    constructor(browserManager: BrowserManager, config: Config, messageBus: MessageBus);
+    build(params: Record<string, unknown>): ToolInvocation<Record<string, unknown>, ToolResult>;
+}
+/**
+ * Creates the analyze_screenshot tool for the browser agent.
+ */
+export declare function createAnalyzeScreenshotTool(browserManager: BrowserManager, config: Config, messageBus: MessageBus): AnalyzeScreenshotTool;
+export {};

package/dist/src/agents/browser/analyzeScreenshot.js

@@ -0,0 +1,183 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * @fileoverview Tool for visual identification via a single model call.
+ *
+ * The semantic browser agent uses this tool when it needs to identify
+ * elements by visual attributes not present in the accessibility tree
+ * (e.g., color, layout, precise coordinates).
+ *
+ * Unlike the semantic agent which works with the accessibility tree,
+ * this tool sends a screenshot to a computer-use model for visual analysis.
+ * It returns the model's analysis (coordinates, element descriptions) back
+ * to the browser agent, which retains full control of subsequent actions.
+ */
+import { DeclarativeTool, BaseToolInvocation, Kind, } from '../../tools/tools.js';
+import { getVisualAgentModel } from './modelAvailability.js';
+import { debugLogger } from '../../utils/debugLogger.js';
+import { LlmRole } from '../../telemetry/llmRole.js';
+/**
+ * System prompt for the visual analysis model call.
+ */
+const VISUAL_SYSTEM_PROMPT = `You are a Visual Analysis Agent. You receive a screenshot of a browser page and an instruction.
+
+Your job is to ANALYZE the screenshot and provide precise information that a browser automation agent can act on.
+
+COORDINATE SYSTEM:
+- Coordinates are pixel-based relative to the viewport
+- (0,0) is top-left of the visible area
+- Estimate element positions from the screenshot
+
+RESPONSE FORMAT:
+- For coordinate identification: provide exact (x, y) pixel coordinates
+- For element identification: describe the element's visual location and appearance
+- For layout analysis: describe the spatial relationships between elements
+- Be concise and actionable — the browser agent will use your response to decide what action to take
+
+IMPORTANT:
+- You are NOT performing actions — you are only providing visual analysis
+- Include coordinates when possible so the caller can use click_at(x, y)
+- If the element is not visible in the screenshot, say so explicitly`;
+/**
+ * Invocation for the analyze_screenshot tool.
+ * Makes a single generateContent call with a screenshot.
+ */
+class AnalyzeScreenshotInvocation extends BaseToolInvocation {
+    browserManager;
+    config;
+    constructor(browserManager, config, params, messageBus) {
+        super(params, messageBus, 'analyze_screenshot', 'Analyze Screenshot');
+        this.browserManager = browserManager;
+        this.config = config;
+    }
+    getDescription() {
+        const instruction = String(this.params['instruction'] ?? '');
+        return `Visual analysis: "${instruction}"`;
+    }
+    async execute(signal) {
+        try {
+            const instruction = String(this.params['instruction'] ?? '');
+            debugLogger.log(`Visual analysis requested: ${instruction}`);
+            // Capture screenshot via MCP tool
+            const screenshotResult = await this.browserManager.callTool('take_screenshot', {});
+            // Extract base64 image data from MCP response.
+            // Search ALL content items for image type — MCP returns [text, image]
+            // where content[0] is a text description and content[1] is the actual PNG.
+            let screenshotBase64 = '';
+            let mimeType = 'image/png';
+            if (screenshotResult.content && Array.isArray(screenshotResult.content)) {
+                for (const item of screenshotResult.content) {
+                    if (item.type === 'image' && item.data) {
+                        screenshotBase64 = item.data;
+                        mimeType = item.mimeType ?? 'image/png';
+                        break;
+                    }
+                }
+            }
+            if (!screenshotBase64) {
+                return {
+                    llmContent: 'Failed to capture screenshot for visual analysis. Use accessibility tree elements instead.',
+                    returnDisplay: 'Screenshot capture failed',
+                    error: { message: 'Screenshot capture failed' },
+                };
+            }
+            // Make a single generateContent call with the visual model
+            const visualModel = getVisualAgentModel(this.config);
+            const contentGenerator = this.config.getContentGenerator();
+            const response = await contentGenerator.generateContent({
+                model: visualModel,
+                config: {
+                    temperature: 0,
+                    topP: 0.95,
+                    systemInstruction: VISUAL_SYSTEM_PROMPT,
+                    abortSignal: signal,
+                },
+                contents: [
+                    {
+                        role: 'user',
+                        parts: [
+                            {
+                                text: `Analyze this screenshot and respond to the following instruction:\n\n${instruction}`,
+                            },
+                            {
+                                inlineData: {
+                                    mimeType,
+                                    data: screenshotBase64,
+                                },
+                            },
+                        ],
+                    },
+                ],
+            }, 'visual-analysis', LlmRole.UTILITY_TOOL);
+            // Extract text from response
+            const responseText = response.candidates?.[0]?.content?.parts
+                ?.filter((p) => p.text)
+                .map((p) => p.text)
+                .join('\n') ?? '';
+            if (!responseText) {
+                return {
+                    llmContent: 'Visual model returned no analysis. Use accessibility tree elements instead.',
+                    returnDisplay: 'Visual analysis returned empty response',
+                    error: { message: 'Empty visual analysis response' },
+                };
+            }
+            debugLogger.log(`Visual analysis complete: ${responseText}`);
+            return {
+                llmContent: `Visual Analysis Result:\n${responseText}`,
+                returnDisplay: `Visual Analysis Result:\n${responseText}`,
+            };
+        }
+        catch (error) {
+            const errorMsg = error instanceof Error ? error.message : String(error);
+            debugLogger.error(`Visual analysis failed: ${errorMsg}`);
+            // Provide a graceful fallback message for model unavailability
+            const isModelError = errorMsg.includes('404') ||
+                errorMsg.includes('403') ||
+                errorMsg.includes('not found') ||
+                errorMsg.includes('permission');
+            const fallbackMsg = isModelError
+                ? 'Visual analysis model is not available. Use accessibility tree elements (uids from take_snapshot) for all interactions instead.'
+                : `Visual analysis failed: ${errorMsg}. Use accessibility tree elements instead.`;
+            return {
+                llmContent: fallbackMsg,
+                returnDisplay: fallbackMsg,
+                error: { message: errorMsg },
+            };
+        }
+    }
+}
+/**
+ * DeclarativeTool for screenshot-based visual analysis.
+ */
+class AnalyzeScreenshotTool extends DeclarativeTool {
+    browserManager;
+    config;
+    constructor(browserManager, config, messageBus) {
+        super('analyze_screenshot', 'analyze_screenshot', 'Analyze the current page visually using a screenshot. Use when you need to identify elements by visual attributes (color, layout, position) not available in the accessibility tree, or when you need precise pixel coordinates for click_at. Returns visual analysis — you perform the actions yourself.', Kind.Other, {
+            type: 'object',
+            properties: {
+                instruction: {
+                    type: 'string',
+                    description: 'What to identify or analyze visually (e.g., "Find the coordinates of the blue submit button", "What is the layout of the navigation menu?").',
+                },
+            },
+            required: ['instruction'],
+        }, messageBus, true, // isOutputMarkdown
+        false);
+        this.browserManager = browserManager;
+        this.config = config;
+    }
+    build(params) {
+        return new AnalyzeScreenshotInvocation(this.browserManager, this.config, params, this.messageBus);
+    }
+}
+/**
+ * Creates the analyze_screenshot tool for the browser agent.
+ */
+export function createAnalyzeScreenshotTool(browserManager, config, messageBus) {
+    return new AnalyzeScreenshotTool(browserManager, config, messageBus);
+}
+//# sourceMappingURL=analyzeScreenshot.js.map

package/dist/src/agents/browser/analyzeScreenshot.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyzeScreenshot.js","sourceRoot":"","sources":["../../../../src/agents/browser/analyzeScreenshot.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;;;;;;;;;;GAWG;AAEH,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,IAAI,GAGL,MAAM,sBAAsB,CAAC;AAI9B,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAErD;;GAEG;AACH,MAAM,oBAAoB,GAAG;;;;;;;;;;;;;;;;;;qEAkBwC,CAAC;AAEtE;;;GAGG;AACH,MAAM,2BAA4B,SAAQ,kBAGzC;IAEoB;IACA;IAFnB,YACmB,cAA8B,EAC9B,MAAc,EAC/B,MAA+B,EAC/B,UAAsB;QAEtB,KAAK,CAAC,MAAM,EAAE,UAAU,EAAE,oBAAoB,EAAE,oBAAoB,CAAC,CAAC;QALrD,mBAAc,GAAd,cAAc,CAAgB;QAC9B,WAAM,GAAN,MAAM,CAAQ;IAKjC,CAAC;IAED,cAAc;QACZ,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7D,OAAO,qBAAqB,WAAW,GAAG,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAmB;QAC/B,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,CAAC;YAE7D,WAAW,CAAC,GAAG,CAAC,8BAA8B,WAAW,EAAE,CAAC,CAAC;YAE7D,kCAAkC;YAClC,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,QAAQ,CACzD,iBAAiB,EACjB,EAAE,CACH,CAAC;YAEF,+CAA+C;YAC/C,sEAAsE;YACtE,2EAA2E;YAC3E,IAAI,gBAAgB,GAAG,EAAE,CAAC;YAC1B,IAAI,QAAQ,GAAG,WAAW,CAAC;YAC3B,IAAI,gBAAgB,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAC;gBACxE,KAAK,MAAM,IAAI,IAAI,gBAAgB,CAAC,OAAO,EAAE,CAAC;oBAC5C,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;wBACvC,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC;wBAC7B,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,WAAW,CAAC;wBACxC,MAAM;oBACR,CAAC;gBACH,CAAC;YACH,CAAC;YAED,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACtB,OAAO;oBACL,UAAU,EACR,4FAA4F;oBAC9F,aAAa,EAAE,2BAA2B;oBAC1C,KAAK,EAAE,EAAE,OAAO,EAAE,2BAA2B,EAAE;iBAChD,CAAC;YACJ,CAAC;YAED,2DAA2D;YAC3D,MAAM,WAAW,GAAG,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrD,MAAM,gBAAgB,GAAG,IAAI,CAAC,MAAM,CAAC,mBAAmB,EAAE,CAAC;YAE3D,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,eAAe,CACrD;gBACE,KAAK,EAAE,WAAW;gBAClB,MAAM,EAAE;oBACN,WAAW,EAAE,CAAC;oBACd,IAAI,EAAE,IAAI;oBACV,iBAAiB,EAAE,oBAAoB;oBACvC,WAAW,EAAE,MAAM;iBACpB;gBACD,QAAQ,EAAE;oBACR;wBACE,IAAI,EAAE,MAAM;wBACZ,KAAK,EAAE;4BACL;gCACE,IAAI,EAAE,wEAAwE,WAAW,EAAE;6BAC5F;4BACD;gCACE,UAAU,EAAE;oCACV,QAAQ;oCACR,IAAI,EAAE,gBAAgB;iCACvB;6BACF;yBACF;qBACF;iBACF;aACF,EACD,iBAAiB,EACjB,OAAO,CAAC,YAAY,CACrB,CAAC;YAEF,6BAA6B;YAC7B,MAAM,YAAY,GAChB,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK;gBACtC,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;iBACtB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;iBAClB,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YAEtB,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,OAAO;oBACL,UAAU,EACR,6EAA6E;oBAC/E,aAAa,EAAE,yCAAyC;oBACxD,KAAK,EAAE,EAAE,OAAO,EAAE,gCAAgC,EAAE;iBACrD,CAAC;YACJ,CAAC;YAED,WAAW,CAAC,GAAG,CAAC,6BAA6B,YAAY,EAAE,CAAC,CAAC;YAE7D,OAAO;gBACL,UAAU,EAAE,4BAA4B,YAAY,EAAE;gBACtD,aAAa,EAAE,4BAA4B,YAAY,EAAE;aAC1D,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,QAAQ,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACxE,WAAW,CAAC,KAAK,CAAC,2BAA2B,QAAQ,EAAE,CAAC,CAAC;YAEzD,+DAA+D;YAC/D,MAAM,YAAY,GAChB,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC;gBACxB,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC;gBACxB,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC;gBAC9B,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;YAElC,MAAM,WAAW,GAAG,YAAY;gBAC9B,CAAC,CAAC,iIAAiI;gBACnI,CAAC,CAAC,2BAA2B,QAAQ,4CAA4C,CAAC;YAEpF,OAAO;gBACL,UAAU,EAAE,WAAW;gBACvB,aAAa,EAAE,WAAW;gBAC1B,KAAK,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;aAC7B,CAAC;QACJ,CAAC;IACH,CAAC;CACF;AAED;;GAEG;AACH,MAAM,qBAAsB,SAAQ,eAGnC;IAEoB;IACA;IAFnB,YACmB,cAA8B,EAC9B,MAAc,EAC/B,UAAsB;QAEtB,KAAK,CACH,oBAAoB,EACpB,oBAAoB,EACpB,2SAA2S,EAC3S,IAAI,CAAC,KAAK,EACV;YACE,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,WAAW,EAAE;oBACX,IAAI,EAAE,QAAQ;oBACd,WAAW,EACT,8IAA8I;iBACjJ;aACF;YACD,QAAQ,EAAE,CAAC,aAAa,CAAC;SAC1B,EACD,UAAU,EACV,IAAI,EAAE,mBAAmB;QACzB,KAAK,CACN,CAAC;QAvBe,mBAAc,GAAd,cAAc,CAAgB;QAC9B,WAAM,GAAN,MAAM,CAAQ;IAuBjC,CAAC;IAED,KAAK,CACH,MAA+B;QAE/B,OAAO,IAAI,2BAA2B,CACpC,IAAI,CAAC,cAAc,EACnB,IAAI,CAAC,MAAM,EACX,MAAM,EACN,IAAI,CAAC,UAAU,CAChB,CAAC;IACJ,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,2BAA2B,CACzC,cAA8B,EAC9B,MAAc,EACd,UAAsB;IAEtB,OAAO,IAAI,qBAAqB,CAAC,cAAc,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;AACvE,CAAC"}

package/dist/src/agents/browser/analyzeScreenshot.test.d.ts

@@ -0,0 +1,6 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+export {};

package/dist/src/agents/browser/analyzeScreenshot.test.js

@@ -0,0 +1,161 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { createAnalyzeScreenshotTool } from './analyzeScreenshot.js';
+const mockMessageBus = {
+    waitForConfirmation: vi.fn().mockResolvedValue({ approved: true }),
+};
+function createMockBrowserManager(callToolResult) {
+    return {
+        callTool: vi.fn().mockResolvedValue(callToolResult ?? {
+            content: [
+                { type: 'text', text: 'Screenshot captured' },
+                {
+                    type: 'image',
+                    data: 'base64encodeddata',
+                    mimeType: 'image/png',
+                },
+            ],
+        }),
+    };
+}
+function createMockConfig(generateContentResult, generateContentError) {
+    const generateContent = generateContentError
+        ? vi.fn().mockRejectedValue(generateContentError)
+        : vi.fn().mockResolvedValue(generateContentResult ?? {
+            candidates: [
+                {
+                    content: {
+                        parts: [
+                            {
+                                text: 'The blue submit button is at coordinates (250, 400).',
+                            },
+                        ],
+                    },
+                },
+            ],
+        });
+    return {
+        getBrowserAgentConfig: vi.fn().mockReturnValue({
+            customConfig: { visualModel: 'test-visual-model' },
+        }),
+        getContentGenerator: vi.fn().mockReturnValue({
+            generateContent,
+        }),
+    };
+}
+describe('analyzeScreenshot', () => {
+    beforeEach(() => {
+        vi.clearAllMocks();
+    });
+    describe('createAnalyzeScreenshotTool', () => {
+        it('creates a tool with the correct name and schema', () => {
+            const browserManager = createMockBrowserManager();
+            const config = createMockConfig();
+            const tool = createAnalyzeScreenshotTool(browserManager, config, mockMessageBus);
+            expect(tool.name).toBe('analyze_screenshot');
+        });
+    });
+    describe('AnalyzeScreenshotInvocation', () => {
+        it('captures a screenshot and returns visual analysis', async () => {
+            const browserManager = createMockBrowserManager();
+            const config = createMockConfig();
+            const tool = createAnalyzeScreenshotTool(browserManager, config, mockMessageBus);
+            const invocation = tool.build({
+                instruction: 'Find the blue submit button',
+            });
+            const result = await invocation.execute(new AbortController().signal);
+            // Verify screenshot was captured
+            expect(browserManager.callTool).toHaveBeenCalledWith('take_screenshot', {});
+            // Verify the visual model was called
+            const contentGenerator = config.getContentGenerator();
+            expect(contentGenerator.generateContent).toHaveBeenCalledWith(expect.objectContaining({
+                model: 'test-visual-model',
+                contents: expect.arrayContaining([
+                    expect.objectContaining({
+                        role: 'user',
+                        parts: expect.arrayContaining([
+                            expect.objectContaining({
+                                inlineData: {
+                                    mimeType: 'image/png',
+                                    data: 'base64encodeddata',
+                                },
+                            }),
+                        ]),
+                    }),
+                ]),
+            }), 'visual-analysis', 'utility_tool');
+            // Verify result
+            expect(result.llmContent).toContain('Visual Analysis Result');
+            expect(result.llmContent).toContain('The blue submit button is at coordinates (250, 400).');
+            expect(result.error).toBeUndefined();
+        });
+        it('returns an error when screenshot capture fails (no image)', async () => {
+            const browserManager = createMockBrowserManager({
+                content: [{ type: 'text', text: 'No screenshot available' }],
+            });
+            const config = createMockConfig();
+            const tool = createAnalyzeScreenshotTool(browserManager, config, mockMessageBus);
+            const invocation = tool.build({
+                instruction: 'Find the button',
+            });
+            const result = await invocation.execute(new AbortController().signal);
+            expect(result.error).toBeDefined();
+            expect(result.llmContent).toContain('Failed to capture screenshot');
+            // Should NOT call the visual model
+            const contentGenerator = config.getContentGenerator();
+            expect(contentGenerator.generateContent).not.toHaveBeenCalled();
+        });
+        it('returns an error when visual model returns empty response', async () => {
+            const browserManager = createMockBrowserManager();
+            const config = createMockConfig({
+                candidates: [{ content: { parts: [] } }],
+            });
+            const tool = createAnalyzeScreenshotTool(browserManager, config, mockMessageBus);
+            const invocation = tool.build({
+                instruction: 'Check the layout',
+            });
+            const result = await invocation.execute(new AbortController().signal);
+            expect(result.error).toBeDefined();
+            expect(result.llmContent).toContain('Visual model returned no analysis');
+        });
+        it('returns a model-unavailability fallback for 404 errors', async () => {
+            const browserManager = createMockBrowserManager();
+            const config = createMockConfig(undefined, new Error('Model not found: 404'));
+            const tool = createAnalyzeScreenshotTool(browserManager, config, mockMessageBus);
+            const invocation = tool.build({
+                instruction: 'Find the red error',
+            });
+            const result = await invocation.execute(new AbortController().signal);
+            expect(result.error).toBeDefined();
+            expect(result.llmContent).toContain('Visual analysis model is not available');
+        });
+        it('returns a model-unavailability fallback for 403 errors', async () => {
+            const browserManager = createMockBrowserManager();
+            const config = createMockConfig(undefined, new Error('permission denied: 403'));
+            const tool = createAnalyzeScreenshotTool(browserManager, config, mockMessageBus);
+            const invocation = tool.build({
+                instruction: 'Identify the element',
+            });
+            const result = await invocation.execute(new AbortController().signal);
+            expect(result.error).toBeDefined();
+            expect(result.llmContent).toContain('Visual analysis model is not available');
+        });
+        it('returns a generic error for non-model errors', async () => {
+            const browserManager = createMockBrowserManager();
+            const config = createMockConfig(undefined, new Error('Network timeout'));
+            const tool = createAnalyzeScreenshotTool(browserManager, config, mockMessageBus);
+            const invocation = tool.build({
+                instruction: 'Find something',
+            });
+            const result = await invocation.execute(new AbortController().signal);
+            expect(result.error).toBeDefined();
+            expect(result.llmContent).toContain('Visual analysis failed');
+            expect(result.llmContent).toContain('Network timeout');
+        });
+    });
+});
+//# sourceMappingURL=analyzeScreenshot.test.js.map

package/dist/src/agents/browser/analyzeScreenshot.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyzeScreenshot.test.js","sourceRoot":"","sources":["../../../../src/agents/browser/analyzeScreenshot.test.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAC9D,OAAO,EAAE,2BAA2B,EAAE,MAAM,wBAAwB,CAAC;AAKrE,MAAM,cAAc,GAAG;IACrB,mBAAmB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;CAC1C,CAAC;AAE3B,SAAS,wBAAwB,CAC/B,cAAkC;IAElC,OAAO;QACL,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CACjC,cAAc,IAAI;YAChB,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,qBAAqB,EAAE;gBAC7C;oBACE,IAAI,EAAE,OAAO;oBACb,IAAI,EAAE,mBAAmB;oBACzB,QAAQ,EAAE,WAAW;iBACtB;aACF;SACF,CACF;KAC2B,CAAC;AACjC,CAAC;AAED,SAAS,gBAAgB,CACvB,qBAA+B,EAC/B,oBAA4B;IAE5B,MAAM,eAAe,GAAG,oBAAoB;QAC1C,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,oBAAoB,CAAC;QACjD,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CACvB,qBAAqB,IAAI;YACvB,UAAU,EAAE;gBACV;oBACE,OAAO,EAAE;wBACP,KAAK,EAAE;4BACL;gCACE,IAAI,EAAE,sDAAsD;6BAC7D;yBACF;qBACF;iBACF;aACF;SACF,CACF,CAAC;IAEN,OAAO;QACL,qBAAqB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC;YAC7C,YAAY,EAAE,EAAE,WAAW,EAAE,mBAAmB,EAAE;SACnD,CAAC;QACF,mBAAmB,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC;YAC3C,eAAe;SAChB,CAAC;KACkB,CAAC;AACzB,CAAC;AAED,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;IACjC,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC;IACrB,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,6BAA6B,EAAE,GAAG,EAAE;QAC3C,EAAE,CAAC,iDAAiD,EAAE,GAAG,EAAE;YACzD,MAAM,cAAc,GAAG,wBAAwB,EAAE,CAAC;YAClD,MAAM,MAAM,GAAG,gBAAgB,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,2BAA2B,CACtC,cAAc,EACd,MAAM,EACN,cAAc,CACf,CAAC;YAEF,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,6BAA6B,EAAE,GAAG,EAAE;QAC3C,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;YACjE,MAAM,cAAc,GAAG,wBAAwB,EAAE,CAAC;YAClD,MAAM,MAAM,GAAG,gBAAgB,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,2BAA2B,CACtC,cAAc,EACd,MAAM,EACN,cAAc,CACf,CAAC;YAEF,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC;gBAC5B,WAAW,EAAE,6BAA6B;aAC3C,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,IAAI,eAAe,EAAE,CAAC,MAAM,CAAC,CAAC;YAEtE,iCAAiC;YACjC,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,oBAAoB,CAClD,iBAAiB,EACjB,EAAE,CACH,CAAC;YAEF,qCAAqC;YACrC,MAAM,gBAAgB,GAAG,MAAM,CAAC,mBAAmB,EAAE,CAAC;YACtD,MAAM,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC,oBAAoB,CAC3D,MAAM,CAAC,gBAAgB,CAAC;gBACtB,KAAK,EAAE,mBAAmB;gBAC1B,QAAQ,EAAE,MAAM,CAAC,eAAe,CAAC;oBAC/B,MAAM,CAAC,gBAAgB,CAAC;wBACtB,IAAI,EAAE,MAAM;wBACZ,KAAK,EAAE,MAAM,CAAC,eAAe,CAAC;4BAC5B,MAAM,CAAC,gBAAgB,CAAC;gCACtB,UAAU,EAAE;oCACV,QAAQ,EAAE,WAAW;oCACrB,IAAI,EAAE,mBAAmB;iCAC1B;6BACF,CAAC;yBACH,CAAC;qBACH,CAAC;iBACH,CAAC;aACH,CAAC,EACF,iBAAiB,EACjB,cAAc,CACf,CAAC;YAEF,gBAAgB;YAChB,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,wBAAwB,CAAC,CAAC;YAC9D,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,SAAS,CACjC,sDAAsD,CACvD,CAAC;YACF,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,aAAa,EAAE,CAAC;QACvC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;YACzE,MAAM,cAAc,GAAG,wBAAwB,CAAC;gBAC9C,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,yBAAyB,EAAE,CAAC;aAC7D,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,gBAAgB,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,2BAA2B,CACtC,cAAc,EACd,MAAM,EACN,cAAc,CACf,CAAC;YAEF,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC;gBAC5B,WAAW,EAAE,iBAAiB;aAC/B,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,IAAI,eAAe,EAAE,CAAC,MAAM,CAAC,CAAC;YAEtE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,8BAA8B,CAAC,CAAC;YACpE,mCAAmC;YACnC,MAAM,gBAAgB,GAAG,MAAM,CAAC,mBAAmB,EAAE,CAAC;YACtD,MAAM,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QAClE,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;YACzE,MAAM,cAAc,GAAG,wBAAwB,EAAE,CAAC;YAClD,MAAM,MAAM,GAAG,gBAAgB,CAAC;gBAC9B,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC;aACzC,CAAC,CAAC;YACH,MAAM,IAAI,GAAG,2BAA2B,CACtC,cAAc,EACd,MAAM,EACN,cAAc,CACf,CAAC;YAEF,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC;gBAC5B,WAAW,EAAE,kBAAkB;aAChC,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,IAAI,eAAe,EAAE,CAAC,MAAM,CAAC,CAAC;YAEtE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,mCAAmC,CAAC,CAAC;QAC3E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wDAAwD,EAAE,KAAK,IAAI,EAAE;YACtE,MAAM,cAAc,GAAG,wBAAwB,EAAE,CAAC;YAClD,MAAM,MAAM,GAAG,gBAAgB,CAC7B,SAAS,EACT,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAClC,CAAC;YACF,MAAM,IAAI,GAAG,2BAA2B,CACtC,cAAc,EACd,MAAM,EACN,cAAc,CACf,CAAC;YAEF,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC;gBAC5B,WAAW,EAAE,oBAAoB;aAClC,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,IAAI,eAAe,EAAE,CAAC,MAAM,CAAC,CAAC;YAEtE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,SAAS,CACjC,wCAAwC,CACzC,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,wDAAwD,EAAE,KAAK,IAAI,EAAE;YACtE,MAAM,cAAc,GAAG,wBAAwB,EAAE,CAAC;YAClD,MAAM,MAAM,GAAG,gBAAgB,CAC7B,SAAS,EACT,IAAI,KAAK,CAAC,wBAAwB,CAAC,CACpC,CAAC;YACF,MAAM,IAAI,GAAG,2BAA2B,CACtC,cAAc,EACd,MAAM,EACN,cAAc,CACf,CAAC;YAEF,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC;gBAC5B,WAAW,EAAE,sBAAsB;aACpC,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,IAAI,eAAe,EAAE,CAAC,MAAM,CAAC,CAAC;YAEtE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,SAAS,CACjC,wCAAwC,CACzC,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;YAC5D,MAAM,cAAc,GAAG,wBAAwB,EAAE,CAAC;YAClD,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,EAAE,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC;YACzE,MAAM,IAAI,GAAG,2BAA2B,CACtC,cAAc,EACd,MAAM,EACN,cAAc,CACf,CAAC;YAEF,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC;gBAC5B,WAAW,EAAE,gBAAgB;aAC9B,CAAC,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,IAAI,eAAe,EAAE,CAAC,MAAM,CAAC,CAAC;YAEtE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,wBAAwB,CAAC,CAAC;YAC9D,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/src/agents/browser/automationOverlay.d.ts

@@ -0,0 +1,26 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+/**
+ * @fileoverview Automation overlay utilities for visual indication during browser automation.
+ *
+ * Provides functions to inject and remove a pulsating blue border overlay
+ * that indicates when the browser is under AI agent control.
+ *
+ * Uses the Web Animations API instead of injected <style> tags so the
+ * animation works on sites with strict Content Security Policies (e.g. google.com).
+ *
+ * The script strings are passed to chrome-devtools-mcp's evaluate_script tool
+ * which expects a plain function expression (NOT an IIFE).
+ */
+import type { BrowserManager } from './browserManager.js';
+/**
+ * Injects the automation overlay into the current page.
+ */
+export declare function injectAutomationOverlay(browserManager: BrowserManager, signal?: AbortSignal): Promise<void>;
+/**
+ * Removes the automation overlay from the current page.
+ */
+export declare function removeAutomationOverlay(browserManager: BrowserManager, signal?: AbortSignal): Promise<void>;