browser-use 0.2.0 → 0.3.0

package/dist/index.d.ts

@@ -14,11 +14,14 @@ export * from './screenshots/service.js';
 export * from './controller/views.js';
 export * from './controller/registry/service.js';
 export * from './controller/service.js';
+export { Tools } from './tools/service.js';
+export type { ToolsOptions, ToolsActParams } from './tools/service.js';
 export * from './filesystem/file-system.js';
 export * from './agent/views.js';
 export * from './telemetry/views.js';
 export * from './telemetry/service.js';
 export * from './llm/messages.js';
+export * from './llm/models.js';
 export * from './llm/views.js';
 export * from './llm/base.js';
 export * from './llm/exceptions.js';
@@ -29,3 +32,7 @@ export * from './agent/prompts.js';
 export * from './agent/message-manager/service.js';
 export * from './agent/service.js';
 export * from './agent/message-manager/utils.js';
+export * from './skills/index.js';
+export * from './sandbox/index.js';
+export * from './code-use/index.js';
+export * from './skill-cli/index.js';

package/dist/index.js

@@ -16,11 +16,13 @@ export * from './screenshots/service.js';
 export * from './controller/views.js';
 export * from './controller/registry/service.js';
 export * from './controller/service.js';
+export { Tools } from './tools/service.js';
 export * from './filesystem/file-system.js';
 export * from './agent/views.js';
 export * from './telemetry/views.js';
 export * from './telemetry/service.js';
 export * from './llm/messages.js';
+export * from './llm/models.js';
 export * from './llm/views.js';
 export * from './llm/base.js';
 export * from './llm/exceptions.js';
@@ -31,3 +33,7 @@ export * from './agent/prompts.js';
 export * from './agent/message-manager/service.js';
 export * from './agent/service.js';
 export * from './agent/message-manager/utils.js';
+export * from './skills/index.js';
+export * from './sandbox/index.js';
+export * from './code-use/index.js';
+export * from './skill-cli/index.js';

package/dist/integrations/gmail/actions.d.ts

@@ -3,10 +3,10 @@
  * Defines agent actions for Gmail integration including 2FA code retrieval,
  * email reading, and authentication management.
  */
-import type { Controller } from '../../controller/service.js';
 import { GmailService } from './service.js';
+import type { Tools } from '../../tools/service.js';
 /**
- * Register Gmail actions with the provided controller
+ * Register Gmail actions with the provided tools registry
  */
-export declare function registerGmailActions(controller: Controller, gmailService?: GmailService | null, accessToken?: string | null): Controller;
+export declare function registerGmailActions(tools: Tools, gmailService?: GmailService | null, accessToken?: string | null): Tools;
 export declare const register_gmail_actions: typeof registerGmailActions;

package/dist/integrations/gmail/actions.js

@@ -25,9 +25,9 @@ const GetRecentEmailsParamsSchema = z.object({
         .describe('Maximum number of emails to retrieve (1-50, default: 3)'),
 });
 /**
- * Register Gmail actions with the provided controller
+ * Register Gmail actions with the provided tools registry
  */
-export function registerGmailActions(controller, gmailService, accessToken) {
+export function registerGmailActions(tools, gmailService, accessToken) {
     // Use provided service or create a new one with access token if provided
     if (gmailService) {
         _gmailService = gmailService;
@@ -39,7 +39,7 @@ export function registerGmailActions(controller, gmailService, accessToken) {
         _gmailService = new GmailService();
     }
     // Register get_recent_emails action
-    controller.registry.action('Get recent emails from the mailbox with a keyword to retrieve verification codes, OTP, 2FA tokens, magic links, or any recent email content. Keep your query a single keyword.', GetRecentEmailsParamsSchema)(async (params) => {
+    tools.registry.action('Get recent emails from the mailbox with a keyword to retrieve verification codes, OTP, 2FA tokens, magic links, or any recent email content. Keep your query a single keyword.', GetRecentEmailsParamsSchema)(async (params) => {
         try {
             if (!_gmailService) {
                 throw new Error('Gmail service not initialized');
@@ -107,7 +107,7 @@ export function registerGmailActions(controller, gmailService, accessToken) {
             });
         }
     });
-    return controller;
+    return tools;
 }
 // Backward compatibility export
 export const register_gmail_actions = registerGmailActions;

package/dist/llm/anthropic/chat.d.ts

@@ -1,14 +1,24 @@
+import { type ClientOptions } from '@anthropic-ai/sdk';
 import type { BaseChatModel, ChatInvokeOptions } from '../base.js';
 import { ChatInvokeCompletion } from '../views.js';
 import { type Message } from '../messages.js';
 export interface ChatAnthropicOptions {
     model?: string;
     apiKey?: string;
+    authToken?: string;
     baseURL?: string;
+    timeout?: number;
     maxTokens?: number;
     temperature?: number | null;
     topP?: number | null;
+    seed?: number | null;
     maxRetries?: number;
+    defaultHeaders?: Record<string, string>;
+    defaultQuery?: Record<string, string | undefined>;
+    fetchImplementation?: ClientOptions['fetch'];
+    fetchOptions?: ClientOptions['fetchOptions'];
+    removeMinItemsFromSchema?: boolean;
+    removeDefaultsFromSchema?: boolean;
 }
 export declare class ChatAnthropic implements BaseChatModel {
     model: string;
@@ -17,9 +27,16 @@ export declare class ChatAnthropic implements BaseChatModel {
     private maxTokens;
     private temperature;
     private topP;
-    constructor(options?: ChatAnthropicOptions);
+    private seed;
+    private removeMinItemsFromSchema;
+    private removeDefaultsFromSchema;
+    constructor(options?: string | ChatAnthropicOptions);
     get name(): string;
     get model_name(): string;
+    private getModelParams;
+    private getZodSchemaCandidate;
+    private parseOutput;
+    private getTextCompletion;
     private getUsage;
     ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise<ChatInvokeCompletion<string>>;
     ainvoke<T>(messages: Message[], output_format: {

package/dist/llm/anthropic/chat.js

@@ -1,9 +1,8 @@
-import Anthropic from '@anthropic-ai/sdk';
-import { zodToJsonSchema } from 'zod-to-json-schema';
+import Anthropic, { APIConnectionError, APIError, RateLimitError, } from '@anthropic-ai/sdk';
 import { ChatInvokeCompletion } from '../views.js';
-import { SystemMessage } from '../messages.js';
 import { AnthropicMessageSerializer } from './serializer.js';
 import { ModelProviderError, ModelRateLimitError } from '../exceptions.js';
+import { SchemaOptimizer, zodSchemaToJsonSchema } from '../schema.js';
 export class ChatAnthropic {
     model;
     provider = 'anthropic';
@@ -11,16 +10,29 @@ export class ChatAnthropic {
     maxTokens;
     temperature;
     topP;
+    seed;
+    removeMinItemsFromSchema;
+    removeDefaultsFromSchema;
     constructor(options = {}) {
-        const { model = 'claude-sonnet-4-20250514', apiKey, baseURL, maxTokens = 8192, temperature = null, topP = null, maxRetries = 10, } = options;
+        const normalizedOptions = typeof options === 'string' ? { model: options } : options;
+        const { model = 'claude-sonnet-4-20250514', apiKey = process.env.ANTHROPIC_API_KEY, authToken = process.env.ANTHROPIC_AUTH_TOKEN, baseURL, timeout, maxTokens = 8192, temperature = null, topP = null, seed = null, maxRetries = 10, defaultHeaders, defaultQuery, fetchImplementation, fetchOptions, removeMinItemsFromSchema = false, removeDefaultsFromSchema = false, } = normalizedOptions;
         this.model = model;
         this.maxTokens = maxTokens;
         this.temperature = temperature;
         this.topP = topP;
+        this.seed = seed;
+        this.removeMinItemsFromSchema = removeMinItemsFromSchema;
+        this.removeDefaultsFromSchema = removeDefaultsFromSchema;
         this.client = new Anthropic({
             apiKey,
+            authToken,
             baseURL,
+            timeout,
             maxRetries,
+            defaultHeaders,
+            defaultQuery,
+            ...(fetchImplementation ? { fetch: fetchImplementation } : {}),
+            ...(fetchOptions ? { fetchOptions } : {}),
         });
     }
     get name() {
@@ -29,6 +41,54 @@ export class ChatAnthropic {
     get model_name() {
         return this.model;
     }
+    getModelParams() {
+        const modelParams = {};
+        if (this.temperature !== null) {
+            modelParams.temperature = this.temperature;
+        }
+        if (this.topP !== null) {
+            modelParams.top_p = this.topP;
+        }
+        if (this.seed !== null) {
+            modelParams.seed = this.seed;
+        }
+        return modelParams;
+    }
+    getZodSchemaCandidate(output_format) {
+        const output = output_format;
+        if (output &&
+            typeof output === 'object' &&
+            typeof output.safeParse === 'function' &&
+            typeof output.parse === 'function') {
+            return output;
+        }
+        if (output &&
+            typeof output === 'object' &&
+            output.schema &&
+            typeof output.schema.safeParse === 'function' &&
+            typeof output.schema.parse === 'function') {
+            return output.schema;
+        }
+        return null;
+    }
+    parseOutput(output_format, payload) {
+        const output = output_format;
+        if (output &&
+            typeof output === 'object' &&
+            output.schema &&
+            typeof output.schema.parse === 'function') {
+            return output.schema.parse(payload);
+        }
+        return output.parse(payload);
+    }
+    getTextCompletion(response) {
+        const textBlock = response.content.find((block) => block.type === 'text');
+        if (textBlock && textBlock.type === 'text') {
+            return textBlock.text;
+        }
+        const firstBlock = response.content[0];
+        return firstBlock ? String(firstBlock) : '';
+    }
     getUsage(response) {
         const cacheReadTokens = response.usage.cache_read_input_tokens ?? 0;
         const cacheCreationTokens = response.usage.cache_creation_input_tokens ?? 0;
@@ -43,84 +103,92 @@ export class ChatAnthropic {
     }
     async ainvoke(messages, output_format, options = {}) {
         const serializer = new AnthropicMessageSerializer();
-        const [anthropicMessages] = serializer.serializeMessages(messages);
-        const systemMessage = messages.find((msg) => msg instanceof SystemMessage);
-        const system = systemMessage ? systemMessage.text : undefined;
+        const [anthropicMessages, systemPrompt] = serializer.serializeMessages(messages);
+        const zodSchemaCandidate = this.getZodSchemaCandidate(output_format);
         let tools = undefined;
         let toolChoice = undefined;
-        if (output_format && 'schema' in output_format && output_format.schema) {
-            // Assuming output_format is a Zod schema wrapper
+        if (output_format && zodSchemaCandidate) {
             try {
-                const jsonSchema = zodToJsonSchema(output_format, {
+                const rawJsonSchema = zodSchemaToJsonSchema(zodSchemaCandidate, {
                     name: 'Response',
                     target: 'jsonSchema7',
                 });
+                const optimizedJsonSchema = SchemaOptimizer.createOptimizedJsonSchema(rawJsonSchema, {
+                    removeMinItems: this.removeMinItemsFromSchema,
+                    removeDefaults: this.removeDefaultsFromSchema,
+                });
+                delete optimizedJsonSchema.title;
+                const toolName = output_format?.name || 'response';
                 tools = [
                     {
-                        name: 'response',
-                        description: 'The response to the user request',
-                        input_schema: jsonSchema,
+                        name: toolName,
+                        description: `Extract information in the format of ${toolName}`,
+                        input_schema: optimizedJsonSchema,
+                        cache_control: { type: 'ephemeral' },
                     },
                 ];
-                toolChoice = { type: 'tool', name: 'response' };
+                toolChoice = { type: 'tool', name: toolName };
             }
             catch (e) {
                 console.warn('Failed to convert output_format to JSON schema for Anthropic', e);
             }
         }
-        // Build model parameters
-        const modelParams = {};
-        if (this.temperature !== null) {
-            modelParams.temperature = this.temperature;
-        }
-        if (this.topP !== null) {
-            modelParams.top_p = this.topP;
+        const requestPayload = {
+            model: this.model,
+            max_tokens: this.maxTokens,
+            messages: anthropicMessages,
+            ...this.getModelParams(),
+        };
+        if (systemPrompt !== undefined) {
+            requestPayload.system = systemPrompt;
         }
-        // Add cache_control to tools if present
-        if (tools && tools.length > 0) {
-            tools = tools.map((tool, index) => {
-                if (index === tools.length - 1) {
-                    return {
-                        ...tool,
-                        cache_control: { type: 'ephemeral' },
-                    };
-                }
-                return tool;
-            });
+        if (tools?.length) {
+            requestPayload.tools = tools;
+            requestPayload.tool_choice = toolChoice;
         }
         try {
-            const response = await this.client.messages.create({
-                model: this.model,
-                max_tokens: this.maxTokens,
-                system: system,
-                messages: anthropicMessages,
-                tools: tools,
-                tool_choice: toolChoice,
-                ...modelParams,
-            }, options.signal ? { signal: options.signal } : undefined);
-            let completion = '';
-            // Handle tool use response
-            const toolUseBlock = response.content.find((block) => block.type === 'tool_use');
-            if (toolUseBlock && output_format) {
-                completion = output_format.parse(toolUseBlock.input);
+            const response = await this.client.messages.create(requestPayload, options.signal ? { signal: options.signal } : undefined);
+            let completion = this.getTextCompletion(response);
+            if (output_format) {
+                const toolUseBlock = response.content.find((block) => block.type === 'tool_use');
+                if (toolUseBlock && toolUseBlock.type === 'tool_use') {
+                    try {
+                        completion = this.parseOutput(output_format, toolUseBlock.input);
+                    }
+                    catch (error) {
+                        if (typeof toolUseBlock.input === 'string') {
+                            completion = this.parseOutput(output_format, JSON.parse(toolUseBlock.input));
+                        }
+                        else {
+                            throw error;
+                        }
+                    }
+                }
+                else if (tools?.length) {
+                    throw new ModelProviderError('Expected tool use in response but none found', 502, this.model);
+                }
+                else {
+                    completion = this.parseOutput(output_format, completion);
+                }
             }
             else {
-                // Fallback to text content
-                const textBlock = response.content.find((block) => block.type === 'text');
-                completion = textBlock ? textBlock.text : '';
+                completion = this.getTextCompletion(response);
             }
             const usage = this.getUsage(response);
-            return new ChatInvokeCompletion(completion, usage);
+            const stopReason = response.stop_reason ?? null;
+            return new ChatInvokeCompletion(completion, usage, null, null, stopReason);
         }
         catch (error) {
-            // Handle Anthropic-specific errors
-            if (error?.status === 429) {
+            if (error instanceof RateLimitError || error?.status === 429) {
                 throw new ModelRateLimitError(error?.message ?? 'Rate limit exceeded', 429, this.model);
             }
-            if (error?.status >= 500) {
-                throw new ModelProviderError(error?.message ?? 'Server error', error.status, this.model);
+            if (error instanceof APIConnectionError) {
+                throw new ModelProviderError(error?.message ?? 'Connection error', 502, this.model);
+            }
+            if (error instanceof APIError) {
+                throw new ModelProviderError(error?.message ?? 'Anthropic API error', error?.status ?? 502, this.model);
             }
-            throw new ModelProviderError(error?.message ?? String(error), error?.status ?? 500, this.model);
+            throw new ModelProviderError(error?.message ?? String(error), error?.status ?? 502, this.model);
         }
     }
 }

package/dist/llm/anthropic/serializer.d.ts

@@ -58,6 +58,8 @@ export declare class AnthropicMessageSerializer {
      * Parse base64 data URL to extract media type and data
      */
     private _parseBase64Url;
+    private _cloneMessage;
+    private _cloneContent;
     /**
      * Clean cache settings so only the last cache=true message remains cached
      *

package/dist/llm/anthropic/serializer.js

@@ -16,7 +16,7 @@
  *   - Subsequent: 1,000 tokens @ $3/M + 9,000 tokens @ $0.30/M = $0.0057
  *   - Savings: 81% cost reduction
  */
-import { AssistantMessage, ContentPartImageParam, ContentPartTextParam, UserMessage, SystemMessage, } from '../messages.js';
+import { AssistantMessage, ContentPartImageParam, ContentPartRefusalParam, ContentPartTextParam, FunctionCall, ImageURL, ToolCall, UserMessage, SystemMessage, } from '../messages.js';
 export class AnthropicMessageSerializer {
     /**
      * Serialize a list of messages, extracting any system message
@@ -26,7 +26,7 @@ export class AnthropicMessageSerializer {
      */
     serializeMessages(messages) {
         // Make deep copies to avoid modifying originals
-        const messagesCopy = messages.map((m) => ({ ...m }));
+        const messagesCopy = messages.map((message) => this._cloneMessage(message));
         // Separate system messages from normal messages
         const normalMessages = [];
         let systemMessage = null;
@@ -79,7 +79,7 @@ export class AnthropicMessageSerializer {
                     content.push({
                         type: 'text',
                         text: message.content,
-                        ...(message.cache
+                        ...(message.cache && !message.tool_calls?.length
                             ? { cache_control: this._serializeCacheControl(true) }
                             : {}),
                     });
@@ -99,11 +99,18 @@ export class AnthropicMessageSerializer {
                 message.tool_calls.forEach((toolCall, idx, arr) => {
                     const isLastToolCall = idx === arr.length - 1;
                     const useCache = message.cache && isLastToolCall;
+                    let toolInput;
+                    try {
+                        toolInput = JSON.parse(toolCall.functionCall.arguments);
+                    }
+                    catch {
+                        toolInput = { arguments: toolCall.functionCall.arguments };
+                    }
                     content.push({
                         type: 'tool_use',
                         id: toolCall.id,
                         name: toolCall.functionCall.name,
-                        input: JSON.parse(toolCall.functionCall.arguments),
+                        input: toolInput,
                         ...(useCache
                             ? { cache_control: this._serializeCacheControl(true) }
                             : {}),
@@ -120,9 +127,19 @@ export class AnthropicMessageSerializer {
                         : {}),
                 });
             }
+            const normalizedContent = (() => {
+                if (message.cache || content.length > 1) {
+                    return content;
+                }
+                const first = content[0];
+                if (first && first.type === 'text' && !first.cache_control) {
+                    return first.text;
+                }
+                return content;
+            })();
             return {
                 role: 'assistant',
-                content: content,
+                content: normalizedContent,
             };
         }
         throw new Error(`Unknown message type or unhandled role: ${message.role}`);
@@ -238,7 +255,7 @@ export class AnthropicMessageSerializer {
         if (!header || !data) {
             throw new Error(`Invalid base64 URL format: ${url}`);
         }
-        let mediaType = header.split(';')[0]?.replace('data:', '') || 'image/png';
+        let mediaType = header.split(';')[0]?.replace('data:', '') || 'image/jpeg';
         // Ensure it's a supported media type
         const supportedTypes = [
             'image/jpeg',
@@ -247,11 +264,66 @@ export class AnthropicMessageSerializer {
             'image/webp',
         ];
         if (!supportedTypes.includes(mediaType)) {
-            // Default to png if not recognized
-            mediaType = 'image/png';
+            // Default to jpeg if not recognized
+            mediaType = 'image/jpeg';
         }
         return [mediaType, data];
     }
+    _cloneMessage(message) {
+        if (message instanceof UserMessage) {
+            const clone = new UserMessage(this._cloneContent(message.content), message.name);
+            clone.cache = message.cache;
+            return clone;
+        }
+        if (message instanceof SystemMessage) {
+            const clone = new SystemMessage(typeof message.content === 'string'
+                ? message.content
+                : message.content.map((part) => new ContentPartTextParam(part.text)), message.name);
+            clone.cache = message.cache;
+            return clone;
+        }
+        if (message instanceof AssistantMessage) {
+            const clone = new AssistantMessage({
+                content: message.content === null
+                    ? null
+                    : typeof message.content === 'string'
+                        ? message.content
+                        : message.content.map((part) => {
+                            if (part instanceof ContentPartTextParam) {
+                                return new ContentPartTextParam(part.text);
+                            }
+                            if (part instanceof ContentPartRefusalParam) {
+                                return new ContentPartRefusalParam(part.refusal);
+                            }
+                            if (part instanceof ContentPartImageParam) {
+                                return new ContentPartImageParam(new ImageURL(part.image_url.url, part.image_url.detail, part.image_url.media_type));
+                            }
+                            return part;
+                        }),
+                tool_calls: message.tool_calls
+                    ? message.tool_calls.map((toolCall) => new ToolCall(toolCall.id, new FunctionCall(toolCall.functionCall.name, toolCall.functionCall.arguments)))
+                    : null,
+                refusal: message.refusal,
+            });
+            clone.cache = message.cache;
+            return clone;
+        }
+        return message;
+    }
+    _cloneContent(content) {
+        if (typeof content === 'string') {
+            return content;
+        }
+        return content.map((part) => {
+            if (part instanceof ContentPartTextParam) {
+                return new ContentPartTextParam(part.text);
+            }
+            if (part instanceof ContentPartRefusalParam) {
+                return new ContentPartRefusalParam(part.refusal);
+            }
+            return new ContentPartImageParam(new ImageURL(part.image_url.url, part.image_url.detail, part.image_url.media_type));
+        });
+    }
     /**
      * Clean cache settings so only the last cache=true message remains cached
      *
@@ -263,7 +335,7 @@ export class AnthropicMessageSerializer {
             return messages;
         }
         // Create deep copies to avoid modifying originals
-        const cleanedMessages = messages.map((msg) => ({ ...msg }));
+        const cleanedMessages = messages.map((msg) => this._cloneMessage(msg));
         // Find the last message with cache=true
         let lastCacheIndex = -1;
         for (let i = cleanedMessages.length - 1; i >= 0; i--) {

package/dist/llm/aws/chat-anthropic.d.ts

@@ -26,6 +26,14 @@ export interface ChatAnthropicBedrockConfig {
     model?: string;
     /** AWS region, defaults to us-east-1 */
     region?: string;
+    /** AWS access key ID */
+    awsAccessKeyId?: string;
+    /** AWS secret access key */
+    awsSecretAccessKey?: string;
+    /** AWS session token */
+    awsSessionToken?: string;
+    /** Retry attempts */
+    maxRetries?: number;
     /** Maximum tokens to generate */
     max_tokens?: number;
     /** Temperature for sampling (0-1) */
@@ -36,6 +44,10 @@ export interface ChatAnthropicBedrockConfig {
     top_k?: number | null;
     /** Stop sequences */
     stop_sequences?: string[] | null;
+    /** Remove minItems from schema for provider compatibility */
+    removeMinItemsFromSchema?: boolean;
+    /** Remove default from schema for provider compatibility */
+    removeDefaultsFromSchema?: boolean;
 }
 export declare class ChatAnthropicBedrock implements BaseChatModel {
     model: string;
@@ -46,10 +58,15 @@ export declare class ChatAnthropicBedrock implements BaseChatModel {
     private top_p;
     private top_k;
     private stop_sequences;
+    private removeMinItemsFromSchema;
+    private removeDefaultsFromSchema;
     constructor(config?: ChatAnthropicBedrockConfig);
     get name(): string;
     get model_name(): string;
     private _getInferenceParams;
+    private getZodSchemaCandidate;
+    private parseOutput;
+    private getTextCompletion;
     ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise<ChatInvokeCompletion<string>>;
     ainvoke<T>(messages: Message[], output_format: {
         parse: (input: string) => T;