browser-use 0.0.1 → 0.0.2

package/dist/llm/anthropic/serializer.js

@@ -0,0 +1,285 @@
+/**
+ * Anthropic Message Serializer with Prompt Caching Support
+ *
+ * This serializer converts custom message types to Anthropic's MessageParam format
+ * and implements Anthropic's Prompt Caching feature to reduce costs by up to 90%.
+ *
+ * Caching Strategy:
+ * - Only the last message with cache=true will have cache_control enabled
+ * - Caching is most effective for system prompts and large conversation histories
+ * - Cache writes cost 25% more, but cache reads cost 90% less
+ *
+ * Example cost savings:
+ * - Without caching: 10,000 tokens @ $3/M = $0.030 per request
+ * - With caching (90% hit rate):
+ *   - First request: 10,000 tokens @ $3.75/M (write) = $0.0375
+ *   - Subsequent: 1,000 tokens @ $3/M + 9,000 tokens @ $0.30/M = $0.0057
+ *   - Savings: 81% cost reduction
+ */
+import { AssistantMessage, ContentPartImageParam, ContentPartTextParam, UserMessage, SystemMessage, } from '../messages.js';
+export class AnthropicMessageSerializer {
+    /**
+     * Serialize a list of messages, extracting any system message
+     *
+     * @param messages - List of messages to serialize
+     * @returns Tuple of [messages, system_message]
+     */
+    serializeMessages(messages) {
+        // Make deep copies to avoid modifying originals
+        const messagesCopy = messages.map((m) => ({ ...m }));
+        // Separate system messages from normal messages
+        const normalMessages = [];
+        let systemMessage = null;
+        for (const message of messagesCopy) {
+            if (message instanceof SystemMessage) {
+                systemMessage = message;
+            }
+            else {
+                normalMessages.push(message);
+            }
+        }
+        // Clean cache messages so only the last cache=true message remains cached
+        const cleanedMessages = this._cleanCacheMessages(normalMessages);
+        // Serialize normal messages
+        const serializedMessages = cleanedMessages.map((msg) => this.serializeMessage(msg));
+        // Serialize system message
+        let serializedSystemMessage = undefined;
+        if (systemMessage) {
+            serializedSystemMessage = this._serializeContentToStr(systemMessage.content, systemMessage.cache);
+        }
+        return [serializedMessages, serializedSystemMessage];
+    }
+    /**
+     * Serialize a single message
+     */
+    serializeMessage(message) {
+        if (message instanceof UserMessage) {
+            return {
+                role: 'user',
+                content: Array.isArray(message.content)
+                    ? message.content.map((part, idx, arr) => {
+                        const isLastPart = idx === arr.length - 1;
+                        const useCache = message.cache && isLastPart;
+                        if (part instanceof ContentPartTextParam) {
+                            return this._serializeContentPartText(part, useCache);
+                        }
+                        if (part instanceof ContentPartImageParam) {
+                            return this._serializeContentPartImage(part);
+                        }
+                        return { type: 'text', text: '' };
+                    })
+                    : this._serializeContent(message.content, message.cache),
+            };
+        }
+        if (message instanceof AssistantMessage) {
+            const content = [];
+            // Add content blocks if present
+            if (message.content) {
+                if (typeof message.content === 'string') {
+                    content.push({
+                        type: 'text',
+                        text: message.content,
+                        ...(message.cache
+                            ? { cache_control: this._serializeCacheControl(true) }
+                            : {}),
+                    });
+                }
+                else if (Array.isArray(message.content)) {
+                    message.content.forEach((part, idx, arr) => {
+                        const isLastPart = idx === arr.length - 1;
+                        const useCache = message.cache && isLastPart && !message.tool_calls?.length;
+                        if (part instanceof ContentPartTextParam) {
+                            content.push(this._serializeContentPartText(part, useCache));
+                        }
+                    });
+                }
+            }
+            // Add tool use blocks if present
+            if (message.tool_calls) {
+                message.tool_calls.forEach((toolCall, idx, arr) => {
+                    const isLastToolCall = idx === arr.length - 1;
+                    const useCache = message.cache && isLastToolCall;
+                    content.push({
+                        type: 'tool_use',
+                        id: toolCall.id,
+                        name: toolCall.functionCall.name,
+                        input: JSON.parse(toolCall.functionCall.arguments),
+                        ...(useCache
+                            ? { cache_control: this._serializeCacheControl(true) }
+                            : {}),
+                    });
+                });
+            }
+            // If no content or tool calls, add empty text block
+            if (!content.length) {
+                content.push({
+                    type: 'text',
+                    text: '',
+                    ...(message.cache
+                        ? { cache_control: this._serializeCacheControl(true) }
+                        : {}),
+                });
+            }
+            return {
+                role: 'assistant',
+                content: content,
+            };
+        }
+        throw new Error(`Unknown message type or unhandled role: ${message.role}`);
+    }
+    /**
+     * Serialize cache control parameter
+     */
+    _serializeCacheControl(useCache) {
+        return useCache ? { type: 'ephemeral' } : undefined;
+    }
+    /**
+     * Serialize text content part with optional caching
+     */
+    _serializeContentPartText(part, useCache) {
+        return {
+            type: 'text',
+            text: part.text,
+            ...(useCache ? { cache_control: this._serializeCacheControl(true) } : {}),
+        };
+    }
+    /**
+     * Serialize image content part
+     */
+    _serializeContentPartImage(part) {
+        const url = part.image_url.url;
+        if (this._isBase64Image(url)) {
+            // Handle base64 encoded images
+            const [mediaType, data] = this._parseBase64Url(url);
+            return {
+                type: 'image',
+                source: {
+                    type: 'base64',
+                    media_type: mediaType,
+                    data: data,
+                },
+            };
+        }
+        else {
+            // Handle URL images
+            return {
+                type: 'image',
+                source: {
+                    type: 'url',
+                    url: url,
+                },
+            };
+        }
+    }
+    /**
+     * Serialize content (string or array) with optional caching
+     */
+    _serializeContent(content, useCache) {
+        if (typeof content === 'string') {
+            if (useCache) {
+                return [
+                    {
+                        type: 'text',
+                        text: content,
+                        cache_control: this._serializeCacheControl(true),
+                    },
+                ];
+            }
+            return content;
+        }
+        return content.map((part, idx, arr) => {
+            const isLastPart = idx === arr.length - 1;
+            const partUseCache = useCache && isLastPart;
+            if (part instanceof ContentPartTextParam) {
+                return this._serializeContentPartText(part, partUseCache);
+            }
+            else if (part instanceof ContentPartImageParam) {
+                return this._serializeContentPartImage(part);
+            }
+            return { type: 'text', text: '' };
+        });
+    }
+    /**
+     * Serialize content to string format (for system messages)
+     */
+    _serializeContentToStr(content, useCache) {
+        if (typeof content === 'string') {
+            if (useCache) {
+                return [
+                    {
+                        type: 'text',
+                        text: content,
+                        cache_control: this._serializeCacheControl(true),
+                    },
+                ];
+            }
+            return content;
+        }
+        return content.map((part, idx, arr) => {
+            const isLastPart = idx === arr.length - 1;
+            const partUseCache = useCache && isLastPart;
+            return this._serializeContentPartText(part, partUseCache);
+        });
+    }
+    /**
+     * Check if URL is a base64 encoded image
+     */
+    _isBase64Image(url) {
+        return url.startsWith('data:image/');
+    }
+    /**
+     * Parse base64 data URL to extract media type and data
+     */
+    _parseBase64Url(url) {
+        if (!url.startsWith('data:')) {
+            throw new Error(`Invalid base64 URL: ${url}`);
+        }
+        const [header, data] = url.split(',', 2);
+        if (!header || !data) {
+            throw new Error(`Invalid base64 URL format: ${url}`);
+        }
+        let mediaType = header.split(';')[0]?.replace('data:', '') || 'image/png';
+        // Ensure it's a supported media type
+        const supportedTypes = [
+            'image/jpeg',
+            'image/png',
+            'image/gif',
+            'image/webp',
+        ];
+        if (!supportedTypes.includes(mediaType)) {
+            // Default to png if not recognized
+            mediaType = 'image/png';
+        }
+        return [mediaType, data];
+    }
+    /**
+     * Clean cache settings so only the last cache=true message remains cached
+     *
+     * Because of how Claude caching works, only the last cache message matters.
+     * This method automatically removes cache=True from all messages except the last one.
+     */
+    _cleanCacheMessages(messages) {
+        if (!messages.length) {
+            return messages;
+        }
+        // Create deep copies to avoid modifying originals
+        const cleanedMessages = messages.map((msg) => ({ ...msg }));
+        // Find the last message with cache=true
+        let lastCacheIndex = -1;
+        for (let i = cleanedMessages.length - 1; i >= 0; i--) {
+            if (cleanedMessages[i].cache) {
+                lastCacheIndex = i;
+                break;
+            }
+        }
+        // If we found a cached message, disable cache for all others
+        if (lastCacheIndex !== -1) {
+            for (let i = 0; i < cleanedMessages.length; i++) {
+                if (i !== lastCacheIndex && cleanedMessages[i].cache) {
+                    cleanedMessages[i].cache = false;
+                }
+            }
+        }
+        return cleanedMessages;
+    }
+}

package/dist/llm/aws/chat-anthropic.d.ts

@@ -0,0 +1,61 @@
+/**
+ * AWS Bedrock Anthropic Claude chat model.
+ *
+ * This is a convenience class that provides Claude-specific defaults
+ * for the AWS Bedrock service. It inherits all functionality from
+ * ChatBedrockConverse but sets Anthropic Claude as the default model
+ * and uses the Anthropic message serializer for better compatibility.
+ *
+ * Usage:
+ * ```typescript
+ * import { ChatAnthropicBedrock } from './llm/aws/chat-anthropic.js';
+ *
+ * const llm = new ChatAnthropicBedrock({
+ *   model: 'anthropic.claude-3-5-sonnet-20241022-v2:0',
+ *   region: 'us-east-1'
+ * });
+ *
+ * const response = await llm.ainvoke(messages);
+ * ```
+ */
+import type { BaseChatModel, ChatInvokeOptions } from '../base.js';
+import { ChatInvokeCompletion } from '../views.js';
+import { type Message } from '../messages.js';
+export interface ChatAnthropicBedrockConfig {
+    /** Model ID, defaults to Claude 3.5 Sonnet */
+    model?: string;
+    /** AWS region, defaults to us-east-1 */
+    region?: string;
+    /** Maximum tokens to generate */
+    max_tokens?: number;
+    /** Temperature for sampling (0-1) */
+    temperature?: number | null;
+    /** Top-p sampling parameter */
+    top_p?: number | null;
+    /** Top-k sampling parameter */
+    top_k?: number | null;
+    /** Stop sequences */
+    stop_sequences?: string[] | null;
+}
+export declare class ChatAnthropicBedrock implements BaseChatModel {
+    model: string;
+    provider: string;
+    private client;
+    private max_tokens;
+    private temperature;
+    private top_p;
+    private top_k;
+    private stop_sequences;
+    constructor(config?: ChatAnthropicBedrockConfig);
+    get name(): string;
+    get model_name(): string;
+    private _getInferenceParams;
+    ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise<ChatInvokeCompletion<string>>;
+    ainvoke<T>(messages: Message[], output_format: {
+        parse: (input: string) => T;
+    }, options?: ChatInvokeOptions): Promise<ChatInvokeCompletion<T>>;
+    /**
+     * Simple Zod to JSON Schema conversion for structured output
+     */
+    private _zodToJsonSchema;
+}

package/dist/llm/aws/chat-anthropic.js

@@ -0,0 +1,176 @@
+/**
+ * AWS Bedrock Anthropic Claude chat model.
+ *
+ * This is a convenience class that provides Claude-specific defaults
+ * for the AWS Bedrock service. It inherits all functionality from
+ * ChatBedrockConverse but sets Anthropic Claude as the default model
+ * and uses the Anthropic message serializer for better compatibility.
+ *
+ * Usage:
+ * ```typescript
+ * import { ChatAnthropicBedrock } from './llm/aws/chat-anthropic.js';
+ *
+ * const llm = new ChatAnthropicBedrock({
+ *   model: 'anthropic.claude-3-5-sonnet-20241022-v2:0',
+ *   region: 'us-east-1'
+ * });
+ *
+ * const response = await llm.ainvoke(messages);
+ * ```
+ */
+import { BedrockRuntimeClient, ConverseCommand, } from '@aws-sdk/client-bedrock-runtime';
+import { ChatInvokeCompletion } from '../views.js';
+import { AnthropicMessageSerializer } from '../anthropic/serializer.js';
+export class ChatAnthropicBedrock {
+    model;
+    provider = 'anthropic_bedrock';
+    client;
+    max_tokens;
+    temperature;
+    top_p;
+    top_k;
+    stop_sequences;
+    constructor(config = {}) {
+        // Anthropic Claude specific defaults
+        this.model = config.model || 'anthropic.claude-3-5-sonnet-20241022-v2:0';
+        this.max_tokens = config.max_tokens || 8192;
+        this.temperature =
+            config.temperature === undefined ? null : config.temperature;
+        this.top_p = config.top_p === undefined ? null : config.top_p;
+        this.top_k = config.top_k === undefined ? null : config.top_k;
+        this.stop_sequences =
+            config.stop_sequences === undefined ? null : config.stop_sequences;
+        const region = config.region || process.env.AWS_REGION || 'us-east-1';
+        this.client = new BedrockRuntimeClient({ region });
+    }
+    get name() {
+        return this.model;
+    }
+    get model_name() {
+        return this.model;
+    }
+    _getInferenceParams() {
+        const params = {
+            maxTokens: this.max_tokens,
+        };
+        if (this.temperature !== null) {
+            params.temperature = this.temperature;
+        }
+        if (this.top_p !== null) {
+            params.topP = this.top_p;
+        }
+        if (this.stop_sequences !== null && this.stop_sequences.length > 0) {
+            params.stopSequences = this.stop_sequences;
+        }
+        return params;
+    }
+    async ainvoke(messages, output_format, options = {}) {
+        // Use Anthropic-specific message serializer
+        const serializer = new AnthropicMessageSerializer();
+        const [anthropicMessages, systemPrompt] = serializer.serializeMessages(messages);
+        // Convert Anthropic messages to Bedrock format
+        const bedrockMessages = anthropicMessages.map((msg) => {
+            const content = Array.isArray(msg.content)
+                ? msg.content.map((block) => {
+                    if (block.type === 'text') {
+                        return { text: block.text };
+                    }
+                    else if (block.type === 'image') {
+                        // Handle image blocks if needed
+                        return { text: '[Image]' };
+                    }
+                    return { text: String(block) };
+                })
+                : [{ text: msg.content }];
+            return {
+                role: msg.role,
+                content,
+            };
+        });
+        // Handle system message
+        const system = systemPrompt
+            ? [
+                {
+                    text: typeof systemPrompt === 'string'
+                        ? systemPrompt
+                        : JSON.stringify(systemPrompt),
+                },
+            ]
+            : undefined;
+        let tools = undefined;
+        let toolConfig = undefined;
+        if (output_format && 'schema' in output_format) {
+            // Structured output using tools
+            try {
+                const schema = output_format.schema?.shape
+                    ? this._zodToJsonSchema(output_format.schema)
+                    : {};
+                tools = [
+                    {
+                        toolSpec: {
+                            name: 'extract_structured_data',
+                            description: 'Extract structured data from the response',
+                            inputSchema: {
+                                json: schema,
+                            },
+                        },
+                    },
+                ];
+                toolConfig = {
+                    tools: tools,
+                    toolChoice: { tool: { name: 'extract_structured_data' } },
+                };
+            }
+            catch (e) {
+                console.warn('Failed to convert output_format to JSON schema', e);
+            }
+        }
+        const command = new ConverseCommand({
+            modelId: this.model,
+            messages: bedrockMessages,
+            system: system,
+            toolConfig: toolConfig,
+            inferenceConfig: this._getInferenceParams(),
+        });
+        const response = await this.client.send(command, options.signal ? { abortSignal: options.signal } : undefined);
+        let completion = '';
+        if (response.output?.message?.content) {
+            // Check for tool use (structured output)
+            const toolUseBlock = response.output.message.content.find((block) => block.toolUse);
+            if (toolUseBlock && toolUseBlock.toolUse && output_format) {
+                completion = output_format.parse(toolUseBlock.toolUse.input);
+            }
+            else {
+                // Fallback to text
+                const textBlock = response.output.message.content.find((block) => block.text);
+                completion = textBlock?.text || '';
+            }
+        }
+        return new ChatInvokeCompletion(completion, {
+            prompt_tokens: response.usage?.inputTokens ?? 0,
+            completion_tokens: response.usage?.outputTokens ?? 0,
+            total_tokens: response.usage?.totalTokens ?? 0,
+        });
+    }
+    /**
+     * Simple Zod to JSON Schema conversion for structured output
+     */
+    _zodToJsonSchema(schema) {
+        // This is a simplified version - you might want to use zod-to-json-schema package
+        try {
+            const { zodToJsonSchema } = require('zod-to-json-schema');
+            return zodToJsonSchema(schema, {
+                name: 'Response',
+                target: 'jsonSchema7',
+            });
+        }
+        catch {
+            // Fallback if zod-to-json-schema is not available
+            return {
+                type: 'object',
+                properties: {},
+                required: [],
+            };
+        }
+    }
+}

package/dist/llm/aws/chat-bedrock.d.ts

@@ -0,0 +1,15 @@
+import type { BaseChatModel, ChatInvokeOptions } from '../base.js';
+import { ChatInvokeCompletion } from '../views.js';
+import { type Message } from '../messages.js';
+export declare class ChatBedrockConverse implements BaseChatModel {
+    model: string;
+    provider: string;
+    private client;
+    constructor(model?: string, region?: string);
+    get name(): string;
+    get model_name(): string;
+    ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise<ChatInvokeCompletion<string>>;
+    ainvoke<T>(messages: Message[], output_format: {
+        parse: (input: string) => T;
+    } | undefined, options?: ChatInvokeOptions): Promise<ChatInvokeCompletion<T>>;
+}

package/dist/llm/aws/chat-bedrock.js

@@ -0,0 +1,80 @@
+import { BedrockRuntimeClient, ConverseCommand, } from '@aws-sdk/client-bedrock-runtime';
+import { zodToJsonSchema } from 'zod-to-json-schema';
+import { ChatInvokeCompletion } from '../views.js';
+import { SystemMessage } from '../messages.js';
+import { AWSBedrockMessageSerializer } from './serializer.js';
+export class ChatBedrockConverse {
+    model;
+    provider = 'aws';
+    client;
+    constructor(model = 'anthropic.claude-3-5-sonnet-20240620-v1:0', region = process.env.AWS_REGION || 'us-east-1') {
+        this.model = model;
+        this.client = new BedrockRuntimeClient({ region });
+    }
+    get name() {
+        return this.model;
+    }
+    get model_name() {
+        return this.model;
+    }
+    async ainvoke(messages, output_format, options = {}) {
+        const serializer = new AWSBedrockMessageSerializer();
+        const bedrockMessages = serializer.serialize(messages);
+        const systemMessage = messages.find((msg) => msg instanceof SystemMessage);
+        const system = systemMessage ? [{ text: systemMessage.text }] : undefined;
+        let tools = undefined;
+        let toolConfig = undefined;
+        if (output_format && 'schema' in output_format && output_format.schema) {
+            try {
+                const jsonSchema = zodToJsonSchema(output_format, {
+                    name: 'Response',
+                    target: 'jsonSchema7',
+                });
+                tools = [
+                    {
+                        toolSpec: {
+                            name: 'response',
+                            description: 'The response to the user request',
+                            inputSchema: {
+                                json: jsonSchema,
+                            },
+                        },
+                    },
+                ];
+                toolConfig = {
+                    tools: tools,
+                    toolChoice: { tool: { name: 'response' } },
+                };
+            }
+            catch (e) {
+                console.warn('Failed to convert output_format to JSON schema for AWS Bedrock', e);
+            }
+        }
+        const command = new ConverseCommand({
+            modelId: this.model,
+            messages: bedrockMessages,
+            system: system,
+            toolConfig: toolConfig,
+        });
+        const response = await this.client.send(command, options.signal ? { abortSignal: options.signal } : undefined);
+        let completion = '';
+        if (response.output?.message?.content) {
+            // Check for tool use
+            const toolUseBlock = response.output.message.content.find((block) => block.toolUse);
+            if (toolUseBlock && toolUseBlock.toolUse && output_format) {
+                completion = output_format.parse(toolUseBlock.toolUse.input);
+            }
+            else {
+                // Fallback to text
+                const textBlock = response.output.message.content.find((block) => block.text);
+                completion = textBlock?.text || '';
+            }
+        }
+        const usage = response.usage || {};
+        return new ChatInvokeCompletion(completion, {
+            prompt_tokens: usage.inputTokens || 0,
+            completion_tokens: usage.outputTokens || 0,
+            total_tokens: usage.totalTokens || 0,
+        });
+    }
+}

package/dist/llm/aws/index.d.ts

@@ -0,0 +1,3 @@
+export * from './chat-bedrock.js';
+export * from './chat-anthropic.js';
+export * from './serializer.js';

package/dist/llm/aws/index.js

@@ -0,0 +1,3 @@
+export * from './chat-bedrock.js';
+export * from './chat-anthropic.js';
+export * from './serializer.js';

package/dist/llm/aws/serializer.d.ts

@@ -0,0 +1,5 @@
+import { type Message } from '../messages.js';
+export declare class AWSBedrockMessageSerializer {
+    serialize(messages: Message[]): any[];
+    private serializeMessage;
+}