browser-use 0.0.1 → 0.1.0

package/dist/llm/aws/serializer.js

@@ -0,0 +1,68 @@
+import { AssistantMessage, ContentPartImageParam, ContentPartTextParam, SystemMessage, UserMessage, } from '../messages.js';
+// AWS Bedrock types are a bit complex and vary by model.
+// We will define a generic structure that fits most Bedrock models (like Claude on Bedrock).
+export class AWSBedrockMessageSerializer {
+    serialize(messages) {
+        return messages
+            .filter((msg) => !(msg instanceof SystemMessage)) // System messages are handled separately usually
+            .map((message) => this.serializeMessage(message));
+    }
+    serializeMessage(message) {
+        if (message instanceof UserMessage) {
+            return {
+                role: 'user',
+                content: Array.isArray(message.content)
+                    ? message.content.map((part) => {
+                        if (part instanceof ContentPartTextParam) {
+                            return { text: part.text };
+                        }
+                        if (part instanceof ContentPartImageParam) {
+                            const mediaType = part.image_url.media_type;
+                            const data = part.image_url.url.split(',')[1];
+                            return {
+                                image: {
+                                    format: mediaType.split('/')[1], // e.g. 'png' from 'image/png'
+                                    source: {
+                                        bytes: Buffer.from(data, 'base64'),
+                                    },
+                                },
+                            };
+                        }
+                        return { text: '' };
+                    })
+                    : [{ text: message.content }],
+            };
+        }
+        if (message instanceof AssistantMessage) {
+            const content = [];
+            if (message.content) {
+                if (typeof message.content === 'string') {
+                    content.push({ text: message.content });
+                }
+                else if (Array.isArray(message.content)) {
+                    message.content.forEach((part) => {
+                        if (part instanceof ContentPartTextParam) {
+                            content.push({ text: part.text });
+                        }
+                    });
+                }
+            }
+            if (message.tool_calls) {
+                message.tool_calls.forEach((toolCall) => {
+                    content.push({
+                        toolUse: {
+                            toolUseId: toolCall.id,
+                            name: toolCall.functionCall.name,
+                            input: JSON.parse(toolCall.functionCall.arguments),
+                        },
+                    });
+                });
+            }
+            return {
+                role: 'assistant',
+                content: content,
+            };
+        }
+        throw new Error(`Unknown message type: ${message.constructor.name}`);
+    }
+}

package/dist/llm/azure/chat.d.ts

@@ -0,0 +1,15 @@
+import type { BaseChatModel, ChatInvokeOptions } from '../base.js';
+import { ChatInvokeCompletion } from '../views.js';
+import type { Message } from '../messages.js';
+export declare class ChatAzure implements BaseChatModel {
+    model: string;
+    provider: string;
+    private client;
+    constructor(model?: string);
+    get name(): string;
+    get model_name(): string;
+    ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise<ChatInvokeCompletion<string>>;
+    ainvoke<T>(messages: Message[], output_format: {
+        parse: (input: string) => T;
+    } | undefined, options?: ChatInvokeOptions): Promise<ChatInvokeCompletion<T>>;
+}

package/dist/llm/azure/chat.js

@@ -0,0 +1,83 @@
+import { AzureOpenAI } from 'openai';
+import { ChatInvokeCompletion } from '../views.js';
+import { OpenAIMessageSerializer } from '../openai/serializer.js';
+export class ChatAzure {
+    model;
+    provider = 'azure';
+    client;
+    constructor(model = 'gpt-4o') {
+        this.model = model;
+        this.client = new AzureOpenAI({
+            apiKey: process.env.AZURE_OPENAI_API_KEY,
+            endpoint: process.env.AZURE_OPENAI_ENDPOINT,
+            apiVersion: process.env.AZURE_OPENAI_API_VERSION || '2024-05-01-preview',
+            deployment: model,
+        });
+    }
+    get name() {
+        return this.model;
+    }
+    get model_name() {
+        return this.model;
+    }
+    async ainvoke(messages, output_format, options = {}) {
+        const serializer = new OpenAIMessageSerializer();
+        const openaiMessages = serializer.serialize(messages);
+        // Use simple json_object format for better compatibility with Azure
+        // json_schema format may not be supported on all Azure API versions/deployments
+        const responseFormat = output_format
+            ? { type: 'json_object' }
+            : undefined;
+        const response = await this.client.chat.completions.create({
+            model: this.model,
+            messages: openaiMessages,
+            response_format: responseFormat,
+        }, options.signal ? { signal: options.signal } : undefined);
+        const content = response.choices[0].message.content || '';
+        let completion = content;
+        if (output_format) {
+            try {
+                // Extract JSON from the response
+                let jsonText = content.trim();
+                // Handle markdown fenced code blocks
+                const fencedMatch = jsonText.match(/```(?:json)?\s*([\s\S]*?)```/);
+                if (fencedMatch && fencedMatch[1]) {
+                    jsonText = fencedMatch[1].trim();
+                }
+                // Extract JSON object/array from the text
+                const firstBrace = jsonText.indexOf('{');
+                const firstBracket = jsonText.indexOf('[');
+                const lastBrace = jsonText.lastIndexOf('}');
+                const lastBracket = jsonText.lastIndexOf(']');
+                // Determine if it's an object or array
+                let startIdx = -1;
+                let endIdx = -1;
+                if (firstBrace !== -1 && (firstBracket === -1 || firstBrace < firstBracket)) {
+                    // It's an object
+                    startIdx = firstBrace;
+                    endIdx = lastBrace;
+                }
+                else if (firstBracket !== -1) {
+                    // It's an array
+                    startIdx = firstBracket;
+                    endIdx = lastBracket;
+                }
+                if (startIdx !== -1 && endIdx !== -1 && endIdx > startIdx) {
+                    jsonText = jsonText.slice(startIdx, endIdx + 1);
+                }
+                const parsedJson = JSON.parse(jsonText);
+                completion = output_format.parse(parsedJson);
+            }
+            catch (e) {
+                console.error('Failed to parse Azure completion:', e);
+                console.error('Raw content:', content.substring(0, 500));
+                throw new Error(`Failed to parse LLM completion as JSON: ${e}`);
+            }
+        }
+        return new ChatInvokeCompletion(completion, {
+            prompt_tokens: response.usage?.prompt_tokens ?? 0,
+            completion_tokens: response.usage?.completion_tokens ?? 0,
+            total_tokens: response.usage?.total_tokens ?? 0,
+        });
+    }
+}

package/dist/llm/azure/index.d.ts

@@ -0,0 +1 @@
+export * from './chat.js';

package/dist/llm/azure/index.js

@@ -0,0 +1 @@
+export * from './chat.js';

package/dist/llm/base.d.ts

@@ -0,0 +1,16 @@
+import type { ChatInvokeCompletion } from './views.js';
+import type { Message } from './messages.js';
+export interface ChatInvokeOptions {
+    signal?: AbortSignal;
+}
+export interface BaseChatModel {
+    model: string;
+    _verified_api_keys?: boolean;
+    get provider(): string;
+    get name(): string;
+    get model_name(): string;
+    ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise<ChatInvokeCompletion<string>>;
+    ainvoke<T>(messages: Message[], output_format: {
+        parse: (input: string) => T;
+    } | undefined, options?: ChatInvokeOptions): Promise<ChatInvokeCompletion<T>>;
+}

package/dist/llm/base.js

@@ -0,0 +1 @@
+export {};

package/dist/llm/deepseek/chat.d.ts

@@ -0,0 +1,15 @@
+import type { BaseChatModel, ChatInvokeOptions } from '../base.js';
+import { ChatInvokeCompletion } from '../views.js';
+import type { Message } from '../messages.js';
+export declare class ChatDeepSeek implements BaseChatModel {
+    model: string;
+    provider: string;
+    private client;
+    constructor(model?: string);
+    get name(): string;
+    get model_name(): string;
+    ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise<ChatInvokeCompletion<string>>;
+    ainvoke<T>(messages: Message[], output_format: {
+        parse: (input: string) => T;
+    } | undefined, options?: ChatInvokeOptions): Promise<ChatInvokeCompletion<T>>;
+}

package/dist/llm/deepseek/chat.js

@@ -0,0 +1,51 @@
+import OpenAI from 'openai';
+import { ChatInvokeCompletion } from '../views.js';
+import { DeepSeekMessageSerializer } from './serializer.js';
+export class ChatDeepSeek {
+    model;
+    provider = 'deepseek';
+    client;
+    constructor(model = 'deepseek-chat') {
+        this.model = model;
+        this.client = new OpenAI({
+            apiKey: process.env.DEEPSEEK_API_KEY,
+            baseURL: 'https://api.deepseek.com',
+        });
+    }
+    get name() {
+        return this.model;
+    }
+    get model_name() {
+        return this.model;
+    }
+    async ainvoke(messages, output_format, options = {}) {
+        const serializer = new DeepSeekMessageSerializer();
+        const deepseekMessages = serializer.serialize(messages);
+        let responseFormat = undefined;
+        if (output_format && 'schema' in output_format && output_format.schema) {
+            // DeepSeek supports json_object
+            responseFormat = { type: 'json_object' };
+        }
+        const response = await this.client.chat.completions.create({
+            model: this.model,
+            messages: deepseekMessages,
+            response_format: responseFormat,
+        }, options.signal ? { signal: options.signal } : undefined);
+        const content = response.choices[0].message.content || '';
+        let completion = content;
+        if (output_format) {
+            try {
+                completion = output_format.parse(JSON.parse(content));
+            }
+            catch (e) {
+                console.error('Failed to parse completion', e);
+                throw e;
+            }
+        }
+        return new ChatInvokeCompletion(completion, {
+            prompt_tokens: response.usage?.prompt_tokens ?? 0,
+            completion_tokens: response.usage?.completion_tokens ?? 0,
+            total_tokens: response.usage?.total_tokens ?? 0,
+        });
+    }
+}

package/dist/llm/deepseek/index.d.ts

@@ -0,0 +1,2 @@
+export * from './chat.js';
+export * from './serializer.js';

package/dist/llm/deepseek/index.js

@@ -0,0 +1,2 @@
+export * from './chat.js';
+export * from './serializer.js';

package/dist/llm/deepseek/serializer.d.ts

@@ -0,0 +1,6 @@
+import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs';
+import { type Message } from '../messages.js';
+export declare class DeepSeekMessageSerializer {
+    serialize(messages: Message[]): ChatCompletionMessageParam[];
+    private serializeMessage;
+}

package/dist/llm/deepseek/serializer.js

@@ -0,0 +1,57 @@
+import { AssistantMessage, ContentPartImageParam, ContentPartTextParam, SystemMessage, UserMessage, } from '../messages.js';
+export class DeepSeekMessageSerializer {
+    serialize(messages) {
+        return messages.map((message) => this.serializeMessage(message));
+    }
+    serializeMessage(message) {
+        if (message instanceof UserMessage) {
+            return {
+                role: 'user',
+                content: Array.isArray(message.content)
+                    ? message.content.map((part) => {
+                        if (part instanceof ContentPartTextParam) {
+                            return { type: 'text', text: part.text };
+                        }
+                        // DeepSeek might not support images in all models, but we'll include it for now
+                        if (part instanceof ContentPartImageParam) {
+                            return {
+                                type: 'image_url',
+                                image_url: {
+                                    url: part.image_url.url,
+                                    detail: part.image_url.detail,
+                                },
+                            };
+                        }
+                        return { type: 'text', text: '' };
+                    })
+                    : message.content,
+                name: message.name || undefined,
+            };
+        }
+        if (message instanceof SystemMessage) {
+            return {
+                role: 'system',
+                content: Array.isArray(message.content)
+                    ? message.content.map((part) => part.text).join('\n')
+                    : message.content,
+                name: message.name || undefined,
+            };
+        }
+        if (message instanceof AssistantMessage) {
+            return {
+                role: 'assistant',
+                content: typeof message.content === 'string' ? message.content : null,
+                // DeepSeek supports tool calls in newer models
+                tool_calls: message.tool_calls?.map((toolCall) => ({
+                    id: toolCall.id,
+                    type: 'function',
+                    function: {
+                        name: toolCall.functionCall.name,
+                        arguments: toolCall.functionCall.arguments,
+                    },
+                })),
+            };
+        }
+        throw new Error(`Unknown message type: ${message.constructor.name}`);
+    }
+}

package/dist/llm/exceptions.d.ts

@@ -0,0 +1,10 @@
+export declare class ModelError extends Error {
+}
+export declare class ModelProviderError extends ModelError {
+    statusCode: number;
+    model: string | null;
+    constructor(message: string, statusCode?: number, model?: string | null);
+}
+export declare class ModelRateLimitError extends ModelProviderError {
+    constructor(message: string, statusCode?: number, model?: string | null);
+}

package/dist/llm/exceptions.js

@@ -0,0 +1,18 @@
+export class ModelError extends Error {
+}
+export class ModelProviderError extends ModelError {
+    statusCode;
+    model;
+    constructor(message, statusCode = 502, model = null) {
+        super(message);
+        this.statusCode = statusCode;
+        this.model = model;
+        this.name = 'ModelProviderError';
+    }
+}
+export class ModelRateLimitError extends ModelProviderError {
+    constructor(message, statusCode = 429, model = null) {
+        super(message, statusCode, model);
+        this.name = 'ModelRateLimitError';
+    }
+}

package/dist/llm/google/chat.d.ts

@@ -0,0 +1,20 @@
+import type { BaseChatModel, ChatInvokeOptions } from '../base.js';
+import { ChatInvokeCompletion } from '../views.js';
+import { type Message } from '../messages.js';
+export declare class ChatGoogle implements BaseChatModel {
+    model: string;
+    provider: string;
+    private client;
+    constructor(model?: string);
+    get name(): string;
+    get model_name(): string;
+    /**
+     * Clean up JSON schema for Google's format
+     * Google API has specific requirements for responseSchema
+     */
+    private _cleanSchemaForGoogle;
+    ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise<ChatInvokeCompletion<string>>;
+    ainvoke<T>(messages: Message[], output_format: {
+        parse: (input: string) => T;
+    } | undefined, options?: ChatInvokeOptions): Promise<ChatInvokeCompletion<T>>;
+}

package/dist/llm/google/chat.js

@@ -0,0 +1,144 @@
+import { GoogleGenAI, } from '@google/genai';
+import { zodToJsonSchema } from 'zod-to-json-schema';
+import { ChatInvokeCompletion } from '../views.js';
+import { SystemMessage } from '../messages.js';
+import { GoogleMessageSerializer } from './serializer.js';
+export class ChatGoogle {
+    model;
+    provider = 'google';
+    client;
+    constructor(model = 'gemini-2.5-flash') {
+        this.model = model;
+        const apiVersion = process.env.GOOGLE_API_VERSION || 'v1';
+        const baseUrl = process.env.GOOGLE_API_BASE_URL;
+        this.client = new GoogleGenAI({
+            apiKey: process.env.GOOGLE_API_KEY || '',
+            ...(baseUrl ? { baseUrl } : {}),
+            ...(apiVersion ? { apiVersion } : {}),
+        });
+    }
+    get name() {
+        return this.model;
+    }
+    get model_name() {
+        return this.model;
+    }
+    /**
+     * Clean up JSON schema for Google's format
+     * Google API has specific requirements for responseSchema
+     */
+    _cleanSchemaForGoogle(schema) {
+        if (!schema || typeof schema !== 'object') {
+            return schema;
+        }
+        const cleaned = {};
+        for (const [key, value] of Object.entries(schema)) {
+            // Skip unsupported keys
+            if (key === '$schema' ||
+                key === 'additionalProperties' ||
+                key === '$ref' ||
+                key === 'definitions') {
+                continue;
+            }
+            if (key === 'properties' && typeof value === 'object') {
+                cleaned.properties = {};
+                for (const [propKey, propValue] of Object.entries(value)) {
+                    cleaned.properties[propKey] = this._cleanSchemaForGoogle(propValue);
+                }
+            }
+            else if (key === 'items' && typeof value === 'object') {
+                cleaned.items = this._cleanSchemaForGoogle(value);
+            }
+            else if (typeof value === 'object' && !Array.isArray(value)) {
+                cleaned[key] = this._cleanSchemaForGoogle(value);
+            }
+            else {
+                cleaned[key] = value;
+            }
+        }
+        return cleaned;
+    }
+    async ainvoke(messages, output_format, options = {}) {
+        const serializer = new GoogleMessageSerializer();
+        const contents = serializer.serialize(messages);
+        const systemMessage = messages.find((msg) => msg instanceof SystemMessage);
+        const systemInstruction = systemMessage ? systemMessage.text : undefined;
+        let tools = undefined;
+        let toolConfig = undefined;
+        // For Google, we need to be more explicit about JSON output
+        // The generationConfig with responseSchema helps enforce JSON structure
+        const generationConfig = {
+            responseMimeType: 'application/json',
+        };
+        // Try to get schema from output_format
+        const schemaForJson = output_format && 'schema' in output_format && output_format.schema
+            ? output_format.schema
+            : null;
+        if (schemaForJson) {
+            try {
+                const jsonSchema = zodToJsonSchema(schemaForJson);
+                // Clean up the schema for Google's format
+                const cleanSchema = this._cleanSchemaForGoogle(jsonSchema);
+                generationConfig.responseSchema = cleanSchema;
+            }
+            catch (e) {
+                console.warn('Failed to set responseSchema', e);
+            }
+        }
+        const request = {
+            model: this.model,
+            contents,
+        };
+        if (systemInstruction) {
+            request.systemInstruction = {
+                role: 'system',
+                parts: [{ text: systemInstruction }],
+            };
+        }
+        if (Object.keys(generationConfig).length > 0) {
+            request.generationConfig = generationConfig;
+        }
+        const result = await this.client.models.generateContent(request, options.signal ? { signal: options.signal } : undefined);
+        // Extract text from first candidate
+        const candidate = result.candidates?.[0];
+        const textParts = candidate?.content?.parts?.filter((p) => p.text) || [];
+        const text = textParts.map((p) => p.text).join('');
+        let completion = text;
+        if (output_format) {
+            try {
+                let parsed = text;
+                if (generationConfig.responseMimeType === 'application/json') {
+                    let jsonText = text.trim();
+                    // Handle markdown code fences like ```json ... ```
+                    const fencedMatch = jsonText.match(/```(?:json)?\s*([\s\S]*?)```/i);
+                    if (fencedMatch && fencedMatch[1]) {
+                        jsonText = fencedMatch[1].trim();
+                    }
+                    // Try to extract JSON object from text
+                    const firstBrace = jsonText.indexOf('{');
+                    const lastBrace = jsonText.lastIndexOf('}');
+                    if (firstBrace !== -1 && lastBrace !== -1 && lastBrace > firstBrace) {
+                        jsonText = jsonText.slice(firstBrace, lastBrace + 1);
+                    }
+                    else {
+                        // If no JSON object found, the model returned plain text
+                        // Try to wrap it in a minimal valid structure
+                        console.warn('Google LLM returned plain text instead of JSON. Raw response:', text.slice(0, 200));
+                        throw new Error(`Expected JSON response but got plain text: "${text.slice(0, 50)}..."`);
+                    }
+                    parsed = JSON.parse(jsonText);
+                }
+                completion = output_format.parse(parsed);
+            }
+            catch (e) {
+                console.error('Failed to parse completion', e);
+                throw e;
+            }
+        }
+        return new ChatInvokeCompletion(completion, {
+            prompt_tokens: result.usageMetadata?.promptTokenCount ?? 0,
+            completion_tokens: result.usageMetadata?.candidatesTokenCount ?? 0,
+            total_tokens: result.usageMetadata?.totalTokenCount ?? 0,
+        });
+    }
+}

package/dist/llm/google/index.d.ts

@@ -0,0 +1,2 @@
+export * from './chat.js';
+export * from './serializer.js';

package/dist/llm/google/index.js

@@ -0,0 +1,2 @@
+export * from './chat.js';
+export * from './serializer.js';

package/dist/llm/google/serializer.d.ts

@@ -0,0 +1,6 @@
+import type { Content } from '@google/genai';
+import { type Message } from '../messages.js';
+export declare class GoogleMessageSerializer {
+    serialize(messages: Message[]): Content[];
+    private serializeMessage;
+}

package/dist/llm/google/serializer.js

@@ -0,0 +1,64 @@
+import { AssistantMessage, ContentPartImageParam, ContentPartTextParam, SystemMessage, UserMessage, } from '../messages.js';
+export class GoogleMessageSerializer {
+    serialize(messages) {
+        return messages
+            .filter((msg) => !(msg instanceof SystemMessage)) // System instructions are passed separately
+            .map((message) => this.serializeMessage(message));
+    }
+    serializeMessage(message) {
+        if (message instanceof UserMessage) {
+            return {
+                role: 'user',
+                parts: Array.isArray(message.content)
+                    ? message.content.map((part) => {
+                        if (part instanceof ContentPartTextParam) {
+                            return { text: part.text };
+                        }
+                        if (part instanceof ContentPartImageParam) {
+                            // Google GenAI expects inlineData for images usually
+                            const data = part.image_url.url.split(',')[1];
+                            const mimeType = part.image_url.media_type;
+                            return {
+                                inlineData: {
+                                    mimeType: mimeType,
+                                    data: data,
+                                },
+                            };
+                        }
+                        return { text: '' };
+                    })
+                    : [{ text: message.content }],
+            };
+        }
+        if (message instanceof AssistantMessage) {
+            const parts = [];
+            if (message.content) {
+                if (typeof message.content === 'string') {
+                    parts.push({ text: message.content });
+                }
+                else if (Array.isArray(message.content)) {
+                    message.content.forEach((part) => {
+                        if (part instanceof ContentPartTextParam) {
+                            parts.push({ text: part.text });
+                        }
+                    });
+                }
+            }
+            if (message.tool_calls) {
+                message.tool_calls.forEach((toolCall) => {
+                    parts.push({
+                        functionCall: {
+                            name: toolCall.functionCall.name,
+                            args: JSON.parse(toolCall.functionCall.arguments),
+                        },
+                    });
+                });
+            }
+            return {
+                role: 'model',
+                parts: parts,
+            };
+        }
+        throw new Error(`Unknown message type: ${message.constructor.name}`);
+    }
+}

package/dist/llm/groq/chat.d.ts

@@ -0,0 +1,15 @@
+import type { BaseChatModel, ChatInvokeOptions } from '../base.js';
+import { ChatInvokeCompletion } from '../views.js';
+import type { Message } from '../messages.js';
+export declare class ChatGroq implements BaseChatModel {
+    model: string;
+    provider: string;
+    private client;
+    constructor(model?: string);
+    get name(): string;
+    get model_name(): string;
+    ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise<ChatInvokeCompletion<string>>;
+    ainvoke<T>(messages: Message[], output_format: {
+        parse: (input: string) => T;
+    } | undefined, options?: ChatInvokeOptions): Promise<ChatInvokeCompletion<T>>;
+}