browser-use 0.2.0 → 0.3.0

package/dist/llm/google/serializer.js

@@ -1,37 +1,89 @@
-import { AssistantMessage, ContentPartImageParam, ContentPartTextParam, SystemMessage, UserMessage, } from '../messages.js';
+import { AssistantMessage, ContentPartImageParam, ContentPartRefusalParam, ContentPartTextParam, SystemMessage, UserMessage, } from '../messages.js';
 export class GoogleMessageSerializer {
     serialize(messages) {
-        return messages
-            .filter((msg) => !(msg instanceof SystemMessage)) // System instructions are passed separately
-            .map((message) => this.serializeMessage(message));
+        return this.serializeWithSystem(messages).contents;
     }
-    serializeMessage(message) {
-        if (message instanceof UserMessage) {
-            return {
+    serializeWithSystem(messages, includeSystemInUser = false) {
+        const contents = [];
+        const systemParts = [];
+        let systemInstruction = null;
+        let injectedSystemIntoUser = false;
+        for (const message of messages) {
+            const role = message?.role;
+            if (message instanceof SystemMessage ||
+                role === 'system' ||
+                role === 'developer') {
+                const text = this.extractMessageText(message);
+                if (text) {
+                    if (includeSystemInUser) {
+                        systemParts.push(text);
+                    }
+                    else {
+                        systemInstruction = text;
+                    }
+                }
+                continue;
+            }
+            if (message instanceof UserMessage) {
+                const prependSystem = includeSystemInUser &&
+                    !injectedSystemIntoUser &&
+                    systemParts.length > 0
+                    ? systemParts.join('\n\n')
+                    : null;
+                contents.push(this.serializeUserMessage(message, prependSystem));
+                if (prependSystem) {
+                    injectedSystemIntoUser = true;
+                }
+                continue;
+            }
+            if (message instanceof AssistantMessage) {
+                contents.push(this.serializeAssistantMessage(message));
+            }
+        }
+        if (includeSystemInUser &&
+            systemParts.length > 0 &&
+            !injectedSystemIntoUser) {
+            contents.unshift({
                 role: 'user',
-                parts: Array.isArray(message.content)
-                    ? message.content.map((part) => {
-                        if (part instanceof ContentPartTextParam) {
-                            return { text: part.text };
-                        }
-                        if (part instanceof ContentPartImageParam) {
-                            // Google GenAI expects inlineData for images usually
-                            const data = part.image_url.url.split(',')[1];
-                            const mimeType = part.image_url.media_type;
-                            return {
-                                inlineData: {
-                                    mimeType: mimeType,
-                                    data: data,
-                                },
-                            };
-                        }
-                        return { text: '' };
-                    })
-                    : [{ text: message.content }],
-            };
+                parts: [{ text: systemParts.join('\n\n') }],
+            });
+        }
+        return {
+            contents,
+            systemInstruction: includeSystemInUser ? null : systemInstruction,
+        };
+    }
+    serializeUserMessage(message, prependSystem = null) {
+        const parts = [];
+        if (prependSystem) {
+            parts.push({ text: prependSystem });
         }
+        if (message instanceof UserMessage) {
+            if (Array.isArray(message.content)) {
+                for (const part of message.content) {
+                    if (part instanceof ContentPartTextParam) {
+                        parts.push({ text: part.text });
+                    }
+                    else if (part instanceof ContentPartRefusalParam) {
+                        parts.push({ text: `[Refusal] ${part.refusal}` });
+                    }
+                    else if (part instanceof ContentPartImageParam) {
+                        parts.push(this.serializeImagePart(part));
+                    }
+                }
+            }
+            else {
+                parts.push({ text: message.content });
+            }
+        }
+        return {
+            role: 'user',
+            parts,
+        };
+    }
+    serializeAssistantMessage(message) {
+        const parts = [];
         if (message instanceof AssistantMessage) {
-            const parts = [];
             if (message.content) {
                 if (typeof message.content === 'string') {
                     parts.push({ text: message.content });
@@ -41,24 +93,79 @@ export class GoogleMessageSerializer {
                         if (part instanceof ContentPartTextParam) {
                             parts.push({ text: part.text });
                         }
+                        else if (part instanceof ContentPartRefusalParam) {
+                            parts.push({ text: `[Refusal] ${part.refusal}` });
+                        }
+                        else if (part instanceof ContentPartImageParam) {
+                            parts.push(this.serializeImagePart(part));
+                        }
                     });
                 }
             }
             if (message.tool_calls) {
                 message.tool_calls.forEach((toolCall) => {
+                    let args;
+                    try {
+                        const parsed = JSON.parse(toolCall.functionCall.arguments);
+                        args =
+                            parsed && typeof parsed === 'object' && !Array.isArray(parsed)
+                                ? parsed
+                                : {};
+                    }
+                    catch {
+                        args = {};
+                    }
                     parts.push({
                         functionCall: {
                             name: toolCall.functionCall.name,
-                            args: JSON.parse(toolCall.functionCall.arguments),
+                            args,
                         },
                     });
                 });
             }
-            return {
-                role: 'model',
-                parts: parts,
-            };
         }
-        throw new Error(`Unknown message type: ${message.constructor.name}`);
+        return {
+            role: 'model',
+            parts: parts,
+        };
+    }
+    serializeImagePart(part) {
+        const imageUrl = part.image_url.url;
+        if (!imageUrl.startsWith('data:')) {
+            return { text: imageUrl };
+        }
+        const commaIndex = imageUrl.indexOf(',');
+        if (commaIndex === -1) {
+            return { text: imageUrl };
+        }
+        const data = imageUrl.slice(commaIndex + 1);
+        return {
+            inlineData: {
+                mimeType: part.image_url.media_type,
+                data,
+            },
+        };
+    }
+    extractMessageText(message) {
+        const content = message?.content;
+        if (typeof content === 'string') {
+            return content;
+        }
+        if (!Array.isArray(content)) {
+            return '';
+        }
+        const parts = [];
+        for (const part of content) {
+            if (part instanceof ContentPartTextParam) {
+                parts.push(part.text);
+            }
+            else if (part &&
+                typeof part === 'object' &&
+                part.type === 'text' &&
+                typeof part.text === 'string') {
+                parts.push(part.text);
+            }
+        }
+        return parts.join('\n');
     }
 }

package/dist/llm/groq/chat.d.ts

@@ -1,13 +1,32 @@
 import type { BaseChatModel, ChatInvokeOptions } from '../base.js';
-import { ChatInvokeCompletion } from '../views.js';
 import type { Message } from '../messages.js';
+import { ChatInvokeCompletion } from '../views.js';
+export interface ChatGroqOptions {
+    model?: string;
+    apiKey?: string;
+    baseURL?: string;
+    temperature?: number | null;
+    serviceTier?: 'auto' | 'on_demand' | 'flex' | null;
+    topP?: number | null;
+    seed?: number | null;
+    maxRetries?: number;
+    removeMinItemsFromSchema?: boolean;
+    removeDefaultsFromSchema?: boolean;
+}
 export declare class ChatGroq implements BaseChatModel {
     model: string;
     provider: string;
     private client;
-    constructor(model?: string);
+    private temperature;
+    private serviceTier;
+    private topP;
+    private seed;
+    private removeMinItemsFromSchema;
+    private removeDefaultsFromSchema;
+    constructor(options?: string | ChatGroqOptions);
     get name(): string;
     get model_name(): string;
+    private getUsage;
     ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise<ChatInvokeCompletion<string>>;
     ainvoke<T>(messages: Message[], output_format: {
         parse: (input: string) => T;

package/dist/llm/groq/chat.js

@@ -1,14 +1,38 @@
 import Groq from 'groq-sdk';
+import { ModelProviderError, ModelRateLimitError } from '../exceptions.js';
+import { SchemaOptimizer, zodSchemaToJsonSchema } from '../schema.js';
 import { ChatInvokeCompletion } from '../views.js';
 import { GroqMessageSerializer } from './serializer.js';
+const JsonSchemaModels = [
+    'meta-llama/llama-4-maverick-17b-128e-instruct',
+    'meta-llama/llama-4-scout-17b-16e-instruct',
+    'openai/gpt-oss-20b',
+    'openai/gpt-oss-120b',
+];
 export class ChatGroq {
     model;
     provider = 'groq';
     client;
-    constructor(model = 'llama-3.1-70b-versatile') {
+    temperature;
+    serviceTier;
+    topP;
+    seed;
+    removeMinItemsFromSchema;
+    removeDefaultsFromSchema;
+    constructor(options = {}) {
+        const normalizedOptions = typeof options === 'string' ? { model: options } : options;
+        const { model = 'llama-3.1-70b-versatile', apiKey = process.env.GROQ_API_KEY, baseURL, temperature = null, serviceTier = null, topP = null, seed = null, maxRetries = 10, removeMinItemsFromSchema = false, removeDefaultsFromSchema = false, } = normalizedOptions;
         this.model = model;
+        this.temperature = temperature;
+        this.serviceTier = serviceTier;
+        this.topP = topP;
+        this.seed = seed;
+        this.removeMinItemsFromSchema = removeMinItemsFromSchema;
+        this.removeDefaultsFromSchema = removeDefaultsFromSchema;
         this.client = new Groq({
-            apiKey: process.env.GROQ_API_KEY,
+            apiKey,
+            baseURL,
+            maxRetries,
         });
     }
     get name() {
@@ -17,36 +41,111 @@ export class ChatGroq {
     get model_name() {
         return this.model;
     }
+    getUsage(response) {
+        if (!response?.usage) {
+            return null;
+        }
+        return {
+            prompt_tokens: response.usage.prompt_tokens,
+            prompt_cached_tokens: null,
+            prompt_cache_creation_tokens: null,
+            prompt_image_tokens: null,
+            completion_tokens: response.usage.completion_tokens,
+            total_tokens: response.usage.total_tokens,
+        };
+    }
     async ainvoke(messages, output_format, options = {}) {
         const serializer = new GroqMessageSerializer();
         const groqMessages = serializer.serialize(messages);
+        const modelParams = {};
+        if (this.temperature !== null) {
+            modelParams.temperature = this.temperature;
+        }
+        if (this.topP !== null) {
+            modelParams.top_p = this.topP;
+        }
+        if (this.seed !== null) {
+            modelParams.seed = this.seed;
+        }
+        if (this.serviceTier !== null) {
+            modelParams.service_tier = this.serviceTier;
+        }
+        const zodSchemaCandidate = (() => {
+            const output = output_format;
+            if (output &&
+                typeof output === 'object' &&
+                typeof output.safeParse === 'function' &&
+                typeof output.parse === 'function') {
+                return output;
+            }
+            if (output &&
+                typeof output === 'object' &&
+                output.schema &&
+                typeof output.schema.safeParse === 'function' &&
+                typeof output.schema.parse === 'function') {
+                return output.schema;
+            }
+            return null;
+        })();
         let responseFormat = undefined;
-        if (output_format && 'schema' in output_format && output_format.schema) {
-            // Groq supports JSON mode, but not full JSON schema validation in the same way as OpenAI yet (or maybe it does now).
-            // For now, we'll enforce JSON mode and prompt engineering or use tools if needed.
-            // But let's try to use json_object which is supported.
-            responseFormat = { type: 'json_object' };
-        }
-        const response = await this.client.chat.completions.create({
-            model: this.model,
-            messages: groqMessages,
-            response_format: responseFormat,
-        }, options.signal ? { signal: options.signal } : undefined);
-        const content = response.choices[0].message.content || '';
-        let completion = content;
-        if (output_format) {
-            try {
-                completion = output_format.parse(JSON.parse(content));
+        if (zodSchemaCandidate) {
+            if (JsonSchemaModels.includes(this.model)) {
+                try {
+                    const rawJsonSchema = zodSchemaToJsonSchema(zodSchemaCandidate, {
+                        name: 'agent_output',
+                        target: 'jsonSchema7',
+                    });
+                    const optimizedJsonSchema = SchemaOptimizer.createOptimizedJsonSchema(rawJsonSchema, {
+                        removeMinItems: this.removeMinItemsFromSchema,
+                        removeDefaults: this.removeDefaultsFromSchema,
+                    });
+                    responseFormat = {
+                        type: 'json_schema',
+                        json_schema: {
+                            name: 'agent_output',
+                            schema: optimizedJsonSchema,
+                        },
+                    };
+                }
+                catch {
+                    responseFormat = { type: 'json_object' };
+                }
             }
-            catch (e) {
-                console.error('Failed to parse completion', e);
-                throw e;
+            else {
+                responseFormat = { type: 'json_object' };
             }
         }
-        return new ChatInvokeCompletion(completion, {
-            prompt_tokens: response.usage?.prompt_tokens ?? 0,
-            completion_tokens: response.usage?.completion_tokens ?? 0,
-            total_tokens: response.usage?.total_tokens ?? 0,
-        });
+        try {
+            const response = await this.client.chat.completions.create({
+                model: this.model,
+                messages: groqMessages,
+                response_format: responseFormat,
+                ...modelParams,
+            }, options.signal ? { signal: options.signal } : undefined);
+            const content = response.choices[0].message.content || '';
+            const usage = this.getUsage(response);
+            const stopReason = response.choices[0].finish_reason ?? null;
+            let completion = content;
+            if (output_format) {
+                const parsedJson = JSON.parse(content);
+                const output = output_format;
+                if (output &&
+                    typeof output === 'object' &&
+                    output.schema &&
+                    typeof output.schema.parse === 'function') {
+                    completion = output.schema.parse(parsedJson);
+                }
+                else {
+                    completion = output.parse(parsedJson);
+                }
+            }
+            return new ChatInvokeCompletion(completion, usage, null, null, stopReason);
+        }
+        catch (error) {
+            if (error?.status === 429) {
+                throw new ModelRateLimitError(error?.message ?? 'Rate limit exceeded', 429, this.model);
+            }
+            throw new ModelProviderError(error?.message ?? String(error), error?.status ?? 500, this.model);
+        }
     }
 }

package/dist/llm/groq/parser.js

@@ -98,7 +98,9 @@ export function tryParseGroqFailedGeneration(error, outputFormat) {
         }
         if (err instanceof SyntaxError) {
             logger.warning(`Failed to parse model output: ${err.message}`);
-            throw new Error(`Could not parse response. ${err.message}`);
+            throw new Error(`Could not parse response. ${err.message}`, {
+                cause: err,
+            });
         }
         const errorMessage = error.message || String(error);
         throw new ParseFailedGenerationError(errorMessage);

package/dist/llm/mistral/chat.d.ts

@@ -0,0 +1,43 @@
+import type { BaseChatModel, ChatInvokeOptions } from '../base.js';
+import type { Message } from '../messages.js';
+import { ChatInvokeCompletion } from '../views.js';
+export interface ChatMistralOptions {
+    model?: string;
+    apiKey?: string;
+    baseURL?: string;
+    timeout?: number | null;
+    defaultHeaders?: Record<string, string> | null;
+    defaultQuery?: Record<string, string | undefined> | null;
+    fetchImplementation?: typeof fetch;
+    fetchOptions?: RequestInit | null;
+    clientParams?: Record<string, unknown> | null;
+    temperature?: number | null;
+    maxTokens?: number | null;
+    topP?: number | null;
+    seed?: number | null;
+    safePrompt?: boolean;
+    maxRetries?: number;
+    removeMinItemsFromSchema?: boolean;
+    removeDefaultsFromSchema?: boolean;
+}
+export declare class ChatMistral implements BaseChatModel {
+    model: string;
+    provider: string;
+    private client;
+    private temperature;
+    private maxTokens;
+    private topP;
+    private seed;
+    private safePrompt;
+    private removeMinItemsFromSchema;
+    private removeDefaultsFromSchema;
+    constructor(options?: string | ChatMistralOptions);
+    get name(): string;
+    get model_name(): string;
+    private getUsage;
+    private getSchemaCandidate;
+    ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise<ChatInvokeCompletion<string>>;
+    ainvoke<T>(messages: Message[], output_format: {
+        parse: (input: string) => T;
+    } | undefined, options?: ChatInvokeOptions): Promise<ChatInvokeCompletion<T>>;
+}

package/dist/llm/mistral/chat.js

@@ -0,0 +1,154 @@
+import OpenAI from 'openai';
+import { ModelProviderError, ModelRateLimitError } from '../exceptions.js';
+import { ChatInvokeCompletion } from '../views.js';
+import { zodSchemaToJsonSchema } from '../schema.js';
+import { OpenAIMessageSerializer } from '../openai/serializer.js';
+import { MistralSchemaOptimizer } from './schema.js';
+export class ChatMistral {
+    model;
+    provider = 'mistral';
+    client;
+    temperature;
+    maxTokens;
+    topP;
+    seed;
+    safePrompt;
+    removeMinItemsFromSchema;
+    removeDefaultsFromSchema;
+    constructor(options = {}) {
+        const normalizedOptions = typeof options === 'string' ? { model: options } : options;
+        const { model = 'mistral-medium-latest', apiKey = process.env.MISTRAL_API_KEY, baseURL = process.env.MISTRAL_BASE_URL || 'https://api.mistral.ai/v1', timeout = null, defaultHeaders = null, defaultQuery = null, fetchImplementation, fetchOptions = null, clientParams = null, temperature = 0.2, maxTokens = 4096, topP = null, seed = null, safePrompt = false, maxRetries = 5, removeMinItemsFromSchema = false, removeDefaultsFromSchema = false, } = normalizedOptions;
+        this.model = model;
+        this.temperature = temperature;
+        this.maxTokens = maxTokens;
+        this.topP = topP;
+        this.seed = seed;
+        this.safePrompt = safePrompt;
+        this.removeMinItemsFromSchema = removeMinItemsFromSchema;
+        this.removeDefaultsFromSchema = removeDefaultsFromSchema;
+        this.client = new OpenAI({
+            apiKey,
+            baseURL,
+            ...(timeout !== null ? { timeout } : {}),
+            maxRetries,
+            defaultHeaders: defaultHeaders ?? undefined,
+            defaultQuery: defaultQuery ?? undefined,
+            fetch: fetchImplementation,
+            fetchOptions: (fetchOptions ?? undefined),
+            ...(clientParams ?? {}),
+        });
+    }
+    get name() {
+        return this.model;
+    }
+    get model_name() {
+        return this.model;
+    }
+    getUsage(response) {
+        if (!response.usage) {
+            return null;
+        }
+        return {
+            prompt_tokens: response.usage.prompt_tokens,
+            prompt_cached_tokens: response.usage.prompt_tokens_details?.cached_tokens ?? null,
+            prompt_cache_creation_tokens: null,
+            prompt_image_tokens: null,
+            completion_tokens: response.usage.completion_tokens,
+            total_tokens: response.usage.total_tokens,
+        };
+    }
+    getSchemaCandidate(output_format) {
+        const output = output_format;
+        if (output &&
+            typeof output === 'object' &&
+            typeof output.safeParse === 'function' &&
+            typeof output.parse === 'function') {
+            return output;
+        }
+        if (output &&
+            typeof output === 'object' &&
+            output.schema &&
+            typeof output.schema.safeParse === 'function' &&
+            typeof output.schema.parse === 'function') {
+            return output.schema;
+        }
+        return null;
+    }
+    async ainvoke(messages, output_format, options = {}) {
+        const serializer = new OpenAIMessageSerializer();
+        const mistralMessages = serializer.serialize(messages);
+        const modelParams = {};
+        if (this.temperature !== null) {
+            modelParams.temperature = this.temperature;
+        }
+        if (this.maxTokens !== null) {
+            modelParams.max_tokens = this.maxTokens;
+        }
+        if (this.topP !== null) {
+            modelParams.top_p = this.topP;
+        }
+        if (this.seed !== null) {
+            modelParams.seed = this.seed;
+        }
+        if (this.safePrompt) {
+            modelParams.safe_prompt = true;
+        }
+        const zodSchemaCandidate = this.getSchemaCandidate(output_format);
+        let responseFormat = undefined;
+        if (zodSchemaCandidate) {
+            try {
+                const rawJsonSchema = zodSchemaToJsonSchema(zodSchemaCandidate, {
+                    name: 'agent_output',
+                    target: 'jsonSchema7',
+                });
+                const optimizedJsonSchema = MistralSchemaOptimizer.createMistralCompatibleSchema(rawJsonSchema, {
+                    removeMinItems: this.removeMinItemsFromSchema,
+                    removeDefaults: this.removeDefaultsFromSchema,
+                });
+                responseFormat = {
+                    type: 'json_schema',
+                    json_schema: {
+                        name: 'agent_output',
+                        schema: optimizedJsonSchema,
+                        strict: true,
+                    },
+                };
+            }
+            catch {
+                responseFormat = undefined;
+            }
+        }
+        try {
+            const response = await this.client.chat.completions.create({
+                model: this.model,
+                messages: mistralMessages,
+                response_format: responseFormat,
+                ...modelParams,
+            }, options.signal ? { signal: options.signal } : undefined);
+            const content = response.choices[0].message.content || '';
+            const usage = this.getUsage(response);
+            const stopReason = response.choices[0].finish_reason ?? null;
+            let completion = content;
+            if (output_format) {
+                const parsedJson = JSON.parse(content);
+                const output = output_format;
+                if (output &&
+                    typeof output === 'object' &&
+                    output.schema &&
+                    typeof output.schema.parse === 'function') {
+                    completion = output.schema.parse(parsedJson);
+                }
+                else {
+                    completion = output.parse(parsedJson);
+                }
+            }
+            return new ChatInvokeCompletion(completion, usage, null, null, stopReason);
+        }
+        catch (error) {
+            if (error?.status === 429) {
+                throw new ModelRateLimitError(error?.message ?? 'Rate limit exceeded', 429, this.model);
+            }
+            throw new ModelProviderError(error?.message ?? String(error), error?.status ?? 500, this.model);
+        }
+    }
+}

package/dist/llm/mistral/index.d.ts

@@ -0,0 +1,2 @@
+export * from './chat.js';
+export * from './schema.js';

package/dist/llm/mistral/index.js

@@ -0,0 +1,2 @@
+export * from './chat.js';
+export * from './schema.js';

package/dist/llm/mistral/schema.d.ts

@@ -0,0 +1,8 @@
+export declare class MistralSchemaOptimizer {
+    static readonly UNSUPPORTED_KEYWORDS: Set<string>;
+    static createMistralCompatibleSchema(rawSchema: Record<string, unknown>, options?: {
+        removeMinItems?: boolean;
+        removeDefaults?: boolean;
+    }): Record<string, unknown>;
+    static stripUnsupportedKeywords(value: unknown): any;
+}

package/dist/llm/mistral/schema.js

@@ -0,0 +1,27 @@
+import { SchemaOptimizer } from '../schema.js';
+export class MistralSchemaOptimizer {
+    static UNSUPPORTED_KEYWORDS = new Set([
+        'minLength',
+        'maxLength',
+        'pattern',
+        'format',
+    ]);
+    static createMistralCompatibleSchema(rawSchema, options = {}) {
+        const baseSchema = SchemaOptimizer.createOptimizedJsonSchema(rawSchema, {
+            removeMinItems: options.removeMinItems ?? false,
+            removeDefaults: options.removeDefaults ?? false,
+        });
+        return this.stripUnsupportedKeywords(baseSchema);
+    }
+    static stripUnsupportedKeywords(value) {
+        if (Array.isArray(value)) {
+            return value.map((item) => this.stripUnsupportedKeywords(item));
+        }
+        if (!value || typeof value !== 'object') {
+            return value;
+        }
+        return Object.fromEntries(Object.entries(value)
+            .filter(([key]) => !this.UNSUPPORTED_KEYWORDS.has(key))
+            .map(([key, item]) => [key, this.stripUnsupportedKeywords(item)]));
+    }
+}

package/dist/llm/models.d.ts

@@ -0,0 +1,2 @@
+import type { BaseChatModel } from './base.js';
+export declare const getLlmByName: (modelName: string) => BaseChatModel;