browser-use 0.2.0 → 0.3.0

package/dist/llm/openai/chat.js

@@ -1,10 +1,10 @@
 import OpenAI from 'openai';
-import { zodToJsonSchema } from 'zod-to-json-schema';
 import { ChatInvokeCompletion } from '../views.js';
 import { OpenAIMessageSerializer } from './serializer.js';
-import { ModelProviderError } from '../exceptions.js';
+import { ModelProviderError, ModelRateLimitError } from '../exceptions.js';
+import { SchemaOptimizer, zodSchemaToJsonSchema } from '../schema.js';
 // Reasoning models that support reasoning_effort parameter
-const ReasoningModels = [
+const DEFAULT_REASONING_MODELS = [
     'o4-mini',
     'o3',
     'o3-mini',
@@ -22,25 +22,43 @@ export class ChatOpenAI {
     temperature;
     frequencyPenalty;
     reasoningEffort;
+    serviceTier;
     maxCompletionTokens;
     seed;
     topP;
     addSchemaToSystemPrompt;
+    dontForceStructuredOutput;
+    removeMinItemsFromSchema;
+    removeDefaultsFromSchema;
+    reasoningModels;
     constructor(options = {}) {
-        const { model = 'gpt-4o', apiKey, organization, baseURL, temperature = 0.2, frequencyPenalty = 0.1, reasoningEffort = 'low', maxCompletionTokens = 8000, maxRetries = 10, seed = null, topP = null, addSchemaToSystemPrompt = false, } = options;
+        const { model = 'gpt-4o', apiKey, organization, project, baseURL, timeout = null, temperature = 0.2, frequencyPenalty = 0.3, reasoningEffort = 'low', serviceTier = null, maxCompletionTokens = 4096, maxRetries = 5, defaultHeaders = null, defaultQuery = null, fetchImplementation, fetchOptions = null, seed = null, topP = null, addSchemaToSystemPrompt = false, dontForceStructuredOutput = false, removeMinItemsFromSchema = false, removeDefaultsFromSchema = false, reasoningModels = DEFAULT_REASONING_MODELS, } = options;
         this.model = model;
         this.temperature = temperature;
         this.frequencyPenalty = frequencyPenalty;
         this.reasoningEffort = reasoningEffort;
+        this.serviceTier = serviceTier;
         this.maxCompletionTokens = maxCompletionTokens;
         this.seed = seed;
         this.topP = topP;
         this.addSchemaToSystemPrompt = addSchemaToSystemPrompt;
+        this.dontForceStructuredOutput = dontForceStructuredOutput;
+        this.removeMinItemsFromSchema = removeMinItemsFromSchema;
+        this.removeDefaultsFromSchema = removeDefaultsFromSchema;
+        this.reasoningModels = reasoningModels
+            ? [...reasoningModels]
+            : reasoningModels;
         this.client = new OpenAI({
             apiKey,
             organization,
+            project,
             baseURL,
+            timeout: timeout ?? undefined,
             maxRetries,
+            defaultHeaders: defaultHeaders ?? undefined,
+            defaultQuery: defaultQuery ?? undefined,
+            fetch: fetchImplementation,
+            fetchOptions: (fetchOptions ?? undefined),
         });
     }
     get name() {
@@ -50,7 +68,7 @@ export class ChatOpenAI {
         return this.model;
     }
     isReasoningModel() {
-        return ReasoningModels.some((m) => this.model.toLowerCase().includes(m.toLowerCase()));
+        return (this.reasoningModels ?? []).some((m) => this.model.toLowerCase().includes(m.toLowerCase()));
     }
     getUsage(response) {
         if (!response.usage)
@@ -96,36 +114,69 @@ export class ChatOpenAI {
         if (this.topP !== null) {
             modelParams.top_p = this.topP;
         }
+        if (this.serviceTier !== null) {
+            modelParams.service_tier = this.serviceTier;
+        }
+        const zodSchemaCandidate = (() => {
+            const output = output_format;
+            if (output &&
+                typeof output === 'object' &&
+                typeof output.safeParse === 'function' &&
+                typeof output.parse === 'function') {
+                return output;
+            }
+            if (output &&
+                typeof output === 'object' &&
+                output.schema &&
+                typeof output.schema.safeParse === 'function' &&
+                typeof output.schema.parse === 'function') {
+                return output.schema;
+            }
+            return null;
+        })();
         let responseFormat = undefined;
-        if (output_format && 'schema' in output_format && output_format.schema) {
-            // Assuming output_format is a Zod schema wrapper or similar that has a schema property
-            // But the interface says { parse: ... }
-            // In the plan, it was passed as a Zod schema directly.
-            // However, the BaseChatModel interface I saw earlier has:
-            // ainvoke<T>(messages: Message[], output_format: { parse: (input: string) => T } | undefined): Promise<ChatInvokeCompletion<T>>;
-            // So I need to handle how to extract the schema if I want to use structured outputs.
-            // If output_format is just a Zod schema, it has a parse method.
-            // Let's assume it's a Zod schema for now, as that's what the plan implies.
-            // We need to cast it to any or check if it's a Zod schema to get the schema for JSON schema generation.
-            // For now, I'll try to use zodToJsonSchema on it if possible.
+        if (zodSchemaCandidate) {
             try {
-                const jsonSchema = zodToJsonSchema(output_format, {
-                    name: 'Response',
+                const rawJsonSchema = zodSchemaToJsonSchema(zodSchemaCandidate, {
+                    name: 'agent_output',
                     target: 'jsonSchema7',
                 });
-                // OpenAI expects a specific format for json_schema
-                responseFormat = {
-                    type: 'json_schema',
-                    json_schema: {
-                        name: 'Response',
-                        schema: jsonSchema,
-                        strict: true,
-                    },
+                const optimizedJsonSchema = SchemaOptimizer.createOptimizedJsonSchema(rawJsonSchema, {
+                    removeMinItems: this.removeMinItemsFromSchema,
+                    removeDefaults: this.removeDefaultsFromSchema,
+                });
+                const responseJsonSchema = {
+                    name: 'agent_output',
+                    schema: optimizedJsonSchema,
+                    strict: true,
                 };
+                if (this.addSchemaToSystemPrompt && openaiMessages.length > 0) {
+                    const firstMessage = openaiMessages[0];
+                    const schemaText = `\n<json_schema>\n` +
+                        `${JSON.stringify(responseJsonSchema, null, 2)}\n` +
+                        `</json_schema>`;
+                    if (firstMessage?.role === 'system') {
+                        if (typeof firstMessage.content === 'string') {
+                            firstMessage.content =
+                                (firstMessage.content ?? '') + schemaText;
+                        }
+                        else if (Array.isArray(firstMessage.content)) {
+                            firstMessage.content = [
+                                ...firstMessage.content,
+                                { type: 'text', text: schemaText },
+                            ];
+                        }
+                    }
+                }
+                if (!this.dontForceStructuredOutput) {
+                    responseFormat = {
+                        type: 'json_schema',
+                        json_schema: responseJsonSchema,
+                    };
+                }
             }
-            catch (e) {
-                // If it's not a Zod schema or fails, we might fallback or just not use response_format
-                console.warn('Failed to convert output_format to JSON schema', e);
+            catch {
+                responseFormat = undefined;
             }
         }
         try {
@@ -137,33 +188,32 @@ export class ChatOpenAI {
             }, options.signal ? { signal: options.signal } : undefined);
             const content = response.choices[0].message.content || '';
             const usage = this.getUsage(response);
+            const stopReason = response.choices[0].finish_reason ?? null;
             let completion = content;
             if (output_format) {
-                try {
-                    // If it's structured output, we need to parse the JSON first
-                    if (responseFormat?.type === 'json_schema') {
-                        const parsedJson = JSON.parse(content);
-                        completion = output_format.parse(parsedJson);
+                if (zodSchemaCandidate) {
+                    const parsedJson = JSON.parse(content);
+                    const output = output_format;
+                    if (output &&
+                        typeof output === 'object' &&
+                        output.schema &&
+                        typeof output.schema.parse === 'function') {
+                        completion = output.schema.parse(parsedJson);
                     }
                     else {
-                        // If it's not structured output but we have a parser (e.g. for simple types or manual parsing)
-                        // But usually for OpenAI we want structured output if a schema is provided.
-                        // If we didn't use json_schema, we might still try to parse if it looks like JSON?
-                        // For now, let's trust the output_format.parse
-                        completion = output_format.parse(content);
+                        completion = output_format.parse(parsedJson);
                     }
                 }
-                catch (e) {
-                    console.error('Failed to parse completion', e);
-                    throw e;
+                else {
+                    completion = output_format.parse(content);
                 }
             }
-            return new ChatInvokeCompletion(completion, usage);
+            return new ChatInvokeCompletion(completion, usage, null, null, stopReason);
         }
         catch (error) {
             // Handle OpenAI-specific errors
             if (error?.status === 429) {
-                throw new ModelProviderError(error?.message ?? 'Rate limit exceeded', 429, this.model);
+                throw new ModelRateLimitError(error?.message ?? 'Rate limit exceeded', 429, this.model);
             }
             if (error?.status >= 500) {
                 throw new ModelProviderError(error?.message ?? 'Server error', error.status, this.model);

package/dist/llm/openai/like.d.ts

@@ -1,4 +1,4 @@
-import { ChatOpenAI } from './chat.js';
+import { ChatOpenAI, type ChatOpenAIOptions } from './chat.js';
 /**
  * A class to interact with any provider using the OpenAI API schema.
  *
@@ -11,7 +11,9 @@ import { ChatOpenAI } from './chat.js';
  */
 export declare class ChatOpenAILike extends ChatOpenAI {
     /**
-     * @param model - The name of the model to use (any OpenAI-compatible model)
+     * @param options - A model name or ChatOpenAI-compatible options
      */
-    constructor(model: string);
+    constructor(options: string | (ChatOpenAIOptions & {
+        model: string;
+    }));
 }

package/dist/llm/openai/like.js

@@ -11,9 +11,13 @@ import { ChatOpenAI } from './chat.js';
  */
 export class ChatOpenAILike extends ChatOpenAI {
     /**
-     * @param model - The name of the model to use (any OpenAI-compatible model)
+     * @param options - A model name or ChatOpenAI-compatible options
      */
-    constructor(model) {
-        super({ model });
+    constructor(options) {
+        if (typeof options === 'string') {
+            super({ model: options });
+            return;
+        }
+        super(options);
     }
 }

package/dist/llm/openai/responses-serializer.d.ts

@@ -0,0 +1,18 @@
+import { type Message } from '../messages.js';
+type ResponsesInputPart = {
+    type: 'input_text';
+    text: string;
+} | {
+    type: 'input_image';
+    image_url: string;
+    detail?: 'auto' | 'low' | 'high';
+};
+export type ResponsesInputMessage = {
+    role: 'user' | 'system' | 'assistant';
+    content: string | ResponsesInputPart[];
+};
+export declare class ResponsesAPIMessageSerializer {
+    serialize(messages: Message[]): ResponsesInputMessage[];
+    private serializeMessage;
+}
+export {};

package/dist/llm/openai/responses-serializer.js

@@ -0,0 +1,72 @@
+import { AssistantMessage, ContentPartImageParam, ContentPartRefusalParam, ContentPartTextParam, SystemMessage, UserMessage, } from '../messages.js';
+export class ResponsesAPIMessageSerializer {
+    serialize(messages) {
+        return messages.map((message) => this.serializeMessage(message));
+    }
+    serializeMessage(message) {
+        if (message instanceof UserMessage) {
+            if (typeof message.content === 'string') {
+                return { role: 'user', content: message.content };
+            }
+            const content = message.content
+                .map((part) => {
+                if (part instanceof ContentPartTextParam) {
+                    return { type: 'input_text', text: part.text };
+                }
+                if (part instanceof ContentPartImageParam) {
+                    return {
+                        type: 'input_image',
+                        image_url: part.image_url.url,
+                        detail: part.image_url.detail,
+                    };
+                }
+                return null;
+            })
+                .filter((part) => part !== null);
+            return { role: 'user', content };
+        }
+        if (message instanceof SystemMessage) {
+            if (typeof message.content === 'string') {
+                return { role: 'system', content: message.content };
+            }
+            return {
+                role: 'system',
+                content: message.content.map((part) => ({
+                    type: 'input_text',
+                    text: part.text,
+                })),
+            };
+        }
+        if (message instanceof AssistantMessage) {
+            if (message.content == null) {
+                if (Array.isArray(message.tool_calls) &&
+                    message.tool_calls.length > 0) {
+                    const toolCallText = message.tool_calls
+                        .map((toolCall) => `[Tool call: ${toolCall.functionCall.name}(${toolCall.functionCall.arguments})]`)
+                        .join('\n');
+                    return { role: 'assistant', content: toolCallText };
+                }
+                return { role: 'assistant', content: '' };
+            }
+            if (typeof message.content === 'string') {
+                return { role: 'assistant', content: message.content };
+            }
+            const content = message.content
+                .map((part) => {
+                if (part instanceof ContentPartTextParam) {
+                    return { type: 'input_text', text: part.text };
+                }
+                if (part instanceof ContentPartRefusalParam) {
+                    return {
+                        type: 'input_text',
+                        text: `[Refusal: ${part.refusal}]`,
+                    };
+                }
+                return null;
+            })
+                .filter((part) => part !== null);
+            return { role: 'assistant', content };
+        }
+        throw new Error(`Unknown message type: ${message?.constructor?.name ?? typeof message}`);
+    }
+}

package/dist/llm/openrouter/chat.d.ts

@@ -1,13 +1,39 @@
 import type { BaseChatModel, ChatInvokeOptions } from '../base.js';
-import { ChatInvokeCompletion } from '../views.js';
 import type { Message } from '../messages.js';
+import { ChatInvokeCompletion } from '../views.js';
+export interface ChatOpenRouterOptions {
+    model?: string;
+    apiKey?: string;
+    baseURL?: string;
+    timeout?: number | null;
+    temperature?: number | null;
+    topP?: number | null;
+    seed?: number | null;
+    maxRetries?: number;
+    defaultHeaders?: Record<string, string> | null;
+    defaultQuery?: Record<string, string | undefined> | null;
+    fetchImplementation?: typeof fetch;
+    fetchOptions?: RequestInit | null;
+    httpReferer?: string | null;
+    extraBody?: Record<string, unknown> | null;
+    removeMinItemsFromSchema?: boolean;
+    removeDefaultsFromSchema?: boolean;
+}
 export declare class ChatOpenRouter implements BaseChatModel {
     model: string;
     provider: string;
     private client;
-    constructor(model?: string);
+    private temperature;
+    private topP;
+    private seed;
+    private httpReferer;
+    private extraBody;
+    private removeMinItemsFromSchema;
+    private removeDefaultsFromSchema;
+    constructor(options?: string | ChatOpenRouterOptions);
     get name(): string;
     get model_name(): string;
+    private getUsage;
     ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise<ChatInvokeCompletion<string>>;
     ainvoke<T>(messages: Message[], output_format: {
         parse: (input: string) => T;

package/dist/llm/openrouter/chat.js

@@ -1,16 +1,39 @@
 import OpenAI from 'openai';
-import { zodToJsonSchema } from 'zod-to-json-schema';
+import { ModelProviderError } from '../exceptions.js';
+import { SchemaOptimizer, zodSchemaToJsonSchema } from '../schema.js';
 import { ChatInvokeCompletion } from '../views.js';
 import { OpenRouterMessageSerializer } from './serializer.js';
 export class ChatOpenRouter {
     model;
     provider = 'openrouter';
     client;
-    constructor(model = 'openai/gpt-4o') {
+    temperature;
+    topP;
+    seed;
+    httpReferer;
+    extraBody;
+    removeMinItemsFromSchema;
+    removeDefaultsFromSchema;
+    constructor(options = {}) {
+        const normalizedOptions = typeof options === 'string' ? { model: options } : options;
+        const { model = 'openai/gpt-4o', apiKey = process.env.OPENROUTER_API_KEY, baseURL = 'https://openrouter.ai/api/v1', timeout = null, temperature = null, topP = null, seed = null, maxRetries = 10, defaultHeaders = null, defaultQuery = null, fetchImplementation, fetchOptions = null, httpReferer = null, extraBody = null, removeMinItemsFromSchema = false, removeDefaultsFromSchema = false, } = normalizedOptions;
         this.model = model;
+        this.temperature = temperature;
+        this.topP = topP;
+        this.seed = seed;
+        this.httpReferer = httpReferer;
+        this.extraBody = extraBody;
+        this.removeMinItemsFromSchema = removeMinItemsFromSchema;
+        this.removeDefaultsFromSchema = removeDefaultsFromSchema;
         this.client = new OpenAI({
-            apiKey: process.env.OPENROUTER_API_KEY,
-            baseURL: 'https://openrouter.ai/api/v1',
+            apiKey,
+            baseURL,
+            timeout: timeout ?? undefined,
+            maxRetries,
+            defaultHeaders: defaultHeaders ?? undefined,
+            defaultQuery: defaultQuery ?? undefined,
+            fetch: fetchImplementation,
+            fetchOptions: (fetchOptions ?? undefined),
         });
     }
     get name() {
@@ -19,56 +42,119 @@ export class ChatOpenRouter {
     get model_name() {
         return this.model;
     }
+    getUsage(response) {
+        if (!response.usage) {
+            return null;
+        }
+        return {
+            prompt_tokens: response.usage.prompt_tokens,
+            prompt_cached_tokens: response.usage.prompt_tokens_details?.cached_tokens ?? null,
+            prompt_cache_creation_tokens: null,
+            prompt_image_tokens: null,
+            completion_tokens: response.usage.completion_tokens,
+            total_tokens: response.usage.total_tokens,
+        };
+    }
     async ainvoke(messages, output_format, options = {}) {
         const serializer = new OpenRouterMessageSerializer();
         const openRouterMessages = serializer.serialize(messages);
+        const modelParams = {};
+        if (this.temperature !== null) {
+            modelParams.temperature = this.temperature;
+        }
+        if (this.topP !== null) {
+            modelParams.top_p = this.topP;
+        }
+        if (this.seed !== null) {
+            modelParams.seed = this.seed;
+        }
+        const zodSchemaCandidate = (() => {
+            const output = output_format;
+            if (output &&
+                typeof output === 'object' &&
+                typeof output.safeParse === 'function' &&
+                typeof output.parse === 'function') {
+                return output;
+            }
+            if (output &&
+                typeof output === 'object' &&
+                output.schema &&
+                typeof output.schema.safeParse === 'function' &&
+                typeof output.schema.parse === 'function') {
+                return output.schema;
+            }
+            return null;
+        })();
         let responseFormat = undefined;
-        if (output_format && 'schema' in output_format && output_format.schema) {
-            // OpenRouter supports structured outputs for some models, but it depends on the underlying provider.
-            // We'll try to use json_schema if possible, or json_object.
+        if (zodSchemaCandidate) {
             try {
-                const jsonSchema = zodToJsonSchema(output_format, {
-                    name: 'Response',
+                const rawJsonSchema = zodSchemaToJsonSchema(zodSchemaCandidate, {
+                    name: 'agent_output',
                     target: 'jsonSchema7',
                 });
+                const optimizedJsonSchema = SchemaOptimizer.createOptimizedJsonSchema(rawJsonSchema, {
+                    removeMinItems: this.removeMinItemsFromSchema,
+                    removeDefaults: this.removeDefaultsFromSchema,
+                });
                 responseFormat = {
                     type: 'json_schema',
                     json_schema: {
-                        name: 'Response',
-                        schema: jsonSchema,
+                        name: 'agent_output',
+                        schema: optimizedJsonSchema,
                         strict: true,
                     },
                 };
             }
-            catch (e) {
-                console.warn('Failed to convert output_format to JSON schema for OpenRouter', e);
+            catch {
+                responseFormat = undefined;
             }
         }
-        const response = await this.client.chat.completions.create({
+        const request = {
             model: this.model,
             messages: openRouterMessages,
             response_format: responseFormat,
-        }, options.signal ? { signal: options.signal } : undefined);
-        const content = response.choices[0].message.content || '';
-        let completion = content;
-        if (output_format) {
-            try {
-                if (responseFormat?.type === 'json_schema') {
-                    completion = output_format.parse(JSON.parse(content));
+            ...modelParams,
+            ...(this.extraBody ?? {}),
+        };
+        if (this.httpReferer) {
+            request.extra_headers = {
+                'HTTP-Referer': this.httpReferer,
+            };
+        }
+        try {
+            const response = await this.client.chat.completions.create(request, options.signal ? { signal: options.signal } : undefined);
+            const content = response.choices[0].message.content || '';
+            const usage = this.getUsage(response);
+            const stopReason = response.choices[0].finish_reason ?? null;
+            let completion = content;
+            if (output_format) {
+                if (zodSchemaCandidate) {
+                    const parsedJson = JSON.parse(content);
+                    const output = output_format;
+                    if (output &&
+                        typeof output === 'object' &&
+                        output.schema &&
+                        typeof output.schema.parse === 'function') {
+                        completion = output.schema.parse(parsedJson);
+                    }
+                    else {
+                        completion = output_format.parse(parsedJson);
+                    }
                 }
                 else {
                     completion = output_format.parse(content);
                 }
             }
-            catch (e) {
-                console.error('Failed to parse completion', e);
-                throw e;
+            return new ChatInvokeCompletion(completion, usage, null, null, stopReason);
+        }
+        catch (error) {
+            if (error?.status === 429) {
+                throw new ModelProviderError(error?.message ?? 'Rate limit exceeded', 429, this.model);
+            }
+            if (error?.status >= 500) {
+                throw new ModelProviderError(error?.message ?? 'Server error', error.status, this.model);
             }
+            throw new ModelProviderError(error?.message ?? String(error), error?.status ?? 500, this.model);
         }
-        return new ChatInvokeCompletion(completion, {
-            prompt_tokens: response.usage?.prompt_tokens ?? 0,
-            completion_tokens: response.usage?.completion_tokens ?? 0,
-            total_tokens: response.usage?.total_tokens ?? 0,
-        });
     }
 }

package/dist/llm/schema.d.ts

@@ -1,6 +1,16 @@
 type JsonSchema = Record<string, unknown>;
+interface ZodJsonSchemaOptions {
+    name?: string;
+    target?: string;
+    [key: string]: unknown;
+}
+export declare const zodSchemaToJsonSchema: (schema: unknown, options?: ZodJsonSchemaOptions) => JsonSchema;
 export declare class SchemaOptimizer {
-    static createOptimizedJsonSchema(schema: JsonSchema): JsonSchema;
+    static createOptimizedJsonSchema(schema: JsonSchema, options?: {
+        removeMinItems?: boolean;
+        removeDefaults?: boolean;
+    }): JsonSchema;
+    static createGeminiOptimizedSchema(schema: JsonSchema): JsonSchema;
     static makeStrictCompatible(schema: any): void;
 }
 export {};