browser-use 0.2.0 → 0.4.0

package/dist/llm/azure/chat.d.ts

@@ -1,15 +1,66 @@
 import type { BaseChatModel, ChatInvokeOptions } from '../base.js';
-import { ChatInvokeCompletion } from '../views.js';
 import type { Message } from '../messages.js';
+import { ChatInvokeCompletion } from '../views.js';
+export interface ChatAzureOptions {
+    model?: string;
+    apiKey?: string;
+    endpoint?: string;
+    baseURL?: string;
+    apiVersion?: string;
+    deployment?: string;
+    azureAdToken?: string | null;
+    azureAdTokenProvider?: (() => Promise<string>) | null;
+    timeout?: number | null;
+    temperature?: number | null;
+    frequencyPenalty?: number | null;
+    reasoningEffort?: 'low' | 'medium' | 'high';
+    serviceTier?: 'auto' | 'default' | 'flex' | 'priority' | 'scale' | null;
+    maxCompletionTokens?: number | null;
+    topP?: number | null;
+    seed?: number | null;
+    maxRetries?: number;
+    defaultHeaders?: Record<string, string> | null;
+    defaultQuery?: Record<string, string | undefined> | null;
+    fetchImplementation?: typeof fetch;
+    fetchOptions?: RequestInit | null;
+    useResponsesApi?: boolean | 'auto';
+    addSchemaToSystemPrompt?: boolean;
+    dontForceStructuredOutput?: boolean;
+    removeMinItemsFromSchema?: boolean;
+    removeDefaultsFromSchema?: boolean;
+}
 export declare class ChatAzure implements BaseChatModel {
     model: string;
     provider: string;
     private client;
-    constructor(model?: string);
+    private temperature;
+    private frequencyPenalty;
+    private reasoningEffort;
+    private serviceTier;
+    private maxCompletionTokens;
+    private topP;
+    private seed;
+    private useResponsesApi;
+    private addSchemaToSystemPrompt;
+    private dontForceStructuredOutput;
+    private removeMinItemsFromSchema;
+    private removeDefaultsFromSchema;
+    constructor(options?: string | ChatAzureOptions);
     get name(): string;
     get model_name(): string;
+    private isReasoningModel;
+    private shouldUseResponsesApi;
+    private getChatUsage;
+    private getResponsesUsage;
+    private getResponseOutputText;
+    private getModelParamsForCompletions;
+    private getModelParamsForResponses;
+    private getZodSchemaCandidate;
+    private applySchemaToSystemMessage;
     ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise<ChatInvokeCompletion<string>>;
     ainvoke<T>(messages: Message[], output_format: {
         parse: (input: string) => T;
     } | undefined, options?: ChatInvokeOptions): Promise<ChatInvokeCompletion<T>>;
+    private invokeChatCompletions;
+    private invokeResponses;
 }

package/dist/llm/azure/chat.js

@@ -1,17 +1,74 @@
 import { AzureOpenAI } from 'openai';
-import { ChatInvokeCompletion } from '../views.js';
+import { ModelProviderError, ModelRateLimitError } from '../exceptions.js';
 import { OpenAIMessageSerializer } from '../openai/serializer.js';
+import { ResponsesAPIMessageSerializer } from '../openai/responses-serializer.js';
+import { SchemaOptimizer, zodSchemaToJsonSchema } from '../schema.js';
+import { ChatInvokeCompletion } from '../views.js';
+const RESPONSES_API_ONLY_MODELS = [
+    'gpt-5.1-codex',
+    'gpt-5.1-codex-mini',
+    'gpt-5.1-codex-max',
+    'gpt-5-codex',
+    'codex-mini-latest',
+    'computer-use-preview',
+];
+const REASONING_MODELS = [
+    'o4-mini',
+    'o3',
+    'o3-mini',
+    'o1',
+    'o1-pro',
+    'o3-pro',
+    'gpt-5',
+    'gpt-5-mini',
+    'gpt-5-nano',
+];
 export class ChatAzure {
     model;
     provider = 'azure';
     client;
-    constructor(model = 'gpt-4o') {
+    temperature;
+    frequencyPenalty;
+    reasoningEffort;
+    serviceTier;
+    maxCompletionTokens;
+    topP;
+    seed;
+    useResponsesApi;
+    addSchemaToSystemPrompt;
+    dontForceStructuredOutput;
+    removeMinItemsFromSchema;
+    removeDefaultsFromSchema;
+    constructor(options = {}) {
+        const normalizedOptions = typeof options === 'string' ? { model: options } : options;
+        const { model = 'gpt-4o', apiKey = process.env.AZURE_OPENAI_API_KEY ?? process.env.AZURE_OPENAI_KEY, endpoint = process.env.AZURE_OPENAI_ENDPOINT, baseURL = undefined, apiVersion = process.env.AZURE_OPENAI_API_VERSION ?? '2024-12-01-preview', deployment = process.env.AZURE_OPENAI_DEPLOYMENT ?? model, azureAdToken = null, azureAdTokenProvider = null, timeout = null, temperature = 0.2, frequencyPenalty = 0.3, reasoningEffort = 'low', serviceTier = null, maxCompletionTokens = 4096, topP = null, seed = null, maxRetries = 5, defaultHeaders = null, defaultQuery = null, fetchImplementation, fetchOptions = null, useResponsesApi = 'auto', addSchemaToSystemPrompt = false, dontForceStructuredOutput = false, removeMinItemsFromSchema = false, removeDefaultsFromSchema = false, } = normalizedOptions;
         this.model = model;
+        this.temperature = temperature;
+        this.frequencyPenalty = frequencyPenalty;
+        this.reasoningEffort = reasoningEffort;
+        this.serviceTier = serviceTier;
+        this.maxCompletionTokens = maxCompletionTokens;
+        this.topP = topP;
+        this.seed = seed;
+        this.useResponsesApi = useResponsesApi;
+        this.addSchemaToSystemPrompt = addSchemaToSystemPrompt;
+        this.dontForceStructuredOutput = dontForceStructuredOutput;
+        this.removeMinItemsFromSchema = removeMinItemsFromSchema;
+        this.removeDefaultsFromSchema = removeDefaultsFromSchema;
         this.client = new AzureOpenAI({
-            apiKey: process.env.AZURE_OPENAI_API_KEY,
-            endpoint: process.env.AZURE_OPENAI_ENDPOINT,
-            apiVersion: process.env.AZURE_OPENAI_API_VERSION || '2024-05-01-preview',
-            deployment: model,
+            apiKey,
+            endpoint,
+            baseURL,
+            apiVersion,
+            deployment,
+            azureADTokenProvider: azureAdTokenProvider ??
+                (azureAdToken ? async () => String(azureAdToken) : undefined),
+            timeout: timeout ?? undefined,
+            maxRetries,
+            defaultHeaders: defaultHeaders ?? undefined,
+            defaultQuery: defaultQuery ?? undefined,
+            fetch: fetchImplementation,
+            fetchOptions: (fetchOptions ?? undefined),
         });
     }
     get name() {
@@ -20,65 +77,320 @@ export class ChatAzure {
     get model_name() {
         return this.model;
     }
+    isReasoningModel() {
+        return REASONING_MODELS.some((m) => this.model.toLowerCase().includes(m.toLowerCase()));
+    }
+    shouldUseResponsesApi() {
+        if (typeof this.useResponsesApi === 'boolean') {
+            return this.useResponsesApi;
+        }
+        return RESPONSES_API_ONLY_MODELS.some((name) => this.model.toLowerCase().includes(name.toLowerCase()));
+    }
+    getChatUsage(response) {
+        if (!response?.usage) {
+            return null;
+        }
+        let completionTokens = response.usage.completion_tokens;
+        const completionDetails = response.usage.completion_tokens_details;
+        if (completionDetails?.reasoning_tokens) {
+            completionTokens += completionDetails.reasoning_tokens;
+        }
+        return {
+            prompt_tokens: response.usage.prompt_tokens,
+            prompt_cached_tokens: response.usage.prompt_tokens_details?.cached_tokens ?? null,
+            prompt_cache_creation_tokens: null,
+            prompt_image_tokens: null,
+            completion_tokens: completionTokens,
+            total_tokens: response.usage.total_tokens,
+        };
+    }
+    getResponsesUsage(response) {
+        if (!response?.usage) {
+            return null;
+        }
+        return {
+            prompt_tokens: response.usage.input_tokens ?? 0,
+            prompt_cached_tokens: response.usage.input_tokens_details?.cached_tokens ?? null,
+            prompt_cache_creation_tokens: null,
+            prompt_image_tokens: null,
+            completion_tokens: response.usage.output_tokens ?? 0,
+            total_tokens: response.usage.total_tokens ?? 0,
+        };
+    }
+    getResponseOutputText(response) {
+        if (typeof response?.output_text === 'string') {
+            return response.output_text;
+        }
+        const outputs = Array.isArray(response?.output) ? response.output : [];
+        for (const item of outputs) {
+            if (Array.isArray(item?.content)) {
+                for (const part of item.content) {
+                    if (typeof part?.text === 'string') {
+                        return part.text;
+                    }
+                    if (typeof part?.output_text === 'string') {
+                        return part.output_text;
+                    }
+                }
+            }
+        }
+        return '';
+    }
+    getModelParamsForCompletions() {
+        const modelParams = {};
+        if (!this.isReasoningModel()) {
+            if (this.temperature !== null) {
+                modelParams.temperature = this.temperature;
+            }
+            if (this.frequencyPenalty !== null) {
+                modelParams.frequency_penalty = this.frequencyPenalty;
+            }
+        }
+        else {
+            modelParams.reasoning_effort = this.reasoningEffort;
+        }
+        if (this.maxCompletionTokens !== null) {
+            modelParams.max_completion_tokens = this.maxCompletionTokens;
+        }
+        if (this.topP !== null) {
+            modelParams.top_p = this.topP;
+        }
+        if (this.seed !== null) {
+            modelParams.seed = this.seed;
+        }
+        if (this.serviceTier !== null) {
+            modelParams.service_tier = this.serviceTier;
+        }
+        return modelParams;
+    }
+    getModelParamsForResponses() {
+        const modelParams = {};
+        if (!this.isReasoningModel()) {
+            if (this.temperature !== null) {
+                modelParams.temperature = this.temperature;
+            }
+            if (this.frequencyPenalty !== null) {
+                modelParams.frequency_penalty = this.frequencyPenalty;
+            }
+        }
+        else {
+            modelParams.reasoning = { effort: this.reasoningEffort };
+        }
+        if (this.maxCompletionTokens !== null) {
+            modelParams.max_output_tokens = this.maxCompletionTokens;
+        }
+        if (this.topP !== null) {
+            modelParams.top_p = this.topP;
+        }
+        if (this.seed !== null) {
+            modelParams.seed = this.seed;
+        }
+        if (this.serviceTier !== null) {
+            modelParams.service_tier = this.serviceTier;
+        }
+        return modelParams;
+    }
+    getZodSchemaCandidate(output_format) {
+        const output = output_format;
+        if (output &&
+            typeof output === 'object' &&
+            typeof output.safeParse === 'function' &&
+            typeof output.parse === 'function') {
+            return output;
+        }
+        if (output &&
+            typeof output === 'object' &&
+            output.schema &&
+            typeof output.schema.safeParse === 'function' &&
+            typeof output.schema.parse === 'function') {
+            return output.schema;
+        }
+        return null;
+    }
+    applySchemaToSystemMessage(openaiMessages, responseJsonSchema) {
+        if (!this.addSchemaToSystemPrompt || openaiMessages.length === 0) {
+            return;
+        }
+        const firstMessage = openaiMessages[0];
+        if (firstMessage?.role !== 'system') {
+            return;
+        }
+        const schemaText = `\n<json_schema>\n` +
+            `${JSON.stringify(responseJsonSchema, null, 2)}\n` +
+            `</json_schema>`;
+        if (typeof firstMessage.content === 'string') {
+            firstMessage.content = (firstMessage.content ?? '') + schemaText;
+            return;
+        }
+        if (Array.isArray(firstMessage.content)) {
+            firstMessage.content = [
+                ...firstMessage.content,
+                { type: 'text', text: schemaText },
+            ];
+        }
+    }
     async ainvoke(messages, output_format, options = {}) {
+        const zodSchemaCandidate = this.getZodSchemaCandidate(output_format);
+        if (this.shouldUseResponsesApi()) {
+            return this.invokeResponses(messages, output_format, zodSchemaCandidate, options);
+        }
+        return this.invokeChatCompletions(messages, output_format, zodSchemaCandidate, options);
+    }
+    async invokeChatCompletions(messages, output_format, zodSchemaCandidate, options) {
         const serializer = new OpenAIMessageSerializer();
         const openaiMessages = serializer.serialize(messages);
-        // Use simple json_object format for better compatibility with Azure
-        // json_schema format may not be supported on all Azure API versions/deployments
-        const responseFormat = output_format
-            ? { type: 'json_object' }
-            : undefined;
-        const response = await this.client.chat.completions.create({
+        let responseFormat = undefined;
+        if (zodSchemaCandidate) {
+            try {
+                const rawJsonSchema = zodSchemaToJsonSchema(zodSchemaCandidate, {
+                    name: 'agent_output',
+                    target: 'jsonSchema7',
+                });
+                const optimizedJsonSchema = SchemaOptimizer.createOptimizedJsonSchema(rawJsonSchema, {
+                    removeMinItems: this.removeMinItemsFromSchema,
+                    removeDefaults: this.removeDefaultsFromSchema,
+                });
+                const responseJsonSchema = {
+                    name: 'agent_output',
+                    schema: optimizedJsonSchema,
+                    strict: true,
+                };
+                this.applySchemaToSystemMessage(openaiMessages, responseJsonSchema);
+                if (!this.dontForceStructuredOutput) {
+                    responseFormat = {
+                        type: 'json_schema',
+                        json_schema: responseJsonSchema,
+                    };
+                }
+            }
+            catch {
+                responseFormat = undefined;
+            }
+        }
+        try {
+            const response = await this.client.chat.completions.create({
+                model: this.model,
+                messages: openaiMessages,
+                response_format: responseFormat,
+                ...this.getModelParamsForCompletions(),
+            }, options.signal ? { signal: options.signal } : undefined);
+            const content = response.choices[0].message.content || '';
+            const usage = this.getChatUsage(response);
+            const stopReason = response.choices[0].finish_reason ?? null;
+            let completion = content;
+            if (output_format) {
+                if (zodSchemaCandidate) {
+                    const parsedJson = JSON.parse(content);
+                    const output = output_format;
+                    if (output &&
+                        typeof output === 'object' &&
+                        output.schema &&
+                        typeof output.schema.parse === 'function') {
+                        completion = output.schema.parse(parsedJson);
+                    }
+                    else {
+                        completion = output.parse(parsedJson);
+                    }
+                }
+                else {
+                    completion = output_format.parse(content);
+                }
+            }
+            return new ChatInvokeCompletion(completion, usage, null, null, stopReason);
+        }
+        catch (error) {
+            if (error?.status === 429) {
+                throw new ModelRateLimitError(error?.message ?? 'Rate limit exceeded', 429, this.model);
+            }
+            throw new ModelProviderError(error?.message ?? String(error), error?.status ?? 500, this.model);
+        }
+    }
+    async invokeResponses(messages, output_format, zodSchemaCandidate, options) {
+        const serializer = new ResponsesAPIMessageSerializer();
+        const inputMessages = serializer.serialize(messages);
+        const request = {
             model: this.model,
-            messages: openaiMessages,
-            response_format: responseFormat,
-        }, options.signal ? { signal: options.signal } : undefined);
-        const content = response.choices[0].message.content || '';
-        let completion = content;
-        if (output_format) {
+            input: inputMessages,
+            ...this.getModelParamsForResponses(),
+        };
+        if (zodSchemaCandidate) {
             try {
-                // Extract JSON from the response
-                let jsonText = content.trim();
-                // Handle markdown fenced code blocks
-                const fencedMatch = jsonText.match(/```(?:json)?\s*([\s\S]*?)```/);
-                if (fencedMatch && fencedMatch[1]) {
-                    jsonText = fencedMatch[1].trim();
+                const rawJsonSchema = zodSchemaToJsonSchema(zodSchemaCandidate, {
+                    name: 'agent_output',
+                    target: 'jsonSchema7',
+                });
+                const optimizedJsonSchema = SchemaOptimizer.createOptimizedJsonSchema(rawJsonSchema, {
+                    removeMinItems: this.removeMinItemsFromSchema,
+                    removeDefaults: this.removeDefaultsFromSchema,
+                });
+                if (this.addSchemaToSystemPrompt &&
+                    inputMessages.length > 0 &&
+                    inputMessages[0]?.role === 'system') {
+                    const schemaText = `\n<json_schema>\n${JSON.stringify(optimizedJsonSchema)}\n</json_schema>`;
+                    const firstInput = inputMessages[0];
+                    const firstContent = firstInput?.content;
+                    let patchedContent = firstContent ?? '';
+                    if (typeof firstContent === 'string') {
+                        patchedContent = firstContent + schemaText;
+                    }
+                    else if (Array.isArray(firstContent)) {
+                        patchedContent = [
+                            ...firstContent,
+                            { type: 'input_text', text: schemaText },
+                        ];
+                    }
+                    inputMessages[0] = {
+                        ...inputMessages[0],
+                        content: patchedContent,
+                    };
+                    request.input = inputMessages;
                 }
-                // Extract JSON object/array from the text
-                const firstBrace = jsonText.indexOf('{');
-                const firstBracket = jsonText.indexOf('[');
-                const lastBrace = jsonText.lastIndexOf('}');
-                const lastBracket = jsonText.lastIndexOf(']');
-                // Determine if it's an object or array
-                let startIdx = -1;
-                let endIdx = -1;
-                if (firstBrace !== -1 &&
-                    (firstBracket === -1 || firstBrace < firstBracket)) {
-                    // It's an object
-                    startIdx = firstBrace;
-                    endIdx = lastBrace;
+                if (!this.dontForceStructuredOutput) {
+                    request.text = {
+                        format: {
+                            type: 'json_schema',
+                            name: 'agent_output',
+                            strict: true,
+                            schema: optimizedJsonSchema,
+                        },
+                    };
                 }
-                else if (firstBracket !== -1) {
-                    // It's an array
-                    startIdx = firstBracket;
-                    endIdx = lastBracket;
+            }
+            catch {
+                // Skip structured output forcing when schema conversion fails.
+            }
+        }
+        try {
+            const response = await this.client.responses.create(request, options.signal ? { signal: options.signal } : undefined);
+            const content = this.getResponseOutputText(response);
+            const usage = this.getResponsesUsage(response);
+            const stopReason = response?.status ?? null;
+            let completion = content;
+            if (output_format) {
+                if (zodSchemaCandidate) {
+                    const parsedJson = JSON.parse(content);
+                    const output = output_format;
+                    if (output &&
+                        typeof output === 'object' &&
+                        output.schema &&
+                        typeof output.schema.parse === 'function') {
+                        completion = output.schema.parse(parsedJson);
+                    }
+                    else {
+                        completion = output.parse(parsedJson);
+                    }
                 }
-                if (startIdx !== -1 && endIdx !== -1 && endIdx > startIdx) {
-                    jsonText = jsonText.slice(startIdx, endIdx + 1);
+                else {
+                    completion = output_format.parse(content);
                 }
-                const parsedJson = JSON.parse(jsonText);
-                completion = output_format.parse(parsedJson);
             }
-            catch (e) {
-                console.error('Failed to parse Azure completion:', e);
-                console.error('Raw content:', content.substring(0, 500));
-                throw new Error(`Failed to parse LLM completion as JSON: ${e}`);
+            return new ChatInvokeCompletion(completion, usage, null, null, stopReason);
+        }
+        catch (error) {
+            if (error?.status === 429) {
+                throw new ModelRateLimitError(error?.message ?? 'Rate limit exceeded', 429, this.model);
             }
+            throw new ModelProviderError(error?.message ?? String(error), error?.status ?? 500, this.model);
         }
-        return new ChatInvokeCompletion(completion, {
-            prompt_tokens: response.usage?.prompt_tokens ?? 0,
-            completion_tokens: response.usage?.completion_tokens ?? 0,
-            total_tokens: response.usage?.total_tokens ?? 0,
-        });
     }
 }

package/dist/llm/base.d.ts

@@ -2,6 +2,8 @@ import type { ChatInvokeCompletion } from './views.js';
 import type { Message } from './messages.js';
 export interface ChatInvokeOptions {
     signal?: AbortSignal;
+    request_type?: string;
+    [key: string]: unknown;
 }
 export interface BaseChatModel {
     model: string;

package/dist/llm/browser-use/chat.d.ts

@@ -0,0 +1,40 @@
+import type { BaseChatModel, ChatInvokeOptions } from '../base.js';
+import type { Message } from '../messages.js';
+import { ChatInvokeCompletion } from '../views.js';
+export interface ChatBrowserUseOptions {
+    model?: string;
+    apiKey?: string;
+    baseUrl?: string;
+    timeout?: number;
+    maxRetries?: number;
+    retryBaseDelay?: number;
+    retryMaxDelay?: number;
+    fast?: boolean;
+    fetchImplementation?: typeof fetch;
+}
+export declare class ChatBrowserUse implements BaseChatModel {
+    model: string;
+    provider: string;
+    private readonly apiKey;
+    private readonly baseUrl;
+    private readonly timeoutMs;
+    private readonly maxRetries;
+    private readonly retryBaseDelay;
+    private readonly retryMaxDelay;
+    private readonly fast;
+    private readonly fetchImplementation;
+    constructor(options?: ChatBrowserUseOptions);
+    get name(): string;
+    get model_name(): string;
+    private getOutputSchema;
+    private parseOutput;
+    private serializeMessage;
+    private getUsage;
+    private raiseHttpError;
+    private isRetryableNetworkError;
+    private makeRequest;
+    ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise<ChatInvokeCompletion<string>>;
+    ainvoke<T>(messages: Message[], output_format: {
+        parse: (input: string) => T;
+    } | undefined, options?: ChatInvokeOptions): Promise<ChatInvokeCompletion<T>>;
+}