browser-use 0.2.0 → 0.3.0

package/dist/llm/aws/chat-anthropic.js

@@ -19,9 +19,10 @@
  * ```
  */
 import { BedrockRuntimeClient, ConverseCommand, } from '@aws-sdk/client-bedrock-runtime';
+import { ModelProviderError, ModelRateLimitError } from '../exceptions.js';
 import { ChatInvokeCompletion } from '../views.js';
 import { AnthropicMessageSerializer } from '../anthropic/serializer.js';
-import { zodToJsonSchema } from 'zod-to-json-schema';
+import { SchemaOptimizer, zodSchemaToJsonSchema } from '../schema.js';
 export class ChatAnthropicBedrock {
     model;
     provider = 'anthropic_bedrock';
@@ -31,6 +32,8 @@ export class ChatAnthropicBedrock {
     top_p;
     top_k;
     stop_sequences;
+    removeMinItemsFromSchema;
+    removeDefaultsFromSchema;
     constructor(config = {}) {
         // Anthropic Claude specific defaults
         this.model = config.model || 'anthropic.claude-3-5-sonnet-20241022-v2:0';
@@ -41,8 +44,24 @@ export class ChatAnthropicBedrock {
         this.top_k = config.top_k === undefined ? null : config.top_k;
         this.stop_sequences =
             config.stop_sequences === undefined ? null : config.stop_sequences;
+        this.removeMinItemsFromSchema = config.removeMinItemsFromSchema ?? false;
+        this.removeDefaultsFromSchema = config.removeDefaultsFromSchema ?? false;
         const region = config.region || process.env.AWS_REGION || 'us-east-1';
-        this.client = new BedrockRuntimeClient({ region });
+        const awsSessionToken = config.awsSessionToken || process.env.AWS_SESSION_TOKEN;
+        const credentials = config.awsAccessKeyId && config.awsSecretAccessKey
+            ? {
+                accessKeyId: config.awsAccessKeyId,
+                secretAccessKey: config.awsSecretAccessKey,
+                ...(awsSessionToken ? { sessionToken: awsSessionToken } : {}),
+            }
+            : undefined;
+        this.client = new BedrockRuntimeClient({
+            region,
+            ...(credentials ? { credentials } : {}),
+            ...(config.maxRetries !== undefined
+                ? { maxAttempts: config.maxRetries }
+                : {}),
+        });
     }
     get name() {
         return this.model;
@@ -65,6 +84,43 @@ export class ChatAnthropicBedrock {
         }
         return params;
     }
+    getZodSchemaCandidate(output_format) {
+        const output = output_format;
+        if (output &&
+            typeof output === 'object' &&
+            typeof output.safeParse === 'function' &&
+            typeof output.parse === 'function') {
+            return output;
+        }
+        if (output &&
+            typeof output === 'object' &&
+            output.schema &&
+            typeof output.schema.safeParse === 'function' &&
+            typeof output.schema.parse === 'function') {
+            return output.schema;
+        }
+        return null;
+    }
+    parseOutput(output_format, payload) {
+        const output = output_format;
+        if (output &&
+            typeof output === 'object' &&
+            output.schema &&
+            typeof output.schema.parse === 'function') {
+            return output.schema.parse(payload);
+        }
+        return output.parse(payload);
+    }
+    getTextCompletion(response) {
+        const contentBlocks = response?.output?.message?.content;
+        if (!Array.isArray(contentBlocks)) {
+            return '';
+        }
+        return contentBlocks
+            .filter((block) => typeof block?.text === 'string')
+            .map((block) => block.text)
+            .join('\n');
+    }
     async ainvoke(messages, output_format, options = {}) {
         // Use Anthropic-specific message serializer
         const serializer = new AnthropicMessageSerializer();
@@ -76,8 +132,26 @@ export class ChatAnthropicBedrock {
                     if (block.type === 'text') {
                         return { text: block.text };
                     }
+                    else if (block.type === 'tool_use') {
+                        return {
+                            toolUse: {
+                                toolUseId: block.id,
+                                name: block.name,
+                                input: block.input,
+                            },
+                        };
+                    }
                     else if (block.type === 'image') {
-                        // Handle image blocks if needed
+                        if (block.source?.type === 'base64') {
+                            return {
+                                image: {
+                                    format: String(block.source.media_type || 'image/jpeg').split('/')[1] ?? 'jpeg',
+                                    source: {
+                                        bytes: Buffer.from(block.source.data ?? '', 'base64'),
+                                    },
+                                },
+                            };
+                        }
                         return { text: '[Image]' };
                     }
                     return { text: String(block) };
@@ -98,15 +172,18 @@ export class ChatAnthropicBedrock {
                 },
             ]
             : undefined;
-        let tools = undefined;
         let toolConfig = undefined;
-        if (output_format && 'schema' in output_format) {
+        const zodSchemaCandidate = this.getZodSchemaCandidate(output_format);
+        if (output_format && zodSchemaCandidate) {
             // Structured output using tools
             try {
-                const schema = output_format.schema?.shape
-                    ? this._zodToJsonSchema(output_format.schema)
-                    : {};
-                tools = [
+                const rawSchema = this._zodToJsonSchema(zodSchemaCandidate);
+                const schema = SchemaOptimizer.createOptimizedJsonSchema(rawSchema, {
+                    removeMinItems: this.removeMinItemsFromSchema,
+                    removeDefaults: this.removeDefaultsFromSchema,
+                });
+                delete schema.title;
+                const tools = [
                     {
                         toolSpec: {
                             name: 'extract_structured_data',
@@ -118,7 +195,7 @@ export class ChatAnthropicBedrock {
                     },
                 ];
                 toolConfig = {
-                    tools: tools,
+                    tools,
                     toolChoice: { tool: { name: 'extract_structured_data' } },
                 };
             }
@@ -133,31 +210,54 @@ export class ChatAnthropicBedrock {
             toolConfig: toolConfig,
             inferenceConfig: this._getInferenceParams(),
         });
-        const response = await this.client.send(command, options.signal ? { abortSignal: options.signal } : undefined);
-        let completion = '';
-        if (response.output?.message?.content) {
-            // Check for tool use (structured output)
-            const toolUseBlock = response.output.message.content.find((block) => block.toolUse);
-            if (toolUseBlock && toolUseBlock.toolUse && output_format) {
-                completion = output_format.parse(toolUseBlock.toolUse.input);
+        try {
+            const response = await this.client.send(command, options.signal ? { abortSignal: options.signal } : undefined);
+            let completion = this.getTextCompletion(response);
+            const contentBlocks = response?.output?.message?.content;
+            const toolUseBlock = Array.isArray(contentBlocks)
+                ? contentBlocks.find((block) => block?.toolUse)
+                : undefined;
+            if (toolUseBlock?.toolUse && output_format) {
+                const input = toolUseBlock.toolUse.input;
+                if (typeof input === 'string') {
+                    completion = this.parseOutput(output_format, JSON.parse(input));
+                }
+                else {
+                    completion = this.parseOutput(output_format, input);
+                }
+            }
+            else if (output_format && toolConfig) {
+                throw new ModelProviderError('Expected tool use in response but none found', 502, this.model);
+            }
+            else if (output_format) {
+                completion = this.parseOutput(output_format, completion);
             }
             else {
-                // Fallback to text
-                const textBlock = response.output.message.content.find((block) => block.text);
-                completion = textBlock?.text || '';
+                completion = this.getTextCompletion(response);
             }
+            const stopReason = response?.stopReason ?? null;
+            return new ChatInvokeCompletion(completion, {
+                prompt_tokens: response.usage?.inputTokens ?? 0,
+                completion_tokens: response.usage?.outputTokens ?? 0,
+                total_tokens: response.usage?.totalTokens ?? 0,
+            }, null, null, stopReason);
+        }
+        catch (error) {
+            const errorName = String(error?.name ?? '');
+            const statusCode = error?.$metadata?.httpStatusCode ?? 502;
+            if (statusCode === 429 ||
+                errorName.includes('Throttling') ||
+                errorName.includes('TooManyRequests')) {
+                throw new ModelRateLimitError(error?.message ?? 'Rate limit exceeded', 429, this.model);
+            }
+            throw new ModelProviderError(error?.message ?? String(error), statusCode, this.model);
         }
-        return new ChatInvokeCompletion(completion, {
-            prompt_tokens: response.usage?.inputTokens ?? 0,
-            completion_tokens: response.usage?.outputTokens ?? 0,
-            total_tokens: response.usage?.totalTokens ?? 0,
-        });
     }
     /**
      * Simple Zod to JSON Schema conversion for structured output
      */
     _zodToJsonSchema(schema) {
-        return zodToJsonSchema(schema, {
+        return zodSchemaToJsonSchema(schema, {
             name: 'Response',
             target: 'jsonSchema7',
         });

package/dist/llm/aws/chat-bedrock.d.ts

@@ -1,13 +1,40 @@
 import type { BaseChatModel, ChatInvokeOptions } from '../base.js';
 import { ChatInvokeCompletion } from '../views.js';
 import { type Message } from '../messages.js';
+export interface ChatBedrockConverseOptions {
+    model?: string;
+    region?: string;
+    awsAccessKeyId?: string;
+    awsSecretAccessKey?: string;
+    awsSessionToken?: string;
+    maxTokens?: number | null;
+    temperature?: number | null;
+    topP?: number | null;
+    seed?: number | null;
+    stopSequences?: string[] | null;
+    maxRetries?: number;
+    removeMinItemsFromSchema?: boolean;
+    removeDefaultsFromSchema?: boolean;
+}
 export declare class ChatBedrockConverse implements BaseChatModel {
     model: string;
     provider: string;
     private client;
-    constructor(model?: string, region?: string);
+    private maxTokens;
+    private temperature;
+    private topP;
+    private seed;
+    private stopSequences;
+    private removeMinItemsFromSchema;
+    private removeDefaultsFromSchema;
+    constructor(modelOrOptions?: string | ChatBedrockConverseOptions, region?: string);
     get name(): string;
     get model_name(): string;
+    private getInferenceConfig;
+    private getUsage;
+    private getZodSchemaCandidate;
+    private parseOutput;
+    private getTextCompletion;
     ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise<ChatInvokeCompletion<string>>;
     ainvoke<T>(messages: Message[], output_format: {
         parse: (input: string) => T;

package/dist/llm/aws/chat-bedrock.js

@@ -1,15 +1,44 @@
 import { BedrockRuntimeClient, ConverseCommand, } from '@aws-sdk/client-bedrock-runtime';
-import { zodToJsonSchema } from 'zod-to-json-schema';
+import { ModelProviderError, ModelRateLimitError } from '../exceptions.js';
 import { ChatInvokeCompletion } from '../views.js';
-import { SystemMessage } from '../messages.js';
+import { SchemaOptimizer, zodSchemaToJsonSchema } from '../schema.js';
 import { AWSBedrockMessageSerializer } from './serializer.js';
 export class ChatBedrockConverse {
     model;
     provider = 'aws';
     client;
-    constructor(model = 'anthropic.claude-3-5-sonnet-20240620-v1:0', region = process.env.AWS_REGION || 'us-east-1') {
+    maxTokens;
+    temperature;
+    topP;
+    seed;
+    stopSequences;
+    removeMinItemsFromSchema;
+    removeDefaultsFromSchema;
+    constructor(modelOrOptions = {}, region) {
+        const normalizedOptions = typeof modelOrOptions === 'string'
+            ? { model: modelOrOptions, region }
+            : modelOrOptions;
+        const { model = 'anthropic.claude-3-5-sonnet-20240620-v1:0', region: bedrockRegion = process.env.AWS_REGION || 'us-east-1', awsAccessKeyId, awsSecretAccessKey, awsSessionToken = process.env.AWS_SESSION_TOKEN, maxTokens = 4096, temperature = null, topP = null, seed = null, stopSequences = null, maxRetries, removeMinItemsFromSchema = false, removeDefaultsFromSchema = false, } = normalizedOptions;
         this.model = model;
-        this.client = new BedrockRuntimeClient({ region });
+        this.maxTokens = maxTokens;
+        this.temperature = temperature;
+        this.topP = topP;
+        this.seed = seed;
+        this.stopSequences = stopSequences;
+        this.removeMinItemsFromSchema = removeMinItemsFromSchema;
+        this.removeDefaultsFromSchema = removeDefaultsFromSchema;
+        const credentials = awsAccessKeyId && awsSecretAccessKey
+            ? {
+                accessKeyId: awsAccessKeyId,
+                secretAccessKey: awsSecretAccessKey,
+                ...(awsSessionToken ? { sessionToken: awsSessionToken } : {}),
+            }
+            : undefined;
+        this.client = new BedrockRuntimeClient({
+            region: bedrockRegion,
+            ...(credentials ? { credentials } : {}),
+            ...(maxRetries !== undefined ? { maxAttempts: maxRetries } : {}),
+        });
     }
     get name() {
         return this.model;
@@ -17,32 +46,102 @@ export class ChatBedrockConverse {
     get model_name() {
         return this.model;
     }
+    getInferenceConfig() {
+        const config = {};
+        if (this.maxTokens !== null) {
+            config.maxTokens = this.maxTokens;
+        }
+        if (this.temperature !== null) {
+            config.temperature = this.temperature;
+        }
+        if (this.topP !== null) {
+            config.topP = this.topP;
+        }
+        if (this.seed !== null) {
+            config.seed = this.seed;
+        }
+        if (this.stopSequences !== null) {
+            config.stopSequences = this.stopSequences;
+        }
+        return config;
+    }
+    getUsage(response) {
+        const usage = response?.usage ?? {};
+        return {
+            prompt_tokens: usage.inputTokens ?? 0,
+            completion_tokens: usage.outputTokens ?? 0,
+            total_tokens: usage.totalTokens ?? 0,
+            prompt_cached_tokens: null,
+            prompt_cache_creation_tokens: null,
+            prompt_image_tokens: null,
+        };
+    }
+    getZodSchemaCandidate(output_format) {
+        const output = output_format;
+        if (output &&
+            typeof output === 'object' &&
+            typeof output.safeParse === 'function' &&
+            typeof output.parse === 'function') {
+            return output;
+        }
+        if (output &&
+            typeof output === 'object' &&
+            output.schema &&
+            typeof output.schema.safeParse === 'function' &&
+            typeof output.schema.parse === 'function') {
+            return output.schema;
+        }
+        return null;
+    }
+    parseOutput(output_format, payload) {
+        const output = output_format;
+        if (output &&
+            typeof output === 'object' &&
+            output.schema &&
+            typeof output.schema.parse === 'function') {
+            return output.schema.parse(payload);
+        }
+        return output.parse(payload);
+    }
+    getTextCompletion(response) {
+        const contentBlocks = response?.output?.message?.content;
+        if (!Array.isArray(contentBlocks)) {
+            return '';
+        }
+        return contentBlocks
+            .filter((block) => typeof block?.text === 'string')
+            .map((block) => block.text)
+            .join('\n');
+    }
     async ainvoke(messages, output_format, options = {}) {
         const serializer = new AWSBedrockMessageSerializer();
-        const bedrockMessages = serializer.serialize(messages);
-        const systemMessage = messages.find((msg) => msg instanceof SystemMessage);
-        const system = systemMessage ? [{ text: systemMessage.text }] : undefined;
-        let tools = undefined;
+        const [bedrockMessages, systemMessage] = serializer.serializeMessages(messages);
+        const zodSchemaCandidate = this.getZodSchemaCandidate(output_format);
         let toolConfig = undefined;
-        if (output_format && 'schema' in output_format && output_format.schema) {
+        if (output_format && zodSchemaCandidate) {
             try {
-                const jsonSchema = zodToJsonSchema(output_format, {
+                const rawJsonSchema = zodSchemaToJsonSchema(zodSchemaCandidate, {
                     name: 'Response',
                     target: 'jsonSchema7',
                 });
-                tools = [
+                const optimizedJsonSchema = SchemaOptimizer.createOptimizedJsonSchema(rawJsonSchema, {
+                    removeMinItems: this.removeMinItemsFromSchema,
+                    removeDefaults: this.removeDefaultsFromSchema,
+                });
+                delete optimizedJsonSchema.title;
+                const tools = [
                     {
                         toolSpec: {
                             name: 'response',
-                            description: 'The response to the user request',
+                            description: 'Extract information in the format of response',
                             inputSchema: {
-                                json: jsonSchema,
+                                json: optimizedJsonSchema,
                             },
                         },
                     },
                 ];
                 toolConfig = {
-                    tools: tools,
+                    tools,
                     toolChoice: { tool: { name: 'response' } },
                 };
             }
@@ -50,31 +149,59 @@ export class ChatBedrockConverse {
                 console.warn('Failed to convert output_format to JSON schema for AWS Bedrock', e);
             }
         }
-        const command = new ConverseCommand({
+        const requestPayload = {
             modelId: this.model,
             messages: bedrockMessages,
-            system: system,
-            toolConfig: toolConfig,
-        });
-        const response = await this.client.send(command, options.signal ? { abortSignal: options.signal } : undefined);
-        let completion = '';
-        if (response.output?.message?.content) {
-            // Check for tool use
-            const toolUseBlock = response.output.message.content.find((block) => block.toolUse);
-            if (toolUseBlock && toolUseBlock.toolUse && output_format) {
-                completion = output_format.parse(toolUseBlock.toolUse.input);
+        };
+        if (systemMessage) {
+            requestPayload.system = systemMessage;
+        }
+        const inferenceConfig = this.getInferenceConfig();
+        if (Object.keys(inferenceConfig).length) {
+            requestPayload.inferenceConfig = inferenceConfig;
+        }
+        if (toolConfig) {
+            requestPayload.toolConfig = toolConfig;
+        }
+        try {
+            const response = await this.client.send(new ConverseCommand(requestPayload), options.signal ? { abortSignal: options.signal } : undefined);
+            let completion = this.getTextCompletion(response);
+            if (output_format) {
+                const contentBlocks = response?.output?.message?.content;
+                const toolUseBlock = Array.isArray(contentBlocks)
+                    ? contentBlocks.find((block) => block?.toolUse)
+                    : undefined;
+                if (toolUseBlock?.toolUse) {
+                    const input = toolUseBlock.toolUse.input;
+                    if (typeof input === 'string') {
+                        completion = this.parseOutput(output_format, JSON.parse(input));
+                    }
+                    else {
+                        completion = this.parseOutput(output_format, input);
+                    }
+                }
+                else if (toolConfig) {
+                    throw new ModelProviderError('Expected tool use in response but none found', 502, this.model);
+                }
+                else {
+                    completion = this.parseOutput(output_format, completion);
+                }
             }
             else {
-                // Fallback to text
-                const textBlock = response.output.message.content.find((block) => block.text);
-                completion = textBlock?.text || '';
+                completion = this.getTextCompletion(response);
             }
+            const stopReason = response?.stopReason ?? null;
+            return new ChatInvokeCompletion(completion, this.getUsage(response), null, null, stopReason);
+        }
+        catch (error) {
+            const errorName = String(error?.name ?? '');
+            const statusCode = error?.$metadata?.httpStatusCode ?? 502;
+            if (statusCode === 429 ||
+                errorName.includes('Throttling') ||
+                errorName.includes('TooManyRequests')) {
+                throw new ModelRateLimitError(error?.message ?? 'Rate limit exceeded', 429, this.model);
+            }
+            throw new ModelProviderError(error?.message ?? String(error), statusCode, this.model);
         }
-        const usage = response.usage || {};
-        return new ChatInvokeCompletion(completion, {
-            prompt_tokens: usage.inputTokens || 0,
-            completion_tokens: usage.outputTokens || 0,
-            total_tokens: usage.totalTokens || 0,
-        });
     }
 }

package/dist/llm/aws/serializer.d.ts

@@ -1,5 +1,17 @@
 import { type Message } from '../messages.js';
+type BedrockContentBlock = Record<string, unknown>;
+type BedrockMessage = {
+    role: 'user' | 'assistant';
+    content: BedrockContentBlock[];
+};
+type BedrockSystemMessage = {
+    text: string;
+}[];
 export declare class AWSBedrockMessageSerializer {
-    serialize(messages: Message[]): any[];
+    serialize(messages: Message[]): BedrockMessage[];
+    serializeMessages(messages: Message[]): [BedrockMessage[], BedrockSystemMessage?];
+    private serializeSystemContent;
+    private serializeImageContent;
     private serializeMessage;
 }
+export {};

package/dist/llm/aws/serializer.js

@@ -1,11 +1,44 @@
-import { AssistantMessage, ContentPartImageParam, ContentPartTextParam, SystemMessage, UserMessage, } from '../messages.js';
-// AWS Bedrock types are a bit complex and vary by model.
-// We will define a generic structure that fits most Bedrock models (like Claude on Bedrock).
+import { AssistantMessage, ContentPartImageParam, ContentPartRefusalParam, ContentPartTextParam, SystemMessage, UserMessage, } from '../messages.js';
 export class AWSBedrockMessageSerializer {
     serialize(messages) {
-        return messages
-            .filter((msg) => !(msg instanceof SystemMessage)) // System messages are handled separately usually
-            .map((message) => this.serializeMessage(message));
+        return this.serializeMessages(messages)[0];
+    }
+    serializeMessages(messages) {
+        const bedrockMessages = [];
+        let systemMessage = undefined;
+        for (const message of messages) {
+            if (message instanceof SystemMessage) {
+                systemMessage = this.serializeSystemContent(message.content);
+            }
+            else {
+                bedrockMessages.push(this.serializeMessage(message));
+            }
+        }
+        return [bedrockMessages, systemMessage];
+    }
+    serializeSystemContent(content) {
+        if (typeof content === 'string') {
+            return [{ text: content }];
+        }
+        return content
+            .filter((part) => part instanceof ContentPartTextParam)
+            .map((part) => ({ text: part.text }));
+    }
+    serializeImageContent(part) {
+        const url = part.image_url.url;
+        if (!url.startsWith('data:')) {
+            throw new Error(`Unsupported image URL format: ${url}`);
+        }
+        const [, payload = ''] = url.split(',', 2);
+        const format = part.image_url.media_type.split('/')[1] ?? 'jpeg';
+        return {
+            image: {
+                format,
+                source: {
+                    bytes: Buffer.from(payload, 'base64'),
+                },
+            },
+        };
     }
     serializeMessage(message) {
         if (message instanceof UserMessage) {
@@ -17,16 +50,7 @@ export class AWSBedrockMessageSerializer {
                             return { text: part.text };
                         }
                         if (part instanceof ContentPartImageParam) {
-                            const mediaType = part.image_url.media_type;
-                            const data = part.image_url.url.split(',')[1];
-                            return {
-                                image: {
-                                    format: mediaType.split('/')[1], // e.g. 'png' from 'image/png'
-                                    source: {
-                                        bytes: Buffer.from(data, 'base64'),
-                                    },
-                                },
-                            };
+                            return this.serializeImageContent(part);
                         }
                         return { text: '' };
                     })
@@ -44,20 +68,35 @@ export class AWSBedrockMessageSerializer {
                         if (part instanceof ContentPartTextParam) {
                             content.push({ text: part.text });
                         }
+                        else if (part instanceof ContentPartRefusalParam) {
+                            content.push({ text: `[Refusal] ${part.refusal}` });
+                        }
                     });
                 }
             }
             if (message.tool_calls) {
                 message.tool_calls.forEach((toolCall) => {
+                    let parsedArguments;
+                    try {
+                        parsedArguments = JSON.parse(toolCall.functionCall.arguments);
+                    }
+                    catch {
+                        parsedArguments = {
+                            arguments: toolCall.functionCall.arguments,
+                        };
+                    }
                     content.push({
                         toolUse: {
                             toolUseId: toolCall.id,
                             name: toolCall.functionCall.name,
-                            input: JSON.parse(toolCall.functionCall.arguments),
+                            input: parsedArguments,
                         },
                     });
                 });
             }
+            if (!content.length) {
+                content.push({ text: '' });
+            }
             return {
                 role: 'assistant',
                 content: content,