browser-use 0.0.1 → 0.1.0

package/dist/llm/groq/chat.js

@@ -0,0 +1,52 @@
+import Groq from 'groq-sdk';
+import { ChatInvokeCompletion } from '../views.js';
+import { GroqMessageSerializer } from './serializer.js';
+export class ChatGroq {
+    model;
+    provider = 'groq';
+    client;
+    constructor(model = 'llama-3.1-70b-versatile') {
+        this.model = model;
+        this.client = new Groq({
+            apiKey: process.env.GROQ_API_KEY,
+        });
+    }
+    get name() {
+        return this.model;
+    }
+    get model_name() {
+        return this.model;
+    }
+    async ainvoke(messages, output_format, options = {}) {
+        const serializer = new GroqMessageSerializer();
+        const groqMessages = serializer.serialize(messages);
+        let responseFormat = undefined;
+        if (output_format && 'schema' in output_format && output_format.schema) {
+            // Groq supports JSON mode, but not full JSON schema validation in the same way as OpenAI yet (or maybe it does now).
+            // For now, we'll enforce JSON mode and prompt engineering or use tools if needed.
+            // But let's try to use json_object which is supported.
+            responseFormat = { type: 'json_object' };
+        }
+        const response = await this.client.chat.completions.create({
+            model: this.model,
+            messages: groqMessages,
+            response_format: responseFormat,
+        }, options.signal ? { signal: options.signal } : undefined);
+        const content = response.choices[0].message.content || '';
+        let completion = content;
+        if (output_format) {
+            try {
+                completion = output_format.parse(JSON.parse(content));
+            }
+            catch (e) {
+                console.error('Failed to parse completion', e);
+                throw e;
+            }
+        }
+        return new ChatInvokeCompletion(completion, {
+            prompt_tokens: response.usage?.prompt_tokens ?? 0,
+            completion_tokens: response.usage?.completion_tokens ?? 0,
+            total_tokens: response.usage?.total_tokens ?? 0,
+        });
+    }
+}

package/dist/llm/groq/index.d.ts

@@ -0,0 +1,3 @@
+export * from './chat.js';
+export * from './serializer.js';
+export * from './parser.js';

package/dist/llm/groq/index.js

@@ -0,0 +1,3 @@
+export * from './chat.js';
+export * from './serializer.js';
+export * from './parser.js';

package/dist/llm/groq/parser.d.ts

@@ -0,0 +1,32 @@
+import type { APIError } from 'groq-sdk';
+export declare class ParseFailedGenerationError extends Error {
+    constructor(message: string);
+}
+/**
+ * Extract JSON from model output, handling both plain JSON and code-block-wrapped JSON.
+ * This is used to parse Groq's failed_generation field when an API error occurs.
+ *
+ * @param error - The Groq API error containing failed_generation
+ * @param outputFormat - An object with a parse method (typically a Zod schema)
+ * @returns The parsed output in the expected format
+ * @throws ParseFailedGenerationError if the failed_generation field is missing
+ * @throws Error if JSON parsing fails
+ */
+export declare function tryParseGroqFailedGeneration<T>(error: APIError & {
+    body?: {
+        error?: {
+            failed_generation?: string;
+        };
+    };
+}, outputFormat: {
+    parse: (input: string) => T;
+}): T;
+/**
+ * Fix control characters in JSON string values to make them valid JSON.
+ * This function escapes literal control characters (newlines, tabs, etc.) that
+ * appear inside JSON string values, while preserving the JSON structure.
+ *
+ * @param content - The JSON string to fix
+ * @returns The fixed JSON string
+ */
+export declare function fixControlCharactersInJson(content: string): string;

package/dist/llm/groq/parser.js

@@ -0,0 +1,189 @@
+import { logger } from '../../logging-config.js';
+export class ParseFailedGenerationError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'ParseFailedGenerationError';
+    }
+}
+/**
+ * Extract JSON from model output, handling both plain JSON and code-block-wrapped JSON.
+ * This is used to parse Groq's failed_generation field when an API error occurs.
+ *
+ * @param error - The Groq API error containing failed_generation
+ * @param outputFormat - An object with a parse method (typically a Zod schema)
+ * @returns The parsed output in the expected format
+ * @throws ParseFailedGenerationError if the failed_generation field is missing
+ * @throws Error if JSON parsing fails
+ */
+export function tryParseGroqFailedGeneration(error, outputFormat) {
+    try {
+        const failedGeneration = error.body?.error?.failed_generation;
+        if (!failedGeneration) {
+            throw new Error('No failed_generation field in error body');
+        }
+        let content = failedGeneration;
+        // If content is wrapped in code blocks, extract just the JSON part
+        if (content.includes('```')) {
+            // Find the JSON content between code blocks
+            const parts = content.split('```');
+            content = parts[1] || content;
+            // Remove language identifier if present (e.g., 'json\n')
+            if (content.includes('\n')) {
+                const [first, ...rest] = content.split('\n');
+                // Check if first line is just a language identifier
+                if (first.trim().length < 20) {
+                    content = rest.join('\n');
+                }
+            }
+        }
+        // Remove html-like tags before the first { and after the last }
+        // This handles cases like <|header_start|>assistant<|header_end|> and <function=AgentOutput>
+        // Only remove content before { if content doesn't already start with {
+        if (!content.trim().startsWith('{')) {
+            content = content.replace(/^.*?(?=\{)/s, '');
+        }
+        // Remove common HTML-like tags and patterns at the end, but be more conservative
+        // Look for patterns like </function>, <|header_start|>, etc. after the JSON
+        content = content.replace(/\}(\s*<[^>]*>.*?$)/s, '}');
+        content = content.replace(/\}(\s*<\|[^|]*\|>.*?$)/s, '}');
+        // Handle extra characters after the JSON, including stray braces
+        // Find the position of the last } that would close the main JSON object
+        content = content.trim();
+        if (content.endsWith('}')) {
+            // Try to parse and see if we get valid JSON
+            try {
+                JSON.parse(content);
+            }
+            catch {
+                // If parsing fails, try to find the correct end of the JSON
+                // by counting braces and removing anything after the balanced JSON
+                let braceCount = 0;
+                let lastValidPos = -1;
+                for (let i = 0; i < content.length; i++) {
+                    const char = content[i];
+                    if (char === '{') {
+                        braceCount++;
+                    }
+                    else if (char === '}') {
+                        braceCount--;
+                        if (braceCount === 0) {
+                            lastValidPos = i + 1;
+                            break;
+                        }
+                    }
+                }
+                if (lastValidPos > 0) {
+                    content = content.substring(0, lastValidPos);
+                }
+            }
+        }
+        // Fix control characters in JSON strings before parsing
+        // This handles cases where literal control characters appear in JSON values
+        content = fixControlCharactersInJson(content);
+        // Parse the cleaned content
+        let resultDict = JSON.parse(content);
+        // Some models occasionally respond with a list containing one dict
+        if (Array.isArray(resultDict) &&
+            resultDict.length === 1 &&
+            typeof resultDict[0] === 'object') {
+            resultDict = resultDict[0];
+        }
+        logger.debug(`Successfully parsed model output: ${JSON.stringify(resultDict)}`);
+        return outputFormat.parse(JSON.stringify(resultDict));
+    }
+    catch (err) {
+        if (err instanceof Error &&
+            err.message.includes('No failed_generation field')) {
+            throw new ParseFailedGenerationError(err.message);
+        }
+        if (err instanceof SyntaxError) {
+            logger.warning(`Failed to parse model output: ${err.message}`);
+            throw new Error(`Could not parse response. ${err.message}`);
+        }
+        const errorMessage = error.message || String(error);
+        throw new ParseFailedGenerationError(errorMessage);
+    }
+}
+/**
+ * Fix control characters in JSON string values to make them valid JSON.
+ * This function escapes literal control characters (newlines, tabs, etc.) that
+ * appear inside JSON string values, while preserving the JSON structure.
+ *
+ * @param content - The JSON string to fix
+ * @returns The fixed JSON string
+ */
+export function fixControlCharactersInJson(content) {
+    try {
+        // First try to parse as-is to see if it's already valid
+        JSON.parse(content);
+        return content;
+    }
+    catch {
+        // Continue to fix the content
+    }
+    // More sophisticated approach: only escape control characters inside string values
+    // while preserving JSON structure formatting
+    const result = [];
+    let i = 0;
+    let inString = false;
+    let escaped = false;
+    while (i < content.length) {
+        const char = content[i];
+        if (!inString) {
+            // Outside of string - check if we're entering a string
+            if (char === '"') {
+                inString = true;
+            }
+            result.push(char);
+        }
+        else {
+            // Inside string - handle escaping and control characters
+            if (escaped) {
+                // Previous character was backslash, so this character is escaped
+                result.push(char);
+                escaped = false;
+            }
+            else if (char === '\\') {
+                // This is an escape character
+                result.push(char);
+                escaped = true;
+            }
+            else if (char === '"') {
+                // End of string
+                result.push(char);
+                inString = false;
+            }
+            else if (char === '\n') {
+                // Literal newline inside string - escape it
+                result.push('\\n');
+            }
+            else if (char === '\r') {
+                // Literal carriage return inside string - escape it
+                result.push('\\r');
+            }
+            else if (char === '\t') {
+                // Literal tab inside string - escape it
+                result.push('\\t');
+            }
+            else if (char === '\b') {
+                // Literal backspace inside string - escape it
+                result.push('\\b');
+            }
+            else if (char === '\f') {
+                // Literal form feed inside string - escape it
+                result.push('\\f');
+            }
+            else if (char.charCodeAt(0) < 32) {
+                // Other control characters inside string - convert to unicode escape
+                const code = char.charCodeAt(0);
+                result.push(`\\u${code.toString(16).padStart(4, '0')}`);
+            }
+            else {
+                // Normal character inside string
+                result.push(char);
+            }
+        }
+        i++;
+    }
+    return result.join('');
+}

package/dist/llm/groq/serializer.d.ts

@@ -0,0 +1,6 @@
+import type { ChatCompletionMessageParam } from 'groq-sdk/resources/chat/completions.mjs';
+import { type Message } from '../messages.js';
+export declare class GroqMessageSerializer {
+    serialize(messages: Message[]): ChatCompletionMessageParam[];
+    private serializeMessage;
+}

package/dist/llm/groq/serializer.js

@@ -0,0 +1,56 @@
+import { AssistantMessage, ContentPartImageParam, ContentPartTextParam, SystemMessage, UserMessage, } from '../messages.js';
+export class GroqMessageSerializer {
+    serialize(messages) {
+        return messages.map((message) => this.serializeMessage(message));
+    }
+    serializeMessage(message) {
+        if (message instanceof UserMessage) {
+            return {
+                role: 'user',
+                content: Array.isArray(message.content)
+                    ? message.content.map((part) => {
+                        if (part instanceof ContentPartTextParam) {
+                            return { type: 'text', text: part.text };
+                        }
+                        if (part instanceof ContentPartImageParam) {
+                            return {
+                                type: 'image_url',
+                                image_url: {
+                                    url: part.image_url.url,
+                                    detail: part.image_url.detail,
+                                },
+                            };
+                        }
+                        return { type: 'text', text: '' };
+                    })
+                    : message.content,
+                name: message.name || undefined,
+            };
+        }
+        if (message instanceof SystemMessage) {
+            return {
+                role: 'system',
+                content: Array.isArray(message.content)
+                    ? message.content.map((part) => part.text).join('\n')
+                    : message.content,
+                name: message.name || undefined,
+            };
+        }
+        if (message instanceof AssistantMessage) {
+            const toolCalls = message.tool_calls?.map((toolCall) => ({
+                id: toolCall.id,
+                type: 'function',
+                function: {
+                    name: toolCall.functionCall.name,
+                    arguments: toolCall.functionCall.arguments,
+                },
+            }));
+            return {
+                role: 'assistant',
+                content: typeof message.content === 'string' ? message.content : null,
+                tool_calls: toolCalls,
+            };
+        }
+        throw new Error(`Unknown message type: ${message.constructor.name}`);
+    }
+}

package/dist/llm/messages.d.ts

@@ -0,0 +1,77 @@
+export type SupportedImageMediaType = 'image/jpeg' | 'image/png' | 'image/gif' | 'image/webp';
+export declare class ContentPartTextParam {
+    text: string;
+    type: 'text';
+    constructor(text: string);
+    toString(): string;
+}
+export declare class ContentPartRefusalParam {
+    refusal: string;
+    type: 'refusal';
+    constructor(refusal: string);
+    toString(): string;
+}
+export declare class ImageURL {
+    url: string;
+    detail: 'auto' | 'low' | 'high';
+    media_type: SupportedImageMediaType;
+    constructor(url: string, detail?: 'auto' | 'low' | 'high', media?: SupportedImageMediaType);
+    toString(): string;
+}
+export declare class ContentPartImageParam {
+    image_url: ImageURL;
+    type: 'image_url';
+    constructor(image_url: ImageURL);
+    toString(): string;
+}
+export declare class FunctionCall {
+    name: string;
+    arguments: string;
+    constructor(name: string, args: string);
+    toString(): string;
+}
+export declare class ToolCall {
+    id: string;
+    functionCall: FunctionCall;
+    type: 'function';
+    constructor(id: string, functionCall: FunctionCall);
+    toString(): string;
+}
+type ContentPart = ContentPartTextParam | ContentPartImageParam | ContentPartRefusalParam;
+export type MessageRole = 'user' | 'system' | 'assistant';
+export declare abstract class MessageBase {
+    cache: boolean;
+    abstract role: MessageRole;
+    constructor(init?: Partial<MessageBase>);
+}
+export declare class UserMessage extends MessageBase {
+    role: MessageRole;
+    content: string | ContentPart[];
+    name: string | null;
+    constructor(content: string | ContentPart[], name?: string | null);
+    get text(): string;
+    toString(): string;
+}
+export declare class SystemMessage extends MessageBase {
+    role: MessageRole;
+    content: string | ContentPartTextParam[];
+    name: string | null;
+    constructor(content: string | ContentPartTextParam[], name?: string | null);
+    get text(): string;
+    toString(): string;
+}
+export declare class AssistantMessage extends MessageBase {
+    role: MessageRole;
+    content: string | ContentPart[] | null;
+    tool_calls: ToolCall[] | null;
+    refusal: string | null;
+    constructor(init: {
+        content?: string | ContentPart[] | null;
+        tool_calls?: ToolCall[] | null;
+        refusal?: string | null;
+    });
+    get text(): string;
+    toString(): string;
+}
+export type Message = UserMessage | SystemMessage | AssistantMessage;
+export {};

package/dist/llm/messages.js

@@ -0,0 +1,157 @@
+const truncate = (text, maxLength = 50) => {
+    if (text.length <= maxLength) {
+        return text;
+    }
+    return `${text.slice(0, maxLength - 3)}...`;
+};
+const formatImageUrl = (url, maxLength = 50) => {
+    if (url.startsWith('data:')) {
+        const mediaType = url.split(';')[0]?.split(':')[1] ?? 'image';
+        return `<base64 ${mediaType}>`;
+    }
+    return truncate(url, maxLength);
+};
+export class ContentPartTextParam {
+    text;
+    type = 'text';
+    constructor(text) {
+        this.text = text;
+    }
+    toString() {
+        return `Text: ${truncate(this.text)}`;
+    }
+}
+export class ContentPartRefusalParam {
+    refusal;
+    type = 'refusal';
+    constructor(refusal) {
+        this.refusal = refusal;
+    }
+    toString() {
+        return `Refusal: ${truncate(this.refusal)}`;
+    }
+}
+export class ImageURL {
+    url;
+    detail;
+    media_type;
+    constructor(url, detail = 'auto', media = 'image/png') {
+        this.url = url;
+        this.detail = detail;
+        this.media_type = media;
+    }
+    toString() {
+        return `🖼️  Image[${this.media_type}, detail=${this.detail}]: ${formatImageUrl(this.url)}`;
+    }
+}
+export class ContentPartImageParam {
+    image_url;
+    type = 'image_url';
+    constructor(image_url) {
+        this.image_url = image_url;
+    }
+    toString() {
+        return this.image_url.toString();
+    }
+}
+export class FunctionCall {
+    name;
+    // @ts-ignore - 'arguments' is a reserved keyword but valid as property name
+    arguments;
+    constructor(name, args) {
+        this.name = name;
+        // @ts-ignore
+        this.arguments = args;
+    }
+    toString() {
+        return `${this.name}(${truncate(this.arguments, 80)})`;
+    }
+}
+export class ToolCall {
+    id;
+    functionCall;
+    type = 'function';
+    constructor(id, functionCall) {
+        this.id = id;
+        this.functionCall = functionCall;
+    }
+    toString() {
+        return `ToolCall[${this.id}]: ${this.functionCall.toString()}`;
+    }
+}
+export class MessageBase {
+    cache = false;
+    constructor(init) {
+        if (init?.cache !== undefined) {
+            this.cache = init.cache;
+        }
+    }
+}
+export class UserMessage extends MessageBase {
+    role = 'user';
+    content;
+    name;
+    constructor(content, name = null) {
+        super();
+        this.content = content;
+        this.name = name;
+    }
+    get text() {
+        if (typeof this.content === 'string') {
+            return this.content;
+        }
+        return this.content
+            .filter((part) => part instanceof ContentPartTextParam)
+            .map((part) => part.text)
+            .join('\n');
+    }
+    toString() {
+        return `UserMessage(content=${this.text})`;
+    }
+}
+export class SystemMessage extends MessageBase {
+    role = 'system';
+    content;
+    name;
+    constructor(content, name = null) {
+        super();
+        this.content = content;
+        this.name = name;
+    }
+    get text() {
+        if (typeof this.content === 'string') {
+            return this.content;
+        }
+        return this.content.map((part) => part.text).join('\n');
+    }
+    toString() {
+        return `SystemMessage(content=${this.text})`;
+    }
+}
+export class AssistantMessage extends MessageBase {
+    role = 'assistant';
+    content;
+    tool_calls;
+    refusal;
+    constructor(init) {
+        super();
+        this.content = init.content ?? null;
+        this.tool_calls = init.tool_calls ?? null;
+        this.refusal = init.refusal ?? null;
+    }
+    get text() {
+        if (typeof this.content === 'string') {
+            return this.content;
+        }
+        if (Array.isArray(this.content)) {
+            return this.content
+                .filter((part) => part instanceof ContentPartTextParam)
+                .map((part) => part.text)
+                .join('\n');
+        }
+        return '';
+    }
+    toString() {
+        return `AssistantMessage(content=${this.text})`;
+    }
+}

package/dist/llm/ollama/chat.d.ts

@@ -0,0 +1,15 @@
+import type { BaseChatModel, ChatInvokeOptions } from '../base.js';
+import { ChatInvokeCompletion } from '../views.js';
+import type { Message } from '../messages.js';
+export declare class ChatOllama implements BaseChatModel {
+    model: string;
+    provider: string;
+    private client;
+    constructor(model?: string, host?: string);
+    get name(): string;
+    get model_name(): string;
+    ainvoke(messages: Message[], output_format?: undefined, options?: ChatInvokeOptions): Promise<ChatInvokeCompletion<string>>;
+    ainvoke<T>(messages: Message[], output_format: {
+        parse: (input: string) => T;
+    } | undefined, options?: ChatInvokeOptions): Promise<ChatInvokeCompletion<T>>;
+}

package/dist/llm/ollama/chat.js

@@ -0,0 +1,77 @@
+import { Ollama } from 'ollama';
+import { ChatInvokeCompletion } from '../views.js';
+import { OllamaMessageSerializer } from './serializer.js';
+export class ChatOllama {
+    model;
+    provider = 'ollama';
+    client;
+    constructor(model = 'qwen2.5:latest', host = 'http://localhost:11434') {
+        this.model = model;
+        this.client = new Ollama({ host });
+    }
+    get name() {
+        return this.model;
+    }
+    get model_name() {
+        return this.model;
+    }
+    async ainvoke(messages, output_format, options = {}) {
+        const serializer = new OllamaMessageSerializer();
+        const ollamaMessages = serializer.serialize(messages);
+        let format = undefined;
+        if (output_format && 'schema' in output_format && output_format.schema) {
+            // Ollama supports 'json' format
+            format = 'json';
+        }
+        const requestPromise = this.client.chat({
+            model: this.model,
+            messages: ollamaMessages,
+            format: format,
+            stream: false,
+        });
+        const abortSignal = options.signal;
+        const response = abortSignal
+            ? await new Promise((resolve, reject) => {
+                const onAbort = () => {
+                    cleanup();
+                    const error = new Error('Operation aborted');
+                    error.name = 'AbortError';
+                    reject(error);
+                };
+                const cleanup = () => {
+                    abortSignal.removeEventListener('abort', onAbort);
+                };
+                if (abortSignal.aborted) {
+                    onAbort();
+                    return;
+                }
+                abortSignal.addEventListener('abort', onAbort, { once: true });
+                requestPromise
+                    .then((result) => {
+                    cleanup();
+                    resolve(result);
+                })
+                    .catch((error) => {
+                    cleanup();
+                    reject(error);
+                });
+            })
+            : await requestPromise;
+        const content = response.message.content;
+        let completion = content;
+        if (output_format) {
+            try {
+                completion = output_format.parse(JSON.parse(content));
+            }
+            catch (e) {
+                console.error('Failed to parse completion', e);
+                throw e;
+            }
+        }
+        return new ChatInvokeCompletion(completion, {
+            prompt_tokens: response.prompt_eval_count ?? 0,
+            completion_tokens: response.eval_count ?? 0,
+            total_tokens: (response.prompt_eval_count ?? 0) + (response.eval_count ?? 0),
+        });
+    }
+}

package/dist/llm/ollama/index.d.ts

@@ -0,0 +1,2 @@
+export * from './chat.js';
+export * from './serializer.js';

package/dist/llm/ollama/index.js

@@ -0,0 +1,2 @@
+export * from './chat.js';
+export * from './serializer.js';