askui 0.24.0 → 0.25.0

package/dist/esm/core/models/anthropic/claude-agent.js

@@ -0,0 +1,196 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { ToolCollection } from './tools/base';
+import { logger } from '../../../lib/logger';
+export class ClaudeAgent {
+    constructor(predictActResponseFunction) {
+        this.predictActResponseFunction = predictActResponseFunction;
+        this.maxTokens = 4096;
+        this.onlyNMostRecentImages = 3;
+        this.imageTruncationThreshold = 10;
+        this.systemPrompt = '';
+        this.model = 'claude-3-5-sonnet-20241022';
+        this.betas = ['computer-use-2024-10-22'];
+        this._toolCollection = undefined;
+        this.tools = [];
+        this.history = {};
+    }
+    setTools(tools) {
+        this._toolCollection = undefined;
+        this.tools = tools;
+    }
+    addTool(tool) {
+        this._toolCollection = undefined;
+        this.tools.push(tool);
+    }
+    listToolNames() {
+        return this.tools.map((tool) => tool.ToolName);
+    }
+    removeToolByName(toolName) {
+        this.tools = this.tools.filter((tool) => tool.ToolName !== toolName);
+    }
+    setSystemPrompt(systemPrompt) {
+        this.systemPrompt = systemPrompt;
+    }
+    IsConfigured() {
+        return this.tools.length > 0 && this.systemPrompt !== '';
+    }
+    get toolCollection() {
+        if (!this._toolCollection) {
+            this._toolCollection = new ToolCollection(this.tools);
+        }
+        return this._toolCollection;
+    }
+    setHistory(key, messages) {
+        this.history[key] = messages;
+    }
+    getHistory(key) {
+        return this.history[key] || [];
+    }
+    act(goal, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!goal.trim()) {
+                throw new Error('Goal cannot be empty');
+            }
+            if (!this.IsConfigured()) {
+                throw new Error('Claude agent is not configured. Please configure the agent first.');
+            }
+            const messages = [];
+            if (options === null || options === void 0 ? void 0 : options.agentHistory) {
+                messages.push(...options.agentHistory);
+            }
+            if (options === null || options === void 0 ? void 0 : options.chatId) {
+                messages.push(...this.getHistory(options.chatId));
+            }
+            // Add the new goal as a user message
+            messages.push({
+                content: goal,
+                role: 'user',
+            });
+            if (this.onlyNMostRecentImages) {
+                ClaudeAgent.filterNMostRecentImages(messages, this.onlyNMostRecentImages, this.imageTruncationThreshold);
+            }
+            while (true) {
+                const response = yield this.predictActResponseFunction({
+                    max_tokens: this.maxTokens,
+                    messages,
+                    model: this.model,
+                    system: this.systemPrompt,
+                    tools: (new ToolCollection(this.tools).toParams()),
+                    betas: this.betas,
+                });
+                messages.push({
+                    content: response.content,
+                    role: 'assistant',
+                });
+                const toolResultContent = [];
+                for (const block of response.content) {
+                    if (block.type === 'tool_use') {
+                        logger.debug(`Agent will execute tool '${block.name}' with input '${JSON.stringify(block.input)}'`);
+                        const toolUseBlock = block;
+                        const result = yield this.toolCollection.run(toolUseBlock.name, toolUseBlock.input);
+                        const toolResult = this.makeApiToolResult(result, toolUseBlock.id);
+                        toolResultContent.push(toolResult);
+                    }
+                    if (block.type === 'thinking') {
+                        logger.debug(`Agent is thinking: ${block.thinking}`);
+                    }
+                    if (block.type === 'text') {
+                        logger.info(`Agent: ${block.text}`);
+                    }
+                }
+                if (toolResultContent.length === 0) {
+                    if (options === null || options === void 0 ? void 0 : options.chatId) {
+                        this.setHistory(options.chatId, messages);
+                    }
+                    return messages;
+                }
+                messages.push({
+                    role: 'user',
+                    content: toolResultContent,
+                });
+            }
+        });
+    }
+    makeApiToolResult(result, toolUseId) {
+        const content = [];
+        let isError = false;
+        if (result.error) {
+            isError = true;
+            content.push({
+                type: 'text',
+                text: this.maybePrependSystemToolResult(result, result.error),
+            });
+        }
+        else {
+            if (result.output) {
+                content.push({
+                    type: 'text',
+                    text: this.maybePrependSystemToolResult(result, result.output),
+                });
+            }
+            if (result.base64Images) {
+                for (const base64Image of result.base64Images) {
+                    content.push({
+                        type: 'image',
+                        source: {
+                            type: 'base64',
+                            media_type: 'image/png',
+                            data: base64Image,
+                        },
+                    });
+                }
+            }
+        }
+        return {
+            type: 'tool_result',
+            tool_use_id: toolUseId,
+            content,
+            is_error: isError,
+        };
+    }
+    maybePrependSystemToolResult(result, text) {
+        if (result.system) {
+            return `<system>${result.system}</system>\n${text}`;
+        }
+        return text;
+    }
+    static filterNMostRecentImages(messages, imagesToKeep, minRemovalThreshold = 10) {
+        const toolResultBlocks = messages
+            .flatMap((message) => (Array.isArray(message.content) ? message.content : []))
+            .filter((item) => typeof item === 'object'
+            && 'type' in item
+            && item.type === 'tool_result');
+        const totalImages = toolResultBlocks.reduce((count, result) => {
+            const content = Array.isArray(result.content) ? result.content : [];
+            return count + content.filter((item) => typeof item === 'object'
+                && 'type' in item
+                && item.type === 'image').length;
+        }, 0);
+        let imagesToRemove = totalImages - imagesToKeep;
+        imagesToRemove -= imagesToRemove % minRemovalThreshold;
+        if (imagesToRemove <= 0)
+            return;
+        for (const result of toolResultBlocks) {
+            if (Array.isArray(result.content)) {
+                const newContent = result.content.filter((item) => {
+                    if ('type' in item && item.type === 'image') {
+                        if (imagesToRemove > 0) {
+                            imagesToRemove -= 1;
+                            return false;
+                        }
+                    }
+                    return true;
+                });
+                result.content = newContent;
+            }
+        }
+    }
+}

package/dist/esm/core/models/anthropic/index.d.ts

@@ -0,0 +1,4 @@
+import { BetaMessageParam } from '@anthropic-ai/sdk/resources/beta/messages';
+export type AgentHistory = BetaMessageParam[];
+export { AskUIAgent } from './askui-agent';
+export { ToolFailure, ToolError, BaseAgentTool } from './tools/base';

package/dist/esm/core/models/anthropic/index.js

@@ -0,0 +1,2 @@
+export { AskUIAgent } from './askui-agent';
+export { ToolFailure, ToolError, BaseAgentTool } from './tools/base';

package/dist/esm/core/models/anthropic/tools/agent-errors.d.ts

@@ -0,0 +1,2 @@
+export declare class AgentError extends Error {
+}

package/dist/esm/core/models/anthropic/tools/agent-errors.js

@@ -0,0 +1,2 @@
+export class AgentError extends Error {
+}

package/dist/esm/core/models/anthropic/tools/base.d.ts

@@ -0,0 +1,24 @@
+export interface ToolResult {
+    output?: string;
+    error?: string;
+    base64Images?: string[];
+    system?: string;
+}
+export declare class ToolFailure implements ToolResult {
+    error: string;
+    constructor(error: string);
+}
+export declare class ToolError extends Error {
+    constructor(message: string);
+}
+export declare abstract class BaseAgentTool {
+    abstract execute(params: Record<string, any>): Promise<ToolResult>;
+    abstract toParams(): any;
+    get ToolName(): string;
+}
+export declare class ToolCollection {
+    private toolMap;
+    constructor(tools: BaseAgentTool[]);
+    toParams(): any[];
+    run(name: string, toolInput: Record<string, any>): Promise<ToolResult>;
+}

package/dist/esm/core/models/anthropic/tools/base.js

@@ -0,0 +1,59 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+/* eslint-disable */
+import { AgentError } from './agent-errors';
+export class ToolFailure {
+    constructor(error) {
+        this.error = error;
+    }
+}
+export class ToolError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'ToolError';
+    }
+}
+export class BaseAgentTool {
+    get ToolName() {
+        return this.constructor.name;
+    }
+}
+export class ToolCollection {
+    constructor(tools) {
+        this.toolMap = new Map();
+        tools.forEach((tool) => {
+            const params = tool.toParams();
+            this.toolMap.set(params.name, tool);
+        });
+    }
+    toParams() {
+        return Array.from(this.toolMap.values()).map((tool) => tool.toParams());
+    }
+    run(name, toolInput) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const tool = this.toolMap.get(name);
+            if (!tool) {
+                return new ToolFailure(`Tool ${name} is invalid`);
+            }
+            try {
+                return yield tool.execute(toolInput);
+            }
+            catch (err) {
+                if (err instanceof AgentError) {
+                    throw err;
+                }
+                if (err instanceof ToolError) {
+                    return new ToolFailure(err.message);
+                }
+                return new ToolFailure(`Unknown error: ${err}`);
+            }
+        });
+    }
+}

package/dist/esm/core/models/anthropic/tools/os-agent-tools.d.ts

@@ -0,0 +1,113 @@
+import { BetaTool } from '@anthropic-ai/sdk/resources/beta/messages/messages';
+import { MODIFIER_KEY, PC_KEY, ANDROID_KEY, PC_AND_MODIFIER_KEY } from '../../../../execution/dsl';
+import { BaseAgentTool, ToolResult } from './base';
+import { ExecutionRuntime } from '../../../../execution/execution-runtime';
+import { ControlCommand } from '../../../ui-control-commands';
+export declare class OsAgentHandler {
+    private AgentOsClient;
+    private TargetResolution;
+    private screenDimensions;
+    constructor(AgentOsClient: ExecutionRuntime, screenDimensions: {
+        width: number;
+        height: number;
+    });
+    static createInstance(AgentOsClient: ExecutionRuntime): Promise<OsAgentHandler>;
+    getTargetResolution(): {
+        width: number;
+        height: number;
+    };
+    setTargetResolution(width: number, height: number): void;
+    takeScreenshot(): Promise<string>;
+    private scaleCoordinates;
+    requestControl(controlCommand: ControlCommand): Promise<void>;
+}
+export declare class ScreenShotTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class MouseMoveTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(command: {
+        x: number;
+        y: number;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class MouseClickTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(command: {
+        button: 'left' | 'right' | 'middle';
+        doubleClick: boolean;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class MouseScrollTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(command: {
+        dx: number;
+        dy: number;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class DesktopKeyPressSequenceTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(command: {
+        key: PC_KEY;
+        firstModifier?: MODIFIER_KEY;
+        secondModifier?: MODIFIER_KEY;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class DesktopSingleKeyPressTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(command: {
+        key: PC_AND_MODIFIER_KEY;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class TypeTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(command: {
+        text: string;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class AndroidSingleKeyPressTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(command: {
+        key: ANDROID_KEY;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class AndroidSequenceKeyPressTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(command: {
+        keys: ANDROID_KEY[];
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class AgentErrorTool extends BaseAgentTool {
+    constructor();
+    execute(command: {
+        error: string;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}
+export declare class ExecuteShellCommandTool extends BaseAgentTool {
+    private osAgentHandler;
+    constructor(osAgentHandler: OsAgentHandler);
+    execute(command: {
+        command: string;
+    }): Promise<ToolResult>;
+    toParams(): BetaTool;
+}