@eko-ai/eko 1.0.1 → 1.0.3

package/dist/extension_content_script.js

@@ -40,10 +40,6 @@ chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) {
                     sendResponse(result);
                     break;
                 }
-                case 'computer:key': {
-                    sendResponse(key(request));
-                    break;
-                }
                 case 'computer:type': {
                     sendResponse(type(request));
                     break;
@@ -64,10 +60,6 @@ chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) {
                     sendResponse(simulateMouseEvent(request, ['mousedown', 'mouseup', 'click', 'mousedown', 'mouseup', 'click', 'dblclick'], 0));
                     break;
                 }
-                case 'computer:left_click_drag': {
-                    sendResponse(left_click_drag(request));
-                    break;
-                }
                 case 'computer:scroll_to': {
                     sendResponse(scroll_to(request));
                     break;
@@ -76,6 +68,14 @@ chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) {
                     sendResponse({ coordinate: [eko.lastMouseX, eko.lastMouseY] });
                     break;
                 }
+                case 'computer:get_dropdown_options': {
+                    sendResponse(get_dropdown_options(request));
+                    break;
+                }
+                case 'computer:select_dropdown_option': {
+                    sendResponse(select_dropdown_option(request));
+                    break;
+                }
             }
         }
         catch (e) {
@@ -85,30 +85,22 @@ chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) {
     })();
     return true;
 });
-function key(request) {
-    const event = new KeyboardEvent(request.keyEventType || 'keydown', {
-        key: request.key,
-        ctrlKey: request.ctrlKey,
-        altKey: request.altKey,
-        shiftKey: request.shiftKey,
-        metaKey: request.metaKey,
-        bubbles: true,
-        cancelable: true,
-    });
-    let coordinate = request.coordinate;
-    let element = (document.activeElement ||
-        document.elementFromPoint(coordinate[0], coordinate[1]));
-    if (element && element.focus) {
-        element.focus();
-    }
-    let result = element === null || element === void 0 ? void 0 : element.dispatchEvent(event);
-    console.log('key', element, request, result);
-    return result;
-}
 function type(request) {
     let text = request.text;
+    let enter = false;
+    if (text.endsWith('\\n')) {
+        enter = true;
+        text = text.substring(0, text.length - 2);
+    }
+    else if (text.endsWith('\n')) {
+        enter = true;
+        text = text.substring(0, text.length - 1);
+    }
     let element;
-    if (request.xpath) {
+    if (request.highlightIndex != null) {
+        element = window.get_highlight_element(request.highlightIndex);
+    }
+    else if (request.xpath) {
         let xpath = request.xpath;
         let result = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
         element = result.singleNodeValue;
@@ -118,7 +110,7 @@ function type(request) {
         element = document.elementFromPoint(coordinate[0], coordinate[1]) || document.activeElement;
     }
     if (!element) {
-        return;
+        return false;
     }
     let input;
     if (element.tagName == 'INPUT' ||
@@ -137,8 +129,20 @@ function type(request) {
         input.value += text;
     }
     let result = input.dispatchEvent(new Event('input', { bubbles: true }));
+    if (enter) {
+        ['keydown', 'keypress', 'keyup'].forEach((eventType) => {
+            const event = new KeyboardEvent(eventType, {
+                key: 'Enter',
+                code: 'Enter',
+                keyCode: 13,
+                bubbles: true,
+                cancelable: true,
+            });
+            input.dispatchEvent(event);
+        });
+    }
     console.log('type', input, request, result);
-    return result;
+    return true;
 }
 function mouse_move(request) {
     let coordinate = request.coordinate;
@@ -155,11 +159,14 @@ function mouse_move(request) {
     });
     let result = document.body.dispatchEvent(event);
     console.log('mouse_move', document.body, request, result);
-    return result;
+    return true;
 }
 function simulateMouseEvent(request, eventTypes, button) {
     let element;
-    if (request.xpath) {
+    if (request.highlightIndex != null) {
+        element = window.get_highlight_element(request.highlightIndex);
+    }
+    else if (request.xpath) {
         let xpath = request.xpath;
         let result = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
         element = result.singleNodeValue;
@@ -168,9 +175,11 @@ function simulateMouseEvent(request, eventTypes, button) {
         let coordinate = request.coordinate;
         element = document.elementFromPoint(coordinate[0], coordinate[1]) || document.body;
     }
+    if (!element) {
+        return false;
+    }
     const x = undefined;
     const y = undefined;
-    let result = false;
     for (let i = 0; i < eventTypes.length; i++) {
         const event = new MouseEvent(eventTypes[i], {
             view: window,
@@ -180,16 +189,28 @@ function simulateMouseEvent(request, eventTypes, button) {
             clientY: y,
             button, // 0 left; 2 right
         });
-        result = element.dispatchEvent(event);
+        let result = element.dispatchEvent(event);
         console.log('simulateMouse', element, { ...request, eventTypes, button }, result);
     }
-    return result;
+    return true;
 }
 function scroll_to(request) {
-    if (request.xpath) {
+    if (request.highlightIndex != null) {
+        let element = window.get_highlight_element(request.highlightIndex);
+        if (!element) {
+            return false;
+        }
+        element.scrollIntoView({
+            behavior: 'smooth'
+        });
+    }
+    else if (request.xpath) {
         let xpath = request.xpath;
         let result = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
         let element = result.singleNodeValue;
+        if (!element) {
+            return false;
+        }
         element.scrollIntoView({
             behavior: 'smooth'
         });
@@ -203,45 +224,53 @@ function scroll_to(request) {
         });
     }
     console.log('scroll_to', request);
+    return true;
 }
-function left_click_drag(request, steps = 10) {
-    const from_coordinate = request.from_coordinate;
-    const to_coordinate = request.to_coordinate;
-    let startX = from_coordinate[0];
-    let startY = from_coordinate[1];
-    let endX = to_coordinate[0];
-    let endY = to_coordinate[1];
-    let element = document.elementFromPoint(startX, startY) || document.body;
-    const mouseDownEvent = new MouseEvent('mousedown', {
-        bubbles: true,
-        cancelable: true,
-        view: window,
-        clientX: startX,
-        clientY: startY,
-        button: 0,
-    });
-    element.dispatchEvent(mouseDownEvent);
-    for (let i = 1; i <= steps; i++) {
-        const intermediateX = startX + (endX - startX) * (i / steps);
-        const intermediateY = startY + (endY - startY) * (i / steps);
-        const dragEvent = new MouseEvent('mousemove', {
-            bubbles: true,
-            cancelable: true,
-            view: window,
-            clientX: intermediateX,
-            clientY: intermediateY,
-            button: 0,
-        });
-        element.dispatchEvent(dragEvent);
+function get_dropdown_options(request) {
+    let select;
+    if (request.highlightIndex != null) {
+        select = window.get_highlight_element(request.highlightIndex);
     }
-    const mouseUpEvent = new MouseEvent('mouseup', {
-        bubbles: true,
-        cancelable: true,
-        view: window,
-        clientX: endX,
-        clientY: endY,
-        button: 0,
-    });
-    console.log('left_click_drag', request);
-    return element.dispatchEvent(mouseUpEvent);
+    else if (request.xpath) {
+        select = document.evaluate(request.xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
+    }
+    if (!select) {
+        return null;
+    }
+    return {
+        options: Array.from(select.options).map((opt) => ({
+            index: opt.index,
+            text: opt.text.trim(),
+            value: opt.value,
+        })),
+        id: select.id,
+        name: select.name,
+    };
+}
+function select_dropdown_option(request) {
+    let select;
+    if (request.highlightIndex != null) {
+        select = window.get_highlight_element(request.highlightIndex);
+    }
+    else if (request.xpath) {
+        select = document.evaluate(request.xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
+    }
+    if (!select || select.tagName.toUpperCase() !== 'SELECT') {
+        return { success: false, error: 'Select not found or invalid element type' };
+    }
+    const option = Array.from(select.options).find((opt) => opt.text.trim() === request.text);
+    if (!option) {
+        return {
+            success: false,
+            error: 'Option not found',
+            availableOptions: Array.from(select.options).map((o) => o.text.trim()),
+        };
+    }
+    select.value = option.value;
+    select.dispatchEvent(new Event('change'));
+    return {
+        success: true,
+        selectedValue: option.value,
+        selectedText: option.text.trim(),
+    };
 }

package/dist/index.cjs.js

@@ -169,9 +169,10 @@ function createReturnTool(outputSchema) {
 }
 class ActionImpl {
     constructor(type, // Only support prompt type
-    name, tools, llmProvider, llmConfig, config) {
+    name, description, tools, llmProvider, llmConfig, config) {
         this.type = type;
         this.name = name;
+        this.description = description;
         this.tools = tools;
         this.llmProvider = llmProvider;
         this.llmConfig = llmConfig;
@@ -195,7 +196,6 @@ class ActionImpl {
         const handler = {
             onContent: (content) => {
                 if (content.trim()) {
-                    console.log('LLM:', content);
                     assistantTextMessage += content;
                 }
             },
@@ -313,12 +313,10 @@ class ActionImpl {
         (_a = context.tools) === null || _a === void 0 ? void 0 : _a.forEach((tool) => toolMap.set(tool.name, tool));
         toolMap.set(returnTool.name, returnTool);
         // Prepare initial messages
-        const messages = input && Object.keys(input).length > 0
-            ? [
-                { role: 'system', content: this.formatSystemPrompt(context) },
-                { role: 'user', content: this.formatUserPrompt(input) },
-            ]
-            : [{ role: 'user', content: this.formatSystemPrompt(context) }];
+        const messages = [
+            { role: 'system', content: this.formatSystemPrompt() },
+            { role: 'user', content: this.formatUserPrompt(context, input) },
+        ];
         console.log('Starting LLM conversation...');
         console.log('Initial messages:', messages);
         console.log('Output schema:', outputSchema);
@@ -343,6 +341,7 @@ class ActionImpl {
             if (!hasToolUse && response) {
                 // LLM sent a message without using tools - request explicit return
                 console.log('No tool use detected, requesting explicit return');
+                console.log('Response:', response);
                 const returnOnlyParams = {
                     ...params,
                     tools: [
@@ -395,32 +394,30 @@ class ActionImpl {
         }
         return output;
     }
-    formatSystemPrompt(context) {
+    formatSystemPrompt() {
+        return `You are a task executor. You need to complete the task specified by the user, using the tools provided. When you need to store results or outputs, use the write_context tool. When you are ready to return the final output, use the return_output tool.
+
+    Remember to:
+    1. Use tools when needed to accomplish the task
+    2. Store important results using write_context, including intermediate and final results
+    3. Think step by step about what needs to be done`;
+    }
+    formatUserPrompt(context, input) {
         // Create a description of the current context
         const contextDescription = Array.from(context.variables.entries())
             .map(([key, value]) => `${key}: ${JSON.stringify(value)}`)
             .join('\n');
-        return `You are executing the action "${this.name}". You have access to the following context:
-
-${contextDescription || 'No context variables set'}
+        return `You are executing the action "${this.name}". The specific instructions are: "${this.description}". You have access to the following context:
 
-You can use the provided tools to accomplish your task. When you need to store results or outputs,
-use the write_context tool to save them to the workflow context.
+    ${contextDescription || 'No context variables set'}
 
-Remember to:
-1. Use tools when needed to accomplish the task
-2. Store important results using write_context
-3. Think step by step about what needs to be done`;
-    }
-    formatUserPrompt(input) {
-        if (typeof input === 'string') {
-            return input;
-        }
-        return JSON.stringify(input, null, 2);
+    You have been provided with the following input:
+    ${(typeof input === 'string' ? input : JSON.stringify(input, null, 2)) || 'No additional input provided'}
+    `;
     }
     // Static factory method
-    static createPromptAction(name, tools, llmProvider, llmConfig) {
-        return new ActionImpl('prompt', name, tools, llmProvider, llmConfig);
+    static createPromptAction(name, description, tools, llmProvider, llmConfig) {
+        return new ActionImpl('prompt', name, description, tools, llmProvider, llmConfig);
     }
 }
 
@@ -444,7 +441,8 @@ Generate a complete workflow that:
 1. Only uses the tools listed above
 2. Properly sequences tool usage based on dependencies
 3. Ensures each action has appropriate input/output schemas
-4. Creates a clear, logical flow to accomplish the user's goal`;
+4. Creates a clear, logical flow to accomplish the user's goal
+5. Includes detailed descriptions for each action, ensuring that the actions, when combined, is a complete solution to the user's problem`;
         },
         formatUserPrompt: (requirement) => `Create a workflow for the following requirement: ${requirement}`,
     };
@@ -564,7 +562,7 @@ class WorkflowGenerator {
         if (Array.isArray(data.nodes)) {
             data.nodes.forEach((nodeData) => {
                 const tools = nodeData.action.tools.map((toolName) => this.toolRegistry.getTool(toolName));
-                const action = ActionImpl.createPromptAction(nodeData.action.name, tools, this.llmProvider, { maxTokens: 1000 });
+                const action = ActionImpl.createPromptAction(nodeData.action.name, nodeData.action.description, tools, this.llmProvider, { maxTokens: 1000 });
                 const node = {
                     id: nodeData.id,
                     name: nodeData.name || nodeData.id,
@@ -3286,16 +3284,21 @@ Anthropic.ModelInfosPage = ModelInfosPage;
 Anthropic.Beta = Beta$1;
 
 class ClaudeProvider {
-    constructor(apiKey, defaultModel, options) {
+    constructor(param, defaultModel, options) {
         this.defaultModel = 'claude-3-5-sonnet-20241022';
         if (defaultModel) {
             this.defaultModel = defaultModel;
         }
-        this.client = new Anthropic({
-            apiKey: apiKey,
-            dangerouslyAllowBrowser: true,
-            ...options,
-        });
+        if (typeof param == 'string') {
+            this.client = new Anthropic({
+                apiKey: param,
+                dangerouslyAllowBrowser: true,
+                ...options,
+            });
+        }
+        else {
+            this.client = new Anthropic(param);
+        }
     }
     processResponse(response) {
         const toolCalls = response.content
@@ -3389,7 +3392,7 @@ class ClaudeProvider {
                             const toolCall = {
                                 id: currentToolUse.id,
                                 name: currentToolUse.name,
-                                input: JSON.parse(currentToolUse.accumulatedJson),
+                                input: JSON.parse(currentToolUse.accumulatedJson || '{}'),
                             };
                             (_d = handler.onToolUse) === null || _d === void 0 ? void 0 : _d.call(handler, toolCall);
                             currentToolUse = null;
@@ -8688,16 +8691,21 @@ OpenAI.BatchesPage = BatchesPage;
 OpenAI.Uploads = Uploads;
 
 class OpenaiProvider {
-    constructor(apiKey, defaultModel, options) {
+    constructor(param, defaultModel, options) {
         this.defaultModel = 'gpt-4o';
         if (defaultModel) {
             this.defaultModel = defaultModel;
         }
-        this.client = new OpenAI({
-            apiKey,
-            dangerouslyAllowBrowser: true,
-            ...options,
-        });
+        if (typeof param == 'string') {
+            this.client = new OpenAI({
+                apiKey: param,
+                dangerouslyAllowBrowser: true,
+                ...options,
+            });
+        }
+        else {
+            this.client = new OpenAI(param);
+        }
     }
     buildParams(messages, params, stream) {
         let tools = undefined;
@@ -8975,7 +8983,7 @@ const workflowSchema = {
                     },
                     action: {
                         type: "object",
-                        required: ["type", "name"],
+                        required: ["type", "name", "description"],
                         properties: {
                             type: {
                                 type: "string",
@@ -8983,6 +8991,7 @@ const workflowSchema = {
                                 enum: ["prompt"],
                             },
                             name: { type: "string" },
+                            description: { type: "string" },
                             params: { type: "object" },
                             tools: {
                                 type: "array",
@@ -9006,16 +9015,10 @@ class ToolRegistry {
         this.tools = new Map();
     }
     registerTool(tool) {
-        if (this.tools.has(tool.name)) {
-            throw new Error(`Tool with name ${tool.name} already registered`);
-        }
         this.tools.set(tool.name, tool);
     }
     unregisterTool(toolName) {
-        if (!this.tools.has(toolName)) {
-            throw new Error(`Tool with name ${toolName} not found`);
-        }
-        this.tools.delete(toolName);
+        return this.tools.delete(toolName);
     }
     getTool(toolName) {
         const tool = this.tools.get(toolName);
@@ -9081,6 +9084,7 @@ class Eko {
         else {
             this.llmProvider = config;
         }
+        Eko.tools.forEach((tool) => this.toolRegistry.registerTool(tool));
     }
     async generateWorkflow(prompt, param) {
         let toolRegistry = this.toolRegistry;
@@ -9107,6 +9111,9 @@ class Eko {
         if (this.toolRegistry.hasTools([toolName])) {
             tool = this.toolRegistry.getTool(toolName);
         }
+        else if (Eko.tools.has(toolName)) {
+            tool = Eko.tools.get(toolName);
+        }
         else {
             throw new Error(`Tool with name ${toolName} not found`);
         }
@@ -9135,6 +9142,7 @@ class Eko {
         this.toolRegistry.unregisterTool(toolName);
     }
 }
+Eko.tools = new Map();
 
 class WorkflowParser {
     /**
@@ -9293,6 +9301,7 @@ class WorkflowParser {
                 action: {
                     type: nodeJson.action.type,
                     name: nodeJson.action.name,
+                    description: nodeJson.action.description,
                     tools: nodeJson.action.tools || [],
                     execute: async (input, context) => {
                         // Default implementation - should be overridden by specific action types
@@ -9323,6 +9332,7 @@ class WorkflowParser {
                 action: {
                     type: node.action.type,
                     name: node.action.name,
+                    description: node.action.description,
                     tools: node.action.tools,
                 },
             })),