@eko-ai/eko 1.0.6 → 1.0.8

package/dist/extension_content_script.js

@@ -40,6 +40,11 @@ chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) {
                     sendResponse(result);
                     break;
                 }
+                case 'request_user_help': {
+                    request_user_help(request.task_id, request.failure_type, request.failure_message);
+                    sendResponse(true);
+                    break;
+                }
                 case 'computer:type': {
                     sendResponse(type(request));
                     break;
@@ -204,7 +209,7 @@ function scroll_to(request) {
             return false;
         }
         element.scrollIntoView({
-            behavior: 'smooth'
+            behavior: 'smooth',
         });
     }
     else if (request.xpath) {
@@ -215,7 +220,7 @@ function scroll_to(request) {
             return false;
         }
         element.scrollIntoView({
-            behavior: 'smooth'
+            behavior: 'smooth',
         });
     }
     else {
@@ -277,3 +282,125 @@ function select_dropdown_option(request) {
         selectedText: option.text.trim(),
     };
 }
+function request_user_help(task_id, failure_type, failure_message) {
+    const domId = 'eko-request-user-help';
+    if (document.getElementById(domId)) {
+        return;
+    }
+    const failureTitleMap = {
+        login_required: 'Login Required',
+        captcha: 'Captcha Detected',
+        blocked: 'Blocked',
+        other: 'Error',
+        rate_limited: 'Rate Limited',
+    };
+    const notification = document.createElement('div');
+    notification.id = domId;
+    notification.style.cssText = `
+    position: fixed;
+    top: 5px;
+    left: 18px;
+    z-index: 9999;
+    background-color: #FEF0ED;
+    color: white;
+    padding: 16px;
+    border-radius: 12px;
+    border: 1px solid #FBB8A5;
+    font-family: Arial, sans-serif;
+    width: 350px;
+    display: flex;
+    flex-direction: row;
+    gap: 10px;
+    cursor: move;
+    user-select: none;
+  `;
+    let isDragging = false;
+    let xOffset = 0;
+    let yOffset = 0;
+    let initialX = 0;
+    let initialY = 0;
+    notification.addEventListener('mousedown', (e) => {
+        isDragging = true;
+        initialX = e.clientX - xOffset;
+        initialY = e.clientY - yOffset;
+        e.preventDefault();
+    });
+    document.addEventListener('mousemove', (e) => {
+        if (!isDragging)
+            return;
+        const currentX = e.clientX - initialX;
+        const currentY = e.clientY - initialY;
+        xOffset = currentX;
+        yOffset = currentY;
+        notification.style.transform = `translate(${xOffset}px, ${yOffset}px)`;
+    });
+    document.addEventListener('mouseup', () => {
+        isDragging = false;
+    });
+    const leftContainer = document.createElement('div');
+    leftContainer.style.cssText = `
+    width: 28px;
+    height: 28px;
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    border-radius: 99px;
+    background: #FDCCCC;
+    justify-content: center;
+  `;
+    leftContainer.innerHTML = ``;
+    const rightContainer = document.createElement('div');
+    rightContainer.style.cssText = `
+    flex: 1;
+    display: flex;
+    flex-direction: column;
+  `;
+    const title = document.createElement('div');
+    title.style.cssText = `
+    font-size: 16px;
+    font-weight: 700;
+    line-height: 22px;
+    color: #DD342D;
+    text-align: left;
+  `;
+    title.innerText = failureTitleMap[failure_type] || failure_type;
+    const message2 = document.createElement('div');
+    message2.style.cssText = `
+    font-size: 16px;
+    font-weight: 400;
+    line-height: 22px;
+    color: #DD342D;
+    text-align: left;
+  `;
+    message2.innerText = failure_message + '\nWhen you resolve the issue, click the button below.';
+    const buttonDiv = document.createElement('div');
+    buttonDiv.style.cssText = `
+    margin-top: 16px;
+    display: flex;
+    flex-direction: row-reverse;
+    justify-content: flex-start;
+    align-items: center;
+  `;
+    const resolvedBut = document.createElement('div');
+    resolvedBut.innerText = 'Resolved';
+    resolvedBut.style.cssText = `
+    border-radius: 8px;
+    background: #DD342D;
+    color: white;
+    padding: 10px;
+    border: none;
+    cursor: pointer;
+  `;
+    resolvedBut.onclick = () => {
+        chrome.runtime.sendMessage({ type: 'issue_resolved', task_id, failure_type }, () => {
+            notification.remove();
+        });
+    };
+    buttonDiv.appendChild(resolvedBut);
+    rightContainer.appendChild(title);
+    rightContainer.appendChild(message2);
+    rightContainer.appendChild(buttonDiv);
+    notification.appendChild(leftContainer);
+    notification.appendChild(rightContainer);
+    document.body.appendChild(notification);
+}

package/dist/index.cjs.js

@@ -243,6 +243,16 @@ class ActionImpl {
                             }
                         }
                         if (context.__skip || context.__abort) {
+                            toolResultMessage = {
+                                role: 'user',
+                                content: [
+                                    {
+                                        type: 'tool_result',
+                                        tool_use_id: toolCall.id,
+                                        content: 'skip',
+                                    },
+                                ],
+                            };
                             return;
                         }
                         // Execute the tool
@@ -303,6 +313,7 @@ class ActionImpl {
                 console.error('Stream Error:', error);
             },
         };
+        this.handleHistoryImageMessages(messages);
         // Wait for stream to complete
         await this.llmProvider.generateStream(messages, params, handler);
         // Wait for tool execution to complete if it was started
@@ -310,7 +321,7 @@ class ActionImpl {
             await toolExecutionPromise;
         }
         if (context.__abort) {
-            throw new Error("Abort");
+            throw new Error('Abort');
         }
         // Add messages in the correct order after everything is complete
         if (assistantTextMessage) {
@@ -324,6 +335,37 @@ class ActionImpl {
         }
         return { response, hasToolUse, roundMessages };
     }
+    handleHistoryImageMessages(messages) {
+        // Remove all images of the historical tool call results, except for the last one.
+        let last_user = true;
+        for (let i = messages.length - 1; i >= 0; i--) {
+            const message = messages[i];
+            if (message.role === 'user') {
+                if (last_user) {
+                    last_user = false;
+                    continue;
+                }
+                if (message.content instanceof Array) {
+                    let content = message.content;
+                    for (let j = 0; j < content.length; j++) {
+                        if (content[j].type === 'tool_result' && content[j].content instanceof Array) {
+                            let tool_content = content[j].content;
+                            if (tool_content.length > 0) {
+                                for (let k = tool_content.length - 1; k >= 0; k--) {
+                                    if (tool_content[k].type === 'image') {
+                                        tool_content.splice(k, 1);
+                                    }
+                                }
+                            }
+                            else if (tool_content[0].type === 'image') {
+                                tool_content = [{ type: 'text', text: 'ok' }];
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
     async execute(input, context, outputSchema) {
         var _a;
         // Create return tool with output schema
@@ -466,6 +508,7 @@ Generate a complete workflow that:
 5. Includes detailed descriptions for each action, ensuring that the actions, when combined, is a complete solution to the user's problem`;
         },
         formatUserPrompt: (requirement) => `Create a workflow for the following requirement: ${requirement}`,
+        modifyUserPrompt: (prompt) => `Modify workflow: ${prompt}`,
     };
 }
 function createWorkflowGenerationTool(registry) {
@@ -540,30 +583,68 @@ class WorkflowGenerator {
     constructor(llmProvider, toolRegistry) {
         this.llmProvider = llmProvider;
         this.toolRegistry = toolRegistry;
+        this.message_history = [];
     }
     async generateWorkflow(prompt) {
+        return this.doGenerateWorkflow(prompt, false);
+    }
+    async modifyWorkflow(prompt) {
+        return this.doGenerateWorkflow(prompt, true);
+    }
+    async doGenerateWorkflow(prompt, modify) {
         // Create prompts with current set of tools
         const prompts = createWorkflowPrompts(this.toolRegistry.getToolDefinitions());
-        const messages = [
-            {
-                role: 'system',
-                content: prompts.formatSystemPrompt()
-            },
-            {
+        let messages = [];
+        if (modify) {
+            messages = this.message_history;
+            messages.push({
                 role: 'user',
-                content: prompts.formatUserPrompt(prompt)
-            }
-        ];
+                content: prompts.modifyUserPrompt(prompt),
+            });
+        }
+        else {
+            messages = this.message_history = [
+                {
+                    role: 'system',
+                    content: prompts.formatSystemPrompt(),
+                },
+                {
+                    role: 'user',
+                    content: prompts.formatUserPrompt(prompt),
+                },
+            ];
+        }
         const params = {
             temperature: 0.7,
             maxTokens: 8192,
             tools: [createWorkflowGenerationTool(this.toolRegistry)],
-            toolChoice: { type: 'tool', name: 'generate_workflow' }
+            toolChoice: { type: 'tool', name: 'generate_workflow' },
         };
         const response = await this.llmProvider.generateText(messages, params);
         if (!response.toolCalls.length || !response.toolCalls[0].input.workflow) {
+            messages.pop();
             throw new Error('Failed to generate workflow: Invalid response from LLM');
         }
+        messages.push({
+            role: 'assistant',
+            content: [
+                {
+                    type: 'tool_use',
+                    id: response.toolCalls[0].id,
+                    name: response.toolCalls[0].name,
+                    input: response.toolCalls[0].input,
+                },
+            ],
+        }, {
+            role: 'user',
+            content: [
+                {
+                    type: 'tool_result',
+                    tool_use_id: response.toolCalls[0].id,
+                    content: 'ok',
+                },
+            ],
+        });
         const workflowData = response.toolCalls[0].input.workflow;
         // Validate all tools exist
         for (const node of workflowData.nodes) {
@@ -590,16 +671,13 @@ class WorkflowGenerator {
                     input: nodeData.input || { type: 'any', schema: {}, value: undefined },
                     output: nodeData.output || { type: 'any', schema: {}, value: undefined },
                     action: action,
-                    dependencies: nodeData.dependencies || []
+                    dependencies: nodeData.dependencies || [],
                 };
                 workflow.addNode(node);
             });
         }
         return workflow;
     }
-    async modifyWorkflow(workflow, prompt) {
-        throw new Error('Not implemented');
-    }
 }
 
 const VERSION$1 = '0.33.1'; // x-release-please-version
@@ -8830,13 +8908,61 @@ class OpenaiProvider {
                             content: content.text,
                         });
                     }
-                    else if (content.type == 'tool_result') {
+                    else if (content.type == 'image') {
                         _messages.push({
-                            role: 'tool',
-                            content: content.content,
-                            tool_call_id: content.tool_call_id || content.tool_use_id,
+                            role: 'user',
+                            content: [
+                                {
+                                    type: 'image_url',
+                                    image_url: {
+                                        url: `data:${content.source.media_type};base64,${content.source.data}`,
+                                    },
+                                },
+                            ],
                         });
                     }
+                    else if (content.type == 'tool_result') {
+                        let _content = [];
+                        if (content.content == 'string') {
+                            _content.push({ type: 'text', text: content.content });
+                        }
+                        else {
+                            for (let k = 0; k < content.content.length; k++) {
+                                let item = content.content[k];
+                                if (item.type == 'text') {
+                                    _content.push({ ...item });
+                                }
+                                else if (item.type == 'image') {
+                                    _content.push({
+                                        type: 'image_url',
+                                        image_url: {
+                                            url: `data:${item.source.media_type};base64,${item.source.data}`,
+                                        },
+                                    });
+                                }
+                            }
+                        }
+                        let hasImage = _content.filter((s) => s.type == 'image_url').length > 0;
+                        if (hasImage) {
+                            // OpenAI does not support images returned by the tool.
+                            _messages.push({
+                                role: 'tool',
+                                content: 'ok',
+                                tool_call_id: content.tool_call_id || content.tool_use_id,
+                            });
+                            _messages.push({
+                                role: 'user',
+                                content: _content,
+                            });
+                        }
+                        else {
+                            _messages.push({
+                                role: 'tool',
+                                content: _content,
+                                tool_call_id: content.tool_call_id || content.tool_use_id,
+                            });
+                        }
+                    }
                 }
             }
             else {
@@ -9112,6 +9238,7 @@ class ToolRegistry {
 class Eko {
     constructor(config) {
         this.toolRegistry = new ToolRegistry();
+        this.workflowGeneratorMap = new Map();
         if (typeof config == 'string') {
             this.llmProvider = new ClaudeProvider(config);
         }
@@ -9148,11 +9275,19 @@ class Eko {
             }
         }
         const generator = new WorkflowGenerator(this.llmProvider, toolRegistry);
-        return await generator.generateWorkflow(prompt);
+        const workflow = await generator.generateWorkflow(prompt);
+        this.workflowGeneratorMap.set(workflow, generator);
+        return workflow;
     }
     async execute(workflow, callback) {
         return await workflow.execute(callback);
     }
+    async modify(workflow, prompt) {
+        const generator = this.workflowGeneratorMap.get(workflow);
+        workflow = await generator.modifyWorkflow(prompt);
+        this.workflowGeneratorMap.set(workflow, generator);
+        return workflow;
+    }
     getTool(toolName) {
         let tool;
         if (this.toolRegistry.hasTools([toolName])) {

package/dist/index.esm.js

@@ -239,6 +239,16 @@ class ActionImpl {
                             }
                         }
                         if (context.__skip || context.__abort) {
+                            toolResultMessage = {
+                                role: 'user',
+                                content: [
+                                    {
+                                        type: 'tool_result',
+                                        tool_use_id: toolCall.id,
+                                        content: 'skip',
+                                    },
+                                ],
+                            };
                             return;
                         }
                         // Execute the tool
@@ -299,6 +309,7 @@ class ActionImpl {
                 console.error('Stream Error:', error);
             },
         };
+        this.handleHistoryImageMessages(messages);
         // Wait for stream to complete
         await this.llmProvider.generateStream(messages, params, handler);
         // Wait for tool execution to complete if it was started
@@ -306,7 +317,7 @@ class ActionImpl {
             await toolExecutionPromise;
         }
         if (context.__abort) {
-            throw new Error("Abort");
+            throw new Error('Abort');
         }
         // Add messages in the correct order after everything is complete
         if (assistantTextMessage) {
@@ -320,6 +331,37 @@ class ActionImpl {
         }
         return { response, hasToolUse, roundMessages };
     }
+    handleHistoryImageMessages(messages) {
+        // Remove all images of the historical tool call results, except for the last one.
+        let last_user = true;
+        for (let i = messages.length - 1; i >= 0; i--) {
+            const message = messages[i];
+            if (message.role === 'user') {
+                if (last_user) {
+                    last_user = false;
+                    continue;
+                }
+                if (message.content instanceof Array) {
+                    let content = message.content;
+                    for (let j = 0; j < content.length; j++) {
+                        if (content[j].type === 'tool_result' && content[j].content instanceof Array) {
+                            let tool_content = content[j].content;
+                            if (tool_content.length > 0) {
+                                for (let k = tool_content.length - 1; k >= 0; k--) {
+                                    if (tool_content[k].type === 'image') {
+                                        tool_content.splice(k, 1);
+                                    }
+                                }
+                            }
+                            else if (tool_content[0].type === 'image') {
+                                tool_content = [{ type: 'text', text: 'ok' }];
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
     async execute(input, context, outputSchema) {
         var _a;
         // Create return tool with output schema
@@ -462,6 +504,7 @@ Generate a complete workflow that:
 5. Includes detailed descriptions for each action, ensuring that the actions, when combined, is a complete solution to the user's problem`;
         },
         formatUserPrompt: (requirement) => `Create a workflow for the following requirement: ${requirement}`,
+        modifyUserPrompt: (prompt) => `Modify workflow: ${prompt}`,
     };
 }
 function createWorkflowGenerationTool(registry) {
@@ -536,30 +579,68 @@ class WorkflowGenerator {
     constructor(llmProvider, toolRegistry) {
         this.llmProvider = llmProvider;
         this.toolRegistry = toolRegistry;
+        this.message_history = [];
     }
     async generateWorkflow(prompt) {
+        return this.doGenerateWorkflow(prompt, false);
+    }
+    async modifyWorkflow(prompt) {
+        return this.doGenerateWorkflow(prompt, true);
+    }
+    async doGenerateWorkflow(prompt, modify) {
         // Create prompts with current set of tools
         const prompts = createWorkflowPrompts(this.toolRegistry.getToolDefinitions());
-        const messages = [
-            {
-                role: 'system',
-                content: prompts.formatSystemPrompt()
-            },
-            {
+        let messages = [];
+        if (modify) {
+            messages = this.message_history;
+            messages.push({
                 role: 'user',
-                content: prompts.formatUserPrompt(prompt)
-            }
-        ];
+                content: prompts.modifyUserPrompt(prompt),
+            });
+        }
+        else {
+            messages = this.message_history = [
+                {
+                    role: 'system',
+                    content: prompts.formatSystemPrompt(),
+                },
+                {
+                    role: 'user',
+                    content: prompts.formatUserPrompt(prompt),
+                },
+            ];
+        }
         const params = {
             temperature: 0.7,
             maxTokens: 8192,
             tools: [createWorkflowGenerationTool(this.toolRegistry)],
-            toolChoice: { type: 'tool', name: 'generate_workflow' }
+            toolChoice: { type: 'tool', name: 'generate_workflow' },
         };
         const response = await this.llmProvider.generateText(messages, params);
         if (!response.toolCalls.length || !response.toolCalls[0].input.workflow) {
+            messages.pop();
             throw new Error('Failed to generate workflow: Invalid response from LLM');
         }
+        messages.push({
+            role: 'assistant',
+            content: [
+                {
+                    type: 'tool_use',
+                    id: response.toolCalls[0].id,
+                    name: response.toolCalls[0].name,
+                    input: response.toolCalls[0].input,
+                },
+            ],
+        }, {
+            role: 'user',
+            content: [
+                {
+                    type: 'tool_result',
+                    tool_use_id: response.toolCalls[0].id,
+                    content: 'ok',
+                },
+            ],
+        });
         const workflowData = response.toolCalls[0].input.workflow;
         // Validate all tools exist
         for (const node of workflowData.nodes) {
@@ -586,16 +667,13 @@ class WorkflowGenerator {
                     input: nodeData.input || { type: 'any', schema: {}, value: undefined },
                     output: nodeData.output || { type: 'any', schema: {}, value: undefined },
                     action: action,
-                    dependencies: nodeData.dependencies || []
+                    dependencies: nodeData.dependencies || [],
                 };
                 workflow.addNode(node);
             });
         }
         return workflow;
     }
-    async modifyWorkflow(workflow, prompt) {
-        throw new Error('Not implemented');
-    }
 }
 
 const VERSION$1 = '0.33.1'; // x-release-please-version
@@ -8826,13 +8904,61 @@ class OpenaiProvider {
                             content: content.text,
                         });
                     }
-                    else if (content.type == 'tool_result') {
+                    else if (content.type == 'image') {
                         _messages.push({
-                            role: 'tool',
-                            content: content.content,
-                            tool_call_id: content.tool_call_id || content.tool_use_id,
+                            role: 'user',
+                            content: [
+                                {
+                                    type: 'image_url',
+                                    image_url: {
+                                        url: `data:${content.source.media_type};base64,${content.source.data}`,
+                                    },
+                                },
+                            ],
                         });
                     }
+                    else if (content.type == 'tool_result') {
+                        let _content = [];
+                        if (content.content == 'string') {
+                            _content.push({ type: 'text', text: content.content });
+                        }
+                        else {
+                            for (let k = 0; k < content.content.length; k++) {
+                                let item = content.content[k];
+                                if (item.type == 'text') {
+                                    _content.push({ ...item });
+                                }
+                                else if (item.type == 'image') {
+                                    _content.push({
+                                        type: 'image_url',
+                                        image_url: {
+                                            url: `data:${item.source.media_type};base64,${item.source.data}`,
+                                        },
+                                    });
+                                }
+                            }
+                        }
+                        let hasImage = _content.filter((s) => s.type == 'image_url').length > 0;
+                        if (hasImage) {
+                            // OpenAI does not support images returned by the tool.
+                            _messages.push({
+                                role: 'tool',
+                                content: 'ok',
+                                tool_call_id: content.tool_call_id || content.tool_use_id,
+                            });
+                            _messages.push({
+                                role: 'user',
+                                content: _content,
+                            });
+                        }
+                        else {
+                            _messages.push({
+                                role: 'tool',
+                                content: _content,
+                                tool_call_id: content.tool_call_id || content.tool_use_id,
+                            });
+                        }
+                    }
                 }
             }
             else {
@@ -9108,6 +9234,7 @@ class ToolRegistry {
 class Eko {
     constructor(config) {
         this.toolRegistry = new ToolRegistry();
+        this.workflowGeneratorMap = new Map();
         if (typeof config == 'string') {
             this.llmProvider = new ClaudeProvider(config);
         }
@@ -9144,11 +9271,19 @@ class Eko {
             }
         }
         const generator = new WorkflowGenerator(this.llmProvider, toolRegistry);
-        return await generator.generateWorkflow(prompt);
+        const workflow = await generator.generateWorkflow(prompt);
+        this.workflowGeneratorMap.set(workflow, generator);
+        return workflow;
     }
     async execute(workflow, callback) {
         return await workflow.execute(callback);
     }
+    async modify(workflow, prompt) {
+        const generator = this.workflowGeneratorMap.get(workflow);
+        workflow = await generator.modifyWorkflow(prompt);
+        this.workflowGeneratorMap.set(workflow, generator);
+        return workflow;
+    }
     getTool(toolName) {
         let tool;
         if (this.toolRegistry.hasTools([toolName])) {