@eko-ai/eko 1.0.7 → 1.0.8

package/dist/extension_content_script.js

@@ -40,6 +40,11 @@ chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) {
                     sendResponse(result);
                     break;
                 }
+                case 'request_user_help': {
+                    request_user_help(request.task_id, request.failure_type, request.failure_message);
+                    sendResponse(true);
+                    break;
+                }
                 case 'computer:type': {
                     sendResponse(type(request));
                     break;
@@ -204,7 +209,7 @@ function scroll_to(request) {
             return false;
         }
         element.scrollIntoView({
-            behavior: 'smooth'
+            behavior: 'smooth',
         });
     }
     else if (request.xpath) {
@@ -215,7 +220,7 @@ function scroll_to(request) {
             return false;
         }
         element.scrollIntoView({
-            behavior: 'smooth'
+            behavior: 'smooth',
         });
     }
     else {
@@ -277,3 +282,125 @@ function select_dropdown_option(request) {
         selectedText: option.text.trim(),
     };
 }
+function request_user_help(task_id, failure_type, failure_message) {
+    const domId = 'eko-request-user-help';
+    if (document.getElementById(domId)) {
+        return;
+    }
+    const failureTitleMap = {
+        login_required: 'Login Required',
+        captcha: 'Captcha Detected',
+        blocked: 'Blocked',
+        other: 'Error',
+        rate_limited: 'Rate Limited',
+    };
+    const notification = document.createElement('div');
+    notification.id = domId;
+    notification.style.cssText = `
+    position: fixed;
+    top: 5px;
+    left: 18px;
+    z-index: 9999;
+    background-color: #FEF0ED;
+    color: white;
+    padding: 16px;
+    border-radius: 12px;
+    border: 1px solid #FBB8A5;
+    font-family: Arial, sans-serif;
+    width: 350px;
+    display: flex;
+    flex-direction: row;
+    gap: 10px;
+    cursor: move;
+    user-select: none;
+  `;
+    let isDragging = false;
+    let xOffset = 0;
+    let yOffset = 0;
+    let initialX = 0;
+    let initialY = 0;
+    notification.addEventListener('mousedown', (e) => {
+        isDragging = true;
+        initialX = e.clientX - xOffset;
+        initialY = e.clientY - yOffset;
+        e.preventDefault();
+    });
+    document.addEventListener('mousemove', (e) => {
+        if (!isDragging)
+            return;
+        const currentX = e.clientX - initialX;
+        const currentY = e.clientY - initialY;
+        xOffset = currentX;
+        yOffset = currentY;
+        notification.style.transform = `translate(${xOffset}px, ${yOffset}px)`;
+    });
+    document.addEventListener('mouseup', () => {
+        isDragging = false;
+    });
+    const leftContainer = document.createElement('div');
+    leftContainer.style.cssText = `
+    width: 28px;
+    height: 28px;
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    border-radius: 99px;
+    background: #FDCCCC;
+    justify-content: center;
+  `;
+    leftContainer.innerHTML = ``;
+    const rightContainer = document.createElement('div');
+    rightContainer.style.cssText = `
+    flex: 1;
+    display: flex;
+    flex-direction: column;
+  `;
+    const title = document.createElement('div');
+    title.style.cssText = `
+    font-size: 16px;
+    font-weight: 700;
+    line-height: 22px;
+    color: #DD342D;
+    text-align: left;
+  `;
+    title.innerText = failureTitleMap[failure_type] || failure_type;
+    const message2 = document.createElement('div');
+    message2.style.cssText = `
+    font-size: 16px;
+    font-weight: 400;
+    line-height: 22px;
+    color: #DD342D;
+    text-align: left;
+  `;
+    message2.innerText = failure_message + '\nWhen you resolve the issue, click the button below.';
+    const buttonDiv = document.createElement('div');
+    buttonDiv.style.cssText = `
+    margin-top: 16px;
+    display: flex;
+    flex-direction: row-reverse;
+    justify-content: flex-start;
+    align-items: center;
+  `;
+    const resolvedBut = document.createElement('div');
+    resolvedBut.innerText = 'Resolved';
+    resolvedBut.style.cssText = `
+    border-radius: 8px;
+    background: #DD342D;
+    color: white;
+    padding: 10px;
+    border: none;
+    cursor: pointer;
+  `;
+    resolvedBut.onclick = () => {
+        chrome.runtime.sendMessage({ type: 'issue_resolved', task_id, failure_type }, () => {
+            notification.remove();
+        });
+    };
+    buttonDiv.appendChild(resolvedBut);
+    rightContainer.appendChild(title);
+    rightContainer.appendChild(message2);
+    rightContainer.appendChild(buttonDiv);
+    notification.appendChild(leftContainer);
+    notification.appendChild(rightContainer);
+    document.body.appendChild(notification);
+}

package/dist/index.cjs.js

@@ -313,6 +313,7 @@ class ActionImpl {
                 console.error('Stream Error:', error);
             },
         };
+        this.handleHistoryImageMessages(messages);
         // Wait for stream to complete
         await this.llmProvider.generateStream(messages, params, handler);
         // Wait for tool execution to complete if it was started
@@ -320,7 +321,7 @@ class ActionImpl {
             await toolExecutionPromise;
         }
         if (context.__abort) {
-            throw new Error("Abort");
+            throw new Error('Abort');
         }
         // Add messages in the correct order after everything is complete
         if (assistantTextMessage) {
@@ -334,6 +335,37 @@ class ActionImpl {
         }
         return { response, hasToolUse, roundMessages };
     }
+    handleHistoryImageMessages(messages) {
+        // Remove all images of the historical tool call results, except for the last one.
+        let last_user = true;
+        for (let i = messages.length - 1; i >= 0; i--) {
+            const message = messages[i];
+            if (message.role === 'user') {
+                if (last_user) {
+                    last_user = false;
+                    continue;
+                }
+                if (message.content instanceof Array) {
+                    let content = message.content;
+                    for (let j = 0; j < content.length; j++) {
+                        if (content[j].type === 'tool_result' && content[j].content instanceof Array) {
+                            let tool_content = content[j].content;
+                            if (tool_content.length > 0) {
+                                for (let k = tool_content.length - 1; k >= 0; k--) {
+                                    if (tool_content[k].type === 'image') {
+                                        tool_content.splice(k, 1);
+                                    }
+                                }
+                            }
+                            else if (tool_content[0].type === 'image') {
+                                tool_content = [{ type: 'text', text: 'ok' }];
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
     async execute(input, context, outputSchema) {
         var _a;
         // Create return tool with output schema
@@ -8876,13 +8908,61 @@ class OpenaiProvider {
                             content: content.text,
                         });
                     }
-                    else if (content.type == 'tool_result') {
+                    else if (content.type == 'image') {
                         _messages.push({
-                            role: 'tool',
-                            content: content.content,
-                            tool_call_id: content.tool_call_id || content.tool_use_id,
+                            role: 'user',
+                            content: [
+                                {
+                                    type: 'image_url',
+                                    image_url: {
+                                        url: `data:${content.source.media_type};base64,${content.source.data}`,
+                                    },
+                                },
+                            ],
                         });
                     }
+                    else if (content.type == 'tool_result') {
+                        let _content = [];
+                        if (content.content == 'string') {
+                            _content.push({ type: 'text', text: content.content });
+                        }
+                        else {
+                            for (let k = 0; k < content.content.length; k++) {
+                                let item = content.content[k];
+                                if (item.type == 'text') {
+                                    _content.push({ ...item });
+                                }
+                                else if (item.type == 'image') {
+                                    _content.push({
+                                        type: 'image_url',
+                                        image_url: {
+                                            url: `data:${item.source.media_type};base64,${item.source.data}`,
+                                        },
+                                    });
+                                }
+                            }
+                        }
+                        let hasImage = _content.filter((s) => s.type == 'image_url').length > 0;
+                        if (hasImage) {
+                            // OpenAI does not support images returned by the tool.
+                            _messages.push({
+                                role: 'tool',
+                                content: 'ok',
+                                tool_call_id: content.tool_call_id || content.tool_use_id,
+                            });
+                            _messages.push({
+                                role: 'user',
+                                content: _content,
+                            });
+                        }
+                        else {
+                            _messages.push({
+                                role: 'tool',
+                                content: _content,
+                                tool_call_id: content.tool_call_id || content.tool_use_id,
+                            });
+                        }
+                    }
                 }
             }
             else {

package/dist/index.esm.js

@@ -309,6 +309,7 @@ class ActionImpl {
                 console.error('Stream Error:', error);
             },
         };
+        this.handleHistoryImageMessages(messages);
         // Wait for stream to complete
         await this.llmProvider.generateStream(messages, params, handler);
         // Wait for tool execution to complete if it was started
@@ -316,7 +317,7 @@ class ActionImpl {
             await toolExecutionPromise;
         }
         if (context.__abort) {
-            throw new Error("Abort");
+            throw new Error('Abort');
         }
         // Add messages in the correct order after everything is complete
         if (assistantTextMessage) {
@@ -330,6 +331,37 @@ class ActionImpl {
         }
         return { response, hasToolUse, roundMessages };
     }
+    handleHistoryImageMessages(messages) {
+        // Remove all images of the historical tool call results, except for the last one.
+        let last_user = true;
+        for (let i = messages.length - 1; i >= 0; i--) {
+            const message = messages[i];
+            if (message.role === 'user') {
+                if (last_user) {
+                    last_user = false;
+                    continue;
+                }
+                if (message.content instanceof Array) {
+                    let content = message.content;
+                    for (let j = 0; j < content.length; j++) {
+                        if (content[j].type === 'tool_result' && content[j].content instanceof Array) {
+                            let tool_content = content[j].content;
+                            if (tool_content.length > 0) {
+                                for (let k = tool_content.length - 1; k >= 0; k--) {
+                                    if (tool_content[k].type === 'image') {
+                                        tool_content.splice(k, 1);
+                                    }
+                                }
+                            }
+                            else if (tool_content[0].type === 'image') {
+                                tool_content = [{ type: 'text', text: 'ok' }];
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
     async execute(input, context, outputSchema) {
         var _a;
         // Create return tool with output schema
@@ -8872,13 +8904,61 @@ class OpenaiProvider {
                             content: content.text,
                         });
                     }
-                    else if (content.type == 'tool_result') {
+                    else if (content.type == 'image') {
                         _messages.push({
-                            role: 'tool',
-                            content: content.content,
-                            tool_call_id: content.tool_call_id || content.tool_use_id,
+                            role: 'user',
+                            content: [
+                                {
+                                    type: 'image_url',
+                                    image_url: {
+                                        url: `data:${content.source.media_type};base64,${content.source.data}`,
+                                    },
+                                },
+                            ],
                         });
                     }
+                    else if (content.type == 'tool_result') {
+                        let _content = [];
+                        if (content.content == 'string') {
+                            _content.push({ type: 'text', text: content.content });
+                        }
+                        else {
+                            for (let k = 0; k < content.content.length; k++) {
+                                let item = content.content[k];
+                                if (item.type == 'text') {
+                                    _content.push({ ...item });
+                                }
+                                else if (item.type == 'image') {
+                                    _content.push({
+                                        type: 'image_url',
+                                        image_url: {
+                                            url: `data:${item.source.media_type};base64,${item.source.data}`,
+                                        },
+                                    });
+                                }
+                            }
+                        }
+                        let hasImage = _content.filter((s) => s.type == 'image_url').length > 0;
+                        if (hasImage) {
+                            // OpenAI does not support images returned by the tool.
+                            _messages.push({
+                                role: 'tool',
+                                content: 'ok',
+                                tool_call_id: content.tool_call_id || content.tool_use_id,
+                            });
+                            _messages.push({
+                                role: 'user',
+                                content: _content,
+                            });
+                        }
+                        else {
+                            _messages.push({
+                                role: 'tool',
+                                content: _content,
+                                tool_call_id: content.tool_call_id || content.tool_use_id,
+                            });
+                        }
+                    }
                 }
             }
             else {

package/dist/models/action.d.ts

@@ -14,6 +14,7 @@ export declare class ActionImpl implements Action {
         maxRounds?: number;
     });
     private executeSingleRound;
+    private handleHistoryImageMessages;
     execute(input: unknown, context: ExecutionContext, outputSchema?: unknown): Promise<unknown>;
     private formatSystemPrompt;
     private formatUserPrompt;

package/dist/nodejs/script/build_dom_tree.d.ts

@@ -0,0 +1 @@
+export function run_build_dom_tree(): void;

package/dist/nodejs/tools/browser_use.d.ts

@@ -0,0 +1,28 @@
+import { BrowserUseParam, BrowserUseResult } from '../../types/tools.types';
+import { Tool, InputSchema, ExecutionContext } from '../../types/action.types';
+/**
+ * Browser Use => `npx playwright install`
+ */
+export declare class BrowserUse implements Tool<BrowserUseParam, BrowserUseResult> {
+    name: string;
+    description: string;
+    input_schema: InputSchema;
+    private browser;
+    private browser_context;
+    private current_page;
+    constructor();
+    /**
+     * browser
+     *
+     * @param {*} params { action: 'input_text', index: 1, text: 'string' }
+     * @returns > { success: true, image?: { type: 'base64', media_type: 'image/jpeg', data: '/9j...' }, text?: string }
+     */
+    execute(context: ExecutionContext, params: BrowserUseParam): Promise<BrowserUseResult>;
+    private open_url;
+    private injectScript;
+    private get_highlight_element;
+    private extractHtmlContent;
+    private get_dropdown_options;
+    private select_dropdown_option;
+    destroy(context: ExecutionContext): void;
+}

package/dist/nodejs/tools/index.d.ts

@@ -1,3 +1,4 @@
 export { CommandExecute } from './command_execute';
 export { FileRead } from './file_read';
 export { FileWrite } from './file_write';
+export { BrowserUse } from './browser_use';