@eko-ai/eko 2.1.2 → 2.1.4

package/dist/index.esm.js

@@ -6,7 +6,7 @@ const config = {
     compressThreshold: 80,
     largeTextLength: 5000,
     fileTextMaxLength: 20000,
-    maxDialogueImgFileNum: 2,
+    maxDialogueImgFileNum: 1,
 };
 
 var LogLevel;
@@ -283,6 +283,9 @@ function sub(str, maxLength, appendPoint = true) {
     return str;
 }
 function fixXmlTag(code) {
+    if (code.indexOf('&') > -1) {
+        code = code.replace(/&(?![a-zA-Z0-9#]+;)/g, '&');
+    }
     function fixDoubleChar(code) {
         const stack = [];
         for (let i = 0; i < code.length; i++) {
@@ -17895,7 +17898,7 @@ class HumanInteractTool {
 confirm: Ask the user to confirm whether to execute an operation, especially when performing dangerous actions such as deleting system files, users will choose Yes or No.
 input: Prompt the user to enter text; for example, when a task is ambiguous, the AI can choose to ask the user for details, and the user can respond by inputting.
 select: Allow the user to make a choice; in situations that require selection, the AI can ask the user to make a decision.
-request_help: Request assistance from the user; for instance, when an operation is blocked, the AI can ask the user for help, such as needing to log into a website or solve a CAPTCHA.`;
+request_help: Request assistance from the user; for instance, when an operation is blocked, the AI can ask the user for help, such as needing to log into a website or solve a CAPTCHA or Scan the QR code.`;
         this.parameters = {
             type: "object",
             properties: {
@@ -18157,6 +18160,22 @@ class VariableStorageTool {
 }
 
 const TOOL_NAME = "watch_trigger";
+const watch_system_prompt = `You are a tool for detecting element changes. Given a task description, compare two images to determine whether the changes described in the task have occurred.
+If the changes have occurred, return an json with \`changed\` set to true and \`changeInfo\` containing a description of the changes. If no changes have occurred, return an object with \`changed\` set to false.
+
+## Example
+User: Monitor new messages in group chat
+### No changes detected
+Output:
+{
+  "changed": false
+}
+### Change detected
+Output:
+{
+  "changed": true,
+  "changeInfo": "New message received in the group chat. The message content is: 'Hello, how are you?'"
+}`;
 class WatchTriggerTool {
     constructor() {
         this.name = TOOL_NAME;
@@ -18221,19 +18240,22 @@ class WatchTriggerTool {
                 ],
             };
         }
-        const screenshot = agentContext.agent["screenshot"];
-        const image1Result = (await screenshot.call(agentContext.agent, agentContext));
-        const image1 = toImage(image1Result.imageBase64);
+        await this.init_eko_observer(agentContext);
+        const image1 = await this.get_screenshot(agentContext);
         const start = new Date().getTime();
         const timeout = (args.timeout || 5) * 60000;
-        const frequency = Math.max(500, (args.frequency = args.frequency || 1) * 1000);
+        const frequency = Math.max(500, (args.frequency || 1) * 1000);
         let rlm = new RetryLanguageModel(agentContext.context.config.llms, agentContext.agent.Llms);
         while (new Date().getTime() - start < timeout) {
             await agentContext.context.checkAborted();
             await new Promise((resolve) => setTimeout(resolve, frequency));
-            const image2Result = (await screenshot.call(agentContext.agent, agentContext));
-            const image2 = toImage(image2Result.imageBase64);
-            const changeResult = await this.is_dom_change(agentContext, rlm, image1, image1Result.imageType, image2, image2Result.imageType, task_description);
+            let changed = await this.has_eko_changed(agentContext);
+            if (changed == "false") {
+                continue;
+            }
+            await this.init_eko_observer(agentContext);
+            const image2 = await this.get_screenshot(agentContext);
+            const changeResult = await this.is_dom_change(agentContext, rlm, image1, image2, task_description);
             if (changeResult.changed) {
                 return {
                     content: [
@@ -18254,41 +18276,73 @@ class WatchTriggerTool {
             ],
         };
     }
-    async is_dom_change(agentContext, rlm, image1, image1Type, image2, image2Type, task_description) {
+    async get_screenshot(agentContext) {
+        const screenshot = agentContext.agent["screenshot"];
+        const imageResult = (await screenshot.call(agentContext.agent, agentContext));
+        const image = toImage(imageResult.imageBase64);
+        return {
+            image: image,
+            imageType: imageResult.imageType,
+        };
+    }
+    async init_eko_observer(agentContext) {
+        try {
+            const screenshot = agentContext.agent["execute_script"];
+            await screenshot.call(agentContext.agent, agentContext, () => {
+                let _window = window;
+                _window.has_eko_changed = false;
+                _window.eko_observer && _window.eko_observer.disconnect();
+                let eko_observer = new MutationObserver(function (mutations) {
+                    _window.has_eko_changed = true;
+                });
+                eko_observer.observe(document.body, {
+                    childList: true,
+                    subtree: true,
+                    attributes: true,
+                    attributeOldValue: true,
+                    characterData: true,
+                    characterDataOldValue: true,
+                });
+                _window.eko_observer = eko_observer;
+            }, []);
+        }
+        catch (error) {
+            console.error("Error initializing Eko observer:", error);
+        }
+    }
+    async has_eko_changed(agentContext) {
+        try {
+            const screenshot = agentContext.agent["execute_script"];
+            let result = (await screenshot.call(agentContext.agent, agentContext, () => {
+                return window.has_eko_changed + "";
+            }, []));
+            return result;
+        }
+        catch (e) {
+            console.error("Error checking Eko change:", e);
+            return "undefined";
+        }
+    }
+    async is_dom_change(agentContext, rlm, image1, image2, task_description) {
         try {
             let request = {
                 messages: [
                     {
                         role: "system",
-                        content: `You are a tool for detecting element changes. Given a task description, compare two images to determine whether the changes described in the task have occurred.
-If the changes have occurred, return an json with \`changed\` set to true and \`changeInfo\` containing a description of the changes. If no changes have occurred, return an object with \`changed\` set to false.
-
-## Example
-User: Monitor new messages in group chat
-### No changes detected
-Output:
-{
-  "changed": false
-}
-### Change detected
-Output:
-{
-  "changed": true,
-  "changeInfo": "New message received in the group chat. The message content is: 'Hello, how are you?'"
-}`,
+                        content: watch_system_prompt,
                     },
                     {
                         role: "user",
                         content: [
                             {
                                 type: "image",
-                                image: image1,
-                                mimeType: image1Type,
+                                image: image1.image,
+                                mimeType: image1.imageType,
                             },
                             {
                                 type: "image",
-                                image: image2,
-                                mimeType: image2Type,
+                                image: image2.image,
+                                mimeType: image2.imageType,
                             },
                             {
                                 type: "text",
@@ -18352,9 +18406,8 @@ const HUMAN_PROMPT = `
 * HUMAN INTERACT
 During the task execution process, you can use the \`${TOOL_NAME$3}\` tool to interact with humans, please call it in the following situations:
 - When performing dangerous operations such as deleting files, confirmation from humans is required.
-- When encountering obstacles while visiting a website, such as requiring user login or captcha, you need to request for manual assistance.
-- When requesting login, please only call the function when a login dialog box is clearly displayed.
-- Try to minimize the use of \`${TOOL_NAME$3}\` tool.
+- When encountering obstacles while accessing websites, such as requiring user login, captcha verification, QR code scanning, or human verification, you need to request manual assistance.
+- Please do not use the \`${TOOL_NAME$3}\` tool frequently.
 `;
 const VARIABLE_PROMPT = `
 * VARIABLE STORAGE
@@ -19336,6 +19389,9 @@ class Eko {
         if (!context) {
             throw new Error("The task does not exist");
         }
+        if (context.paused) {
+            context.paused = false;
+        }
         if (context.controller.signal.aborted) {
             context.controller = new AbortController();
         }
@@ -20409,7 +20465,8 @@ function run_build_dom_tree() {
         return { element_str, selector_map };
     }
     function get_highlight_element(highlightIndex) {
-        return window.clickable_elements[highlightIndex];
+        let element = document.querySelector(`[eko-user-highlight-id="eko-highlight-${highlightIndex}"]`);
+        return element || window.clickable_elements[highlightIndex];
     }
     function remove_highlight() {
         let highlight = document.getElementById('eko-highlight-container');
@@ -21000,6 +21057,7 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
    - Only use indexes that exist in the provided element list
    - Each element has a unique index number (e.g., "[33]:<button>")
    - Elements marked with "[]:" are non-interactive (for context only)
+   - Use the latest element index, do not rely on historical outdated element indexes
 * ERROR HANDLING:
    - If no suitable elements exist, use other functions to complete the task
    - If stuck, try alternative approaches, don't refuse tasks
@@ -21057,7 +21115,8 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
         }
         if (extract_page_content) {
             let page_content = await this.extract_page_content(agentContext);
-            return "The current page content has been extracted, latest page content:\n" + page_content;
+            return ("The current page content has been extracted, latest page content:\n" +
+                page_content);
         }
     }
     async hover_to_element(agentContext, index) {
@@ -21413,27 +21472,34 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
                 ],
             });
         }
-        if (messages.length > 10) {
-            // compressed pseudoHtml
-            for (let i = 2; i < messages.length - 3; i++) {
-                let message = messages[i];
-                if (message.role == "user" && message.content.length == 2) {
-                    let content = message.content;
-                    for (let j = 0; j < content.length; j++) {
-                        let _content = content[j];
-                        if (_content.type == "text" &&
-                            _content.text.startsWith(pseudoHtmlDescription)) {
-                            _content.text = this.removePseudoHtmlAttr(_content.text, [
-                                "class",
-                                "src",
-                                "href",
-                            ]);
-                        }
+        super.handleMessages(agentContext, messages, tools);
+        this.handlePseudoHtmlText(messages, pseudoHtmlDescription);
+    }
+    handlePseudoHtmlText(messages, pseudoHtmlDescription) {
+        for (let i = 0; i < messages.length; i++) {
+            let message = messages[i];
+            if (message.role !== "user" || message.content.length <= 1) {
+                continue;
+            }
+            let content = message.content;
+            for (let j = 0; j < content.length; j++) {
+                let _content = content[j];
+                if (_content.type == "text" &&
+                    _content.text.startsWith(pseudoHtmlDescription)) {
+                    if (i >= 2 && i < messages.length - 3) {
+                        _content.text = this.removePseudoHtmlAttr(_content.text, [
+                            "class",
+                            "src",
+                            "href",
+                        ]);
                     }
                 }
             }
+            if (content[0].text == "[image]" &&
+                content[1].text == "[image]") {
+                content.splice(0, 1);
+            }
         }
-        super.handleMessages(agentContext, messages, tools);
     }
     removePseudoHtmlAttr(pseudoHtml, remove_attrs) {
         return pseudoHtml
@@ -21442,6 +21508,7 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
             if (!line.startsWith("[") || line.indexOf("]:<") == -1) {
                 return line;
             }
+            line = line.substring(line.indexOf("]:<") + 2);
             for (let i = 0; i < remove_attrs.length; i++) {
                 let sIdx = line.indexOf(remove_attrs[i] + '="');
                 if (sIdx == -1) {
@@ -21453,12 +21520,9 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
                 }
                 line =
                     line.substring(0, sIdx) +
-                        line
-                            .substring(eIdx + 1)
-                            .trim()
-                            .replace('" >', '">');
+                        line.substring(eIdx + 1).trim();
             }
-            return line;
+            return line.replace('" >', '">').replace(" >", ">");
         })
             .join("\n");
     }