@eko-ai/eko 2.1.2 → 2.1.3

package/dist/index.cjs.js

@@ -17930,7 +17930,7 @@ class HumanInteractTool {
 confirm: Ask the user to confirm whether to execute an operation, especially when performing dangerous actions such as deleting system files, users will choose Yes or No.
 input: Prompt the user to enter text; for example, when a task is ambiguous, the AI can choose to ask the user for details, and the user can respond by inputting.
 select: Allow the user to make a choice; in situations that require selection, the AI can ask the user to make a decision.
-request_help: Request assistance from the user; for instance, when an operation is blocked, the AI can ask the user for help, such as needing to log into a website or solve a CAPTCHA.`;
+request_help: Request assistance from the user; for instance, when an operation is blocked, the AI can ask the user for help, such as needing to log into a website or solve a CAPTCHA or Scan the QR code.`;
         this.parameters = {
             type: "object",
             properties: {
@@ -18192,6 +18192,22 @@ class VariableStorageTool {
 }
 
 const TOOL_NAME = "watch_trigger";
+const watch_system_prompt = `You are a tool for detecting element changes. Given a task description, compare two images to determine whether the changes described in the task have occurred.
+If the changes have occurred, return an json with \`changed\` set to true and \`changeInfo\` containing a description of the changes. If no changes have occurred, return an object with \`changed\` set to false.
+
+## Example
+User: Monitor new messages in group chat
+### No changes detected
+Output:
+{
+  "changed": false
+}
+### Change detected
+Output:
+{
+  "changed": true,
+  "changeInfo": "New message received in the group chat. The message content is: 'Hello, how are you?'"
+}`;
 class WatchTriggerTool {
     constructor() {
         this.name = TOOL_NAME;
@@ -18256,19 +18272,22 @@ class WatchTriggerTool {
                 ],
             };
         }
-        const screenshot = agentContext.agent["screenshot"];
-        const image1Result = (await screenshot.call(agentContext.agent, agentContext));
-        const image1 = toImage(image1Result.imageBase64);
+        await this.init_eko_observer(agentContext);
+        const image1 = await this.get_screenshot(agentContext);
         const start = new Date().getTime();
         const timeout = (args.timeout || 5) * 60000;
-        const frequency = Math.max(500, (args.frequency = args.frequency || 1) * 1000);
+        const frequency = Math.max(500, (args.frequency || 1) * 1000);
         let rlm = new RetryLanguageModel(agentContext.context.config.llms, agentContext.agent.Llms);
         while (new Date().getTime() - start < timeout) {
             await agentContext.context.checkAborted();
             await new Promise((resolve) => setTimeout(resolve, frequency));
-            const image2Result = (await screenshot.call(agentContext.agent, agentContext));
-            const image2 = toImage(image2Result.imageBase64);
-            const changeResult = await this.is_dom_change(agentContext, rlm, image1, image1Result.imageType, image2, image2Result.imageType, task_description);
+            let changed = await this.has_eko_changed(agentContext);
+            if (changed == "false") {
+                continue;
+            }
+            await this.init_eko_observer(agentContext);
+            const image2 = await this.get_screenshot(agentContext);
+            const changeResult = await this.is_dom_change(agentContext, rlm, image1, image2, task_description);
             if (changeResult.changed) {
                 return {
                     content: [
@@ -18289,41 +18308,73 @@ class WatchTriggerTool {
             ],
         };
     }
-    async is_dom_change(agentContext, rlm, image1, image1Type, image2, image2Type, task_description) {
+    async get_screenshot(agentContext) {
+        const screenshot = agentContext.agent["screenshot"];
+        const imageResult = (await screenshot.call(agentContext.agent, agentContext));
+        const image = toImage(imageResult.imageBase64);
+        return {
+            image: image,
+            imageType: imageResult.imageType,
+        };
+    }
+    async init_eko_observer(agentContext) {
+        try {
+            const screenshot = agentContext.agent["execute_script"];
+            await screenshot.call(agentContext.agent, agentContext, () => {
+                let _window = window;
+                _window.has_eko_changed = false;
+                _window.eko_observer && _window.eko_observer.disconnect();
+                let eko_observer = new MutationObserver(function (mutations) {
+                    _window.has_eko_changed = true;
+                });
+                eko_observer.observe(document.body, {
+                    childList: true,
+                    subtree: true,
+                    attributes: true,
+                    attributeOldValue: true,
+                    characterData: true,
+                    characterDataOldValue: true,
+                });
+                _window.eko_observer = eko_observer;
+            }, []);
+        }
+        catch (error) {
+            console.error("Error initializing Eko observer:", error);
+        }
+    }
+    async has_eko_changed(agentContext) {
+        try {
+            const screenshot = agentContext.agent["execute_script"];
+            let result = (await screenshot.call(agentContext.agent, agentContext, () => {
+                return window.has_eko_changed + "";
+            }, []));
+            return result;
+        }
+        catch (e) {
+            console.error("Error checking Eko change:", e);
+            return "undefined";
+        }
+    }
+    async is_dom_change(agentContext, rlm, image1, image2, task_description) {
         try {
             let request = {
                 messages: [
                     {
                         role: "system",
-                        content: `You are a tool for detecting element changes. Given a task description, compare two images to determine whether the changes described in the task have occurred.
-If the changes have occurred, return an json with \`changed\` set to true and \`changeInfo\` containing a description of the changes. If no changes have occurred, return an object with \`changed\` set to false.
-
-## Example
-User: Monitor new messages in group chat
-### No changes detected
-Output:
-{
-  "changed": false
-}
-### Change detected
-Output:
-{
-  "changed": true,
-  "changeInfo": "New message received in the group chat. The message content is: 'Hello, how are you?'"
-}`,
+                        content: watch_system_prompt,
                     },
                     {
                         role: "user",
                         content: [
                             {
                                 type: "image",
-                                image: image1,
-                                mimeType: image1Type,
+                                image: image1.image,
+                                mimeType: image1.imageType,
                             },
                             {
                                 type: "image",
-                                image: image2,
-                                mimeType: image2Type,
+                                image: image2.image,
+                                mimeType: image2.imageType,
                             },
                             {
                                 type: "text",
@@ -18387,9 +18438,8 @@ const HUMAN_PROMPT = `
 * HUMAN INTERACT
 During the task execution process, you can use the \`${TOOL_NAME$3}\` tool to interact with humans, please call it in the following situations:
 - When performing dangerous operations such as deleting files, confirmation from humans is required.
-- When encountering obstacles while visiting a website, such as requiring user login or captcha, you need to request for manual assistance.
-- When requesting login, please only call the function when a login dialog box is clearly displayed.
-- Try to minimize the use of \`${TOOL_NAME$3}\` tool.
+- When encountering obstacles while accessing websites, such as requiring user login, captcha verification, QR code scanning, or human verification, you need to request manual assistance.
+- Please do not use the \`${TOOL_NAME$3}\` tool frequently.
 `;
 const VARIABLE_PROMPT = `
 * VARIABLE STORAGE