@eko-ai/eko 2.1.1 → 2.1.2

package/dist/index.cjs.js

@@ -7,10 +7,11 @@ var buffer = require('buffer');
 const config = {
     name: "Eko",
     platform: "mac",
-    maxReactNum: 200,
+    maxReactNum: 500,
     maxTokens: 16000,
     compressThreshold: 80,
     largeTextLength: 5000,
+    fileTextMaxLength: 20000,
     maxDialogueImgFileNum: 2,
 };
 
@@ -232,6 +233,7 @@ function mergeTools(tools1, tools2) {
         map[tool.name] = tool;
         return map;
     }, {});
+    let names = [];
     for (let i = 0; i < tools1.length; i++) {
         let tool1 = tools1[i];
         let tool2 = toolMap2[tool1.name];
@@ -245,8 +247,9 @@ function mergeTools(tools1, tools2) {
     }
     for (let i = 0; i < tools2.length; i++) {
         let tool2 = tools2[i];
-        if (toolMap2[tool2.name]) {
+        if (toolMap2[tool2.name] && names.indexOf(tool2.name) === -1) {
             tools.push(tool2);
+            names.push(tool2.name);
         }
     }
     return tools;
@@ -17519,10 +17522,17 @@ function extractAgentXmlNode(agentXml, nodeId) {
     let nodesNode = doc.getElementsByTagName("nodes");
     if (nodesNode.length > 0) {
         let nodes = nodesNode[0].childNodes;
+        let _nodeId = 0;
         for (let i = 0; i < nodes.length; i++) {
             let node = nodes[i];
-            if (node.nodeType == 1 && node.getAttribute("id") == nodeId + "") {
-                return node;
+            if (node.nodeType == 1) {
+                if (node.getAttribute("id") == null || node.getAttribute("id") == "") {
+                    node.setAttribute("id", _nodeId + "");
+                }
+                _nodeId++;
+                if (node.getAttribute("id") == nodeId + "") {
+                    return node;
+                }
             }
         }
     }
@@ -17856,7 +17866,7 @@ const TOOL_NAME$4 = "foreach_task";
 class ForeachTaskTool {
     constructor() {
         this.name = TOOL_NAME$4;
-        this.description = `When executing the \`forEach\` node, please use it to complete the tasks corresponding to that forEach node, which will complete all tasks under the entire forEach node.`;
+        this.description = `When executing the \`forEach\` node, please use the current tool for counting to ensure tasks are executed sequentially, the tool needs to be called with each loop iteration.`;
         this.parameters = {
             type: "object",
             properties: {
@@ -17864,12 +17874,19 @@ class ForeachTaskTool {
                     type: "number",
                     description: "forEach node ID.",
                 },
+                progress: {
+                    type: "string",
+                    description: "Current execution progress.",
+                },
+                next_step: {
+                    type: "string",
+                    description: "Next task description.",
+                },
             },
-            required: ["nodeId"],
+            required: ["nodeId", "progress", "next_step"],
         };
     }
     async execute(args, agentContext) {
-        // 调用 forEach Agent 单独逻辑, 根据上下文判断并循环执行
         let nodeId = args.nodeId;
         let agentXml = agentContext.agentChain.agent.xml;
         let node = extractAgentXmlNode(agentXml, nodeId);
@@ -17880,10 +17897,27 @@ class ForeachTaskTool {
             throw new Error("Node ID is not a forEach node: " + nodeId);
         }
         let items = node.getAttribute("items");
+        let varValue = null;
+        let resultText = "Recorded";
         if (items && items != "list") {
-            agentContext.context.variables.get(items.trim());
+            varValue = agentContext.context.variables.get(items.trim());
+            if (varValue) {
+                let key = "foreach_" + nodeId;
+                let loop_count = agentContext.variables.get(key) || 0;
+                if (loop_count % 5 == 0) {
+                    resultText = `Variable information associated with the current loop task.\nvariable_name: ${items.trim()}\nvariable_value: ${varValue}`;
+                }
+                agentContext.variables.set(key, ++loop_count);
+            }
         }
-        return null;
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: resultText,
+                },
+            ],
+        };
     }
 }
 
@@ -17955,6 +17989,11 @@ request_help: Request assistance from the user; for instance, when an operation
                     break;
                 case "request_help":
                     if (callback.onHumanHelp) {
+                        if (args.helpType == "request_login" &&
+                            (await this.checkIsLogined(agentContext))) {
+                            resultText = "Already logged in";
+                            break;
+                        }
                         let result = await callback.onHumanHelp(agentContext, (args.helpType || "request_assistance"), args.prompt);
                         resultText = `request_help result: ${result ? "Solved" : "Unresolved"}`;
                     }
@@ -17983,6 +18022,42 @@ request_help: Request assistance from the user; for instance, when an operation
             };
         }
     }
+    async checkIsLogined(agentContext) {
+        let screenshot = agentContext.agent["screenshot"];
+        if (!screenshot) {
+            return false;
+        }
+        try {
+            let imageResult = (await screenshot.call(agentContext.agent, agentContext));
+            let rlm = new RetryLanguageModel(agentContext.context.config.llms, agentContext.agent.Llms);
+            let image = toImage(imageResult.imageBase64);
+            let request = {
+                messages: [
+                    {
+                        role: "user",
+                        content: [
+                            {
+                                type: "image",
+                                image: image,
+                                mimeType: imageResult.imageType,
+                            },
+                            {
+                                type: "text",
+                                text: "Check if the current website is logged in. If not logged in, output `NOT_LOGIN`. If logged in, output `LOGGED_IN`. Output directly without explanation.",
+                            },
+                        ],
+                    },
+                ],
+                abortSignal: agentContext.context.controller.signal,
+            };
+            let result = await rlm.call(request);
+            return result.text && result.text.indexOf("LOGGED_IN") > -1;
+        }
+        catch (error) {
+            console.error("Error auto checking login status:", error);
+            return false;
+        }
+    }
 }
 
 const TOOL_NAME$2 = "task_node_status";
@@ -18120,21 +18195,156 @@ const TOOL_NAME = "watch_trigger";
 class WatchTriggerTool {
     constructor() {
         this.name = TOOL_NAME;
-        this.description = `When executing the \`watch\` node, please use it to complete the tasks corresponding to that watch node. It will complete all tasks under the entire watch node.`;
+        this.description = `When executing the \`watch\` node, please use it to monitor DOM element changes, it will block the listener until the element changes or times out.`;
         this.parameters = {
             type: "object",
             properties: {
                 nodeId: {
                     type: "number",
-                    description: "forEach node ID."
+                    description: "watch node ID.",
+                },
+                watch_area: {
+                    type: "array",
+                    description: "Element changes in monitoring area, eg: [x, y, width, height].",
+                    items: {
+                        type: "number",
+                    },
+                },
+                watch_index: {
+                    type: "array",
+                    description: "The index of elements to be monitoring multiple elements simultaneously.",
+                    items: {
+                        type: "number",
+                    },
+                },
+                frequency: {
+                    type: "number",
+                    description: "Check frequency, how many seconds between each check, default 1 seconds.",
+                    default: 1,
+                    minimum: 0.5,
+                    maximum: 30,
+                },
+                timeout: {
+                    type: "number",
+                    description: "Timeout in minute, default 5 minutes.",
+                    default: 5,
+                    minimum: 1,
+                    maximum: 30,
                 },
             },
             required: ["nodeId"],
         };
     }
     async execute(args, agentContext) {
-        // TODO Listen for changes to the DOM or file, and execute nodes
-        return null;
+        let nodeId = args.nodeId;
+        let agentXml = agentContext.agentChain.agent.xml;
+        let node = extractAgentXmlNode(agentXml, nodeId);
+        if (node == null) {
+            throw new Error("Node ID does not exist: " + nodeId);
+        }
+        if (node.tagName !== "watch") {
+            throw new Error("Node ID is not a watch node: " + nodeId);
+        }
+        let task_description = node.getElementsByTagName("description")[0]?.textContent || "";
+        if (!task_description) {
+            return {
+                content: [
+                    {
+                        type: "text",
+                        text: "The watch node does not have a description, skip.",
+                    },
+                ],
+            };
+        }
+        const screenshot = agentContext.agent["screenshot"];
+        const image1Result = (await screenshot.call(agentContext.agent, agentContext));
+        const image1 = toImage(image1Result.imageBase64);
+        const start = new Date().getTime();
+        const timeout = (args.timeout || 5) * 60000;
+        const frequency = Math.max(500, (args.frequency = args.frequency || 1) * 1000);
+        let rlm = new RetryLanguageModel(agentContext.context.config.llms, agentContext.agent.Llms);
+        while (new Date().getTime() - start < timeout) {
+            await agentContext.context.checkAborted();
+            await new Promise((resolve) => setTimeout(resolve, frequency));
+            const image2Result = (await screenshot.call(agentContext.agent, agentContext));
+            const image2 = toImage(image2Result.imageBase64);
+            const changeResult = await this.is_dom_change(agentContext, rlm, image1, image1Result.imageType, image2, image2Result.imageType, task_description);
+            if (changeResult.changed) {
+                return {
+                    content: [
+                        {
+                            type: "text",
+                            text: changeResult.changeInfo || "DOM change detected.",
+                        },
+                    ],
+                };
+            }
+        }
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: "Timeout reached, no DOM changes detected.",
+                },
+            ],
+        };
+    }
+    async is_dom_change(agentContext, rlm, image1, image1Type, image2, image2Type, task_description) {
+        try {
+            let request = {
+                messages: [
+                    {
+                        role: "system",
+                        content: `You are a tool for detecting element changes. Given a task description, compare two images to determine whether the changes described in the task have occurred.
+If the changes have occurred, return an json with \`changed\` set to true and \`changeInfo\` containing a description of the changes. If no changes have occurred, return an object with \`changed\` set to false.
+
+## Example
+User: Monitor new messages in group chat
+### No changes detected
+Output:
+{
+  "changed": false
+}
+### Change detected
+Output:
+{
+  "changed": true,
+  "changeInfo": "New message received in the group chat. The message content is: 'Hello, how are you?'"
+}`,
+                    },
+                    {
+                        role: "user",
+                        content: [
+                            {
+                                type: "image",
+                                image: image1,
+                                mimeType: image1Type,
+                            },
+                            {
+                                type: "image",
+                                image: image2,
+                                mimeType: image2Type,
+                            },
+                            {
+                                type: "text",
+                                text: task_description,
+                            },
+                        ],
+                    },
+                ],
+                abortSignal: agentContext.context.controller.signal,
+            };
+            const result = await rlm.call(request);
+            let resultText = result.text || "{}";
+            resultText = resultText.substring(resultText.indexOf("{"), resultText.lastIndexOf("}") + 1);
+            return JSON.parse(resultText);
+        }
+        catch (error) {
+            Log.error("Error in is_dom_change:", error);
+        }
+        return {
+            changed: false,
+        };
     }
 }
 
@@ -18196,7 +18406,7 @@ repetitive tasks, when executing to the forEach node, require the use of the \`$
 `;
 const WATCH_NODE = `
     <!-- monitor task node, the loop attribute specifies whether to listen in a loop or listen once -->
-    <watch event="dom or file" loop="true">
+    <watch event="dom" loop="true">
       <description>Monitor task description</description>
       <trigger>
         <node>Trigger step node</node>
@@ -18205,7 +18415,7 @@ const WATCH_NODE = `
     </watch>`;
 const WATCH_PROMPT = `
 * watch node
-monitor changes in webpage DOM or file content, when executing to the watch node, require the use of the \`${TOOL_NAME}\` tool.
+monitor changes in webpage DOM elements, when executing to the watch node, require the use of the \`${TOOL_NAME}\` tool.
 `;
 function getAgentSystemPrompt(agent, agentNode, context, tools, extSysPrompt) {
     let prompt = "";
@@ -18416,6 +18626,14 @@ class Agent {
         if (hasVariable) {
             tools.push(new VariableStorageTool());
         }
+        let hasForeach = agentNodeXml.indexOf("</forEach>") > -1;
+        if (hasForeach) {
+            tools.push(new ForeachTaskTool());
+        }
+        let hasWatch = agentNodeXml.indexOf("</watch>") > -1;
+        if (hasWatch) {
+            tools.push(new WatchTriggerTool());
+        }
         let toolNames = this.tools.map((tool) => tool.name);
         return tools.filter((tool) => toolNames.indexOf(tool.name) == -1);
     }
@@ -18582,6 +18800,9 @@ class Agent {
     addTool(tool) {
         this.tools.push(tool);
     }
+    get Llms() {
+        return this.llms;
+    }
     get Name() {
         return this.name;
     }
@@ -18822,8 +19043,8 @@ Your task is to understand the user's requirements, dynamically plan the user's
         <forEach items="list or variable name">
           <node>forEach step node</node>
         </forEach>
-        <!-- When you need to monitor changes in webpage DOM or file content, you can use \`Watch\`, the loop attribute specifies whether to listen in a loop or listen once. -->
-        <watch event="dom or file" loop="true">
+        <!-- When you need to monitor changes in webpage DOM elements, you can use \`Watch\`, the loop attribute specifies whether to listen in a loop or listen once. -->
+        <watch event="dom" loop="true">
           <description>Monitor task description</description>
           <trigger>
             <node>Trigger step node</node>
@@ -19498,6 +19719,9 @@ class BaseFileAgent extends Agent {
     }
     async do_file_read(agentContext, path, write_variable) {
         let file_context = await this.file_read(agentContext, path);
+        if (file_context && file_context.length > config.fileTextMaxLength) {
+            file_context = file_context.substring(0, config.fileTextMaxLength) + "...";
+        }
         if (write_variable) {
             agentContext.context.variables.set(write_variable, file_context);
         }
@@ -19993,7 +20217,7 @@ This is a computer GUI interface, observe the execution through screenshots, and
                             description: "Duration in millisecond",
                             default: 500,
                             minimum: 200,
-                            maximum: 2000,
+                            maximum: 10000,
                         },
                     },
                     required: ["duration"],
@@ -20856,17 +21080,19 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
         await sleep(200);
         if (!extract_page_content) {
             const tools = this.toolUseNames(agentContext.agentChain.agentRequest?.messages);
-            if (tools.length > 3 &&
-                tools[tools.length - 1] == "scroll_mouse_wheel" &&
-                tools[tools.length - 2] == "scroll_mouse_wheel" &&
-                tools[tools.length - 3] == "scroll_mouse_wheel") {
-                let page_content = await this.extract_page_content(agentContext);
-                return "The current page content has been extracted, latest page content:\n" + page_content;
+            let scroll_count = 0;
+            for (let i = tools.length - 1; i >= Math.max(tools.length - 8, 0); i--) {
+                if (tools[i] == "scroll_mouse_wheel") {
+                    scroll_count++;
+                }
+            }
+            if (scroll_count >= 3) {
+                extract_page_content = true;
             }
         }
         if (extract_page_content) {
             let page_content = await this.extract_page_content(agentContext);
-            return "This is the latest page content:\n" + page_content;
+            return "The current page content has been extracted, latest page content:\n" + page_content;
         }
     }
     async hover_to_element(agentContext, index) {
@@ -21011,23 +21237,30 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
                     return await this.callInnerTool(() => this.click_element(agentContext, args.index, (args.num_clicks || 1), (args.button || "left")));
                 },
             },
+            /*
             {
-                name: "scroll_to_element",
-                description: "Scroll to the element",
-                parameters: {
-                    type: "object",
-                    properties: {
-                        index: {
-                            type: "number",
-                            description: "The index of the element to input text into",
-                        },
-                    },
-                    required: ["index"],
-                },
-                execute: async (args, agentContext) => {
-                    return await this.callInnerTool(() => this.scroll_to_element(agentContext, args.index));
+              name: "scroll_to_element",
+              description: "Scroll to the element",
+              parameters: {
+                type: "object",
+                properties: {
+                  index: {
+                    type: "number",
+                    description: "The index of the element to input text into",
+                  },
                 },
+                required: ["index"],
+              },
+              execute: async (
+                args: Record<string, unknown>,
+                agentContext: AgentContext
+              ): Promise<ToolResult> => {
+                return await this.callInnerTool(() =>
+                  this.scroll_to_element(agentContext, args.index as number)
+                );
+              },
             },
+            */
             {
                 name: "scroll_mouse_wheel",
                 description: "Scroll the mouse wheel at current position, only scroll when you need to load more content",
@@ -21165,7 +21398,7 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
                             description: "Duration in millisecond",
                             default: 500,
                             minimum: 200,
-                            maximum: 2000,
+                            maximum: 10000,
                         },
                     },
                     required: ["duration"],
@@ -21176,6 +21409,9 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
             },
         ];
     }
+    async double_screenshots(agentContext, messages, tools) {
+        return true;
+    }
     async handleMessages(agentContext, messages, tools) {
         const pseudoHtmlDescription = "This is the latest screenshot and page element information.\nindex and element:\n";
         let lastTool = this.lastToolResult(messages);
@@ -21184,16 +21420,27 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
             lastTool.toolName !== "get_all_tabs" &&
             lastTool.toolName !== "variable_storage") {
             await sleep(300);
+            let image_contents = [];
+            if (await this.double_screenshots(agentContext, messages, tools)) {
+                let imageResult = await this.screenshot(agentContext);
+                let image = toImage(imageResult.imageBase64);
+                image_contents.push({
+                    type: "image",
+                    image: image,
+                    mimeType: imageResult.imageType,
+                });
+            }
             let result = await this.screenshot_and_html(agentContext);
             let image = toImage(result.imageBase64);
+            image_contents.push({
+                type: "image",
+                image: image,
+                mimeType: result.imageType,
+            });
             messages.push({
                 role: "user",
                 content: [
-                    {
-                        type: "image",
-                        image: image,
-                        mimeType: result.imageType,
-                    },
+                    ...image_contents,
                     {
                         type: "text",
                         text: pseudoHtmlDescription + result.pseudoHtml,
@@ -21239,7 +21486,12 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
                 if (eIdx == -1) {
                     continue;
                 }
-                line = line.substring(0, sIdx) + line.substring(eIdx + 1).trim().replace('" >', '">');
+                line =
+                    line.substring(0, sIdx) +
+                        line
+                            .substring(eIdx + 1)
+                            .trim()
+                            .replace('" >', '">');
             }
             return line;
         })
@@ -21406,7 +21658,7 @@ function scroll_by(params) {
     }
     function findNodes(element = document, nodes = []) {
         for (const node of Array.from(element.querySelectorAll("*"))) {
-            if (node.tagName === 'IFRAME' && node.contentDocument) {
+            if (node.tagName === "IFRAME" && node.contentDocument) {
                 findNodes(node.contentDocument, nodes);
             }
             else {
@@ -21427,7 +21679,9 @@ function scroll_by(params) {
             elements = allElements.filter((el) => {
                 const style = window.getComputedStyle(el);
                 const overflowY = style.getPropertyValue("overflow-y");
-                return (overflowY === "auto" || overflowY === "scroll" || el.scrollHeight > el.clientHeight);
+                return (overflowY === "auto" ||
+                    overflowY === "scroll" ||
+                    el.scrollHeight > el.clientHeight);
             });
         }
         return elements;
@@ -21730,7 +21984,7 @@ class BaseBrowserScreenAgent extends BaseBrowserAgent {
                             description: "Duration in millisecond",
                             default: 500,
                             minimum: 200,
-                            maximum: 2000,
+                            maximum: 10000,
                         },
                     },
                     required: ["duration"],