@eko-ai/eko 2.1.2 → 2.1.4

package/dist/agent/browser/browser_labels.d.ts

@@ -20,6 +20,7 @@ export default abstract class BaseBrowserLabelsAgent extends BaseBrowserAgent {
     private buildInitTools;
     protected double_screenshots(agentContext: AgentContext, messages: LanguageModelV1Prompt, tools: Tool[]): Promise<boolean>;
     protected handleMessages(agentContext: AgentContext, messages: LanguageModelV1Prompt, tools: Tool[]): Promise<void>;
+    private handlePseudoHtmlText;
     private removePseudoHtmlAttr;
 }
 export { BaseBrowserLabelsAgent };

package/dist/agent/browser/browser_labels.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"browser_labels.d.ts","sourceRoot":"","sources":["../../../src/agent/browser/browser_labels.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,OAAO,EAAE,gBAAgB,EAAc,MAAM,gBAAgB,CAAC;AAC9D,OAAO,EAEL,qBAAqB,EACtB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,IAAI,EAAc,UAAU,EAAE,MAAM,aAAa,CAAC;AAG3D,MAAM,CAAC,OAAO,CAAC,QAAQ,OAAO,sBAAuB,SAAQ,gBAAgB;gBAC/D,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE,SAAS,CAAC,EAAE,UAAU;cAqCvD,UAAU,CACxB,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,OAAO,GACb,OAAO,CAAC,IAAI,CAAC;cAOA,aAAa,CAC3B,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,GAClC,OAAO,CAAC,IAAI,CAAC;cAMA,iBAAiB,CAC/B,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,IAAI,CAAC;cAaA,kBAAkB,CAChC,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,oBAAoB,EAAE,OAAO,GAC5B,OAAO,CAAC,GAAG,CAAC;cAuBC,gBAAgB,CAC9B,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,IAAI,CAAC;cAIA,kBAAkB,CAChC,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,GAAG,CAAC;cAMC,aAAa,CAC3B,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,GAAG,CAAC;cAMC,mBAAmB,CAAC,YAAY,EAAE,YAAY,GAAG,OAAO,CAAC;QACvE,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,YAAY,GAAG,WAAW,CAAC;QACtC,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IAwCF,SAAS,CAAC,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAInD,OAAO,CAAC,cAAc;cA+VN,kBAAkB,CAChC,YAAY,EAAE,YAAY,EAC1B,QAAQ,EAAE,qBAAqB,EAC/B,KAAK,EAAE,IAAI,EAAE,GACZ,OAAO,CAAC,OAAO,CAAC;cAIH,cAAc,CAC5B,YAAY,EAAE,YAAY,EAC1B,QAAQ,EAAE,qBAAqB,EAC/B,KAAK,EAAE,IAAI,EAAE,GACZ,OAAO,CAAC,IAAI,CAAC;IAiEhB,OAAO,CAAC,oBAAoB;CA8B7B;AAqQD,OAAO,EAAE,sBAAsB,EAAE,CAAC"}
+{"version":3,"file":"browser_labels.d.ts","sourceRoot":"","sources":["../../../src/agent/browser/browser_labels.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,OAAO,EAAE,gBAAgB,EAAc,MAAM,gBAAgB,CAAC;AAC9D,OAAO,EAEL,qBAAqB,EACtB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,IAAI,EAAc,UAAU,EAAE,MAAM,aAAa,CAAC;AAG3D,MAAM,CAAC,OAAO,CAAC,QAAQ,OAAO,sBAAuB,SAAQ,gBAAgB;gBAC/D,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE,SAAS,CAAC,EAAE,UAAU;cAsCvD,UAAU,CACxB,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,OAAO,GACb,OAAO,CAAC,IAAI,CAAC;cAOA,aAAa,CAC3B,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,GAClC,OAAO,CAAC,IAAI,CAAC;cAMA,iBAAiB,CAC/B,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,IAAI,CAAC;cAaA,kBAAkB,CAChC,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,oBAAoB,EAAE,OAAO,GAC5B,OAAO,CAAC,GAAG,CAAC;cA0BC,gBAAgB,CAC9B,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,IAAI,CAAC;cAIA,kBAAkB,CAChC,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,GAAG,CAAC;cAMC,aAAa,CAC3B,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,GAAG,CAAC;cAMC,mBAAmB,CAAC,YAAY,EAAE,YAAY,GAAG,OAAO,CAAC;QACvE,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,YAAY,GAAG,WAAW,CAAC;QACtC,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IAwCF,SAAS,CAAC,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAInD,OAAO,CAAC,cAAc;cA+VN,kBAAkB,CAChC,YAAY,EAAE,YAAY,EAC1B,QAAQ,EAAE,qBAAqB,EAC/B,KAAK,EAAE,IAAI,EAAE,GACZ,OAAO,CAAC,OAAO,CAAC;cAIH,cAAc,CAC5B,YAAY,EAAE,YAAY,EAC1B,QAAQ,EAAE,qBAAqB,EAC/B,KAAK,EAAE,IAAI,EAAE,GACZ,OAAO,CAAC,IAAI,CAAC;IA2ChB,OAAO,CAAC,oBAAoB;IAkC5B,OAAO,CAAC,oBAAoB;CA4B7B;AAqQD,OAAO,EAAE,sBAAsB,EAAE,CAAC"}

package/dist/agent/browser/build_dom_tree.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"build_dom_tree.d.ts","sourceRoot":"","sources":["../../../src/agent/browser/build_dom_tree.ts"],"names":[],"mappings":"AACA,wBAAgB,kBAAkB,SA+pBjC"}
+{"version":3,"file":"build_dom_tree.d.ts","sourceRoot":"","sources":["../../../src/agent/browser/build_dom_tree.ts"],"names":[],"mappings":"AACA,wBAAgB,kBAAkB,SAgqBjC"}

package/dist/common/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/common/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,2BAA2B,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AAEjC,wBAAgB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAEjD;AAED,wBAAgB,MAAM,IAAI,MAAM,CAM/B;AAED,wBAAgB,YAAY,CAAC,CAAC,SAAS,OAAO,CAAC,GAAG,CAAC,EACjD,GAAG,EAAE,MAAM,CAAC,EACZ,OAAO,EAAE,MAAM,EACf,cAAc,CAAC,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,IAAI,GACnC,OAAO,CAAC,CAAC,CAAC,CAgBZ;AAED,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,UAAU,GACf,2BAA2B,CA8B7B;AAED,wBAAgB,OAAO,CAAC,SAAS,EAAE,MAAM,GAAG,UAAU,GAAG,GAAG,CAsB3D;AAED,wBAAgB,UAAU,CAAC,CAAC,SAAS,IAAI,GAAG,2BAA2B,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAyBtG;AAED,wBAAgB,WAAW,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,KAAK,EAAE,CAuBvE;AAED,wBAAgB,GAAG,CACjB,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,EACjB,WAAW,GAAE,OAAc,GAC1B,MAAM,CAQR;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,UA6ErC"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/common/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,2BAA2B,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AAEjC,wBAAgB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAEjD;AAED,wBAAgB,MAAM,IAAI,MAAM,CAM/B;AAED,wBAAgB,YAAY,CAAC,CAAC,SAAS,OAAO,CAAC,GAAG,CAAC,EACjD,GAAG,EAAE,MAAM,CAAC,EACZ,OAAO,EAAE,MAAM,EACf,cAAc,CAAC,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,IAAI,GACnC,OAAO,CAAC,CAAC,CAAC,CAgBZ;AAED,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,UAAU,GACf,2BAA2B,CA8B7B;AAED,wBAAgB,OAAO,CAAC,SAAS,EAAE,MAAM,GAAG,UAAU,GAAG,GAAG,CAsB3D;AAED,wBAAgB,UAAU,CAAC,CAAC,SAAS,IAAI,GAAG,2BAA2B,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAyBtG;AAED,wBAAgB,WAAW,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,KAAK,EAAE,CAuBvE;AAED,wBAAgB,GAAG,CACjB,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,EACjB,WAAW,GAAE,OAAc,GAC1B,MAAM,CAQR;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,UAgFrC"}

package/dist/core/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/core/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,OAAO,MAAM,WAAW,CAAC;AAChC,OAAO,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AAKjC,qBAAa,GAAG;IACd,OAAO,CAAC,MAAM,CAAY;IAC1B,OAAO,CAAC,OAAO,CAAuB;gBAE1B,MAAM,EAAE,SAAS;IAKhB,QAAQ,CACnB,UAAU,EAAE,MAAM,EAClB,MAAM,GAAE,MAAiB,EACzB,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAClC,OAAO,CAAC,QAAQ,CAAC;IAwBP,MAAM,CACjB,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,GACvB,OAAO,CAAC,QAAQ,CAAC;IAcP,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAoB3C,GAAG,CACd,UAAU,EAAE,MAAM,EAClB,MAAM,GAAE,MAAiB,EACzB,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAClC,OAAO,CAAC,SAAS,CAAC;IAKR,WAAW,CACtB,QAAQ,EAAE,QAAQ,EAClB,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAClC,OAAO,CAAC,OAAO,CAAC;YAoBL,aAAa;IAkCpB,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS;IAI5C,YAAY,IAAI,MAAM,EAAE;IAIxB,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO;IAInC,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO;IAUlC,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO;IAUnD,QAAQ,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI;CAIpC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/core/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AACrE,OAAO,OAAO,MAAM,WAAW,CAAC;AAChC,OAAO,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AAKjC,qBAAa,GAAG;IACd,OAAO,CAAC,MAAM,CAAY;IAC1B,OAAO,CAAC,OAAO,CAAuB;gBAE1B,MAAM,EAAE,SAAS;IAKhB,QAAQ,CACnB,UAAU,EAAE,MAAM,EAClB,MAAM,GAAE,MAAiB,EACzB,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAClC,OAAO,CAAC,QAAQ,CAAC;IAwBP,MAAM,CACjB,MAAM,EAAE,MAAM,EACd,gBAAgB,EAAE,MAAM,GACvB,OAAO,CAAC,QAAQ,CAAC;IAcP,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAuB3C,GAAG,CACd,UAAU,EAAE,MAAM,EAClB,MAAM,GAAE,MAAiB,EACzB,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAClC,OAAO,CAAC,SAAS,CAAC;IAKR,WAAW,CACtB,QAAQ,EAAE,QAAQ,EAClB,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAClC,OAAO,CAAC,OAAO,CAAC;YAoBL,aAAa;IAkCpB,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS;IAI5C,YAAY,IAAI,MAAM,EAAE;IAIxB,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO;IAInC,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO;IAUlC,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,GAAG,OAAO;IAUnD,QAAQ,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI;CAIpC"}

package/dist/index.cjs.js

@@ -12,7 +12,7 @@ const config = {
     compressThreshold: 80,
     largeTextLength: 5000,
     fileTextMaxLength: 20000,
-    maxDialogueImgFileNum: 2,
+    maxDialogueImgFileNum: 1,
 };
 
 var LogLevel;
@@ -289,6 +289,9 @@ function sub(str, maxLength, appendPoint = true) {
     return str;
 }
 function fixXmlTag(code) {
+    if (code.indexOf('&') > -1) {
+        code = code.replace(/&(?![a-zA-Z0-9#]+;)/g, '&');
+    }
     function fixDoubleChar(code) {
         const stack = [];
         for (let i = 0; i < code.length; i++) {
@@ -17930,7 +17933,7 @@ class HumanInteractTool {
 confirm: Ask the user to confirm whether to execute an operation, especially when performing dangerous actions such as deleting system files, users will choose Yes or No.
 input: Prompt the user to enter text; for example, when a task is ambiguous, the AI can choose to ask the user for details, and the user can respond by inputting.
 select: Allow the user to make a choice; in situations that require selection, the AI can ask the user to make a decision.
-request_help: Request assistance from the user; for instance, when an operation is blocked, the AI can ask the user for help, such as needing to log into a website or solve a CAPTCHA.`;
+request_help: Request assistance from the user; for instance, when an operation is blocked, the AI can ask the user for help, such as needing to log into a website or solve a CAPTCHA or Scan the QR code.`;
         this.parameters = {
             type: "object",
             properties: {
@@ -18192,6 +18195,22 @@ class VariableStorageTool {
 }
 
 const TOOL_NAME = "watch_trigger";
+const watch_system_prompt = `You are a tool for detecting element changes. Given a task description, compare two images to determine whether the changes described in the task have occurred.
+If the changes have occurred, return an json with \`changed\` set to true and \`changeInfo\` containing a description of the changes. If no changes have occurred, return an object with \`changed\` set to false.
+
+## Example
+User: Monitor new messages in group chat
+### No changes detected
+Output:
+{
+  "changed": false
+}
+### Change detected
+Output:
+{
+  "changed": true,
+  "changeInfo": "New message received in the group chat. The message content is: 'Hello, how are you?'"
+}`;
 class WatchTriggerTool {
     constructor() {
         this.name = TOOL_NAME;
@@ -18256,19 +18275,22 @@ class WatchTriggerTool {
                 ],
             };
         }
-        const screenshot = agentContext.agent["screenshot"];
-        const image1Result = (await screenshot.call(agentContext.agent, agentContext));
-        const image1 = toImage(image1Result.imageBase64);
+        await this.init_eko_observer(agentContext);
+        const image1 = await this.get_screenshot(agentContext);
         const start = new Date().getTime();
         const timeout = (args.timeout || 5) * 60000;
-        const frequency = Math.max(500, (args.frequency = args.frequency || 1) * 1000);
+        const frequency = Math.max(500, (args.frequency || 1) * 1000);
         let rlm = new RetryLanguageModel(agentContext.context.config.llms, agentContext.agent.Llms);
         while (new Date().getTime() - start < timeout) {
             await agentContext.context.checkAborted();
             await new Promise((resolve) => setTimeout(resolve, frequency));
-            const image2Result = (await screenshot.call(agentContext.agent, agentContext));
-            const image2 = toImage(image2Result.imageBase64);
-            const changeResult = await this.is_dom_change(agentContext, rlm, image1, image1Result.imageType, image2, image2Result.imageType, task_description);
+            let changed = await this.has_eko_changed(agentContext);
+            if (changed == "false") {
+                continue;
+            }
+            await this.init_eko_observer(agentContext);
+            const image2 = await this.get_screenshot(agentContext);
+            const changeResult = await this.is_dom_change(agentContext, rlm, image1, image2, task_description);
             if (changeResult.changed) {
                 return {
                     content: [
@@ -18289,41 +18311,73 @@ class WatchTriggerTool {
             ],
         };
     }
-    async is_dom_change(agentContext, rlm, image1, image1Type, image2, image2Type, task_description) {
+    async get_screenshot(agentContext) {
+        const screenshot = agentContext.agent["screenshot"];
+        const imageResult = (await screenshot.call(agentContext.agent, agentContext));
+        const image = toImage(imageResult.imageBase64);
+        return {
+            image: image,
+            imageType: imageResult.imageType,
+        };
+    }
+    async init_eko_observer(agentContext) {
+        try {
+            const screenshot = agentContext.agent["execute_script"];
+            await screenshot.call(agentContext.agent, agentContext, () => {
+                let _window = window;
+                _window.has_eko_changed = false;
+                _window.eko_observer && _window.eko_observer.disconnect();
+                let eko_observer = new MutationObserver(function (mutations) {
+                    _window.has_eko_changed = true;
+                });
+                eko_observer.observe(document.body, {
+                    childList: true,
+                    subtree: true,
+                    attributes: true,
+                    attributeOldValue: true,
+                    characterData: true,
+                    characterDataOldValue: true,
+                });
+                _window.eko_observer = eko_observer;
+            }, []);
+        }
+        catch (error) {
+            console.error("Error initializing Eko observer:", error);
+        }
+    }
+    async has_eko_changed(agentContext) {
+        try {
+            const screenshot = agentContext.agent["execute_script"];
+            let result = (await screenshot.call(agentContext.agent, agentContext, () => {
+                return window.has_eko_changed + "";
+            }, []));
+            return result;
+        }
+        catch (e) {
+            console.error("Error checking Eko change:", e);
+            return "undefined";
+        }
+    }
+    async is_dom_change(agentContext, rlm, image1, image2, task_description) {
         try {
             let request = {
                 messages: [
                     {
                         role: "system",
-                        content: `You are a tool for detecting element changes. Given a task description, compare two images to determine whether the changes described in the task have occurred.
-If the changes have occurred, return an json with \`changed\` set to true and \`changeInfo\` containing a description of the changes. If no changes have occurred, return an object with \`changed\` set to false.
-
-## Example
-User: Monitor new messages in group chat
-### No changes detected
-Output:
-{
-  "changed": false
-}
-### Change detected
-Output:
-{
-  "changed": true,
-  "changeInfo": "New message received in the group chat. The message content is: 'Hello, how are you?'"
-}`,
+                        content: watch_system_prompt,
                     },
                     {
                         role: "user",
                         content: [
                             {
                                 type: "image",
-                                image: image1,
-                                mimeType: image1Type,
+                                image: image1.image,
+                                mimeType: image1.imageType,
                             },
                             {
                                 type: "image",
-                                image: image2,
-                                mimeType: image2Type,
+                                image: image2.image,
+                                mimeType: image2.imageType,
                             },
                             {
                                 type: "text",
@@ -18387,9 +18441,8 @@ const HUMAN_PROMPT = `
 * HUMAN INTERACT
 During the task execution process, you can use the \`${TOOL_NAME$3}\` tool to interact with humans, please call it in the following situations:
 - When performing dangerous operations such as deleting files, confirmation from humans is required.
-- When encountering obstacles while visiting a website, such as requiring user login or captcha, you need to request for manual assistance.
-- When requesting login, please only call the function when a login dialog box is clearly displayed.
-- Try to minimize the use of \`${TOOL_NAME$3}\` tool.
+- When encountering obstacles while accessing websites, such as requiring user login, captcha verification, QR code scanning, or human verification, you need to request manual assistance.
+- Please do not use the \`${TOOL_NAME$3}\` tool frequently.
 `;
 const VARIABLE_PROMPT = `
 * VARIABLE STORAGE
@@ -19371,6 +19424,9 @@ class Eko {
         if (!context) {
             throw new Error("The task does not exist");
         }
+        if (context.paused) {
+            context.paused = false;
+        }
         if (context.controller.signal.aborted) {
             context.controller = new AbortController();
         }
@@ -20444,7 +20500,8 @@ function run_build_dom_tree() {
         return { element_str, selector_map };
     }
     function get_highlight_element(highlightIndex) {
-        return window.clickable_elements[highlightIndex];
+        let element = document.querySelector(`[eko-user-highlight-id="eko-highlight-${highlightIndex}"]`);
+        return element || window.clickable_elements[highlightIndex];
     }
     function remove_highlight() {
         let highlight = document.getElementById('eko-highlight-container');
@@ -21035,6 +21092,7 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
    - Only use indexes that exist in the provided element list
    - Each element has a unique index number (e.g., "[33]:<button>")
    - Elements marked with "[]:" are non-interactive (for context only)
+   - Use the latest element index, do not rely on historical outdated element indexes
 * ERROR HANDLING:
    - If no suitable elements exist, use other functions to complete the task
    - If stuck, try alternative approaches, don't refuse tasks
@@ -21092,7 +21150,8 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
         }
         if (extract_page_content) {
             let page_content = await this.extract_page_content(agentContext);
-            return "The current page content has been extracted, latest page content:\n" + page_content;
+            return ("The current page content has been extracted, latest page content:\n" +
+                page_content);
         }
     }
     async hover_to_element(agentContext, index) {
@@ -21448,27 +21507,34 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
                 ],
             });
         }
-        if (messages.length > 10) {
-            // compressed pseudoHtml
-            for (let i = 2; i < messages.length - 3; i++) {
-                let message = messages[i];
-                if (message.role == "user" && message.content.length == 2) {
-                    let content = message.content;
-                    for (let j = 0; j < content.length; j++) {
-                        let _content = content[j];
-                        if (_content.type == "text" &&
-                            _content.text.startsWith(pseudoHtmlDescription)) {
-                            _content.text = this.removePseudoHtmlAttr(_content.text, [
-                                "class",
-                                "src",
-                                "href",
-                            ]);
-                        }
+        super.handleMessages(agentContext, messages, tools);
+        this.handlePseudoHtmlText(messages, pseudoHtmlDescription);
+    }
+    handlePseudoHtmlText(messages, pseudoHtmlDescription) {
+        for (let i = 0; i < messages.length; i++) {
+            let message = messages[i];
+            if (message.role !== "user" || message.content.length <= 1) {
+                continue;
+            }
+            let content = message.content;
+            for (let j = 0; j < content.length; j++) {
+                let _content = content[j];
+                if (_content.type == "text" &&
+                    _content.text.startsWith(pseudoHtmlDescription)) {
+                    if (i >= 2 && i < messages.length - 3) {
+                        _content.text = this.removePseudoHtmlAttr(_content.text, [
+                            "class",
+                            "src",
+                            "href",
+                        ]);
                     }
                 }
             }
+            if (content[0].text == "[image]" &&
+                content[1].text == "[image]") {
+                content.splice(0, 1);
+            }
         }
-        super.handleMessages(agentContext, messages, tools);
     }
     removePseudoHtmlAttr(pseudoHtml, remove_attrs) {
         return pseudoHtml
@@ -21477,6 +21543,7 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
             if (!line.startsWith("[") || line.indexOf("]:<") == -1) {
                 return line;
             }
+            line = line.substring(line.indexOf("]:<") + 2);
             for (let i = 0; i < remove_attrs.length; i++) {
                 let sIdx = line.indexOf(remove_attrs[i] + '="');
                 if (sIdx == -1) {
@@ -21488,12 +21555,9 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
                 }
                 line =
                     line.substring(0, sIdx) +
-                        line
-                            .substring(eIdx + 1)
-                            .trim()
-                            .replace('" >', '">');
+                        line.substring(eIdx + 1).trim();
             }
-            return line;
+            return line.replace('" >', '">').replace(" >", ">");
         })
             .join("\n");
     }