misoai-web 1.0.2 → 1.0.4

package/dist/lib/chrome-extension.js

@@ -264,7 +264,9 @@ var ScriptPlayer = class {
           typeof prompt === "string",
           "prompt for aiAction must be a string"
         );
-        await agent.aiAction(prompt);
+        await agent.aiAction(prompt, {
+          cacheable: actionTask.cacheable
+        });
       } else if ("aiAssert" in flowItem) {
         const assertTask = flowItem;
         const prompt = assertTask.aiAssert;
@@ -466,8 +468,24 @@ function interpolateEnvVars(content) {
   });
 }
 function parseYamlScript(content, filePath, ignoreCheckingTarget) {
-  const interpolatedContent = interpolateEnvVars(content);
-  const obj = import_js_yaml2.default.load(interpolatedContent);
+  let processedContent = content;
+  if (content.indexOf("android") !== -1 && content.match(/deviceId:\s*(\d+)/)) {
+    let matchedDeviceId;
+    processedContent = content.replace(
+      /deviceId:\s*(\d+)/g,
+      (match, deviceId) => {
+        matchedDeviceId = deviceId;
+        return `deviceId: '${deviceId}'`;
+      }
+    );
+    console.warn(
+      `please use string-style deviceId in yaml script, for example: deviceId: "${matchedDeviceId}"`
+    );
+  }
+  const interpolatedContent = interpolateEnvVars(processedContent);
+  const obj = import_js_yaml2.default.load(interpolatedContent, {
+    schema: import_js_yaml2.default.JSON_SCHEMA
+  });
   const pathTip = filePath ? `, failed to load ${filePath}` : "";
   const android = typeof obj.android !== "undefined" ? Object.assign({}, obj.android || {}) : void 0;
   const webConfig = obj.web || obj.target;
@@ -508,7 +526,6 @@ var import_misoai_core = require("misoai-core");
 var import_ai_model2 = require("misoai-core/ai-model");
 var import_utils5 = require("misoai-core/utils");
 var import_constants = require("misoai-shared/constants");
-var import_fs = require("misoai-shared/fs");
 var import_logger = require("misoai-shared/logger");
 var import_utils6 = require("misoai-shared/utils");
 
@@ -649,16 +666,18 @@ var PageTaskExecutor = class {
       );
       if (info?.id) {
         elementId = info.id;
+      } else {
+        debug(
+          "no element id found for position node, will not update cache",
+          element
+        );
       }
     }
     if (!elementId) {
       return void 0;
     }
     try {
-      const elementInfosScriptContent = (0, import_fs.getElementInfosScriptContent)();
-      const result = await this.page.evaluateJavaScript?.(
-        `${elementInfosScriptContent}midscene_element_inspector.getXpathsById('${elementId}')`
-      );
+      const result = await this.page.getXpathsById(elementId);
       return result;
     } catch (error) {
       debug("getXpathsById error: ", error);
@@ -697,7 +716,7 @@ var PageTaskExecutor = class {
     };
     return taskWithScreenshot;
   }
-  async convertPlanToExecutable(plans) {
+  async convertPlanToExecutable(plans, opts) {
     const tasks = [];
     plans.forEach((plan2) => {
       if (plan2.type === "Locate") {
@@ -707,7 +726,10 @@ var PageTaskExecutor = class {
         const taskFind = {
           type: "Insight",
           subType: "Locate",
-          param: plan2.locate || void 0,
+          param: plan2.locate ? {
+            ...plan2.locate,
+            cacheable: opts?.cacheable
+          } : void 0,
           thought: plan2.thought,
           locate: plan2.locate,
           executor: async (param, taskContext) => {
@@ -744,19 +766,21 @@ var PageTaskExecutor = class {
             let elementFromCache = null;
             try {
               if (xpaths?.length && this.taskCache?.isCacheResultUsed && param?.cacheable !== false) {
-                const elementInfosScriptContent = (0, import_fs.getElementInfosScriptContent)();
-                const element2 = await this.page.evaluateJavaScript?.(
-                  `${elementInfosScriptContent}midscene_element_inspector.getElementInfoByXpath('${xpaths[0]}')`
-                );
-                if (element2?.id) {
-                  elementFromCache = element2;
-                  debug("cache hit, prompt: %s", cachePrompt);
-                  cacheHitFlag = true;
-                  debug(
-                    "found a new new element with same xpath, xpath: %s, id: %s",
-                    xpaths[0],
-                    element2?.id
+                for (let i = 0; i < xpaths.length; i++) {
+                  const element2 = await this.page.getElementInfoByXpath(
+                    xpaths[i]
                   );
+                  if (element2?.id) {
+                    elementFromCache = element2;
+                    debug("cache hit, prompt: %s", cachePrompt);
+                    cacheHitFlag = true;
+                    debug(
+                      "found a new new element with same xpath, xpath: %s, id: %s",
+                      xpaths[i],
+                      element2?.id
+                    );
+                    break;
+                  }
                 }
               }
             } catch (error) {
@@ -769,12 +793,14 @@ var PageTaskExecutor = class {
               context: pageContext
             })).element;
             const aiCost = Date.now() - startTime;
+            let currentXpaths;
             if (element && this.taskCache && !cacheHitFlag && param?.cacheable !== false) {
               const elementXpaths = await this.getElementXpath(
                 pageContext,
                 element
               );
-              if (elementXpaths) {
+              if (elementXpaths?.length) {
+                currentXpaths = elementXpaths;
                 this.taskCache.updateOrAppendCacheRecord(
                   {
                     type: "locate",
@@ -784,7 +810,11 @@ var PageTaskExecutor = class {
                   locateCacheRecord
                 );
               } else {
-                debug("no xpaths found, will not update cache", cachePrompt);
+                debug(
+                  "no xpaths found, will not update cache",
+                  cachePrompt,
+                  elementXpaths
+                );
               }
             }
             if (!element) {
@@ -796,7 +826,9 @@ var PageTaskExecutor = class {
               },
               pageContext,
               cache: {
-                hit: cacheHitFlag
+                hit: cacheHitFlag,
+                originalXpaths: xpaths,
+                currentXpaths
               },
               aiCost
             };
@@ -1164,6 +1196,7 @@ var PageTaskExecutor = class {
           sleep: sleep3
         } = planResult;
         executorContext.task.log = {
+          ...executorContext.task.log || {},
           rawResponse
         };
         executorContext.task.usage = usage;
@@ -1288,11 +1321,11 @@ var PageTaskExecutor = class {
     };
     return task;
   }
-  async runPlans(title, plans) {
+  async runPlans(title, plans, opts) {
     const taskExecutor = new import_misoai_core.Executor(title, {
       onTaskStart: this.onTaskStartCallback
     });
-    const { tasks } = await this.convertPlanToExecutable(plans);
+    const { tasks } = await this.convertPlanToExecutable(plans, opts);
     await taskExecutor.append(tasks);
     const result = await taskExecutor.flush();
     return {
@@ -1300,7 +1333,7 @@ var PageTaskExecutor = class {
       executor: taskExecutor
     };
   }
-  async action(userPrompt, actionContext) {
+  async action(userPrompt, actionContext, opts) {
     const taskExecutor = new import_misoai_core.Executor(taskTitleStr("Action", userPrompt), {
       onTaskStart: this.onTaskStartCallback
     });
@@ -1325,7 +1358,7 @@ var PageTaskExecutor = class {
       yamlFlow.push(...planResult.yamlFlow || []);
       let executables;
       try {
-        executables = await this.convertPlanToExecutable(plans);
+        executables = await this.convertPlanToExecutable(plans, opts);
         taskExecutor.append(executables.tasks);
       } catch (error) {
         return this.appendErrorPlan(
@@ -1363,7 +1396,7 @@ var PageTaskExecutor = class {
       executor: taskExecutor
     };
   }
-  async actionToGoal(userPrompt) {
+  async actionToGoal(userPrompt, opts) {
     const taskExecutor = new import_misoai_core.Executor(taskTitleStr("Action", userPrompt), {
       onTaskStart: this.onTaskStartCallback
     });
@@ -1387,7 +1420,7 @@ var PageTaskExecutor = class {
       yamlFlow.push(...output.yamlFlow || []);
       let executables;
       try {
-        executables = await this.convertPlanToExecutable(plans);
+        executables = await this.convertPlanToExecutable(plans, opts);
         taskExecutor.append(executables.tasks);
       } catch (error) {
         return this.appendErrorPlan(
@@ -1692,7 +1725,7 @@ var import_js_yaml3 = __toESM(require("js-yaml"));
 var import_semver = __toESM(require("semver"));
 
 // package.json
-var version = "1.0.2";
+var version = "1.0.4";
 
 // src/common/task-cache.ts
 var debug3 = (0, import_logger3.getDebug)("cache");
@@ -2035,9 +2068,9 @@ var PageAgent = class {
   buildDetailedLocateParam(locatePrompt, opt) {
     (0, import_utils12.assert)(locatePrompt, "missing locate prompt");
     if (typeof opt === "object") {
-      const prompt = opt.prompt || locatePrompt;
-      const deepThink = opt.deepThink || false;
-      const cacheable = opt.cacheable || true;
+      const prompt = opt.prompt ?? locatePrompt;
+      const deepThink = opt.deepThink ?? false;
+      const cacheable = opt.cacheable ?? true;
       return {
         prompt,
         deepThink,
@@ -2056,7 +2089,8 @@ var PageAgent = class {
     const plans = buildPlans("Tap", detailedLocateParam);
     const { executor, output } = await this.taskExecutor.runPlans(
       taskTitleStr("Tap", locateParamStr(detailedLocateParam)),
-      plans
+      plans,
+      { cacheable: opt?.cacheable }
     );
     const metadata = this.afterTaskRunning(executor);
     return {
@@ -2072,7 +2106,8 @@ var PageAgent = class {
     const plans = buildPlans("Hover", detailedLocateParam);
     const { executor, output } = await this.taskExecutor.runPlans(
       taskTitleStr("Hover", locateParamStr(detailedLocateParam)),
-      plans
+      plans,
+      { cacheable: opt?.cacheable }
     );
     const metadata = this.afterTaskRunning(executor);
     return {
@@ -2095,7 +2130,8 @@ var PageAgent = class {
     });
     const { executor, output } = await this.taskExecutor.runPlans(
       taskTitleStr("Input", locateParamStr(detailedLocateParam)),
-      plans
+      plans,
+      { cacheable: opt?.cacheable }
     );
     const metadata = this.afterTaskRunning(executor);
     return {
@@ -2111,7 +2147,8 @@ var PageAgent = class {
     });
     const { executor, output } = await this.taskExecutor.runPlans(
       taskTitleStr("KeyboardPress", locateParamStr(detailedLocateParam)),
-      plans
+      plans,
+      { cacheable: opt?.cacheable }
     );
     const metadata = this.afterTaskRunning(executor);
     return {
@@ -2125,7 +2162,8 @@ var PageAgent = class {
     const paramInTitle = locatePrompt ? `${locateParamStr(detailedLocateParam)} - ${scrollParamStr(scrollParam)}` : scrollParamStr(scrollParam);
     const { executor, output } = await this.taskExecutor.runPlans(
       taskTitleStr("Scroll", paramInTitle),
-      plans
+      plans,
+      { cacheable: opt?.cacheable }
     );
     const metadata = this.afterTaskRunning(executor);
     return {
@@ -2134,6 +2172,19 @@ var PageAgent = class {
     };
   }
   async aiAction(taskPrompt, opt) {
+    try {
+      const aiModel = await import("misoai-core/ai-model");
+      const contextStore = aiModel.getContextStore();
+      const processedPrompt = contextStore.replaceAllReferences(taskPrompt, "action");
+      contextStore.addStep({
+        type: "action",
+        summary: `Action: ${processedPrompt}`,
+        prompt: processedPrompt
+      });
+      taskPrompt = processedPrompt;
+    } catch (error) {
+      debug4("Context store not available:", error);
+    }
     const cacheable = opt?.cacheable;
     const isVlmUiTars = (0, import_env2.vlLocateMode)() === "vlm-ui-tars";
     const matchedCache = isVlmUiTars || cacheable === false ? void 0 : this.taskCache?.matchPlanCache(taskPrompt);
@@ -2151,7 +2202,9 @@ var PageAgent = class {
         metadata: metadata2
       };
     }
-    const { output, executor } = await (isVlmUiTars ? this.taskExecutor.actionToGoal(taskPrompt) : this.taskExecutor.action(taskPrompt, this.opts.aiActionContext));
+    const { output, executor } = await (isVlmUiTars ? this.taskExecutor.actionToGoal(taskPrompt, { cacheable }) : this.taskExecutor.action(taskPrompt, this.opts.aiActionContext, {
+      cacheable
+    }));
     if (this.taskCache && output?.yamlFlow && cacheable !== false) {
       const yamlContent = {
         tasks: [
@@ -2178,7 +2231,63 @@ var PageAgent = class {
     };
   }
   async aiQuery(demand) {
-    const { output, executor } = await this.taskExecutor.query(demand);
+    let processedDemand = demand;
+    let storageKey;
+    try {
+      const aiModel = await import("misoai-core/ai-model");
+      const contextStore = aiModel.getContextStore();
+      if (typeof demand === "string") {
+        const storageInstruction = contextStore.parseStorageInstruction(demand);
+        if (storageInstruction) {
+          storageKey = storageInstruction.key;
+          processedDemand = storageInstruction.cleanText;
+          contextStore._pendingAliases = storageInstruction.aliases;
+        } else {
+          const storageMatch = demand.match(/store\s+(?:as\s+)?(\w+)/i);
+          if (storageMatch) {
+            storageKey = storageMatch[1];
+            processedDemand = demand.replace(/,?\s*store\s+(?:as\s+)?\w+/i, "").trim();
+          }
+        }
+      }
+    } catch (error) {
+      debug4("Context store not available:", error);
+    }
+    const { output, executor } = await this.taskExecutor.query(processedDemand);
+    if (storageKey && output) {
+      try {
+        const aiModel = await import("misoai-core/ai-model");
+        const contextStore = aiModel.getContextStore();
+        const pendingAliases = contextStore._pendingAliases;
+        if (pendingAliases) {
+          contextStore.storeDataWithAliases(storageKey, output, pendingAliases, typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand));
+          delete contextStore._pendingAliases;
+        } else {
+          contextStore.storeData(storageKey, output);
+        }
+        contextStore.addStep({
+          type: "query",
+          summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)} (stored as ${storageKey})`,
+          data: output,
+          prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
+        });
+      } catch (error) {
+        debug4("Failed to store query result:", error);
+      }
+    } else {
+      try {
+        const aiModel = await import("misoai-core/ai-model");
+        const contextStore = aiModel.getContextStore();
+        contextStore.addStep({
+          type: "query",
+          summary: `Query: ${typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)}`,
+          data: output,
+          prompt: typeof processedDemand === "string" ? processedDemand : JSON.stringify(processedDemand)
+        });
+      } catch (error) {
+        debug4("Failed to add query step:", error);
+      }
+    }
     const metadata = this.afterTaskRunning(executor);
     return {
       result: output,
@@ -2273,7 +2382,8 @@ var PageAgent = class {
     const plans = buildPlans("Locate", detailedLocateParam);
     const { executor, output } = await this.taskExecutor.runPlans(
       taskTitleStr("Locate", locateParamStr(detailedLocateParam)),
-      plans
+      plans,
+      { cacheable: opt?.cacheable }
     );
     const metadata = this.afterTaskRunning(executor);
     const { element } = output;
@@ -2287,6 +2397,19 @@ var PageAgent = class {
     };
   }
   async aiAssert(assertion, msg, opt) {
+    let processedAssertion = assertion;
+    try {
+      const aiModel = await import("misoai-core/ai-model");
+      const contextStore = aiModel.getContextStore();
+      processedAssertion = contextStore.replaceAllReferences(assertion, "assertion");
+      contextStore.addStep({
+        type: "assertion",
+        summary: `Assertion: ${processedAssertion}`,
+        prompt: processedAssertion
+      });
+    } catch (error) {
+      debug4("Context store not available:", error);
+    }
     let currentUrl = "";
     if (this.page.url) {
       try {
@@ -2294,7 +2417,7 @@ var PageAgent = class {
       } catch (e) {
       }
     }
-    const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${assertion}` : assertion;
+    const assertionWithContext = currentUrl ? `For the page at URL "${currentUrl}", ${processedAssertion}` : processedAssertion;
     const { output, executor } = await this.taskExecutor.assert(assertionWithContext);
     const metadata = this.afterTaskRunning(executor, true);
     if (output && opt?.keepRawResponse) {
@@ -2725,30 +2848,54 @@ function sleep2(ms) {
 var ChromeExtensionProxyPage = class {
   constructor(forceSameTabNavigation) {
     this.pageType = "chrome-extension-proxy";
-    this.version = "1.0.2";
+    this.version = "1.0.4";
     this.activeTabId = null;
     this.tabIdOfDebuggerAttached = null;
     this.attachingDebugger = null;
     this.destroyed = false;
+    this.isMobileEmulation = null;
     this.latestMouseX = 100;
     this.latestMouseY = 100;
     this.mouse = {
       click: async (x, y) => {
         await this.mouse.move(x, y);
-        await this.sendCommandToDebugger("Input.dispatchMouseEvent", {
-          type: "mousePressed",
-          x,
-          y,
-          button: "left",
-          clickCount: 1
-        });
-        await this.sendCommandToDebugger("Input.dispatchMouseEvent", {
-          type: "mouseReleased",
-          x,
-          y,
-          button: "left",
-          clickCount: 1
-        });
+        if (this.isMobileEmulation === null) {
+          const result = await this.sendCommandToDebugger("Runtime.evaluate", {
+            expression: `(() => {
+            return /Android|iPhone|iPad|iPod|Mobile/i.test(navigator.userAgent);
+          })()`,
+            returnByValue: true
+          });
+          this.isMobileEmulation = result?.result?.value;
+        }
+        if (this.isMobileEmulation) {
+          const touchPoints = [{ x: Math.round(x), y: Math.round(y) }];
+          await this.sendCommandToDebugger("Input.dispatchTouchEvent", {
+            type: "touchStart",
+            touchPoints,
+            modifiers: 0
+          });
+          await this.sendCommandToDebugger("Input.dispatchTouchEvent", {
+            type: "touchEnd",
+            touchPoints: [],
+            modifiers: 0
+          });
+        } else {
+          await this.sendCommandToDebugger("Input.dispatchMouseEvent", {
+            type: "mousePressed",
+            x,
+            y,
+            button: "left",
+            clickCount: 1
+          });
+          await this.sendCommandToDebugger("Input.dispatchMouseEvent", {
+            type: "mouseReleased",
+            x,
+            y,
+            button: "left",
+            clickCount: 1
+          });
+        }
       },
       wheel: async (deltaX, deltaY, startX, startY) => {
         const finalX = startX || this.latestMouseX;
@@ -2981,6 +3128,7 @@ var ChromeExtensionProxyPage = class {
       expression: script
     });
     const expression = () => {
+      window.midscene_element_inspector.setNodeHashCacheListOnWindow();
       return {
         tree: window.midscene_element_inspector.webExtractNodeTree(),
         size: {
@@ -3034,6 +3182,28 @@ var ChromeExtensionProxyPage = class {
     const tree = await this.getElementsNodeTree();
     return (0, import_extractor2.treeToList)(tree);
   }
+  async getXpathsById(id) {
+    const script = await getHtmlElementScript();
+    await this.sendCommandToDebugger("Runtime.evaluate", {
+      expression: script
+    });
+    const result = await this.sendCommandToDebugger("Runtime.evaluate", {
+      expression: `window.midscene_element_inspector.getXpathsById('${id}')`,
+      returnByValue: true
+    });
+    return result.result.value;
+  }
+  async getElementInfoByXpath(xpath) {
+    const script = await getHtmlElementScript();
+    await this.sendCommandToDebugger("Runtime.evaluate", {
+      expression: script
+    });
+    const result = await this.sendCommandToDebugger("Runtime.evaluate", {
+      expression: `window.midscene_element_inspector.getElementInfoByXpath('${xpath}')`,
+      returnByValue: true
+    });
+    return result.result.value;
+  }
   async getElementsNodeTree() {
     await this.hideMousePointer();
     const content = await this.getPageContentByCDP();