@midscene/web 0.17.2 → 0.17.3-beta-20250526042111.0

package/dist/es/agent.js

@@ -90,7 +90,9 @@ var ScriptPlayer = class {
           typeof prompt === "string",
           "prompt for aiAction must be a string"
         );
-        await agent.aiAction(prompt);
+        await agent.aiAction(prompt, {
+          cacheable: actionTask.cacheable
+        });
       } else if ("aiAssert" in flowItem) {
         const assertTask = flowItem;
         const prompt = assertTask.aiAssert;
@@ -365,7 +367,6 @@ import {
 } from "@midscene/core/ai-model";
 import { sleep } from "@midscene/core/utils";
 import { NodeType } from "@midscene/shared/constants";
-import { getElementInfosScriptContent } from "@midscene/shared/fs";
 import { getDebug } from "@midscene/shared/logger";
 import { assert as assert4 } from "@midscene/shared/utils";
 
@@ -602,16 +603,18 @@ var PageTaskExecutor = class {
       );
       if (info?.id) {
         elementId = info.id;
+      } else {
+        debug(
+          "no element id found for position node, will not update cache",
+          element
+        );
       }
     }
     if (!elementId) {
       return void 0;
     }
     try {
-      const elementInfosScriptContent = getElementInfosScriptContent();
-      const result = await this.page.evaluateJavaScript?.(
-        `${elementInfosScriptContent}midscene_element_inspector.getXpathsById('${elementId}')`
-      );
+      const result = await this.page.getXpathsById(elementId);
       return result;
     } catch (error) {
       debug("getXpathsById error: ", error);
@@ -650,7 +653,7 @@ var PageTaskExecutor = class {
     };
     return taskWithScreenshot;
   }
-  async convertPlanToExecutable(plans) {
+  async convertPlanToExecutable(plans, opts) {
     const tasks = [];
     plans.forEach((plan2) => {
       if (plan2.type === "Locate") {
@@ -660,7 +663,10 @@ var PageTaskExecutor = class {
         const taskFind = {
           type: "Insight",
           subType: "Locate",
-          param: plan2.locate || void 0,
+          param: plan2.locate ? {
+            ...plan2.locate,
+            cacheable: opts?.cacheable
+          } : void 0,
           thought: plan2.thought,
           locate: plan2.locate,
           executor: async (param, taskContext) => {
@@ -697,19 +703,21 @@ var PageTaskExecutor = class {
             let elementFromCache = null;
             try {
               if (xpaths?.length && this.taskCache?.isCacheResultUsed && param?.cacheable !== false) {
-                const elementInfosScriptContent = getElementInfosScriptContent();
-                const element2 = await this.page.evaluateJavaScript?.(
-                  `${elementInfosScriptContent}midscene_element_inspector.getElementInfoByXpath('${xpaths[0]}')`
-                );
-                if (element2?.id) {
-                  elementFromCache = element2;
-                  debug("cache hit, prompt: %s", cachePrompt);
-                  cacheHitFlag = true;
-                  debug(
-                    "found a new new element with same xpath, xpath: %s, id: %s",
-                    xpaths[0],
-                    element2?.id
+                for (let i = 0; i < xpaths.length; i++) {
+                  const element2 = await this.page.getElementInfoByXpath(
+                    xpaths[i]
                   );
+                  if (element2?.id) {
+                    elementFromCache = element2;
+                    debug("cache hit, prompt: %s", cachePrompt);
+                    cacheHitFlag = true;
+                    debug(
+                      "found a new new element with same xpath, xpath: %s, id: %s",
+                      xpaths[i],
+                      element2?.id
+                    );
+                    break;
+                  }
                 }
               }
             } catch (error) {
@@ -722,12 +730,14 @@ var PageTaskExecutor = class {
               context: pageContext
             })).element;
             const aiCost = Date.now() - startTime;
+            let currentXpaths;
             if (element && this.taskCache && !cacheHitFlag && param?.cacheable !== false) {
               const elementXpaths = await this.getElementXpath(
                 pageContext,
                 element
               );
-              if (elementXpaths) {
+              if (elementXpaths?.length) {
+                currentXpaths = elementXpaths;
                 this.taskCache.updateOrAppendCacheRecord(
                   {
                     type: "locate",
@@ -737,7 +747,11 @@ var PageTaskExecutor = class {
                   locateCacheRecord
                 );
               } else {
-                debug("no xpaths found, will not update cache", cachePrompt);
+                debug(
+                  "no xpaths found, will not update cache",
+                  cachePrompt,
+                  elementXpaths
+                );
               }
             }
             if (!element) {
@@ -749,7 +763,9 @@ var PageTaskExecutor = class {
               },
               pageContext,
               cache: {
-                hit: cacheHitFlag
+                hit: cacheHitFlag,
+                originalXpaths: xpaths,
+                currentXpaths
               },
               aiCost
             };
@@ -1241,11 +1257,11 @@ var PageTaskExecutor = class {
     };
     return task;
   }
-  async runPlans(title, plans) {
+  async runPlans(title, plans, opts) {
     const taskExecutor = new Executor(title, {
       onTaskStart: this.onTaskStartCallback
     });
-    const { tasks } = await this.convertPlanToExecutable(plans);
+    const { tasks } = await this.convertPlanToExecutable(plans, opts);
     await taskExecutor.append(tasks);
     const result = await taskExecutor.flush();
     return {
@@ -1253,7 +1269,7 @@ var PageTaskExecutor = class {
       executor: taskExecutor
     };
   }
-  async action(userPrompt, actionContext) {
+  async action(userPrompt, actionContext, opts) {
     const taskExecutor = new Executor(taskTitleStr("Action", userPrompt), {
       onTaskStart: this.onTaskStartCallback
     });
@@ -1278,7 +1294,7 @@ var PageTaskExecutor = class {
       yamlFlow.push(...planResult.yamlFlow || []);
       let executables;
       try {
-        executables = await this.convertPlanToExecutable(plans);
+        executables = await this.convertPlanToExecutable(plans, opts);
         taskExecutor.append(executables.tasks);
       } catch (error) {
         return this.appendErrorPlan(
@@ -1316,7 +1332,7 @@ var PageTaskExecutor = class {
       executor: taskExecutor
     };
   }
-  async actionToGoal(userPrompt) {
+  async actionToGoal(userPrompt, opts) {
     const taskExecutor = new Executor(taskTitleStr("Action", userPrompt), {
       onTaskStart: this.onTaskStartCallback
     });
@@ -1340,7 +1356,7 @@ var PageTaskExecutor = class {
       yamlFlow.push(...output.yamlFlow || []);
       let executables;
       try {
-        executables = await this.convertPlanToExecutable(plans);
+        executables = await this.convertPlanToExecutable(plans, opts);
         taskExecutor.append(executables.tasks);
       } catch (error) {
         return this.appendErrorPlan(
@@ -1645,7 +1661,7 @@ import yaml3 from "js-yaml";
 import semver from "semver";
 
 // package.json
-var version = "0.17.2";
+var version = "0.17.3-beta-20250526042111.0";
 
 // src/common/task-cache.ts
 var debug3 = getDebug3("cache");
@@ -1923,9 +1939,9 @@ ${errorTask?.errorStack}`);
   buildDetailedLocateParam(locatePrompt, opt) {
     assert7(locatePrompt, "missing locate prompt");
     if (typeof opt === "object") {
-      const prompt = opt.prompt || locatePrompt;
-      const deepThink = opt.deepThink || false;
-      const cacheable = opt.cacheable || true;
+      const prompt = opt.prompt ?? locatePrompt;
+      const deepThink = opt.deepThink ?? false;
+      const cacheable = opt.cacheable ?? true;
       return {
         prompt,
         deepThink,
@@ -1944,7 +1960,8 @@ ${errorTask?.errorStack}`);
     const plans = buildPlans("Tap", detailedLocateParam);
     const { executor, output } = await this.taskExecutor.runPlans(
       taskTitleStr("Tap", locateParamStr(detailedLocateParam)),
-      plans
+      plans,
+      { cacheable: opt?.cacheable }
     );
     this.afterTaskRunning(executor);
     return output;
@@ -1957,7 +1974,8 @@ ${errorTask?.errorStack}`);
     const plans = buildPlans("Hover", detailedLocateParam);
     const { executor, output } = await this.taskExecutor.runPlans(
       taskTitleStr("Hover", locateParamStr(detailedLocateParam)),
-      plans
+      plans,
+      { cacheable: opt?.cacheable }
     );
     this.afterTaskRunning(executor);
     return output;
@@ -1977,7 +1995,8 @@ ${errorTask?.errorStack}`);
     });
     const { executor, output } = await this.taskExecutor.runPlans(
       taskTitleStr("Input", locateParamStr(detailedLocateParam)),
-      plans
+      plans,
+      { cacheable: opt?.cacheable }
     );
     this.afterTaskRunning(executor);
     return output;
@@ -1990,7 +2009,8 @@ ${errorTask?.errorStack}`);
     });
     const { executor, output } = await this.taskExecutor.runPlans(
       taskTitleStr("KeyboardPress", locateParamStr(detailedLocateParam)),
-      plans
+      plans,
+      { cacheable: opt?.cacheable }
     );
     this.afterTaskRunning(executor);
     return output;
@@ -2001,7 +2021,8 @@ ${errorTask?.errorStack}`);
     const paramInTitle = locatePrompt ? `${locateParamStr(detailedLocateParam)} - ${scrollParamStr(scrollParam)}` : scrollParamStr(scrollParam);
     const { executor, output } = await this.taskExecutor.runPlans(
       taskTitleStr("Scroll", paramInTitle),
-      plans
+      plans,
+      { cacheable: opt?.cacheable }
     );
     this.afterTaskRunning(executor);
     return output;
@@ -2020,7 +2041,9 @@ ${errorTask?.errorStack}`);
       const yaml5 = matchedCache.cacheContent?.yamlWorkflow;
       return this.runYaml(yaml5);
     }
-    const { output, executor } = await (isVlmUiTars ? this.taskExecutor.actionToGoal(taskPrompt) : this.taskExecutor.action(taskPrompt, this.opts.aiActionContext));
+    const { output, executor } = await (isVlmUiTars ? this.taskExecutor.actionToGoal(taskPrompt, { cacheable }) : this.taskExecutor.action(taskPrompt, this.opts.aiActionContext, {
+      cacheable
+    }));
     if (this.taskCache && output?.yamlFlow && cacheable !== false) {
       const yamlContent = {
         tasks: [
@@ -2129,7 +2152,8 @@ ${errorTask?.errorStack}`);
     const plans = buildPlans("Locate", detailedLocateParam);
     const { executor, output } = await this.taskExecutor.runPlans(
       taskTitleStr("Locate", locateParamStr(detailedLocateParam)),
-      plans
+      plans,
+      { cacheable: opt?.cacheable }
     );
     this.afterTaskRunning(executor);
     const { element } = output;