@midscene/web 0.19.1 → 0.20.0

package/dist/lib/index.js

@@ -190,7 +190,7 @@ var ScriptPlayer = class {
           domIncluded: numberTask.domIncluded,
           screenshotIncluded: numberTask.screenshotIncluded
         };
-        (0, import_utils.assert)(prompt, "missing prompt for number");
+        (0, import_utils.assert)(prompt, "missing prompt for aiNumber");
         (0, import_utils.assert)(
           typeof prompt === "string",
           "prompt for number must be a string"
@@ -204,7 +204,7 @@ var ScriptPlayer = class {
           domIncluded: stringTask.domIncluded,
           screenshotIncluded: stringTask.screenshotIncluded
         };
-        (0, import_utils.assert)(prompt, "missing prompt for string");
+        (0, import_utils.assert)(prompt, "missing prompt for aiNumber");
         (0, import_utils.assert)(
           typeof prompt === "string",
           "prompt for string must be a string"
@@ -218,13 +218,20 @@ var ScriptPlayer = class {
           domIncluded: booleanTask.domIncluded,
           screenshotIncluded: booleanTask.screenshotIncluded
         };
-        (0, import_utils.assert)(prompt, "missing prompt for boolean");
+        (0, import_utils.assert)(prompt, "missing prompt for aiBoolean");
         (0, import_utils.assert)(
           typeof prompt === "string",
           "prompt for boolean must be a string"
         );
         const booleanResult = await agent.aiBoolean(prompt, options);
         this.setResult(booleanTask.name, booleanResult);
+      } else if ("aiAsk" in flowItem) {
+        const askTask = flowItem;
+        const prompt = askTask.aiAsk;
+        (0, import_utils.assert)(prompt, "missing prompt for aiAsk");
+        (0, import_utils.assert)(typeof prompt === "string", "prompt for aiAsk must be a string");
+        const askResult = await agent.aiAsk(prompt);
+        this.setResult(askTask.name, askResult);
       } else if ("aiLocate" in flowItem) {
         const locateTask = flowItem;
         const prompt = locateTask.aiLocate;
@@ -233,7 +240,7 @@ var ScriptPlayer = class {
           typeof prompt === "string",
           "prompt for aiLocate must be a string"
         );
-        const locateResult = await agent.aiLocate(prompt);
+        const locateResult = await agent.aiLocate(prompt, locateTask);
         this.setResult(locateTask.name, locateResult);
       } else if ("aiWaitFor" in flowItem) {
         const waitForTask = flowItem;
@@ -442,10 +449,10 @@ var import_utils12 = require("@midscene/shared/utils");
 // src/common/tasks.ts
 var import_core = require("@midscene/core");
 var import_ai_model2 = require("@midscene/core/ai-model");
-var import_utils5 = require("@midscene/core/utils");
+var import_utils7 = require("@midscene/core/utils");
 var import_constants = require("@midscene/shared/constants");
-var import_logger = require("@midscene/shared/logger");
-var import_utils6 = require("@midscene/shared/utils");
+var import_logger2 = require("@midscene/shared/logger");
+var import_utils8 = require("@midscene/shared/utils");
 
 // src/common/ui-utils.ts
 function typeStr(task) {
@@ -522,11 +529,11 @@ function paramStr(task) {
 
 // src/common/utils.ts
 var import_ai_model = require("@midscene/core/ai-model");
-var import_utils3 = require("@midscene/core/utils");
+var import_utils5 = require("@midscene/core/utils");
 var import_env = require("@midscene/shared/env");
 var import_extractor = require("@midscene/shared/extractor");
 var import_img = require("@midscene/shared/img");
-var import_utils4 = require("@midscene/shared/utils");
+var import_utils6 = require("@midscene/shared/utils");
 var import_dayjs = __toESM(require("dayjs"));
 
 // src/web-element.ts
@@ -554,14 +561,189 @@ var WebElementInfo = class {
   }
 };
 
+// src/common/task-cache.ts
+var import_node_assert = __toESM(require("assert"));
+var import_node_fs2 = require("fs");
+var import_node_path2 = require("path");
+var import_common2 = require("@midscene/shared/common");
+var import_logger = require("@midscene/shared/logger");
+var import_utils3 = require("@midscene/shared/utils");
+var import_js_yaml3 = __toESM(require("js-yaml"));
+var import_semver = __toESM(require("semver"));
+
+// package.json
+var version = "0.20.0";
+
+// src/common/task-cache.ts
+var debug = (0, import_logger.getDebug)("cache");
+var lowestSupportedMidsceneVersion = "0.16.10";
+var cacheFileExt = ".cache.yaml";
+var TaskCache = class {
+  // Track matched records
+  constructor(cacheId, isCacheResultUsed, cacheFilePath) {
+    this.matchedCacheIndices = /* @__PURE__ */ new Set();
+    (0, import_node_assert.default)(cacheId, "cacheId is required");
+    this.cacheId = replaceIllegalPathCharsAndSpace(cacheId);
+    this.cacheFilePath = import_utils3.ifInBrowser ? void 0 : cacheFilePath || (0, import_node_path2.join)((0, import_common2.getMidsceneRunSubDir)("cache"), `${this.cacheId}${cacheFileExt}`);
+    this.isCacheResultUsed = isCacheResultUsed;
+    let cacheContent;
+    if (this.cacheFilePath) {
+      cacheContent = this.loadCacheFromFile();
+    }
+    if (!cacheContent) {
+      cacheContent = {
+        midsceneVersion: version,
+        cacheId: this.cacheId,
+        caches: []
+      };
+    }
+    this.cache = cacheContent;
+    this.cacheOriginalLength = this.cache.caches.length;
+  }
+  matchCache(prompt, type) {
+    for (let i = 0; i < this.cacheOriginalLength; i++) {
+      const item = this.cache.caches[i];
+      const key = `${type}:${prompt}:${i}`;
+      if (item.type === type && item.prompt === prompt && !this.matchedCacheIndices.has(key)) {
+        this.matchedCacheIndices.add(key);
+        debug(
+          "cache found and marked as used, type: %s, prompt: %s, index: %d",
+          type,
+          prompt,
+          i
+        );
+        return {
+          cacheContent: item,
+          updateFn: (cb) => {
+            debug(
+              "will call updateFn to update cache, type: %s, prompt: %s, index: %d",
+              type,
+              prompt,
+              i
+            );
+            cb(item);
+            debug(
+              "cache updated, will flush to file, type: %s, prompt: %s, index: %d",
+              type,
+              prompt,
+              i
+            );
+            this.flushCacheToFile();
+          }
+        };
+      }
+    }
+    debug("no unused cache found, type: %s, prompt: %s", type, prompt);
+    return void 0;
+  }
+  matchPlanCache(prompt) {
+    return this.matchCache(prompt, "plan");
+  }
+  matchLocateCache(prompt) {
+    return this.matchCache(prompt, "locate");
+  }
+  appendCache(cache) {
+    debug("will append cache", cache);
+    this.cache.caches.push(cache);
+    this.flushCacheToFile();
+  }
+  loadCacheFromFile() {
+    const cacheFile = this.cacheFilePath;
+    (0, import_node_assert.default)(cacheFile, "cache file path is required");
+    if (!(0, import_node_fs2.existsSync)(cacheFile)) {
+      debug("no cache file found, path: %s", cacheFile);
+      return void 0;
+    }
+    const jsonTypeCacheFile = cacheFile.replace(cacheFileExt, ".json");
+    if ((0, import_node_fs2.existsSync)(jsonTypeCacheFile) && this.isCacheResultUsed) {
+      console.warn(
+        `An outdated cache file from an earlier version of Midscene has been detected. Since version 0.17, we have implemented an improved caching strategy. Please delete the old file located at: ${jsonTypeCacheFile}.`
+      );
+      return void 0;
+    }
+    try {
+      const data = (0, import_node_fs2.readFileSync)(cacheFile, "utf8");
+      const jsonData = import_js_yaml3.default.load(data);
+      if (!version) {
+        debug("no midscene version info, will not read cache from file");
+        return void 0;
+      }
+      if (import_semver.default.lt(jsonData.midsceneVersion, lowestSupportedMidsceneVersion) && !jsonData.midsceneVersion.includes("beta")) {
+        console.warn(
+          `You are using an old version of Midscene cache file, and we cannot match any info from it. Starting from Midscene v0.17, we changed our strategy to use xpath for cache info, providing better performance.
+Please delete the existing cache and rebuild it. Sorry for the inconvenience.
+cache file: ${cacheFile}`
+        );
+        return void 0;
+      }
+      debug(
+        "cache loaded from file, path: %s, cache version: %s, record length: %s",
+        cacheFile,
+        jsonData.midsceneVersion,
+        jsonData.caches.length
+      );
+      jsonData.midsceneVersion = version;
+      return jsonData;
+    } catch (err) {
+      debug(
+        "cache file exists but load failed, path: %s, error: %s",
+        cacheFile,
+        err
+      );
+      return void 0;
+    }
+  }
+  flushCacheToFile() {
+    if (!version) {
+      debug("no midscene version info, will not write cache to file");
+      return;
+    }
+    if (!this.cacheFilePath) {
+      debug("no cache file path, will not write cache to file");
+      return;
+    }
+    try {
+      const dir = (0, import_node_path2.dirname)(this.cacheFilePath);
+      if (!(0, import_node_fs2.existsSync)(dir)) {
+        (0, import_node_fs2.mkdirSync)(dir, { recursive: true });
+        debug("created cache directory: %s", dir);
+      }
+      const yamlData = import_js_yaml3.default.dump(this.cache);
+      (0, import_node_fs2.writeFileSync)(this.cacheFilePath, yamlData);
+      debug("cache flushed to file: %s", this.cacheFilePath);
+    } catch (err) {
+      debug(
+        "write cache to file failed, path: %s, error: %s",
+        this.cacheFilePath,
+        err
+      );
+    }
+  }
+  updateOrAppendCacheRecord(newRecord, cachedRecord) {
+    if (cachedRecord) {
+      if (newRecord.type === "plan") {
+        cachedRecord.updateFn((cache) => {
+          cache.yamlWorkflow = newRecord.yamlWorkflow;
+        });
+      } else {
+        cachedRecord.updateFn((cache) => {
+          cache.xpaths = newRecord.xpaths;
+        });
+      }
+    } else {
+      this.appendCache(newRecord);
+    }
+  }
+};
+
 // src/common/utils.ts
 async function parseContextFromWebPage(page, _opt) {
-  (0, import_utils4.assert)(page, "page is required");
+  (0, import_utils6.assert)(page, "page is required");
   if (page._forceUsePageContext) {
     return await page._forceUsePageContext();
   }
   const url = await page.url();
-  (0, import_utils3.uploadTestInfoToServer)({ testUrl: url });
+  (0, import_utils5.uploadTestInfoToServer)({ testUrl: url });
   let screenshotBase64;
   let tree;
   await Promise.all([
@@ -583,7 +765,7 @@ async function parseContextFromWebPage(page, _opt) {
       isVisible
     });
   });
-  (0, import_utils4.assert)(screenshotBase64, "screenshotBase64 is required");
+  (0, import_utils6.assert)(screenshotBase64, "screenshotBase64 is required");
   const size = await page.size();
   if (size.dpr && size.dpr > 1) {
     screenshotBase64 = await (0, import_img.resizeImgBase64)(screenshotBase64, {
@@ -601,11 +783,11 @@ async function parseContextFromWebPage(page, _opt) {
 function reportFileName(tag = "web") {
   const reportTagName = (0, import_env.getAIConfig)(import_env.MIDSCENE_REPORT_TAG_NAME);
   const dateTimeInFileName = (0, import_dayjs.default)().format("YYYY-MM-DD_HH-mm-ss");
-  const uniqueId = (0, import_utils4.uuid)().substring(0, 8);
+  const uniqueId = (0, import_utils6.uuid)().substring(0, 8);
   return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;
 }
 function printReportMsg(filepath) {
-  (0, import_utils4.logMsg)(`Midscene - report file updated: ${filepath}`);
+  (0, import_utils6.logMsg)(`Midscene - report file updated: ${filepath}`);
 }
 function replaceIllegalPathCharsAndSpace(str) {
   return str.replace(/[:*?"<>| ]/g, "-");
@@ -658,6 +840,28 @@ function matchElementFromPlan(planLocateParam, tree) {
   }
   return void 0;
 }
+async function matchElementFromCache(taskExecutor, xpaths, cachePrompt, cacheable) {
+  try {
+    if (xpaths?.length && taskExecutor.taskCache?.isCacheResultUsed && cacheable !== false) {
+      for (let i = 0; i < xpaths.length; i++) {
+        const element = await taskExecutor.page.getElementInfoByXpath(
+          xpaths[i]
+        );
+        if (element?.id) {
+          debug("cache hit, prompt: %s", cachePrompt);
+          debug(
+            "found a new new element with same xpath, xpath: %s, id: %s",
+            xpaths[i],
+            element?.id
+          );
+          return element;
+        }
+      }
+    }
+  } catch (error) {
+    debug("get element info by xpath error: ", error);
+  }
+}
 function trimContextByViewport(execution) {
   function filterVisibleTree(node) {
     if (!node)
@@ -696,7 +900,7 @@ function trimContextByViewport(execution) {
 }
 
 // src/common/tasks.ts
-var debug = (0, import_logger.getDebug)("page-task-executor");
+var debug2 = (0, import_logger2.getDebug)("page-task-executor");
 var replanningCountLimit = 10;
 var isAndroidPage = (page) => {
   return page.pageType === "android";
@@ -737,7 +941,7 @@ var PageTaskExecutor = class {
       if (info?.id) {
         elementId = info.id;
       } else {
-        debug(
+        debug2(
           "no element id found for position node, will not update cache",
           element
         );
@@ -750,7 +954,7 @@ var PageTaskExecutor = class {
       const result = await this.page.getXpathsById(elementId);
       return result;
     } catch (error) {
-      debug("getXpathsById error: ", error);
+      debug2("getXpathsById error: ", error);
     }
   }
   prependExecutorWithScreenshot(taskApply, appendAfterExecution = false) {
@@ -766,7 +970,7 @@ var PageTaskExecutor = class {
         if (taskApply.type === "Action") {
           await Promise.all([
             (async () => {
-              await (0, import_utils5.sleep)(100);
+              await (0, import_utils7.sleep)(100);
               if (this.page.waitUntilNetworkIdle) {
                 try {
                   await this.page.waitUntilNetworkIdle();
@@ -774,7 +978,7 @@ var PageTaskExecutor = class {
                 }
               }
             })(),
-            (0, import_utils5.sleep)(200)
+            (0, import_utils7.sleep)(200)
           ]);
         }
         if (appendAfterExecution) {
@@ -804,7 +1008,7 @@ var PageTaskExecutor = class {
           locate: plan2.locate,
           executor: async (param, taskContext) => {
             const { task } = taskContext;
-            (0, import_utils6.assert)(
+            (0, import_utils8.assert)(
               param?.prompt || param?.id || param?.bbox,
               "No prompt or id or position or bbox to locate"
             );
@@ -829,39 +1033,29 @@ var PageTaskExecutor = class {
               timing: "before Insight"
             };
             task.recorder = [recordItem];
-            let cacheHitFlag = false;
+            const elementFromXpath = param.xpath ? await this.page.getElementInfoByXpath(param.xpath) : void 0;
+            const userExpectedPathHitFlag = !!elementFromXpath;
             const cachePrompt = param.prompt;
             const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);
             const xpaths = locateCacheRecord?.cacheContent?.xpaths;
-            let elementFromCache = null;
-            try {
-              if (xpaths?.length && this.taskCache?.isCacheResultUsed && param?.cacheable !== false) {
-                for (let i = 0; i < xpaths.length; i++) {
-                  const element2 = await this.page.getElementInfoByXpath(
-                    xpaths[i]
-                  );
-                  if (element2?.id) {
-                    elementFromCache = element2;
-                    debug("cache hit, prompt: %s", cachePrompt);
-                    cacheHitFlag = true;
-                    debug(
-                      "found a new new element with same xpath, xpath: %s, id: %s",
-                      xpaths[i],
-                      element2?.id
-                    );
-                    break;
-                  }
-                }
-              }
-            } catch (error) {
-              debug("get element info by xpath error: ", error);
-            }
-            const startTime = Date.now();
-            const element = elementFromCache || // try to match element from cache
-            matchElementFromPlan(param, pageContext.tree) || // try to match element from plan
-            (await this.insight.locate(param, {
+            const elementFromCache = userExpectedPathHitFlag ? null : await matchElementFromCache(
+              this,
+              xpaths,
+              cachePrompt,
+              param.cacheable
+            );
+            const cacheHitFlag = !!elementFromCache;
+            const elementFromPlan = !userExpectedPathHitFlag && !cacheHitFlag ? matchElementFromPlan(param, pageContext.tree) : void 0;
+            const planHitFlag = !!elementFromPlan;
+            const elementFromAiLocate = !userExpectedPathHitFlag && !cacheHitFlag && !planHitFlag ? (await this.insight.locate(param, {
+              // fallback to ai locate
               context: pageContext
-            })).element;
+            })).element : void 0;
+            const aiLocateHitFlag = !!elementFromAiLocate;
+            const element = elementFromXpath || // highest priority
+            elementFromCache || // second priority
+            elementFromPlan || // third priority
+            elementFromAiLocate;
             let currentXpaths;
             if (element && this.taskCache && !cacheHitFlag && param?.cacheable !== false) {
               const elementXpaths = await this.getElementXpath(
@@ -879,7 +1073,7 @@ var PageTaskExecutor = class {
                   locateCacheRecord
                 );
               } else {
-                debug(
+                debug2(
                   "no xpaths found, will not update cache",
                   cachePrompt,
                   elementXpaths
@@ -889,16 +1083,44 @@ var PageTaskExecutor = class {
             if (!element) {
               throw new Error(`Element not found: ${param.prompt}`);
             }
+            let hitBy;
+            if (userExpectedPathHitFlag) {
+              hitBy = {
+                from: "User expected path",
+                context: {
+                  xpath: param.xpath
+                }
+              };
+            } else if (cacheHitFlag) {
+              hitBy = {
+                from: "Cache",
+                context: {
+                  xpathsFromCache: xpaths,
+                  xpathsToSave: currentXpaths
+                }
+              };
+            } else if (planHitFlag) {
+              hitBy = {
+                from: "Planning",
+                context: {
+                  id: elementFromPlan?.id,
+                  bbox: elementFromPlan?.bbox
+                }
+              };
+            } else if (aiLocateHitFlag) {
+              hitBy = {
+                from: "AI model",
+                context: {
+                  prompt: param.prompt
+                }
+              };
+            }
             return {
               output: {
                 element
               },
               pageContext,
-              cache: {
-                hit: cacheHitFlag,
-                originalXpaths: xpaths,
-                currentXpaths
-              }
+              hitBy
             };
           }
         };
@@ -994,7 +1216,7 @@ var PageTaskExecutor = class {
           thought: plan2.thought,
           locate: plan2.locate,
           executor: async (param, { element }) => {
-            (0, import_utils6.assert)(element, "Element not found, cannot tap");
+            (0, import_utils8.assert)(element, "Element not found, cannot tap");
             await this.page.mouse.click(element.center[0], element.center[1]);
           }
         };
@@ -1006,7 +1228,7 @@ var PageTaskExecutor = class {
           thought: plan2.thought,
           locate: plan2.locate,
           executor: async (param, { element }) => {
-            (0, import_utils6.assert)(element, "Element not found, cannot right click");
+            (0, import_utils8.assert)(element, "Element not found, cannot right click");
             await this.page.mouse.click(
               element.center[0],
               element.center[1],
@@ -1023,7 +1245,7 @@ var PageTaskExecutor = class {
           thought: plan2.thought,
           locate: plan2.locate,
           executor: async (taskParam) => {
-            (0, import_utils6.assert)(
+            (0, import_utils8.assert)(
               taskParam?.start_box && taskParam?.end_box,
               "No start_box or end_box to drag"
             );
@@ -1038,7 +1260,7 @@ var PageTaskExecutor = class {
           thought: plan2.thought,
           locate: plan2.locate,
           executor: async (param, { element }) => {
-            (0, import_utils6.assert)(element, "Element not found, cannot hover");
+            (0, import_utils8.assert)(element, "Element not found, cannot hover");
             await this.page.mouse.move(element.center[0], element.center[1]);
           }
         };
@@ -1090,7 +1312,7 @@ var PageTaskExecutor = class {
                   `Unknown scroll direction: ${taskParam.direction}`
                 );
               }
-              await (0, import_utils5.sleep)(500);
+              await (0, import_utils7.sleep)(500);
             } else {
               throw new Error(
                 `Unknown scroll event type: ${scrollToEventName}, taskParam: ${JSON.stringify(
@@ -1109,7 +1331,7 @@ var PageTaskExecutor = class {
           thought: plan2.thought,
           locate: plan2.locate,
           executor: async (taskParam) => {
-            await (0, import_utils5.sleep)(taskParam?.timeMs || 3e3);
+            await (0, import_utils7.sleep)(taskParam?.timeMs || 3e3);
           }
         };
         tasks.push(taskActionSleep);
@@ -1157,7 +1379,7 @@ var PageTaskExecutor = class {
           thought: plan2.thought,
           locate: plan2.locate,
           executor: async (param) => {
-            (0, import_utils6.assert)(
+            (0, import_utils8.assert)(
               isAndroidPage(this.page),
               "Cannot use home button on non-Android devices"
             );
@@ -1173,7 +1395,7 @@ var PageTaskExecutor = class {
           thought: plan2.thought,
           locate: plan2.locate,
           executor: async (param) => {
-            (0, import_utils6.assert)(
+            (0, import_utils8.assert)(
               isAndroidPage(this.page),
               "Cannot use back button on non-Android devices"
             );
@@ -1189,7 +1411,7 @@ var PageTaskExecutor = class {
           thought: plan2.thought,
           locate: plan2.locate,
           executor: async (param) => {
-            (0, import_utils6.assert)(
+            (0, import_utils8.assert)(
               isAndroidPage(this.page),
               "Cannot use recent apps button on non-Android devices"
             );
@@ -1340,7 +1562,7 @@ var PageTaskExecutor = class {
           }
         }
         if (finalActions.length === 0) {
-          (0, import_utils6.assert)(
+          (0, import_utils8.assert)(
             !more_actions_needed_by_instruction || sleep3,
             error ? `Failed to plan: ${error}` : planParsingError || "No plan found"
           );
@@ -1578,7 +1800,7 @@ var PageTaskExecutor = class {
         );
         let outputResult = data;
         if (ifTypeRestricted) {
-          (0, import_utils6.assert)(data?.result !== void 0, "No result in query data");
+          (0, import_utils8.assert)(data?.result !== void 0, "No result in query data");
           outputResult = data.result;
         }
         return {
@@ -1674,9 +1896,9 @@ var PageTaskExecutor = class {
       onTaskStart: this.onTaskStartCallback
     });
     const { timeoutMs, checkIntervalMs } = opt;
-    (0, import_utils6.assert)(assertion, "No assertion for waitFor");
-    (0, import_utils6.assert)(timeoutMs, "No timeoutMs for waitFor");
-    (0, import_utils6.assert)(checkIntervalMs, "No checkIntervalMs for waitFor");
+    (0, import_utils8.assert)(assertion, "No assertion for waitFor");
+    (0, import_utils8.assert)(timeoutMs, "No timeoutMs for waitFor");
+    (0, import_utils8.assert)(checkIntervalMs, "No checkIntervalMs for waitFor");
     const overallStartTime = Date.now();
     let startTime = Date.now();
     let errorThought = "";
@@ -1730,9 +1952,9 @@ var PageTaskExecutor = class {
 };
 
 // src/common/plan-builder.ts
-var import_logger2 = require("@midscene/shared/logger");
-var import_utils8 = require("@midscene/shared/utils");
-var debug2 = (0, import_logger2.getDebug)("plan-builder");
+var import_logger3 = require("@midscene/shared/logger");
+var import_utils10 = require("@midscene/shared/utils");
+var debug3 = (0, import_logger3.getDebug)("plan-builder");
 function buildPlans(type, locateParam, param) {
   let returnPlans = [];
   const locatePlan = locateParam ? {
@@ -1742,8 +1964,8 @@ function buildPlans(type, locateParam, param) {
     thought: ""
   } : null;
   if (type === "Tap" || type === "Hover" || type === "RightClick") {
-    (0, import_utils8.assert)(locateParam, `missing locate info for action "${type}"`);
-    (0, import_utils8.assert)(locatePlan, `missing locate info for action "${type}"`);
+    (0, import_utils10.assert)(locateParam, `missing locate info for action "${type}"`);
+    (0, import_utils10.assert)(locatePlan, `missing locate info for action "${type}"`);
     const tapPlan = {
       type,
       param: null,
@@ -1754,9 +1976,9 @@ function buildPlans(type, locateParam, param) {
   }
   if (type === "Input" || type === "KeyboardPress") {
     if (type === "Input") {
-      (0, import_utils8.assert)(locateParam, `missing locate info for action "${type}"`);
+      (0, import_utils10.assert)(locateParam, `missing locate info for action "${type}"`);
     }
-    (0, import_utils8.assert)(param, `missing param for action "${type}"`);
+    (0, import_utils10.assert)(param, `missing param for action "${type}"`);
     const inputPlan = {
       type,
       param,
@@ -1770,7 +1992,7 @@ function buildPlans(type, locateParam, param) {
     }
   }
   if (type === "Scroll") {
-    (0, import_utils8.assert)(param, `missing param for action "${type}"`);
+    (0, import_utils10.assert)(param, `missing param for action "${type}"`);
     const scrollPlan = {
       type,
       param,
@@ -1784,7 +2006,7 @@ function buildPlans(type, locateParam, param) {
     }
   }
   if (type === "Sleep") {
-    (0, import_utils8.assert)(param, `missing param for action "${type}"`);
+    (0, import_utils10.assert)(param, `missing param for action "${type}"`);
     const sleepPlan = {
       type,
       param,
@@ -1794,7 +2016,7 @@ function buildPlans(type, locateParam, param) {
     returnPlans = [sleepPlan];
   }
   if (type === "Locate") {
-    (0, import_utils8.assert)(locateParam, `missing locate info for action "${type}"`);
+    (0, import_utils10.assert)(locateParam, `missing locate info for action "${type}"`);
     const locatePlan2 = {
       type,
       param: locateParam,
@@ -1804,187 +2026,12 @@ function buildPlans(type, locateParam, param) {
     returnPlans = [locatePlan2];
   }
   if (returnPlans) {
-    debug2("buildPlans", returnPlans);
+    debug3("buildPlans", returnPlans);
     return returnPlans;
   }
   throw new Error(`Not supported type: ${type}`);
 }
 
-// src/common/task-cache.ts
-var import_node_assert = __toESM(require("assert"));
-var import_node_fs2 = require("fs");
-var import_node_path2 = require("path");
-var import_common2 = require("@midscene/shared/common");
-var import_logger3 = require("@midscene/shared/logger");
-var import_utils9 = require("@midscene/shared/utils");
-var import_js_yaml3 = __toESM(require("js-yaml"));
-var import_semver = __toESM(require("semver"));
-
-// package.json
-var version = "0.19.1";
-
-// src/common/task-cache.ts
-var debug3 = (0, import_logger3.getDebug)("cache");
-var lowestSupportedMidsceneVersion = "0.16.10";
-var cacheFileExt = ".cache.yaml";
-var TaskCache = class {
-  // Track matched records
-  constructor(cacheId, isCacheResultUsed, cacheFilePath) {
-    this.matchedCacheIndices = /* @__PURE__ */ new Set();
-    (0, import_node_assert.default)(cacheId, "cacheId is required");
-    this.cacheId = replaceIllegalPathCharsAndSpace(cacheId);
-    this.cacheFilePath = import_utils9.ifInBrowser ? void 0 : cacheFilePath || (0, import_node_path2.join)((0, import_common2.getMidsceneRunSubDir)("cache"), `${this.cacheId}${cacheFileExt}`);
-    this.isCacheResultUsed = isCacheResultUsed;
-    let cacheContent;
-    if (this.cacheFilePath) {
-      cacheContent = this.loadCacheFromFile();
-    }
-    if (!cacheContent) {
-      cacheContent = {
-        midsceneVersion: version,
-        cacheId: this.cacheId,
-        caches: []
-      };
-    }
-    this.cache = cacheContent;
-    this.cacheOriginalLength = this.cache.caches.length;
-  }
-  matchCache(prompt, type) {
-    for (let i = 0; i < this.cacheOriginalLength; i++) {
-      const item = this.cache.caches[i];
-      const key = `${type}:${prompt}:${i}`;
-      if (item.type === type && item.prompt === prompt && !this.matchedCacheIndices.has(key)) {
-        this.matchedCacheIndices.add(key);
-        debug3(
-          "cache found and marked as used, type: %s, prompt: %s, index: %d",
-          type,
-          prompt,
-          i
-        );
-        return {
-          cacheContent: item,
-          updateFn: (cb) => {
-            debug3(
-              "will call updateFn to update cache, type: %s, prompt: %s, index: %d",
-              type,
-              prompt,
-              i
-            );
-            cb(item);
-            debug3(
-              "cache updated, will flush to file, type: %s, prompt: %s, index: %d",
-              type,
-              prompt,
-              i
-            );
-            this.flushCacheToFile();
-          }
-        };
-      }
-    }
-    debug3("no unused cache found, type: %s, prompt: %s", type, prompt);
-    return void 0;
-  }
-  matchPlanCache(prompt) {
-    return this.matchCache(prompt, "plan");
-  }
-  matchLocateCache(prompt) {
-    return this.matchCache(prompt, "locate");
-  }
-  appendCache(cache) {
-    debug3("will append cache", cache);
-    this.cache.caches.push(cache);
-    this.flushCacheToFile();
-  }
-  loadCacheFromFile() {
-    const cacheFile = this.cacheFilePath;
-    (0, import_node_assert.default)(cacheFile, "cache file path is required");
-    if (!(0, import_node_fs2.existsSync)(cacheFile)) {
-      debug3("no cache file found, path: %s", cacheFile);
-      return void 0;
-    }
-    const jsonTypeCacheFile = cacheFile.replace(cacheFileExt, ".json");
-    if ((0, import_node_fs2.existsSync)(jsonTypeCacheFile) && this.isCacheResultUsed) {
-      console.warn(
-        `An outdated cache file from an earlier version of Midscene has been detected. Since version 0.17, we have implemented an improved caching strategy. Please delete the old file located at: ${jsonTypeCacheFile}.`
-      );
-      return void 0;
-    }
-    try {
-      const data = (0, import_node_fs2.readFileSync)(cacheFile, "utf8");
-      const jsonData = import_js_yaml3.default.load(data);
-      if (!version) {
-        debug3("no midscene version info, will not read cache from file");
-        return void 0;
-      }
-      if (import_semver.default.lt(jsonData.midsceneVersion, lowestSupportedMidsceneVersion) && !jsonData.midsceneVersion.includes("beta")) {
-        console.warn(
-          `You are using an old version of Midscene cache file, and we cannot match any info from it. Starting from Midscene v0.17, we changed our strategy to use xpath for cache info, providing better performance.
-Please delete the existing cache and rebuild it. Sorry for the inconvenience.
-cache file: ${cacheFile}`
-        );
-        return void 0;
-      }
-      debug3(
-        "cache loaded from file, path: %s, cache version: %s, record length: %s",
-        cacheFile,
-        jsonData.midsceneVersion,
-        jsonData.caches.length
-      );
-      jsonData.midsceneVersion = version;
-      return jsonData;
-    } catch (err) {
-      debug3(
-        "cache file exists but load failed, path: %s, error: %s",
-        cacheFile,
-        err
-      );
-      return void 0;
-    }
-  }
-  flushCacheToFile() {
-    if (!version) {
-      debug3("no midscene version info, will not write cache to file");
-      return;
-    }
-    if (!this.cacheFilePath) {
-      debug3("no cache file path, will not write cache to file");
-      return;
-    }
-    try {
-      const dir = (0, import_node_path2.dirname)(this.cacheFilePath);
-      if (!(0, import_node_fs2.existsSync)(dir)) {
-        (0, import_node_fs2.mkdirSync)(dir, { recursive: true });
-        debug3("created cache directory: %s", dir);
-      }
-      const yamlData = import_js_yaml3.default.dump(this.cache);
-      (0, import_node_fs2.writeFileSync)(this.cacheFilePath, yamlData);
-      debug3("cache flushed to file: %s", this.cacheFilePath);
-    } catch (err) {
-      debug3(
-        "write cache to file failed, path: %s, error: %s",
-        this.cacheFilePath,
-        err
-      );
-    }
-  }
-  updateOrAppendCacheRecord(newRecord, cachedRecord) {
-    if (cachedRecord) {
-      if (newRecord.type === "plan") {
-        cachedRecord.updateFn((cache) => {
-          cache.yamlWorkflow = newRecord.yamlWorkflow;
-        });
-      } else {
-        cachedRecord.updateFn((cache) => {
-          cache.xpaths = newRecord.xpaths;
-        });
-      }
-    } else {
-      this.appendCache(newRecord);
-    }
-  }
-};
-
 // src/common/agent.ts
 var debug4 = (0, import_logger4.getDebug)("web-integration");
 var distanceOfTwoPoints = (p1, p2) => {
@@ -2113,10 +2160,12 @@ ${errorTask?.errorStack}`);
       const prompt = opt.prompt ?? locatePrompt;
       const deepThink = opt.deepThink ?? false;
       const cacheable = opt.cacheable ?? true;
+      const xpath = opt.xpath;
       return {
         prompt,
         deepThink,
-        cacheable
+        cacheable,
+        xpath
       };
     }
     return {
@@ -2274,6 +2323,9 @@ ${errorTask?.errorStack}`);
     this.afterTaskRunning(executor);
     return output;
   }
+  async aiAsk(prompt, opt = defaultInsightExtractOption) {
+    return this.aiString(prompt, opt);
+  }
   async describeElementAtPoint(center, opt) {
     const { verifyPrompt = true, retryLimit = 3 } = opt || {};
     let success = false;
@@ -2996,6 +3048,14 @@ var PlaywrightAiFixture = (options) => {
         use,
         aiActionType: "aiBoolean"
       });
+    },
+    aiAsk: async ({ page }, use, testInfo) => {
+      await generateAiFunction({
+        page,
+        testInfo,
+        use,
+        aiActionType: "aiAsk"
+      });
     }
   };
 };