@midscene/web 0.19.1 → 0.20.0

package/dist/es/index.js

@@ -149,7 +149,7 @@ var ScriptPlayer = class {
           domIncluded: numberTask.domIncluded,
           screenshotIncluded: numberTask.screenshotIncluded
         };
-        assert(prompt, "missing prompt for number");
+        assert(prompt, "missing prompt for aiNumber");
         assert(
           typeof prompt === "string",
           "prompt for number must be a string"
@@ -163,7 +163,7 @@ var ScriptPlayer = class {
           domIncluded: stringTask.domIncluded,
           screenshotIncluded: stringTask.screenshotIncluded
         };
-        assert(prompt, "missing prompt for string");
+        assert(prompt, "missing prompt for aiNumber");
         assert(
           typeof prompt === "string",
           "prompt for string must be a string"
@@ -177,13 +177,20 @@ var ScriptPlayer = class {
           domIncluded: booleanTask.domIncluded,
           screenshotIncluded: booleanTask.screenshotIncluded
         };
-        assert(prompt, "missing prompt for boolean");
+        assert(prompt, "missing prompt for aiBoolean");
         assert(
           typeof prompt === "string",
           "prompt for boolean must be a string"
         );
         const booleanResult = await agent.aiBoolean(prompt, options);
         this.setResult(booleanTask.name, booleanResult);
+      } else if ("aiAsk" in flowItem) {
+        const askTask = flowItem;
+        const prompt = askTask.aiAsk;
+        assert(prompt, "missing prompt for aiAsk");
+        assert(typeof prompt === "string", "prompt for aiAsk must be a string");
+        const askResult = await agent.aiAsk(prompt);
+        this.setResult(askTask.name, askResult);
       } else if ("aiLocate" in flowItem) {
         const locateTask = flowItem;
         const prompt = locateTask.aiLocate;
@@ -192,7 +199,7 @@ var ScriptPlayer = class {
           typeof prompt === "string",
           "prompt for aiLocate must be a string"
         );
-        const locateResult = await agent.aiLocate(prompt);
+        const locateResult = await agent.aiLocate(prompt, locateTask);
         this.setResult(locateTask.name, locateResult);
       } else if ("aiWaitFor" in flowItem) {
         const waitForTask = flowItem;
@@ -418,8 +425,8 @@ import {
 } from "@midscene/core/ai-model";
 import { sleep } from "@midscene/core/utils";
 import { NodeType } from "@midscene/shared/constants";
-import { getDebug } from "@midscene/shared/logger";
-import { assert as assert4 } from "@midscene/shared/utils";
+import { getDebug as getDebug2 } from "@midscene/shared/logger";
+import { assert as assert5 } from "@midscene/shared/utils";
 
 // src/common/ui-utils.ts
 function typeStr(task) {
@@ -504,7 +511,7 @@ import {
   traverseTree
 } from "@midscene/shared/extractor";
 import { resizeImgBase64 } from "@midscene/shared/img";
-import { assert as assert3, logMsg, uuid } from "@midscene/shared/utils";
+import { assert as assert4, logMsg, uuid } from "@midscene/shared/utils";
 import dayjs from "dayjs";
 
 // src/web-element.ts
@@ -532,9 +539,184 @@ var WebElementInfo = class {
   }
 };
 
+// src/common/task-cache.ts
+import assert3 from "assert";
+import { existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync, writeFileSync as writeFileSync2 } from "fs";
+import { dirname as dirname2, join as join2 } from "path";
+import { getMidsceneRunSubDir as getMidsceneRunSubDir2 } from "@midscene/shared/common";
+import { getDebug } from "@midscene/shared/logger";
+import { ifInBrowser as ifInBrowser2 } from "@midscene/shared/utils";
+import yaml3 from "js-yaml";
+import semver from "semver";
+
+// package.json
+var version = "0.20.0";
+
+// src/common/task-cache.ts
+var debug = getDebug("cache");
+var lowestSupportedMidsceneVersion = "0.16.10";
+var cacheFileExt = ".cache.yaml";
+var TaskCache = class {
+  // Track matched records
+  constructor(cacheId, isCacheResultUsed, cacheFilePath) {
+    this.matchedCacheIndices = /* @__PURE__ */ new Set();
+    assert3(cacheId, "cacheId is required");
+    this.cacheId = replaceIllegalPathCharsAndSpace(cacheId);
+    this.cacheFilePath = ifInBrowser2 ? void 0 : cacheFilePath || join2(getMidsceneRunSubDir2("cache"), `${this.cacheId}${cacheFileExt}`);
+    this.isCacheResultUsed = isCacheResultUsed;
+    let cacheContent;
+    if (this.cacheFilePath) {
+      cacheContent = this.loadCacheFromFile();
+    }
+    if (!cacheContent) {
+      cacheContent = {
+        midsceneVersion: version,
+        cacheId: this.cacheId,
+        caches: []
+      };
+    }
+    this.cache = cacheContent;
+    this.cacheOriginalLength = this.cache.caches.length;
+  }
+  matchCache(prompt, type) {
+    for (let i = 0; i < this.cacheOriginalLength; i++) {
+      const item = this.cache.caches[i];
+      const key = `${type}:${prompt}:${i}`;
+      if (item.type === type && item.prompt === prompt && !this.matchedCacheIndices.has(key)) {
+        this.matchedCacheIndices.add(key);
+        debug(
+          "cache found and marked as used, type: %s, prompt: %s, index: %d",
+          type,
+          prompt,
+          i
+        );
+        return {
+          cacheContent: item,
+          updateFn: (cb) => {
+            debug(
+              "will call updateFn to update cache, type: %s, prompt: %s, index: %d",
+              type,
+              prompt,
+              i
+            );
+            cb(item);
+            debug(
+              "cache updated, will flush to file, type: %s, prompt: %s, index: %d",
+              type,
+              prompt,
+              i
+            );
+            this.flushCacheToFile();
+          }
+        };
+      }
+    }
+    debug("no unused cache found, type: %s, prompt: %s", type, prompt);
+    return void 0;
+  }
+  matchPlanCache(prompt) {
+    return this.matchCache(prompt, "plan");
+  }
+  matchLocateCache(prompt) {
+    return this.matchCache(prompt, "locate");
+  }
+  appendCache(cache) {
+    debug("will append cache", cache);
+    this.cache.caches.push(cache);
+    this.flushCacheToFile();
+  }
+  loadCacheFromFile() {
+    const cacheFile = this.cacheFilePath;
+    assert3(cacheFile, "cache file path is required");
+    if (!existsSync2(cacheFile)) {
+      debug("no cache file found, path: %s", cacheFile);
+      return void 0;
+    }
+    const jsonTypeCacheFile = cacheFile.replace(cacheFileExt, ".json");
+    if (existsSync2(jsonTypeCacheFile) && this.isCacheResultUsed) {
+      console.warn(
+        `An outdated cache file from an earlier version of Midscene has been detected. Since version 0.17, we have implemented an improved caching strategy. Please delete the old file located at: ${jsonTypeCacheFile}.`
+      );
+      return void 0;
+    }
+    try {
+      const data = readFileSync(cacheFile, "utf8");
+      const jsonData = yaml3.load(data);
+      if (!version) {
+        debug("no midscene version info, will not read cache from file");
+        return void 0;
+      }
+      if (semver.lt(jsonData.midsceneVersion, lowestSupportedMidsceneVersion) && !jsonData.midsceneVersion.includes("beta")) {
+        console.warn(
+          `You are using an old version of Midscene cache file, and we cannot match any info from it. Starting from Midscene v0.17, we changed our strategy to use xpath for cache info, providing better performance.
+Please delete the existing cache and rebuild it. Sorry for the inconvenience.
+cache file: ${cacheFile}`
+        );
+        return void 0;
+      }
+      debug(
+        "cache loaded from file, path: %s, cache version: %s, record length: %s",
+        cacheFile,
+        jsonData.midsceneVersion,
+        jsonData.caches.length
+      );
+      jsonData.midsceneVersion = version;
+      return jsonData;
+    } catch (err) {
+      debug(
+        "cache file exists but load failed, path: %s, error: %s",
+        cacheFile,
+        err
+      );
+      return void 0;
+    }
+  }
+  flushCacheToFile() {
+    if (!version) {
+      debug("no midscene version info, will not write cache to file");
+      return;
+    }
+    if (!this.cacheFilePath) {
+      debug("no cache file path, will not write cache to file");
+      return;
+    }
+    try {
+      const dir = dirname2(this.cacheFilePath);
+      if (!existsSync2(dir)) {
+        mkdirSync2(dir, { recursive: true });
+        debug("created cache directory: %s", dir);
+      }
+      const yamlData = yaml3.dump(this.cache);
+      writeFileSync2(this.cacheFilePath, yamlData);
+      debug("cache flushed to file: %s", this.cacheFilePath);
+    } catch (err) {
+      debug(
+        "write cache to file failed, path: %s, error: %s",
+        this.cacheFilePath,
+        err
+      );
+    }
+  }
+  updateOrAppendCacheRecord(newRecord, cachedRecord) {
+    if (cachedRecord) {
+      if (newRecord.type === "plan") {
+        cachedRecord.updateFn((cache) => {
+          cache.yamlWorkflow = newRecord.yamlWorkflow;
+        });
+      } else {
+        cachedRecord.updateFn((cache) => {
+          cache.xpaths = newRecord.xpaths;
+        });
+      }
+    } else {
+      this.appendCache(newRecord);
+    }
+  }
+};
+
 // src/common/utils.ts
 async function parseContextFromWebPage(page, _opt) {
-  assert3(page, "page is required");
+  assert4(page, "page is required");
   if (page._forceUsePageContext) {
     return await page._forceUsePageContext();
   }
@@ -561,7 +743,7 @@ async function parseContextFromWebPage(page, _opt) {
       isVisible
     });
   });
-  assert3(screenshotBase64, "screenshotBase64 is required");
+  assert4(screenshotBase64, "screenshotBase64 is required");
   const size = await page.size();
   if (size.dpr && size.dpr > 1) {
     screenshotBase64 = await resizeImgBase64(screenshotBase64, {
@@ -636,6 +818,28 @@ function matchElementFromPlan(planLocateParam, tree) {
   }
   return void 0;
 }
+async function matchElementFromCache(taskExecutor, xpaths, cachePrompt, cacheable) {
+  try {
+    if (xpaths?.length && taskExecutor.taskCache?.isCacheResultUsed && cacheable !== false) {
+      for (let i = 0; i < xpaths.length; i++) {
+        const element = await taskExecutor.page.getElementInfoByXpath(
+          xpaths[i]
+        );
+        if (element?.id) {
+          debug("cache hit, prompt: %s", cachePrompt);
+          debug(
+            "found a new new element with same xpath, xpath: %s, id: %s",
+            xpaths[i],
+            element?.id
+          );
+          return element;
+        }
+      }
+    }
+  } catch (error) {
+    debug("get element info by xpath error: ", error);
+  }
+}
 function trimContextByViewport(execution) {
   function filterVisibleTree(node) {
     if (!node)
@@ -674,7 +878,7 @@ function trimContextByViewport(execution) {
 }
 
 // src/common/tasks.ts
-var debug = getDebug("page-task-executor");
+var debug2 = getDebug2("page-task-executor");
 var replanningCountLimit = 10;
 var isAndroidPage = (page) => {
   return page.pageType === "android";
@@ -715,7 +919,7 @@ var PageTaskExecutor = class {
       if (info?.id) {
         elementId = info.id;
       } else {
-        debug(
+        debug2(
           "no element id found for position node, will not update cache",
           element
         );
@@ -728,7 +932,7 @@ var PageTaskExecutor = class {
       const result = await this.page.getXpathsById(elementId);
       return result;
     } catch (error) {
-      debug("getXpathsById error: ", error);
+      debug2("getXpathsById error: ", error);
     }
   }
   prependExecutorWithScreenshot(taskApply, appendAfterExecution = false) {
@@ -782,7 +986,7 @@ var PageTaskExecutor = class {
           locate: plan2.locate,
           executor: async (param, taskContext) => {
             const { task } = taskContext;
-            assert4(
+            assert5(
               param?.prompt || param?.id || param?.bbox,
               "No prompt or id or position or bbox to locate"
             );
@@ -807,39 +1011,29 @@ var PageTaskExecutor = class {
               timing: "before Insight"
             };
             task.recorder = [recordItem];
-            let cacheHitFlag = false;
+            const elementFromXpath = param.xpath ? await this.page.getElementInfoByXpath(param.xpath) : void 0;
+            const userExpectedPathHitFlag = !!elementFromXpath;
             const cachePrompt = param.prompt;
             const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);
             const xpaths = locateCacheRecord?.cacheContent?.xpaths;
-            let elementFromCache = null;
-            try {
-              if (xpaths?.length && this.taskCache?.isCacheResultUsed && param?.cacheable !== false) {
-                for (let i = 0; i < xpaths.length; i++) {
-                  const element2 = await this.page.getElementInfoByXpath(
-                    xpaths[i]
-                  );
-                  if (element2?.id) {
-                    elementFromCache = element2;
-                    debug("cache hit, prompt: %s", cachePrompt);
-                    cacheHitFlag = true;
-                    debug(
-                      "found a new new element with same xpath, xpath: %s, id: %s",
-                      xpaths[i],
-                      element2?.id
-                    );
-                    break;
-                  }
-                }
-              }
-            } catch (error) {
-              debug("get element info by xpath error: ", error);
-            }
-            const startTime = Date.now();
-            const element = elementFromCache || // try to match element from cache
-            matchElementFromPlan(param, pageContext.tree) || // try to match element from plan
-            (await this.insight.locate(param, {
+            const elementFromCache = userExpectedPathHitFlag ? null : await matchElementFromCache(
+              this,
+              xpaths,
+              cachePrompt,
+              param.cacheable
+            );
+            const cacheHitFlag = !!elementFromCache;
+            const elementFromPlan = !userExpectedPathHitFlag && !cacheHitFlag ? matchElementFromPlan(param, pageContext.tree) : void 0;
+            const planHitFlag = !!elementFromPlan;
+            const elementFromAiLocate = !userExpectedPathHitFlag && !cacheHitFlag && !planHitFlag ? (await this.insight.locate(param, {
+              // fallback to ai locate
               context: pageContext
-            })).element;
+            })).element : void 0;
+            const aiLocateHitFlag = !!elementFromAiLocate;
+            const element = elementFromXpath || // highest priority
+            elementFromCache || // second priority
+            elementFromPlan || // third priority
+            elementFromAiLocate;
             let currentXpaths;
             if (element && this.taskCache && !cacheHitFlag && param?.cacheable !== false) {
               const elementXpaths = await this.getElementXpath(
@@ -857,7 +1051,7 @@ var PageTaskExecutor = class {
                   locateCacheRecord
                 );
               } else {
-                debug(
+                debug2(
                   "no xpaths found, will not update cache",
                   cachePrompt,
                   elementXpaths
@@ -867,16 +1061,44 @@ var PageTaskExecutor = class {
             if (!element) {
               throw new Error(`Element not found: ${param.prompt}`);
             }
+            let hitBy;
+            if (userExpectedPathHitFlag) {
+              hitBy = {
+                from: "User expected path",
+                context: {
+                  xpath: param.xpath
+                }
+              };
+            } else if (cacheHitFlag) {
+              hitBy = {
+                from: "Cache",
+                context: {
+                  xpathsFromCache: xpaths,
+                  xpathsToSave: currentXpaths
+                }
+              };
+            } else if (planHitFlag) {
+              hitBy = {
+                from: "Planning",
+                context: {
+                  id: elementFromPlan?.id,
+                  bbox: elementFromPlan?.bbox
+                }
+              };
+            } else if (aiLocateHitFlag) {
+              hitBy = {
+                from: "AI model",
+                context: {
+                  prompt: param.prompt
+                }
+              };
+            }
             return {
               output: {
                 element
               },
               pageContext,
-              cache: {
-                hit: cacheHitFlag,
-                originalXpaths: xpaths,
-                currentXpaths
-              }
+              hitBy
             };
           }
         };
@@ -972,7 +1194,7 @@ var PageTaskExecutor = class {
           thought: plan2.thought,
           locate: plan2.locate,
           executor: async (param, { element }) => {
-            assert4(element, "Element not found, cannot tap");
+            assert5(element, "Element not found, cannot tap");
             await this.page.mouse.click(element.center[0], element.center[1]);
           }
         };
@@ -984,7 +1206,7 @@ var PageTaskExecutor = class {
           thought: plan2.thought,
           locate: plan2.locate,
           executor: async (param, { element }) => {
-            assert4(element, "Element not found, cannot right click");
+            assert5(element, "Element not found, cannot right click");
             await this.page.mouse.click(
               element.center[0],
               element.center[1],
@@ -1001,7 +1223,7 @@ var PageTaskExecutor = class {
           thought: plan2.thought,
           locate: plan2.locate,
           executor: async (taskParam) => {
-            assert4(
+            assert5(
               taskParam?.start_box && taskParam?.end_box,
               "No start_box or end_box to drag"
             );
@@ -1016,7 +1238,7 @@ var PageTaskExecutor = class {
           thought: plan2.thought,
           locate: plan2.locate,
           executor: async (param, { element }) => {
-            assert4(element, "Element not found, cannot hover");
+            assert5(element, "Element not found, cannot hover");
             await this.page.mouse.move(element.center[0], element.center[1]);
           }
         };
@@ -1135,7 +1357,7 @@ var PageTaskExecutor = class {
           thought: plan2.thought,
           locate: plan2.locate,
           executor: async (param) => {
-            assert4(
+            assert5(
               isAndroidPage(this.page),
               "Cannot use home button on non-Android devices"
             );
@@ -1151,7 +1373,7 @@ var PageTaskExecutor = class {
           thought: plan2.thought,
           locate: plan2.locate,
           executor: async (param) => {
-            assert4(
+            assert5(
               isAndroidPage(this.page),
               "Cannot use back button on non-Android devices"
             );
@@ -1167,7 +1389,7 @@ var PageTaskExecutor = class {
           thought: plan2.thought,
           locate: plan2.locate,
           executor: async (param) => {
-            assert4(
+            assert5(
               isAndroidPage(this.page),
               "Cannot use recent apps button on non-Android devices"
             );
@@ -1318,7 +1540,7 @@ var PageTaskExecutor = class {
           }
         }
         if (finalActions.length === 0) {
-          assert4(
+          assert5(
             !more_actions_needed_by_instruction || sleep3,
             error ? `Failed to plan: ${error}` : planParsingError || "No plan found"
           );
@@ -1556,7 +1778,7 @@ var PageTaskExecutor = class {
         );
         let outputResult = data;
         if (ifTypeRestricted) {
-          assert4(data?.result !== void 0, "No result in query data");
+          assert5(data?.result !== void 0, "No result in query data");
           outputResult = data.result;
         }
         return {
@@ -1652,9 +1874,9 @@ var PageTaskExecutor = class {
       onTaskStart: this.onTaskStartCallback
     });
     const { timeoutMs, checkIntervalMs } = opt;
-    assert4(assertion, "No assertion for waitFor");
-    assert4(timeoutMs, "No timeoutMs for waitFor");
-    assert4(checkIntervalMs, "No checkIntervalMs for waitFor");
+    assert5(assertion, "No assertion for waitFor");
+    assert5(timeoutMs, "No timeoutMs for waitFor");
+    assert5(checkIntervalMs, "No checkIntervalMs for waitFor");
     const overallStartTime = Date.now();
     let startTime = Date.now();
     let errorThought = "";
@@ -1708,9 +1930,9 @@ var PageTaskExecutor = class {
 };
 
 // src/common/plan-builder.ts
-import { getDebug as getDebug2 } from "@midscene/shared/logger";
-import { assert as assert5 } from "@midscene/shared/utils";
-var debug2 = getDebug2("plan-builder");
+import { getDebug as getDebug3 } from "@midscene/shared/logger";
+import { assert as assert6 } from "@midscene/shared/utils";
+var debug3 = getDebug3("plan-builder");
 function buildPlans(type, locateParam, param) {
   let returnPlans = [];
   const locatePlan = locateParam ? {
@@ -1720,8 +1942,8 @@ function buildPlans(type, locateParam, param) {
     thought: ""
   } : null;
   if (type === "Tap" || type === "Hover" || type === "RightClick") {
-    assert5(locateParam, `missing locate info for action "${type}"`);
-    assert5(locatePlan, `missing locate info for action "${type}"`);
+    assert6(locateParam, `missing locate info for action "${type}"`);
+    assert6(locatePlan, `missing locate info for action "${type}"`);
     const tapPlan = {
       type,
       param: null,
@@ -1732,9 +1954,9 @@ function buildPlans(type, locateParam, param) {
   }
   if (type === "Input" || type === "KeyboardPress") {
     if (type === "Input") {
-      assert5(locateParam, `missing locate info for action "${type}"`);
+      assert6(locateParam, `missing locate info for action "${type}"`);
     }
-    assert5(param, `missing param for action "${type}"`);
+    assert6(param, `missing param for action "${type}"`);
     const inputPlan = {
       type,
       param,
@@ -1748,7 +1970,7 @@ function buildPlans(type, locateParam, param) {
     }
   }
   if (type === "Scroll") {
-    assert5(param, `missing param for action "${type}"`);
+    assert6(param, `missing param for action "${type}"`);
     const scrollPlan = {
       type,
       param,
@@ -1762,7 +1984,7 @@ function buildPlans(type, locateParam, param) {
     }
   }
   if (type === "Sleep") {
-    assert5(param, `missing param for action "${type}"`);
+    assert6(param, `missing param for action "${type}"`);
     const sleepPlan = {
       type,
       param,
@@ -1772,7 +1994,7 @@ function buildPlans(type, locateParam, param) {
     returnPlans = [sleepPlan];
   }
   if (type === "Locate") {
-    assert5(locateParam, `missing locate info for action "${type}"`);
+    assert6(locateParam, `missing locate info for action "${type}"`);
     const locatePlan2 = {
       type,
       param: locateParam,
@@ -1782,187 +2004,12 @@ function buildPlans(type, locateParam, param) {
     returnPlans = [locatePlan2];
   }
   if (returnPlans) {
-    debug2("buildPlans", returnPlans);
+    debug3("buildPlans", returnPlans);
     return returnPlans;
   }
   throw new Error(`Not supported type: ${type}`);
 }
 
-// src/common/task-cache.ts
-import assert6 from "assert";
-import { existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync, writeFileSync as writeFileSync2 } from "fs";
-import { dirname as dirname2, join as join2 } from "path";
-import { getMidsceneRunSubDir as getMidsceneRunSubDir2 } from "@midscene/shared/common";
-import { getDebug as getDebug3 } from "@midscene/shared/logger";
-import { ifInBrowser as ifInBrowser2 } from "@midscene/shared/utils";
-import yaml3 from "js-yaml";
-import semver from "semver";
-
-// package.json
-var version = "0.19.1";
-
-// src/common/task-cache.ts
-var debug3 = getDebug3("cache");
-var lowestSupportedMidsceneVersion = "0.16.10";
-var cacheFileExt = ".cache.yaml";
-var TaskCache = class {
-  // Track matched records
-  constructor(cacheId, isCacheResultUsed, cacheFilePath) {
-    this.matchedCacheIndices = /* @__PURE__ */ new Set();
-    assert6(cacheId, "cacheId is required");
-    this.cacheId = replaceIllegalPathCharsAndSpace(cacheId);
-    this.cacheFilePath = ifInBrowser2 ? void 0 : cacheFilePath || join2(getMidsceneRunSubDir2("cache"), `${this.cacheId}${cacheFileExt}`);
-    this.isCacheResultUsed = isCacheResultUsed;
-    let cacheContent;
-    if (this.cacheFilePath) {
-      cacheContent = this.loadCacheFromFile();
-    }
-    if (!cacheContent) {
-      cacheContent = {
-        midsceneVersion: version,
-        cacheId: this.cacheId,
-        caches: []
-      };
-    }
-    this.cache = cacheContent;
-    this.cacheOriginalLength = this.cache.caches.length;
-  }
-  matchCache(prompt, type) {
-    for (let i = 0; i < this.cacheOriginalLength; i++) {
-      const item = this.cache.caches[i];
-      const key = `${type}:${prompt}:${i}`;
-      if (item.type === type && item.prompt === prompt && !this.matchedCacheIndices.has(key)) {
-        this.matchedCacheIndices.add(key);
-        debug3(
-          "cache found and marked as used, type: %s, prompt: %s, index: %d",
-          type,
-          prompt,
-          i
-        );
-        return {
-          cacheContent: item,
-          updateFn: (cb) => {
-            debug3(
-              "will call updateFn to update cache, type: %s, prompt: %s, index: %d",
-              type,
-              prompt,
-              i
-            );
-            cb(item);
-            debug3(
-              "cache updated, will flush to file, type: %s, prompt: %s, index: %d",
-              type,
-              prompt,
-              i
-            );
-            this.flushCacheToFile();
-          }
-        };
-      }
-    }
-    debug3("no unused cache found, type: %s, prompt: %s", type, prompt);
-    return void 0;
-  }
-  matchPlanCache(prompt) {
-    return this.matchCache(prompt, "plan");
-  }
-  matchLocateCache(prompt) {
-    return this.matchCache(prompt, "locate");
-  }
-  appendCache(cache) {
-    debug3("will append cache", cache);
-    this.cache.caches.push(cache);
-    this.flushCacheToFile();
-  }
-  loadCacheFromFile() {
-    const cacheFile = this.cacheFilePath;
-    assert6(cacheFile, "cache file path is required");
-    if (!existsSync2(cacheFile)) {
-      debug3("no cache file found, path: %s", cacheFile);
-      return void 0;
-    }
-    const jsonTypeCacheFile = cacheFile.replace(cacheFileExt, ".json");
-    if (existsSync2(jsonTypeCacheFile) && this.isCacheResultUsed) {
-      console.warn(
-        `An outdated cache file from an earlier version of Midscene has been detected. Since version 0.17, we have implemented an improved caching strategy. Please delete the old file located at: ${jsonTypeCacheFile}.`
-      );
-      return void 0;
-    }
-    try {
-      const data = readFileSync(cacheFile, "utf8");
-      const jsonData = yaml3.load(data);
-      if (!version) {
-        debug3("no midscene version info, will not read cache from file");
-        return void 0;
-      }
-      if (semver.lt(jsonData.midsceneVersion, lowestSupportedMidsceneVersion) && !jsonData.midsceneVersion.includes("beta")) {
-        console.warn(
-          `You are using an old version of Midscene cache file, and we cannot match any info from it. Starting from Midscene v0.17, we changed our strategy to use xpath for cache info, providing better performance.
-Please delete the existing cache and rebuild it. Sorry for the inconvenience.
-cache file: ${cacheFile}`
-        );
-        return void 0;
-      }
-      debug3(
-        "cache loaded from file, path: %s, cache version: %s, record length: %s",
-        cacheFile,
-        jsonData.midsceneVersion,
-        jsonData.caches.length
-      );
-      jsonData.midsceneVersion = version;
-      return jsonData;
-    } catch (err) {
-      debug3(
-        "cache file exists but load failed, path: %s, error: %s",
-        cacheFile,
-        err
-      );
-      return void 0;
-    }
-  }
-  flushCacheToFile() {
-    if (!version) {
-      debug3("no midscene version info, will not write cache to file");
-      return;
-    }
-    if (!this.cacheFilePath) {
-      debug3("no cache file path, will not write cache to file");
-      return;
-    }
-    try {
-      const dir = dirname2(this.cacheFilePath);
-      if (!existsSync2(dir)) {
-        mkdirSync2(dir, { recursive: true });
-        debug3("created cache directory: %s", dir);
-      }
-      const yamlData = yaml3.dump(this.cache);
-      writeFileSync2(this.cacheFilePath, yamlData);
-      debug3("cache flushed to file: %s", this.cacheFilePath);
-    } catch (err) {
-      debug3(
-        "write cache to file failed, path: %s, error: %s",
-        this.cacheFilePath,
-        err
-      );
-    }
-  }
-  updateOrAppendCacheRecord(newRecord, cachedRecord) {
-    if (cachedRecord) {
-      if (newRecord.type === "plan") {
-        cachedRecord.updateFn((cache) => {
-          cache.yamlWorkflow = newRecord.yamlWorkflow;
-        });
-      } else {
-        cachedRecord.updateFn((cache) => {
-          cache.xpaths = newRecord.xpaths;
-        });
-      }
-    } else {
-      this.appendCache(newRecord);
-    }
-  }
-};
-
 // src/common/agent.ts
 var debug4 = getDebug4("web-integration");
 var distanceOfTwoPoints = (p1, p2) => {
@@ -2091,10 +2138,12 @@ ${errorTask?.errorStack}`);
       const prompt = opt.prompt ?? locatePrompt;
       const deepThink = opt.deepThink ?? false;
       const cacheable = opt.cacheable ?? true;
+      const xpath = opt.xpath;
       return {
         prompt,
         deepThink,
-        cacheable
+        cacheable,
+        xpath
       };
     }
     return {
@@ -2252,6 +2301,9 @@ ${errorTask?.errorStack}`);
     this.afterTaskRunning(executor);
     return output;
   }
+  async aiAsk(prompt, opt = defaultInsightExtractOption) {
+    return this.aiString(prompt, opt);
+  }
   async describeElementAtPoint(center, opt) {
     const { verifyPrompt = true, retryLimit = 3 } = opt || {};
     let success = false;
@@ -2977,6 +3029,14 @@ var PlaywrightAiFixture = (options) => {
         use,
         aiActionType: "aiBoolean"
       });
+    },
+    aiAsk: async ({ page }, use, testInfo) => {
+      await generateAiFunction({
+        page,
+        testInfo,
+        use,
+        aiActionType: "aiAsk"
+      });
     }
   };
 };