@midscene/web 0.12.7 → 0.12.8-beta-20250317112356.0

package/dist/es/appium.js

@@ -219,10 +219,9 @@ function parseYamlScript(content, filePath, ignoreCheckingTarget) {
 
 // src/common/agent.ts
 import {
-  MATCH_BY_POSITION,
   MIDSCENE_USE_VLM_UI_TARS as MIDSCENE_USE_VLM_UI_TARS2,
   getAIConfig as getAIConfig3,
-  getAIConfigInBoolean as getAIConfigInBoolean2
+  vlLocateMode
 } from "@midscene/core/env";
 import {
   groupedActionDumpFileExt,
@@ -274,12 +273,12 @@ var TaskCache = class {
       tasks: newCacheGroup
     });
     return {
-      readCache: (pageContext, type, actionPrompt) => {
+      matchCache: (pageContext, type, actionPrompt) => {
         if (index === -1) {
           return false;
         }
         if (type === "plan") {
-          return this.readCache(
+          return this.matchCache(
             pageContext,
             type,
             actionPrompt,
@@ -287,14 +286,14 @@ var TaskCache = class {
           );
         }
         if (type === "ui-tars-plan") {
-          return this.readCache(
+          return this.matchCache(
             pageContext,
             type,
             actionPrompt,
             aiTasks[index].tasks
           );
         }
-        return this.readCache(
+        return this.matchCache(
           pageContext,
           type,
           actionPrompt,
@@ -303,11 +302,16 @@ var TaskCache = class {
       },
       saveCache: (cache) => {
         newCacheGroup.push(cache);
+        debug(
+          "saving cache to file, type: %s, cacheId: %s",
+          cache.type,
+          this.cacheId
+        );
         this.writeCacheToFile();
       }
     };
   }
-  readCache(pageContext, type, userPrompt, cacheGroup) {
+  matchCache(pageContext, type, userPrompt, cacheGroup) {
     debug(
       "will read cache, type: %s, prompt: %s, cacheGroupLength: %s",
       type,
@@ -327,17 +331,20 @@ var TaskCache = class {
         userPrompt,
         taskRes?.response
       );
-      if (taskRes?.type === "locate" && !taskRes.response?.elements.every((element) => {
-        const findIndex = pageContext.content.findIndex(
-          (contentElement) => contentElement.id === element.id
+      if (taskRes?.type === "locate") {
+        const id = taskRes.response?.elements[0].id;
+        if (!id) {
+          debug("no id in cached response");
+          return false;
+        }
+        const foundInContext = pageContext.content.find(
+          (contentElement) => contentElement.id === id
         );
-        if (findIndex === -1) {
+        if (!foundInContext) {
+          debug("cannot match element with same id in current page");
           return false;
         }
-        return true;
-      })) {
-        debug("cannot find element with same id in current page");
-        return false;
+        return taskRes.response;
       }
       if (taskRes && taskRes.type === type && taskRes.prompt === userPrompt) {
         const contextEqual = this.pageContextEqual(
@@ -384,7 +391,10 @@ var TaskCache = class {
       return void 0;
     }
     const cacheFile = join2(getLogDirByType("cache"), `${this.cacheId}.json`);
-    if (getAIConfigInBoolean("MIDSCENE_CACHE") && existsSync2(cacheFile)) {
+    if (!getAIConfigInBoolean("MIDSCENE_CACHE")) {
+      return void 0;
+    }
+    if (existsSync2(cacheFile)) {
       try {
         const data = readFileSync(cacheFile, "utf8");
         const jsonData = JSON.parse(data);
@@ -396,16 +406,28 @@ var TaskCache = class {
         if (jsonDataPkgVersion[0] !== midscenePkgInfoPkgVersion[0] || jsonDataPkgVersion[1] !== midscenePkgInfoPkgVersion[1]) {
           return void 0;
         }
+        debug("read cache from file: %s", cacheFile);
         return jsonData;
       } catch (err) {
+        debug(
+          "cache file exists but parse failed, path: %s, error: %s",
+          cacheFile,
+          err
+        );
         return void 0;
       }
     }
+    debug("no cache file found, path: %s", cacheFile);
     return void 0;
   }
   writeCacheToFile() {
     const midscenePkgInfo = getRunningPkgInfo();
-    if (!midscenePkgInfo || !this.cacheId) {
+    if (!midscenePkgInfo) {
+      debug("no midscene pkg info, will not write cache to file");
+      return;
+    }
+    if (!this.cacheId) {
+      debug("no cache id, will not write cache to file");
       return;
     }
     if (!ifInBrowser) {
@@ -549,7 +571,7 @@ var PageTaskExecutor = class {
           executor: async (param, taskContext) => {
             const { task } = taskContext;
             assert3(
-              param?.prompt || param?.id || param?.position || param?.bbox,
+              param?.prompt || param?.id || param?.bbox,
               "No prompt or id or position or bbox to locate"
             );
             let insightDump;
@@ -567,40 +589,45 @@ var PageTaskExecutor = class {
               screenshot: pageContext.screenshotBase64,
               timing: "before locate"
             };
-            const locateCache = cacheGroup?.readCache(
+            const cachePrompt = `${param.prompt} @ ${param.searchArea || ""}`;
+            const locateCache = cacheGroup?.matchCache(
               pageContext,
               "locate",
-              param.prompt
+              cachePrompt
             );
-            let locateResult;
-            const callAI = this.insight.aiVendorFn;
+            const idInCache = locateCache?.elements?.[0]?.id;
+            let cacheHitFlag = false;
+            let quickAnswerId = param?.id;
+            if (!quickAnswerId && idInCache) {
+              quickAnswerId = idInCache;
+            }
             const quickAnswer = {
-              id: param?.id,
-              position: param?.position,
+              id: quickAnswerId,
               bbox: param?.bbox
             };
             const startTime = Date.now();
-            const element = await this.insight.locate(param.prompt, {
-              quickAnswer,
-              callAI: async (...message) => {
-                if (locateCache) {
-                  locateResult = locateCache;
-                  return Promise.resolve({ content: locateCache });
-                }
-                const { content: aiResult, usage: usage2 } = await callAI(...message);
-                return { content: aiResult, usage: usage2 };
-              }
+            const element = await this.insight.locate(param, {
+              quickAnswer
             });
             const aiCost = Date.now() - startTime;
-            if (locateResult) {
+            if (element && element.id === quickAnswerId) {
+              cacheHitFlag = true;
+            }
+            if (element) {
               cacheGroup?.saveCache({
                 type: "locate",
                 pageContext: {
                   url: pageContext.url,
                   size: pageContext.size
                 },
-                prompt: param.prompt,
-                response: locateResult
+                prompt: cachePrompt,
+                response: {
+                  elements: [
+                    {
+                      id: element.id
+                    }
+                  ]
+                }
               });
             }
             if (!element) {
@@ -618,7 +645,7 @@ var PageTaskExecutor = class {
                 dump: insightDump
               },
               cache: {
-                hit: Boolean(locateCache)
+                hit: cacheHitFlag
               },
               recorder: [recordItem],
               aiCost,
@@ -884,13 +911,22 @@ var PageTaskExecutor = class {
         };
         executorContext.task.recorder = [recordItem];
         executorContext.task.pageContext = pageContext;
-        const planCache = cacheGroup.readCache(
+        const cachePrompt = `${param.userInstruction} @ ${param.log || ""}`;
+        const planCache = cacheGroup.matchCache(
           pageContext,
           "plan",
-          param.userInstruction
+          cachePrompt
         );
         let planResult;
         if (planCache) {
+          if ("actions" in planCache && Array.isArray(planCache.actions)) {
+            planCache.actions = planCache.actions.map((action) => {
+              if (action.locate) {
+                delete action.locate.bbox;
+              }
+              return action;
+            });
+          }
           planResult = planCache;
         } else {
           planResult = await plan(param.userInstruction, {
@@ -953,7 +989,7 @@ var PageTaskExecutor = class {
         }
         if (finalActions.length === 0) {
           assert3(
-            !more_actions_needed_by_instruction,
+            !more_actions_needed_by_instruction || sleep2,
             error ? `Failed to plan: ${error}` : planParsingError || "No plan found"
           );
         }
@@ -963,7 +999,7 @@ var PageTaskExecutor = class {
             url: pageContext.url,
             size: pageContext.size
           },
-          prompt: userInstruction,
+          prompt: cachePrompt,
           response: planResult
         });
         return {
@@ -1014,7 +1050,7 @@ var PageTaskExecutor = class {
           ]
         });
         const startTime = Date.now();
-        const planCache = cacheGroup.readCache(
+        const planCache = cacheGroup.matchCache(
           pageContext,
           "ui-tars-plan",
           userInstruction
@@ -1061,6 +1097,18 @@ var PageTaskExecutor = class {
     };
     return task;
   }
+  async runPlans(title, plans, options) {
+    const taskExecutor = new Executor(title, {
+      onTaskStart: options?.onTaskStart
+    });
+    const { tasks } = await this.convertPlanToExecutable(plans);
+    await taskExecutor.append(tasks);
+    const result = await taskExecutor.flush();
+    return {
+      output: result,
+      executor: taskExecutor
+    };
+  }
   async action(userPrompt, options) {
     const taskExecutor = new Executor(userPrompt, {
       onTaskStart: options?.onTaskStart
@@ -1355,6 +1403,75 @@ var WebElementInfo = class {
   }
 };
 
+// src/common/plan-builder.ts
+import { assert as assert4, getDebug as getDebug2 } from "@midscene/shared/utils";
+var debug2 = getDebug2("plan-builder");
+function buildPlans(type, locate, param) {
+  let returnPlans = [];
+  const locatePlan = locate ? {
+    type: "Locate",
+    locate,
+    param: locate,
+    thought: ""
+  } : null;
+  if (type === "Tap" || type === "Hover") {
+    assert4(locatePlan, `missing locate info for action "${type}"`);
+    const tapPlan = {
+      type,
+      param: null,
+      thought: "",
+      locate
+    };
+    returnPlans = [locatePlan, tapPlan];
+  }
+  if (type === "Input" || type === "KeyboardPress") {
+    if (type === "Input") {
+      assert4(locatePlan, `missing locate info for action "${type}"`);
+    }
+    assert4(param, `missing param for action "${type}"`);
+    const inputPlan = {
+      type,
+      param,
+      thought: "",
+      locate
+    };
+    if (locatePlan) {
+      returnPlans = [locatePlan, inputPlan];
+    } else {
+      returnPlans = [inputPlan];
+    }
+  }
+  if (type === "Scroll") {
+    assert4(param, `missing param for action "${type}"`);
+    const scrollPlan = {
+      type,
+      param,
+      thought: "",
+      locate
+    };
+    if (locatePlan) {
+      returnPlans = [locatePlan, scrollPlan];
+    } else {
+      returnPlans = [scrollPlan];
+    }
+  }
+  if (type === "Sleep") {
+    assert4(param, `missing param for action "${type}"`);
+    const sleepPlan = {
+      type,
+      param,
+      thought: "",
+      locate: null
+    };
+    returnPlans = [sleepPlan];
+  }
+  if (returnPlans) {
+    debug2("buildPlans", returnPlans);
+    return returnPlans;
+  }
+  throw new Error(`Not supported type: ${type}`);
+}
+
 // src/common/utils.ts
 import {
   MIDSCENE_REPORT_TAG_NAME,
@@ -1365,10 +1482,10 @@ import { uploadTestInfoToServer } from "@midscene/core/utils";
 import { NodeType } from "@midscene/shared/constants";
 import { traverseTree, treeToList } from "@midscene/shared/extractor";
 import { compositeElementInfoImg, resizeImgBase64 } from "@midscene/shared/img";
-import { assert as assert4, uuid } from "@midscene/shared/utils";
+import { assert as assert5, uuid } from "@midscene/shared/utils";
 import dayjs from "dayjs";
 async function parseContextFromWebPage(page, _opt) {
-  assert4(page, "page is required");
+  assert5(page, "page is required");
   if (page._forceUsePageContext) {
     return await page._forceUsePageContext();
   }
@@ -1396,7 +1513,7 @@ async function parseContextFromWebPage(page, _opt) {
     });
   });
   const elementsInfo = treeToList(webTree);
-  assert4(screenshotBase64, "screenshotBase64 is required");
+  assert5(screenshotBase64, "screenshotBase64 is required");
   const elementsPositionInfoWithoutText = elementsInfo.filter(
     (elementInfo) => {
       if (elementInfo.attributes.nodeType === NodeType.TEXT) {
@@ -1488,7 +1605,7 @@ var PageAgent = class {
       });
     }
     return await parseContextFromWebPage(this.page, {
-      ignoreMarker: getAIConfigInBoolean2(MATCH_BY_POSITION)
+      ignoreMarker: !!vlLocateMode()
     });
   }
   resetDump() {
@@ -1535,50 +1652,105 @@ var PageAgent = class {
       }
     }
   }
-  async aiAction(taskPrompt) {
-    if (getAIConfig3(MIDSCENE_USE_VLM_UI_TARS2)) {
-      const { executor } = await this.taskExecutor.actionToGoal(taskPrompt, {
-        onTaskStart: this.callbackOnTaskStartTip.bind(this)
-      });
-      this.appendExecutionDump(executor.dump());
-      this.writeOutActionDumps();
-      if (executor.isInErrorState()) {
-        const errorTask = executor.latestErrorTask();
-        throw new Error(`${errorTask?.error}
+  afterTaskRunning(executor, doNotThrowError = false) {
+    this.appendExecutionDump(executor.dump());
+    this.writeOutActionDumps();
+    if (executor.isInErrorState() && !doNotThrowError) {
+      const errorTask = executor.latestErrorTask();
+      throw new Error(`${errorTask?.error}
 ${errorTask?.errorStack}`);
+    }
+  }
+  async aiTap(targetPrompt, searchArea) {
+    const plans = buildPlans("Tap", {
+      prompt: targetPrompt,
+      searchArea
+    });
+    const { executor, output } = await this.taskExecutor.runPlans(
+      `Tap ${targetPrompt}`,
+      plans
+    );
+    this.afterTaskRunning(executor);
+    return output;
+  }
+  async aiHover(taskPrompt) {
+    const plans = buildPlans("Hover", {
+      prompt: taskPrompt
+    });
+    const { executor, output } = await this.taskExecutor.runPlans(
+      `Hover ${taskPrompt}`,
+      plans
+    );
+    this.afterTaskRunning(executor);
+    return output;
+  }
+  async aiInput(where, value) {
+    const plans = buildPlans(
+      "Input",
+      {
+        prompt: where
+      },
+      {
+        value
       }
-    } else {
-      const { executor } = await this.taskExecutor.action(taskPrompt, {
-        onTaskStart: this.callbackOnTaskStartTip.bind(this)
-      });
-      this.appendExecutionDump(executor.dump());
-      this.writeOutActionDumps();
-      if (executor.isInErrorState()) {
-        const errorTask = executor.latestErrorTask();
-        throw new Error(`${errorTask?.error}
-${errorTask?.errorStack}`);
+    );
+    const { executor, output } = await this.taskExecutor.runPlans(
+      `Input ${where} - ${value}`,
+      plans
+    );
+    this.afterTaskRunning(executor);
+    return output;
+  }
+  async aiKeyboardPress(where, value) {
+    const plans = buildPlans(
+      "KeyboardPress",
+      {
+        prompt: where
+      },
+      {
+        value
       }
-    }
+    );
+    const { executor, output } = await this.taskExecutor.runPlans(
+      `KeyboardPress ${where} - ${value}`,
+      plans
+    );
+    this.afterTaskRunning(executor);
+    return output;
+  }
+  // async aiScroll(where: string, param: PlanningActionParamScroll) {
+  //   const plans = buildPlans(
+  //     'Scroll',
+  //     {
+  //       prompt: where,
+  //     },
+  //     param,
+  //   );
+  //   const { executor, output } = await this.taskExecutor.runPlans(
+  //     `Scroll ${where} - ${paramStr(param)}`,
+  //     plans,
+  //   );
+  // }
+  async aiAction(taskPrompt) {
+    const { executor } = await (getAIConfig3(MIDSCENE_USE_VLM_UI_TARS2) ? this.taskExecutor.actionToGoal(taskPrompt, {
+      onTaskStart: this.callbackOnTaskStartTip.bind(this)
+    }) : this.taskExecutor.action(taskPrompt, {
+      onTaskStart: this.callbackOnTaskStartTip.bind(this)
+    }));
+    this.afterTaskRunning(executor);
   }
   async aiQuery(demand) {
     const { output, executor } = await this.taskExecutor.query(demand, {
       onTaskStart: this.callbackOnTaskStartTip.bind(this)
     });
-    this.appendExecutionDump(executor.dump());
-    this.writeOutActionDumps();
-    if (executor.isInErrorState()) {
-      const errorTask = executor.latestErrorTask();
-      throw new Error(`${errorTask?.error}
-${errorTask?.errorStack}`);
-    }
+    this.afterTaskRunning(executor);
     return output;
   }
   async aiAssert(assertion, msg, opt) {
     const { output, executor } = await this.taskExecutor.assert(assertion, {
       onTaskStart: this.callbackOnTaskStartTip.bind(this)
     });
-    this.appendExecutionDump(executor.dump());
-    this.writeOutActionDumps();
+    this.afterTaskRunning(executor, true);
     if (output && opt?.keepRawResponse) {
       return output;
     }