@midscene/web 0.12.8 → 0.12.9-beta-20250320033304.0

package/dist/es/appium.js

@@ -2,7 +2,6 @@
 import {
   Insight
 } from "@midscene/core";
-import { NodeType as NodeType2 } from "@midscene/shared/constants";
 
 // src/yaml/player.ts
 import { existsSync, mkdirSync, writeFileSync } from "fs";
@@ -124,6 +123,24 @@ var ScriptPlayer = class {
           `ms for sleep must be greater than 0, but got ${ms}`
         );
         await new Promise((resolve) => setTimeout(resolve, msNumber));
+      } else if (flowItem.aiTap) {
+        const tapTask = flowItem;
+        await agent.aiTap(tapTask.aiTap);
+      } else if (flowItem.aiHover) {
+        const hoverTask = flowItem;
+        await agent.aiHover(hoverTask.aiHover);
+      } else if (flowItem.aiInput) {
+        const inputTask = flowItem;
+        await agent.aiInput(inputTask.aiInput, inputTask.locate);
+      } else if (flowItem.aiKeyboardPress) {
+        const keyboardPressTask = flowItem;
+        await agent.aiKeyboardPress(
+          keyboardPressTask.aiKeyboardPress,
+          keyboardPressTask.locate
+        );
+      } else if (flowItem.aiScroll) {
+        const scrollTask = flowItem;
+        await agent.aiScroll(scrollTask.aiScroll, scrollTask.locate);
       } else {
         throw new Error(`unknown flowItem: ${JSON.stringify(flowItem)}`);
       }
@@ -138,6 +155,11 @@ var ScriptPlayer = class {
     try {
       const { agent: newAgent, freeFn: newFreeFn } = await this.setupAgent(target);
       agent = newAgent;
+      agent.onTaskStartTip = (tip) => {
+        if (this.status === "running") {
+          this.agentStatusTip = tip;
+        }
+      };
       freeFn = newFreeFn;
     } catch (e) {
       this.setPlayerStatus("error", e);
@@ -171,6 +193,7 @@ var ScriptPlayer = class {
     } else {
       this.setPlayerStatus("done");
     }
+    this.agentStatusTip = "";
     for (const fn of freeFn) {
       try {
         await fn.fn();
@@ -220,7 +243,7 @@ function parseYamlScript(content, filePath, ignoreCheckingTarget) {
 // src/common/agent.ts
 import {
   MIDSCENE_USE_VLM_UI_TARS as MIDSCENE_USE_VLM_UI_TARS2,
-  getAIConfig as getAIConfig3,
+  getAIConfigInBoolean as getAIConfigInBoolean3,
   vlLocateMode
 } from "@midscene/core/env";
 import {
@@ -229,6 +252,7 @@ import {
   stringifyDumpData as stringifyDumpData2,
   writeLogFile as writeLogFile2
 } from "@midscene/core/utils";
+import { assert as assert6 } from "@midscene/shared/utils";
 
 // src/common/tasks.ts
 import {
@@ -469,6 +493,30 @@ function getKeyCommands(value) {
     return acc.concat([{ key: k }]);
   }, []);
 }
+function locateParamStr(locate) {
+  if (!locate) {
+    return "";
+  }
+  if (typeof locate === "string") {
+    return locate;
+  }
+  if (!locate.searchArea) {
+    if (locate.deepThink) {
+      return `${locate.prompt} (deep think)`;
+    }
+    return locate.prompt;
+  }
+  return `${locate.prompt} @ ${locate.searchArea}`;
+}
+function scrollParamStr(scrollParam) {
+  if (!scrollParam) {
+    return "";
+  }
+  return `${scrollParam.direction || "down"}, ${scrollParam.scrollType || "once"}, ${scrollParam.distance || "distance-not-set"}`;
+}
+function taskTitleStr(type, prompt) {
+  return `${type} - ${prompt}`;
+}
 function paramStr(task) {
   let value;
   if (task.type === "Planning") {
@@ -478,19 +526,22 @@ function paramStr(task) {
     value = task?.param?.prompt || task?.param?.id || task?.param?.dataDemand || task?.param?.assertion;
   }
   if (task.type === "Action") {
-    const sleepMs = task?.param?.timeMs;
-    const scrollType = task?.param?.scrollType;
-    if (sleepMs) {
-      value = `${sleepMs}ms`;
-    } else if (scrollType) {
-      const scrollDirection = task?.param?.direction;
-      const scrollDistance = task?.param?.distance;
-      value = `${scrollDirection || "down"}, ${scrollType || "once"}, ${scrollDistance || "distance-not-set"}`;
-    } else {
-      value = task?.param?.value || task?.param?.scrollType;
+    const locate = task?.locate;
+    const locateStr = locate ? locateParamStr(locate) : "";
+    value = task.thought || "";
+    if (typeof task?.param?.timeMs === "number") {
+      value = `${task?.param?.timeMs}ms`;
+    } else if (typeof task?.param?.scrollType === "string") {
+      value = scrollParamStr(task?.param);
+    } else if (typeof task?.param?.value !== "undefined") {
+      value = task?.param?.value;
     }
-    if (!value) {
-      value = task.thought;
+    if (locateStr) {
+      if (value) {
+        value = `${locateStr} - ${value}`;
+      } else {
+        value = locateStr;
+      }
     }
   }
   if (typeof value === "undefined")
@@ -508,6 +559,7 @@ var PageTaskExecutor = class {
     this.taskCache = new TaskCache({
       cacheId: opts?.cacheId
     });
+    this.onTaskStartCallback = opts?.onTaskStart;
   }
   async recordScreenshot(timing) {
     const base64 = await this.page.screenshotBase64();
@@ -606,7 +658,7 @@ var PageTaskExecutor = class {
               bbox: param?.bbox
             };
             const startTime = Date.now();
-            const element = await this.insight.locate(param, {
+            const { element } = await this.insight.locate(param, {
               quickAnswer
             });
             const aiCost = Date.now() - startTime;
@@ -1097,9 +1149,9 @@ var PageTaskExecutor = class {
     };
     return task;
   }
-  async runPlans(title, plans, options) {
+  async runPlans(title, plans) {
     const taskExecutor = new Executor(title, {
-      onTaskStart: options?.onTaskStart
+      onTaskStart: this.onTaskStartCallback
     });
     const { tasks } = await this.convertPlanToExecutable(plans);
     await taskExecutor.append(tasks);
@@ -1109,9 +1161,9 @@ var PageTaskExecutor = class {
       executor: taskExecutor
     };
   }
-  async action(userPrompt, options) {
-    const taskExecutor = new Executor(userPrompt, {
-      onTaskStart: options?.onTaskStart
+  async action(userPrompt) {
+    const taskExecutor = new Executor(taskTitleStr("Action", userPrompt), {
+      onTaskStart: this.onTaskStartCallback
     });
     const cacheGroup = this.taskCache.getCacheGroupByPrompt(userPrompt);
     let planningTask = this.planningTaskFromPrompt(userPrompt, cacheGroup);
@@ -1170,9 +1222,9 @@ var PageTaskExecutor = class {
       executor: taskExecutor
     };
   }
-  async actionToGoal(userPrompt, options) {
-    const taskExecutor = new Executor(userPrompt, {
-      onTaskStart: options?.onTaskStart
+  async actionToGoal(userPrompt) {
+    const taskExecutor = new Executor(taskTitleStr("Action", userPrompt), {
+      onTaskStart: this.onTaskStartCallback
     });
     this.conversationHistory = [];
     const cacheGroup = this.taskCache.getCacheGroupByPrompt(userPrompt);
@@ -1222,10 +1274,10 @@ var PageTaskExecutor = class {
       executor: taskExecutor
     };
   }
-  async query(demand, options) {
+  async query(demand) {
     const description = typeof demand === "string" ? demand : JSON.stringify(demand);
-    const taskExecutor = new Executor(description, {
-      onTaskStart: options?.onTaskStart
+    const taskExecutor = new Executor(taskTitleStr("Query", description), {
+      onTaskStart: this.onTaskStartCallback
     });
     const queryTask = {
       type: "Insight",
@@ -1257,10 +1309,10 @@ var PageTaskExecutor = class {
       executor: taskExecutor
     };
   }
-  async assert(assertion, options) {
+  async assert(assertion) {
     const description = `assert: ${assertion}`;
-    const taskExecutor = new Executor(description, {
-      onTaskStart: options?.onTaskStart
+    const taskExecutor = new Executor(taskTitleStr("Assert", description), {
+      onTaskStart: this.onTaskStartCallback
     });
     const assertionPlan = {
       type: "Assert",
@@ -1320,8 +1372,8 @@ var PageTaskExecutor = class {
   }
   async waitFor(assertion, opt) {
     const description = `waitFor: ${assertion}`;
-    const taskExecutor = new Executor(description, {
-      onTaskStart: opt.onTaskStart
+    const taskExecutor = new Executor(taskTitleStr("WaitFor", description), {
+      onTaskStart: this.onTaskStartCallback
     });
     const { timeoutMs, checkIntervalMs } = opt;
     assert3(assertion, "No assertion for waitFor");
@@ -1379,61 +1431,39 @@ var PageTaskExecutor = class {
   }
 };
 
-// src/web-element.ts
-var WebElementInfo = class {
-  constructor({
-    content,
-    rect,
-    // page,
-    locator,
-    id,
-    attributes,
-    indexId
-  }) {
-    this.content = content;
-    this.rect = rect;
-    this.center = [
-      Math.floor(rect.left + rect.width / 2),
-      Math.floor(rect.top + rect.height / 2)
-    ];
-    this.locator = locator;
-    this.id = id;
-    this.attributes = attributes;
-    this.indexId = indexId;
-  }
-};
-
 // src/common/plan-builder.ts
 import { assert as assert4, getDebug as getDebug2 } from "@midscene/shared/utils";
 var debug2 = getDebug2("plan-builder");
 function buildPlans(type, locate, param) {
   let returnPlans = [];
-  const locatePlan = locate ? {
+  const locateParam = typeof locate === "string" ? { prompt: locate } : locate ?? null;
+  const locatePlan = locateParam ? {
     type: "Locate",
-    locate,
-    param: locate,
+    locate: locateParam,
+    param: locateParam,
     thought: ""
   } : null;
   if (type === "Tap" || type === "Hover") {
+    assert4(locate && locateParam, `missing locate info for action "${type}"`);
     assert4(locatePlan, `missing locate info for action "${type}"`);
     const tapPlan = {
       type,
       param: null,
       thought: "",
-      locate
+      locate: locateParam
     };
     returnPlans = [locatePlan, tapPlan];
   }
   if (type === "Input" || type === "KeyboardPress") {
     if (type === "Input") {
-      assert4(locatePlan, `missing locate info for action "${type}"`);
+      assert4(locate && locateParam, `missing locate info for action "${type}"`);
     }
     assert4(param, `missing param for action "${type}"`);
     const inputPlan = {
       type,
       param,
       thought: "",
-      locate
+      locate: locateParam
     };
     if (locatePlan) {
       returnPlans = [locatePlan, inputPlan];
@@ -1447,7 +1477,7 @@ function buildPlans(type, locate, param) {
       type,
       param,
       thought: "",
-      locate
+      locate: locateParam
     };
     if (locatePlan) {
       returnPlans = [locatePlan, scrollPlan];
@@ -1476,7 +1506,8 @@ function buildPlans(type, locate, param) {
 import {
   MIDSCENE_REPORT_TAG_NAME,
   MIDSCENE_USE_VLM_UI_TARS,
-  getAIConfig as getAIConfig2
+  getAIConfig as getAIConfig2,
+  getAIConfigInBoolean as getAIConfigInBoolean2
 } from "@midscene/core/env";
 import { uploadTestInfoToServer } from "@midscene/core/utils";
 import { NodeType } from "@midscene/shared/constants";
@@ -1484,6 +1515,32 @@ import { traverseTree, treeToList } from "@midscene/shared/extractor";
 import { compositeElementInfoImg, resizeImgBase64 } from "@midscene/shared/img";
 import { assert as assert5, uuid } from "@midscene/shared/utils";
 import dayjs from "dayjs";
+
+// src/web-element.ts
+var WebElementInfo = class {
+  constructor({
+    content,
+    rect,
+    // page,
+    locator,
+    id,
+    attributes,
+    indexId
+  }) {
+    this.content = content;
+    this.rect = rect;
+    this.center = [
+      Math.floor(rect.left + rect.width / 2),
+      Math.floor(rect.top + rect.height / 2)
+    ];
+    this.locator = locator;
+    this.id = id;
+    this.attributes = attributes;
+    this.indexId = indexId;
+  }
+};
+
+// src/common/utils.ts
 async function parseContextFromWebPage(page, _opt) {
   assert5(page, "page is required");
   if (page._forceUsePageContext) {
@@ -1530,7 +1587,7 @@ async function parseContextFromWebPage(page, _opt) {
     });
   }
   let screenshotBase64WithElementMarker = screenshotBase64;
-  if (!getAIConfig2(MIDSCENE_USE_VLM_UI_TARS)) {
+  if (!getAIConfigInBoolean2(MIDSCENE_USE_VLM_UI_TARS)) {
     if (_opt?.ignoreMarker) {
       screenshotBase64WithElementMarker = screenshotBase64;
     } else {
@@ -1576,24 +1633,15 @@ var PageAgent = class {
       },
       opts || {}
     );
+    this.onTaskStartTip = this.opts.onTaskStartTip;
     this.insight = new Insight(
       async (action) => {
         return this.getUIContext(action);
-      },
-      {
-        generateElement: ({ content, rect }) => new WebElementInfo({
-          content: content || "",
-          rect,
-          id: "",
-          attributes: {
-            nodeType: NodeType2.CONTAINER
-          },
-          indexId: 0
-        })
       }
     );
     this.taskExecutor = new PageTaskExecutor(this.page, this.insight, {
-      cacheId: opts?.cacheId
+      cacheId: opts?.cacheId,
+      onTaskStart: this.callbackOnTaskStartTip.bind(this)
     });
     this.dump = this.resetDump();
     this.reportFileName = reportFileName(opts?.testId || "web");
@@ -1642,14 +1690,10 @@ var PageAgent = class {
     }
   }
   async callbackOnTaskStartTip(task) {
-    if (this.opts.onTaskStartTip) {
-      const param = paramStr(task);
-      if (param) {
-        const tip = `${typeStr(task)} - ${param}`;
-        await this.opts.onTaskStartTip(tip);
-      } else {
-        await this.opts.onTaskStartTip(typeStr(task));
-      }
+    const param = paramStr(task);
+    const tip = param ? `${typeStr(task)} - ${param}` : typeStr(task);
+    if (this.onTaskStartTip) {
+      await this.onTaskStartTip(tip);
     }
   }
   afterTaskRunning(executor, doNotThrowError = false) {
@@ -1661,95 +1705,72 @@ var PageAgent = class {
 ${errorTask?.errorStack}`);
     }
   }
-  async aiTap(targetPrompt, searchArea) {
-    const plans = buildPlans("Tap", {
-      prompt: targetPrompt,
-      searchArea
-    });
+  async aiTap(locate) {
+    const plans = buildPlans("Tap", locate);
     const { executor, output } = await this.taskExecutor.runPlans(
-      `Tap ${targetPrompt}`,
+      taskTitleStr("Tap", locateParamStr(locate)),
       plans
     );
     this.afterTaskRunning(executor);
     return output;
   }
-  async aiHover(taskPrompt) {
-    const plans = buildPlans("Hover", {
-      prompt: taskPrompt
-    });
+  async aiHover(locate) {
+    const plans = buildPlans("Hover", locate);
     const { executor, output } = await this.taskExecutor.runPlans(
-      `Hover ${taskPrompt}`,
+      taskTitleStr("Hover", locateParamStr(locate)),
       plans
     );
     this.afterTaskRunning(executor);
     return output;
   }
-  async aiInput(where, value) {
-    const plans = buildPlans(
-      "Input",
-      {
-        prompt: where
-      },
-      {
-        value
-      }
+  async aiInput(value, locate) {
+    assert6(
+      typeof value === "string",
+      "input value must be a string, use empty string if you want to clear the input"
     );
+    const plans = buildPlans("Input", locate, {
+      value
+    });
     const { executor, output } = await this.taskExecutor.runPlans(
-      `Input ${where} - ${value}`,
+      taskTitleStr("Input", `${locateParamStr(locate)} - ${value}`),
       plans
     );
     this.afterTaskRunning(executor);
     return output;
   }
-  async aiKeyboardPress(where, value) {
-    const plans = buildPlans(
-      "KeyboardPress",
-      {
-        prompt: where
-      },
-      {
-        value
-      }
+  async aiKeyboardPress(keyName, locate) {
+    assert6(keyName, "missing keyName for keyboard press");
+    const plans = buildPlans("KeyboardPress", locate, {
+      value: keyName
+    });
+    const { executor, output } = await this.taskExecutor.runPlans(
+      taskTitleStr("KeyboardPress", `${locateParamStr(locate)} - ${keyName}`),
+      plans
     );
+    this.afterTaskRunning(executor);
+    return output;
+  }
+  async aiScroll(scrollParam, locate) {
+    const plans = buildPlans("Scroll", locate, scrollParam);
+    const paramInTitle = locate ? `${locateParamStr(locate)} - ${scrollParamStr(scrollParam)}` : scrollParamStr(scrollParam);
     const { executor, output } = await this.taskExecutor.runPlans(
-      `KeyboardPress ${where} - ${value}`,
+      taskTitleStr("Scroll", paramInTitle),
       plans
     );
     this.afterTaskRunning(executor);
     return output;
   }
-  // async aiScroll(where: string, param: PlanningActionParamScroll) {
-  //   const plans = buildPlans(
-  //     'Scroll',
-  //     {
-  //       prompt: where,
-  //     },
-  //     param,
-  //   );
-  //   const { executor, output } = await this.taskExecutor.runPlans(
-  //     `Scroll ${where} - ${paramStr(param)}`,
-  //     plans,
-  //   );
-  // }
   async aiAction(taskPrompt) {
-    const { executor } = await (getAIConfig3(MIDSCENE_USE_VLM_UI_TARS2) ? this.taskExecutor.actionToGoal(taskPrompt, {
-      onTaskStart: this.callbackOnTaskStartTip.bind(this)
-    }) : this.taskExecutor.action(taskPrompt, {
-      onTaskStart: this.callbackOnTaskStartTip.bind(this)
-    }));
+    const { executor } = await (getAIConfigInBoolean3(MIDSCENE_USE_VLM_UI_TARS2) ? this.taskExecutor.actionToGoal(taskPrompt) : this.taskExecutor.action(taskPrompt));
     this.afterTaskRunning(executor);
   }
   async aiQuery(demand) {
-    const { output, executor } = await this.taskExecutor.query(demand, {
-      onTaskStart: this.callbackOnTaskStartTip.bind(this)
-    });
+    const { output, executor } = await this.taskExecutor.query(demand);
     this.afterTaskRunning(executor);
     return output;
   }
   async aiAssert(assertion, msg, opt) {
-    const { output, executor } = await this.taskExecutor.assert(assertion, {
-      onTaskStart: this.callbackOnTaskStartTip.bind(this)
-    });
+    const { output, executor } = await this.taskExecutor.assert(assertion);
     this.afterTaskRunning(executor, true);
     if (output && opt?.keepRawResponse) {
       return output;
@@ -1765,8 +1786,7 @@ ${reasonMsg}`);
     const { executor } = await this.taskExecutor.waitFor(assertion, {
       timeoutMs: opt?.timeoutMs || 15 * 1e3,
       checkIntervalMs: opt?.checkIntervalMs || 3 * 1e3,
-      assertion,
-      onTaskStart: this.callbackOnTaskStartTip.bind(this)
+      assertion
     });
     this.appendExecutionDump(executor.dump());
     this.writeOutActionDumps();