@midscene/web 0.16.4 → 0.16.6-beta-20250429073247.0

package/dist/es/agent.js

@@ -109,6 +109,46 @@ var ScriptPlayer = class {
         );
         const queryResult = await agent.aiQuery(prompt);
         this.setResult(queryTask.name, queryResult);
+      } else if (flowItem.aiNumber) {
+        const numberTask = flowItem;
+        const prompt = numberTask.aiNumber;
+        assert(prompt, "missing prompt for number");
+        assert(
+          typeof prompt === "string",
+          "prompt for number must be a string"
+        );
+        const numberResult = await agent.aiNumber(prompt);
+        this.setResult(numberTask.name, numberResult);
+      } else if (flowItem.aiString) {
+        const stringTask = flowItem;
+        const prompt = stringTask.aiString;
+        assert(prompt, "missing prompt for string");
+        assert(
+          typeof prompt === "string",
+          "prompt for string must be a string"
+        );
+        const stringResult = await agent.aiString(prompt);
+        this.setResult(stringTask.name, stringResult);
+      } else if (flowItem.aiBoolean) {
+        const booleanTask = flowItem;
+        const prompt = booleanTask.aiBoolean;
+        assert(prompt, "missing prompt for boolean");
+        assert(
+          typeof prompt === "string",
+          "prompt for boolean must be a string"
+        );
+        const booleanResult = await agent.aiBoolean(prompt);
+        this.setResult(booleanTask.name, booleanResult);
+      } else if (flowItem.aiLocate) {
+        const locateTask = flowItem;
+        const prompt = locateTask.aiLocate;
+        assert(prompt, "missing prompt for aiLocate");
+        assert(
+          typeof prompt === "string",
+          "prompt for aiLocate must be a string"
+        );
+        const locateResult = await agent.aiLocate(prompt);
+        this.setResult(locateTask.name, locateResult);
       } else if (flowItem.aiWaitFor) {
         const waitForTask = flowItem;
         const prompt = waitForTask.aiWaitFor;
@@ -296,10 +336,7 @@ import { sleep } from "@midscene/core/utils";
 import { UITarsModelVersion } from "@midscene/shared/env";
 import { uiTarsModelVersion } from "@midscene/shared/env";
 import { vlLocateMode } from "@midscene/shared/env";
-import {
-  imageInfo,
-  resizeImgBase64 as resizeImgBase642
-} from "@midscene/shared/img";
+import { imageInfo, resizeImgBase64 as resizeImgBase642 } from "@midscene/shared/img";
 import { getDebug as getDebug2 } from "@midscene/shared/logger";
 import { assert as assert4 } from "@midscene/shared/utils";
 
@@ -1485,17 +1522,23 @@ var PageTaskExecutor = class {
       executor: taskExecutor
     };
   }
-  async query(demand) {
-    const description = typeof demand === "string" ? demand : JSON.stringify(demand);
-    const taskExecutor = new Executor(taskTitleStr("Query", description), {
-      onTaskStart: this.onTaskStartCallback
-    });
+  async createTypeQueryTask(type, demand) {
+    const taskExecutor = new Executor(
+      taskTitleStr(
+        type,
+        typeof demand === "string" ? demand : JSON.stringify(demand)
+      ),
+      {
+        onTaskStart: this.onTaskStartCallback
+      }
+    );
     const queryTask = {
       type: "Insight",
-      subType: "Query",
+      subType: type,
       locate: null,
       param: {
         dataDemand: demand
+        // for user param presentation in report right sidebar
       },
       executor: async (param) => {
         let insightDump;
@@ -1503,11 +1546,21 @@ var PageTaskExecutor = class {
           insightDump = dump;
         };
         this.insight.onceDumpUpdatedFn = dumpCollector;
-        const { data, usage } = await this.insight.extract(
-          param.dataDemand
-        );
+        const ifTypeRestricted = type !== "Query";
+        let demandInput = demand;
+        if (ifTypeRestricted) {
+          demandInput = {
+            result: `${type}, ${demand}`
+          };
+        }
+        const { data, usage } = await this.insight.extract(demandInput);
+        let outputResult = data;
+        if (ifTypeRestricted) {
+          assert4(data?.result !== void 0, "No result in query data");
+          outputResult = data.result;
+        }
         return {
-          output: data,
+          output: outputResult,
           log: { dump: insightDump },
           usage
         };
@@ -1520,6 +1573,18 @@ var PageTaskExecutor = class {
       executor: taskExecutor
     };
   }
+  async query(demand) {
+    return this.createTypeQueryTask("Query", demand);
+  }
+  async boolean(prompt) {
+    return this.createTypeQueryTask("Boolean", prompt);
+  }
+  async number(prompt) {
+    return this.createTypeQueryTask("Number", prompt);
+  }
+  async string(prompt) {
+    return this.createTypeQueryTask("String", prompt);
+  }
   async assert(assertion) {
     const description = `assert: ${assertion}`;
     const taskExecutor = new Executor(taskTitleStr("Assert", description), {
@@ -1706,6 +1771,16 @@ function buildPlans(type, locateParam, param) {
     };
     returnPlans = [sleepPlan];
   }
+  if (type === "Locate") {
+    assert5(locateParam, `missing locate info for action "${type}"`);
+    const locatePlan2 = {
+      type,
+      param: locateParam,
+      locate: locateParam,
+      thought: ""
+    };
+    returnPlans = [locatePlan2];
+  }
   if (returnPlans) {
     debug3("buildPlans", returnPlans);
     return returnPlans;
@@ -1905,6 +1980,35 @@ ${errorTask?.errorStack}`);
     this.afterTaskRunning(executor);
     return output;
   }
+  async aiBoolean(prompt) {
+    const { output, executor } = await this.taskExecutor.boolean(prompt);
+    this.afterTaskRunning(executor);
+    return output;
+  }
+  async aiNumber(prompt) {
+    const { output, executor } = await this.taskExecutor.number(prompt);
+    this.afterTaskRunning(executor);
+    return output;
+  }
+  async aiString(prompt) {
+    const { output, executor } = await this.taskExecutor.string(prompt);
+    this.afterTaskRunning(executor);
+    return output;
+  }
+  async aiLocate(prompt, opt) {
+    const detailedLocateParam = this.buildDetailedLocateParam(prompt, opt);
+    const plans = buildPlans("Locate", detailedLocateParam);
+    const { executor, output } = await this.taskExecutor.runPlans(
+      taskTitleStr("Locate", locateParamStr(detailedLocateParam)),
+      plans
+    );
+    this.afterTaskRunning(executor);
+    const { element } = output;
+    return {
+      rect: element?.rect,
+      center: element?.center
+    };
+  }
   async aiAssert(assertion, msg, opt) {
     const { output, executor } = await this.taskExecutor.assert(assertion);
     this.afterTaskRunning(executor, true);