@midscene/web 0.16.5 → 0.16.6

package/dist/es/agent.js

@@ -296,10 +296,7 @@ import { sleep } from "@midscene/core/utils";
 import { UITarsModelVersion } from "@midscene/shared/env";
 import { uiTarsModelVersion } from "@midscene/shared/env";
 import { vlLocateMode } from "@midscene/shared/env";
-import {
-  imageInfo,
-  resizeImgBase64 as resizeImgBase642
-} from "@midscene/shared/img";
+import { imageInfo, resizeImgBase64 as resizeImgBase642 } from "@midscene/shared/img";
 import { getDebug as getDebug2 } from "@midscene/shared/logger";
 import { assert as assert4 } from "@midscene/shared/utils";
 
@@ -1485,17 +1482,23 @@ var PageTaskExecutor = class {
       executor: taskExecutor
     };
   }
-  async query(demand) {
-    const description = typeof demand === "string" ? demand : JSON.stringify(demand);
-    const taskExecutor = new Executor(taskTitleStr("Query", description), {
-      onTaskStart: this.onTaskStartCallback
-    });
+  async createTypeQueryTask(type, demand) {
+    const taskExecutor = new Executor(
+      taskTitleStr(
+        type,
+        typeof demand === "string" ? demand : JSON.stringify(demand)
+      ),
+      {
+        onTaskStart: this.onTaskStartCallback
+      }
+    );
     const queryTask = {
       type: "Insight",
-      subType: "Query",
+      subType: type,
       locate: null,
       param: {
         dataDemand: demand
+        // for user param presentation in report right sidebar
       },
       executor: async (param) => {
         let insightDump;
@@ -1503,11 +1506,21 @@ var PageTaskExecutor = class {
           insightDump = dump;
         };
         this.insight.onceDumpUpdatedFn = dumpCollector;
-        const { data, usage } = await this.insight.extract(
-          param.dataDemand
-        );
+        const ifTypeRestricted = type !== "Query";
+        let demandInput = demand;
+        if (ifTypeRestricted) {
+          demandInput = {
+            result: `${type}, ${demand}`
+          };
+        }
+        const { data, usage } = await this.insight.extract(demandInput);
+        let outputResult = data;
+        if (ifTypeRestricted) {
+          assert4(data?.result !== void 0, "No result in query data");
+          outputResult = data.result;
+        }
         return {
-          output: data,
+          output: outputResult,
           log: { dump: insightDump },
           usage
         };
@@ -1520,6 +1533,18 @@ var PageTaskExecutor = class {
       executor: taskExecutor
     };
   }
+  async query(demand) {
+    return this.createTypeQueryTask("Query", demand);
+  }
+  async boolean(prompt) {
+    return this.createTypeQueryTask("Boolean", prompt);
+  }
+  async number(prompt) {
+    return this.createTypeQueryTask("Number", prompt);
+  }
+  async string(prompt) {
+    return this.createTypeQueryTask("String", prompt);
+  }
   async assert(assertion) {
     const description = `assert: ${assertion}`;
     const taskExecutor = new Executor(taskTitleStr("Assert", description), {
@@ -1706,6 +1731,16 @@ function buildPlans(type, locateParam, param) {
     };
     returnPlans = [sleepPlan];
   }
+  if (type === "Locate") {
+    assert5(locateParam, `missing locate info for action "${type}"`);
+    const locatePlan2 = {
+      type,
+      param: locateParam,
+      locate: locateParam,
+      thought: ""
+    };
+    returnPlans = [locatePlan2];
+  }
   if (returnPlans) {
     debug3("buildPlans", returnPlans);
     return returnPlans;
@@ -1905,6 +1940,35 @@ ${errorTask?.errorStack}`);
     this.afterTaskRunning(executor);
     return output;
   }
+  async aiBoolean(prompt) {
+    const { output, executor } = await this.taskExecutor.boolean(prompt);
+    this.afterTaskRunning(executor);
+    return output;
+  }
+  async aiNumber(prompt) {
+    const { output, executor } = await this.taskExecutor.number(prompt);
+    this.afterTaskRunning(executor);
+    return output;
+  }
+  async aiString(prompt) {
+    const { output, executor } = await this.taskExecutor.string(prompt);
+    this.afterTaskRunning(executor);
+    return output;
+  }
+  async aiLocate(prompt, opt) {
+    const detailedLocateParam = this.buildDetailedLocateParam(prompt, opt);
+    const plans = buildPlans("Locate", detailedLocateParam);
+    const { executor, output } = await this.taskExecutor.runPlans(
+      taskTitleStr("Locate", locateParamStr(detailedLocateParam)),
+      plans
+    );
+    this.afterTaskRunning(executor);
+    const { element } = output;
+    return {
+      rect: element?.rect,
+      center: element?.center
+    };
+  }
   async aiAssert(assertion, msg, opt) {
     const { output, executor } = await this.taskExecutor.assert(assertion);
     this.afterTaskRunning(executor, true);