@midscene/web 0.16.5 → 0.16.6

package/dist/es/chrome-extension.js

@@ -411,10 +411,7 @@ import { sleep } from "@midscene/core/utils";
 import { UITarsModelVersion } from "@midscene/shared/env";
 import { uiTarsModelVersion } from "@midscene/shared/env";
 import { vlLocateMode } from "@midscene/shared/env";
-import {
-  imageInfo,
-  resizeImgBase64 as resizeImgBase642
-} from "@midscene/shared/img";
+import { imageInfo, resizeImgBase64 as resizeImgBase642 } from "@midscene/shared/img";
 import { getDebug as getDebug2 } from "@midscene/shared/logger";
 import { assert as assert4 } from "@midscene/shared/utils";
 
@@ -1530,17 +1527,23 @@ var PageTaskExecutor = class {
       executor: taskExecutor
     };
   }
-  async query(demand) {
-    const description = typeof demand === "string" ? demand : JSON.stringify(demand);
-    const taskExecutor = new Executor(taskTitleStr("Query", description), {
-      onTaskStart: this.onTaskStartCallback
-    });
+  async createTypeQueryTask(type, demand) {
+    const taskExecutor = new Executor(
+      taskTitleStr(
+        type,
+        typeof demand === "string" ? demand : JSON.stringify(demand)
+      ),
+      {
+        onTaskStart: this.onTaskStartCallback
+      }
+    );
     const queryTask = {
       type: "Insight",
-      subType: "Query",
+      subType: type,
       locate: null,
       param: {
         dataDemand: demand
+        // for user param presentation in report right sidebar
       },
       executor: async (param) => {
         let insightDump;
@@ -1548,11 +1551,21 @@ var PageTaskExecutor = class {
           insightDump = dump;
         };
         this.insight.onceDumpUpdatedFn = dumpCollector;
-        const { data, usage } = await this.insight.extract(
-          param.dataDemand
-        );
+        const ifTypeRestricted = type !== "Query";
+        let demandInput = demand;
+        if (ifTypeRestricted) {
+          demandInput = {
+            result: `${type}, ${demand}`
+          };
+        }
+        const { data, usage } = await this.insight.extract(demandInput);
+        let outputResult = data;
+        if (ifTypeRestricted) {
+          assert4(data?.result !== void 0, "No result in query data");
+          outputResult = data.result;
+        }
         return {
-          output: data,
+          output: outputResult,
           log: { dump: insightDump },
           usage
         };
@@ -1565,6 +1578,18 @@ var PageTaskExecutor = class {
       executor: taskExecutor
     };
   }
+  async query(demand) {
+    return this.createTypeQueryTask("Query", demand);
+  }
+  async boolean(prompt) {
+    return this.createTypeQueryTask("Boolean", prompt);
+  }
+  async number(prompt) {
+    return this.createTypeQueryTask("Number", prompt);
+  }
+  async string(prompt) {
+    return this.createTypeQueryTask("String", prompt);
+  }
   async assert(assertion) {
     const description = `assert: ${assertion}`;
     const taskExecutor = new Executor(taskTitleStr("Assert", description), {
@@ -1751,6 +1776,16 @@ function buildPlans(type, locateParam, param) {
     };
     returnPlans = [sleepPlan];
   }
+  if (type === "Locate") {
+    assert5(locateParam, `missing locate info for action "${type}"`);
+    const locatePlan2 = {
+      type,
+      param: locateParam,
+      locate: locateParam,
+      thought: ""
+    };
+    returnPlans = [locatePlan2];
+  }
   if (returnPlans) {
     debug3("buildPlans", returnPlans);
     return returnPlans;
@@ -1950,6 +1985,35 @@ ${errorTask?.errorStack}`);
     this.afterTaskRunning(executor);
     return output;
   }
+  async aiBoolean(prompt) {
+    const { output, executor } = await this.taskExecutor.boolean(prompt);
+    this.afterTaskRunning(executor);
+    return output;
+  }
+  async aiNumber(prompt) {
+    const { output, executor } = await this.taskExecutor.number(prompt);
+    this.afterTaskRunning(executor);
+    return output;
+  }
+  async aiString(prompt) {
+    const { output, executor } = await this.taskExecutor.string(prompt);
+    this.afterTaskRunning(executor);
+    return output;
+  }
+  async aiLocate(prompt, opt) {
+    const detailedLocateParam = this.buildDetailedLocateParam(prompt, opt);
+    const plans = buildPlans("Locate", detailedLocateParam);
+    const { executor, output } = await this.taskExecutor.runPlans(
+      taskTitleStr("Locate", locateParamStr(detailedLocateParam)),
+      plans
+    );
+    this.afterTaskRunning(executor);
+    const { element } = output;
+    return {
+      rect: element?.rect,
+      center: element?.center
+    };
+  }
   async aiAssert(assertion, msg, opt) {
     const { output, executor } = await this.taskExecutor.assert(assertion);
     this.afterTaskRunning(executor, true);
@@ -2240,7 +2304,7 @@ function sleep2(ms) {
 var ChromeExtensionProxyPage = class {
   constructor(forceSameTabNavigation) {
     this.pageType = "chrome-extension-proxy";
-    this.version = "0.16.5";
+    this.version = "0.16.6";
     this.activeTabId = null;
     this.tabIdOfDebuggerAttached = null;
     this.attachingDebugger = null;