@midscene/web 0.11.3 → 0.11.4-beta-20250220011346.0

package/dist/es/appium.js

@@ -452,11 +452,15 @@ var ScriptPlayer = class {
       } else if (flowItem.sleep) {
         const sleepTask = flowItem;
         const ms = sleepTask.sleep;
+        let msNumber = ms;
+        if (typeof ms === "string") {
+          msNumber = Number.parseInt(ms, 10);
+        }
         (0, import_node_assert.default)(
-          ms && ms > 0,
+          msNumber && msNumber > 0,
           `ms for sleep must be greater than 0, but got ${ms}`
         );
-        await new Promise((resolve) => setTimeout(resolve, ms));
+        await new Promise((resolve) => setTimeout(resolve, msNumber));
       } else {
         throw new Error(`unknown flowItem: ${JSON.stringify(flowItem)}`);
       }
@@ -3243,13 +3247,15 @@ function parseYamlScript(content, filePath, ignoreCheckingTarget) {
 }
 
 // src/common/agent.ts
-var import_env3 = require("@midscene/core/env");
+var import_env5 = require("@midscene/core/env");
 var import_utils6 = require("@midscene/core/utils");
 
 // src/common/tasks.ts
 var import_node_assert3 = __toESM(require("assert"));
 var import_core = require("@midscene/core");
 var import_ai_model = require("@midscene/core/ai-model");
+var import_env2 = require("@midscene/core/env");
+var import_env3 = require("@midscene/core/env");
 var import_utils3 = require("@midscene/core/utils");
 
 // src/common/task-cache.ts
@@ -3422,7 +3428,7 @@ function paramStr(task) {
   var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k;
   let value;
   if (task.type === "Planning") {
-    value = (_a = task == null ? void 0 : task.param) == null ? void 0 : _a.userPrompt;
+    value = (_a = task == null ? void 0 : task.param) == null ? void 0 : _a.userInstruction;
   }
   if (task.type === "Insight") {
     value = ((_b = task == null ? void 0 : task.param) == null ? void 0 : _b.prompt) || ((_c = task == null ? void 0 : task.param) == null ? void 0 : _c.id) || ((_d = task == null ? void 0 : task.param) == null ? void 0 : _d.dataDemand) || ((_e = task == null ? void 0 : task.param) == null ? void 0 : _e.assertion);
@@ -3449,6 +3455,7 @@ function paramStr(task) {
 }
 
 // src/common/tasks.ts
+var replanningCountLimit = 10;
 var PageTaskExecutor = class {
   constructor(page, insight, opts) {
     this.conversationHistory = [];
@@ -3507,9 +3514,9 @@ var PageTaskExecutor = class {
   async convertPlanToExecutable(plans, cacheGroup) {
     const tasks = [];
     plans.forEach((plan2) => {
-      var _a, _b;
+      var _a, _b, _c;
       if (plan2.type === "Locate") {
-        if (((_a = plan2.locate) == null ? void 0 : _a.id) === null || ((_b = plan2.locate) == null ? void 0 : _b.id) === "null") {
+        if (plan2.locate === null || ((_a = plan2.locate) == null ? void 0 : _a.id) === null || ((_b = plan2.locate) == null ? void 0 : _b.id) === "null") {
           return;
         }
         const taskFind = {
@@ -3521,12 +3528,15 @@ var PageTaskExecutor = class {
           executor: async (param, taskContext) => {
             const { task } = taskContext;
             (0, import_node_assert3.default)(
-              (param == null ? void 0 : param.prompt) || (param == null ? void 0 : param.id) || (param == null ? void 0 : param.position),
-              "No prompt or id or position to locate"
+              (param == null ? void 0 : param.prompt) || (param == null ? void 0 : param.id) || (param == null ? void 0 : param.position) || (param == null ? void 0 : param.bbox),
+              "No prompt or id or position or bbox to locate"
             );
             let insightDump;
+            let usage;
             const dumpCollector = (dump2) => {
+              var _a2;
               insightDump = dump2;
+              usage = (_a2 = dump2 == null ? void 0 : dump2.taskInfo) == null ? void 0 : _a2.usage;
             };
             this.insight.onceDumpUpdatedFn = dumpCollector;
             const shotTime = Date.now();
@@ -3546,7 +3556,8 @@ var PageTaskExecutor = class {
             const callAI = this.insight.aiVendorFn;
             const quickAnswer = {
               id: param == null ? void 0 : param.id,
-              position: param == null ? void 0 : param.position
+              position: param == null ? void 0 : param.position,
+              bbox: param == null ? void 0 : param.bbox
             };
             const startTime = Date.now();
             const element = await this.insight.locate(param.prompt, {
@@ -3556,8 +3567,8 @@ var PageTaskExecutor = class {
                   locateResult = locateCache;
                   return Promise.resolve({ content: locateCache });
                 }
-                const { content: aiResult, usage } = await callAI(...message);
-                return { content: aiResult, usage };
+                const { content: aiResult, usage: usage2 } = await callAI(...message);
+                return { content: aiResult, usage: usage2 };
               }
             });
             const aiCost = Date.now() - startTime;
@@ -3590,7 +3601,8 @@ var PageTaskExecutor = class {
                 hit: Boolean(locateCache)
               },
               recorder: [recordItem],
-              aiCost
+              aiCost,
+              usage
             };
           }
         };
@@ -3789,12 +3801,12 @@ var PageTaskExecutor = class {
           }
         };
         tasks.push(taskActionError);
-      } else if (plan2.type === "FalsyConditionStatement") {
+      } else if (plan2.type === "ExpectedFalsyCondition") {
         const taskActionFalsyConditionStatement = {
           type: "Action",
-          subType: "FalsyConditionStatement",
+          subType: "ExpectedFalsyCondition",
           param: null,
-          thought: plan2.thought,
+          thought: (_c = plan2.param) == null ? void 0 : _c.reason,
           locate: plan2.locate,
           executor: async () => {
           }
@@ -3830,14 +3842,13 @@ var PageTaskExecutor = class {
       tasks: wrappedTasks
     };
   }
-  planningTaskFromPrompt(userPrompt, cacheGroup, whatHaveDone, originalPrompt) {
+  planningTaskFromPrompt(userInstruction, cacheGroup, log) {
     const task = {
       type: "Planning",
       locate: null,
       param: {
-        userPrompt,
-        whatHaveDone,
-        originalPrompt
+        userInstruction,
+        log
       },
       executor: async (param, executorContext) => {
         const shotTime = Date.now();
@@ -3850,80 +3861,101 @@ var PageTaskExecutor = class {
         };
         executorContext.task.recorder = [recordItem];
         executorContext.task.pageContext = pageContext;
-        const planCache = cacheGroup.readCache(pageContext, "plan", userPrompt);
+        const qwenMode = (0, import_env2.getAIConfigInBoolean)(import_env3.MIDSCENE_USE_QWEN_VL);
+        const planCache = cacheGroup.readCache(
+          pageContext,
+          "plan",
+          param.userInstruction
+        );
         let planResult;
-        if (planCache) {
+        if (planCache && !qwenMode) {
           planResult = planCache;
         } else {
-          planResult = await (0, import_core.plan)(param.userPrompt, {
+          planResult = await (0, import_core.plan)(param.userInstruction, {
             context: pageContext,
-            whatHaveDone: param.whatHaveDone,
-            originalPrompt: param.originalPrompt
+            log: param.log
           });
         }
-        const { actions, furtherPlan, taskWillBeAccomplished, error } = planResult;
+        const { actions, log: log2, finish, error, usage, rawResponse, sleep: sleep3 } = planResult;
         let stopCollecting = false;
+        let bboxCollected = false;
+        let planParsingError = "";
         const finalActions = actions.reduce(
           (acc, planningAction) => {
             if (stopCollecting) {
               return acc;
             }
             if (planningAction.locate) {
+              if (bboxCollected && planningAction.locate.bbox) {
+                delete planningAction.locate.bbox;
+              }
+              if (planningAction.locate.bbox) {
+                bboxCollected = true;
+              }
               acc.push({
                 type: "Locate",
                 locate: planningAction.locate,
-                // remove id from planning, since the result is not accurate
-                // locate: {
-                //   prompt: planningAction.locate.prompt,
-                // },
                 param: null,
                 thought: planningAction.locate.prompt
               });
             } else if (["Tap", "Hover", "Input"].includes(planningAction.type)) {
+              planParsingError = `invalid planning response: ${JSON.stringify(planningAction)}`;
               stopCollecting = true;
               return acc;
             }
             acc.push(planningAction);
+            if (planResult.sleep) {
+              acc.push({
+                type: "Sleep",
+                param: {
+                  timeMs: planResult.sleep
+                },
+                locate: null
+              });
+            }
             return acc;
           },
           []
         );
-        (0, import_node_assert3.default)(
-          finalActions.length > 0,
-          error ? `No plan: ${error}` : "No plans found"
-        );
+        if (finalActions.length === 0) {
+          (0, import_node_assert3.default)(
+            finish,
+            error ? `Failed to plan: ${error}` : planParsingError || "No plan found"
+          );
+        }
         cacheGroup.saveCache({
           type: "plan",
           pageContext: {
             url: pageContext.url,
             size: pageContext.size
           },
-          prompt: userPrompt,
+          prompt: userInstruction,
           response: planResult
         });
         return {
           output: {
             actions: finalActions,
-            taskWillBeAccomplished: false,
-            furtherPlan
+            finish,
+            log: log2
           },
           cache: {
             hit: Boolean(planCache)
           },
           pageContext,
-          // ?
-          recorder: [recordItem]
+          recorder: [recordItem],
+          usage,
+          rawResponse
         };
       }
     };
     return task;
   }
-  planningTaskToGoal(userPrompt, cacheGroup) {
+  planningTaskToGoal(userInstruction, cacheGroup) {
     const task = {
       type: "Planning",
       locate: null,
       param: {
-        userPrompt
+        userInstruction
       },
       executor: async (param, executorContext) => {
         var _a;
@@ -3952,14 +3984,14 @@ var PageTaskExecutor = class {
         const planCache = cacheGroup.readCache(
           pageContext,
           "ui-tars-plan",
-          userPrompt
+          userInstruction
         );
         let planResult;
         if (planCache) {
           planResult = planCache;
         } else {
           planResult = await (0, import_ai_model.vlmPlanning)({
-            userInstruction: param.userPrompt,
+            userInstruction: param.userInstruction,
             conversationHistory: this.conversationHistory,
             size: pageContext.size
           });
@@ -3970,7 +4002,7 @@ var PageTaskExecutor = class {
             url: pageContext.url,
             size: pageContext.size
           },
-          prompt: userPrompt,
+          prompt: userInstruction,
           response: planResult
         });
         const aiCost = Date.now() - startTime;
@@ -3984,11 +4016,8 @@ var PageTaskExecutor = class {
             actions,
             thought: (_a = actions[0]) == null ? void 0 : _a.thought,
             actionType: actions[0].type,
-            taskWillBeAccomplished: false,
-            furtherPlan: {
-              whatToDoNext: "",
-              whatHaveDone: ""
-            }
+            finish: false,
+            log: ""
           },
           cache: {
             hit: Boolean(planCache)
@@ -4000,23 +4029,19 @@ var PageTaskExecutor = class {
     return task;
   }
   async action(userPrompt, options) {
-    var _a;
     const taskExecutor = new import_core.Executor(userPrompt, void 0, void 0, {
       onTaskStart: options == null ? void 0 : options.onTaskStart
     });
     const cacheGroup = this.taskCache.getCacheGroupByPrompt(userPrompt);
-    const originalPrompt = userPrompt;
-    let planningTask = this.planningTaskFromPrompt(originalPrompt, cacheGroup);
+    let planningTask = this.planningTaskFromPrompt(userPrompt, cacheGroup);
     let result;
     let replanCount = 0;
+    const logLog = [];
     while (planningTask) {
-      if (replanCount > 5) {
+      if (replanCount > replanningCountLimit) {
         const errorMsg = "Replanning too many times, please split the task into multiple steps";
         return this.appendErrorPlan(taskExecutor, errorMsg);
       }
-      if (replanCount > 0) {
-        await (0, import_utils3.sleep)(300);
-      }
       await taskExecutor.append(planningTask);
       const planResult = await taskExecutor.flush();
       if (taskExecutor.isInErrorState()) {
@@ -4045,18 +4070,19 @@ var PageTaskExecutor = class {
           executor: taskExecutor
         };
       }
-      if ((_a = planResult.furtherPlan) == null ? void 0 : _a.whatToDoNext) {
-        planningTask = this.planningTaskFromPrompt(
-          planResult.furtherPlan.whatToDoNext,
-          cacheGroup,
-          planResult.furtherPlan.whatHaveDone,
-          originalPrompt
-        );
-        replanCount++;
-      } else {
+      if (planResult == null ? void 0 : planResult.log) {
+        logLog.push(planResult.log);
+      }
+      if (planResult.finish) {
         planningTask = null;
         break;
       }
+      planningTask = this.planningTaskFromPrompt(
+        userPrompt,
+        cacheGroup,
+        logLog.join("\n")
+      );
+      replanCount++;
     }
     return {
       output: result,
@@ -4210,7 +4236,9 @@ var PageTaskExecutor = class {
   }
   async waitFor(assertion, opt) {
     const description = `waitFor: ${assertion}`;
-    const taskExecutor = new import_core.Executor(description);
+    const taskExecutor = new import_core.Executor(description, void 0, void 0, {
+      onTaskStart: opt.onTaskStart
+    });
     const { timeoutMs, checkIntervalMs } = opt;
     (0, import_node_assert3.default)(assertion, "No assertion for waitFor");
     (0, import_node_assert3.default)(timeoutMs, "No timeoutMs for waitFor");
@@ -4295,7 +4323,7 @@ var WebElementInfo = class {
 var import_node_assert4 = __toESM(require("assert"));
 var import_node_fs3 = require("fs");
 var import_node_path3 = __toESM(require("path"));
-var import_env2 = require("@midscene/core/env");
+var import_env4 = require("@midscene/core/env");
 var import_utils4 = require("@midscene/core/utils");
 var import_constants = require("@midscene/shared/constants");
 var import_extractor = require("@midscene/shared/extractor");
@@ -4342,14 +4370,16 @@ async function parseContextFromWebPage(page, _opt) {
     }
   );
   const size = await page.size();
+  if (size.dpr && size.dpr > 1) {
+    screenshotBase64 = await (0, import_img.resizeImgBase64)(screenshotBase64, {
+      width: size.width,
+      height: size.height
+    });
+  }
   let screenshotBase64WithElementMarker = screenshotBase64;
-  if (!(0, import_env2.getAIConfig)(import_env2.MIDSCENE_USE_VLM_UI_TARS)) {
+  if (!(0, import_env4.getAIConfig)(import_env4.MIDSCENE_USE_VLM_UI_TARS)) {
     if (_opt == null ? void 0 : _opt.ignoreMarker) {
-      screenshotBase64WithElementMarker = await (0, import_img.compositeElementInfoImg)({
-        inputImgBase64: screenshotBase64,
-        elementsPositionInfo: [],
-        size
-      });
+      screenshotBase64WithElementMarker = screenshotBase64;
     } else {
       screenshotBase64WithElementMarker = await (0, import_img.compositeElementInfoImg)({
         inputImgBase64: screenshotBase64,
@@ -4368,7 +4398,7 @@ async function parseContextFromWebPage(page, _opt) {
   };
 }
 function reportFileName(tag = "web") {
-  const reportTagName = (0, import_env2.getAIConfig)(import_env2.MIDSCENE_REPORT_TAG_NAME);
+  const reportTagName = (0, import_env4.getAIConfig)(import_env4.MIDSCENE_REPORT_TAG_NAME);
   const dateTimeInFileName = (0, import_dayjs.default)().format("YYYY-MM-DD_HH-mm-ss-SSS");
   return `${reportTagName || tag}-${dateTimeInFileName}`;
 }
@@ -4421,7 +4451,9 @@ var PageAgent = class {
         ignoreMarker: true
       });
     }
-    return await parseContextFromWebPage(this.page);
+    return await parseContextFromWebPage(this.page, {
+      ignoreMarker: (0, import_env5.getAIConfigInBoolean)(import_env5.MATCH_BY_POSITION)
+    });
   }
   resetDump() {
     this.dump = {
@@ -4468,7 +4500,7 @@ var PageAgent = class {
     }
   }
   async aiAction(taskPrompt) {
-    if ((0, import_env3.getAIConfig)(import_env3.MIDSCENE_USE_VLM_UI_TARS)) {
+    if ((0, import_env5.getAIConfig)(import_env5.MIDSCENE_USE_VLM_UI_TARS)) {
       const { executor } = await this.taskExecutor.actionToGoal(taskPrompt, {
         onTaskStart: this.callbackOnTaskStartTip.bind(this)
       });
@@ -4526,7 +4558,8 @@ ${reasonMsg}`);
     const { executor } = await this.taskExecutor.waitFor(assertion, {
       timeoutMs: (opt == null ? void 0 : opt.timeoutMs) || 15 * 1e3,
       checkIntervalMs: (opt == null ? void 0 : opt.checkIntervalMs) || 3 * 1e3,
-      assertion
+      assertion,
+      onTaskStart: this.callbackOnTaskStartTip.bind(this)
     });
     this.appendExecutionDump(executor.dump());
     this.writeOutActionDumps();

package/dist/es/bridge-mode-browser.js

@@ -291,11 +291,13 @@ function sleep(ms) {
 var ChromeExtensionProxyPage = class {
   constructor(forceSameTabNavigation) {
     this.pageType = "chrome-extension-proxy";
-    this.version = "0.11.3";
+    this.version = "0.11.4-beta-20250220011346.0";
     this.activeTabId = null;
     this.tabIdOfDebuggerAttached = null;
     this.attachingDebugger = null;
     this.destroyed = false;
+    this.latestMouseX = 50;
+    this.latestMouseY = 50;
     this.mouse = {
       click: async (x, y) => {
         await this.showMousePointer(x, y);
@@ -315,8 +317,8 @@ var ChromeExtensionProxyPage = class {
         });
       },
       wheel: async (deltaX, deltaY, startX, startY) => {
-        const finalX = startX || 50;
-        const finalY = startY || 50;
+        const finalX = startX || this.latestMouseX;
+        const finalY = startY || this.latestMouseY;
         await this.showMousePointer(finalX, finalY);
         await this.sendCommandToDebugger("Input.dispatchMouseEvent", {
           type: "mouseWheel",
@@ -325,6 +327,8 @@ var ChromeExtensionProxyPage = class {
           deltaX,
           deltaY
         });
+        this.latestMouseX = finalX;
+        this.latestMouseY = finalY;
       },
       move: async (x, y) => {
         await this.showMousePointer(x, y);
@@ -333,6 +337,8 @@ var ChromeExtensionProxyPage = class {
           x,
           y
         });
+        this.latestMouseX = x;
+        this.latestMouseY = y;
       },
       drag: async (from, to) => {
         await this.mouse.move(from.x, from.y);
@@ -690,7 +696,7 @@ var BridgeClient = class {
       this.socket = (0, import_socket.io)(this.endpoint, {
         reconnection: false,
         query: {
-          version: "0.11.3"
+          version: "0.11.4-beta-20250220011346.0"
         }
       });
       const timeout = setTimeout(() => {
@@ -812,7 +818,7 @@ var ChromeExtensionPageBrowserSide = class extends ChromeExtensionProxyPage {
     );
     await this.bridgeClient.connect();
     this.onLogMessage(
-      `Bridge connected, cli-side version v${this.bridgeClient.serverVersion}, browser-side version v${"0.11.3"}`,
+      `Bridge connected, cli-side version v${this.bridgeClient.serverVersion}, browser-side version v${"0.11.4-beta-20250220011346.0"}`,
       "log"
     );
   }