@midscene/web 0.8.6 → 0.8.7

package/dist/es/playground.js

@@ -831,26 +831,42 @@ var PageTaskExecutor = class {
           param: plan2.param,
           thought: plan2.thought,
           locate: plan2.locate,
-          executor: async (taskParam) => {
-            const scrollToEventName = taskParam.scrollType;
-            switch (scrollToEventName) {
-              case "scrollUntilTop":
-                await this.page.scrollUntilTop();
-                break;
-              case "scrollUntilBottom":
-                await this.page.scrollUntilBottom();
-                break;
-              case "scrollUpOneScreen":
-                await this.page.scrollUpOneScreen();
-                break;
-              case "scrollDownOneScreen":
-                await this.page.scrollDownOneScreen();
-                break;
-              default:
-                console.error(
-                  "Unknown scroll event type:",
-                  scrollToEventName
+          executor: async (taskParam, { element }) => {
+            if (element) {
+              await this.page.mouse.move(
+                element.center[0],
+                element.center[1]
+              );
+            }
+            const scrollToEventName = taskParam == null ? void 0 : taskParam.scrollType;
+            if (scrollToEventName === "untilTop") {
+              await this.page.scrollUntilTop();
+            } else if (scrollToEventName === "untilBottom") {
+              await this.page.scrollUntilBottom();
+            } else if (scrollToEventName === "untilRight") {
+              await this.page.scrollUntilRight();
+            } else if (scrollToEventName === "untilLeft") {
+              await this.page.scrollUntilLeft();
+            } else if (scrollToEventName === "once") {
+              if (taskParam.direction === "down") {
+                await this.page.scrollDown(taskParam.distance || void 0);
+              } else if (taskParam.direction === "up") {
+                await this.page.scrollUp(taskParam.distance || void 0);
+              } else if (taskParam.direction === "left") {
+                await this.page.scrollLeft(taskParam.distance || void 0);
+              } else if (taskParam.direction === "right") {
+                await this.page.scrollRight(taskParam.distance || void 0);
+              } else {
+                throw new Error(
+                  `Unknown scroll direction: ${taskParam.direction}`
                 );
+              }
+            } else {
+              throw new Error(
+                `Unknown scroll event type: ${scrollToEventName}, taskParam: ${JSON.stringify(
+                  taskParam
+                )}`
+              );
             }
           }
         };
@@ -879,6 +895,17 @@ var PageTaskExecutor = class {
           }
         };
         tasks.push(taskActionError);
+      } else if (plan2.type === "FalsyConditionStatement") {
+        const taskActionFalsyConditionStatement = {
+          type: "Action",
+          subType: "FalsyConditionStatement",
+          param: null,
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async () => {
+          }
+        };
+        tasks.push(taskActionFalsyConditionStatement);
       } else {
         throw new Error(`Unknown or unsupported task type: ${plan2.type}`);
       }
@@ -998,6 +1025,9 @@ var PageTaskExecutor = class {
         const errorMsg = "Replanning too many times, please split the task into multiple steps";
         return this.appendErrorPlan(taskExecutor, errorMsg);
       }
+      if (replanCount > 0) {
+        await (0, import_utils5.sleep)(300);
+      }
       await taskExecutor.append(planningTask);
       const planResult = await taskExecutor.flush();
       if (taskExecutor.isInErrorState()) {
@@ -1367,11 +1397,23 @@ var StaticPage = class {
   async scrollUntilBottom() {
     return ThrowNotImplemented("scrollUntilBottom");
   }
-  async scrollUpOneScreen() {
-    return ThrowNotImplemented("scrollUpOneScreen");
+  async scrollUntilLeft() {
+    return ThrowNotImplemented("scrollUntilLeft");
+  }
+  async scrollUntilRight() {
+    return ThrowNotImplemented("scrollUntilRight");
+  }
+  async scrollUp(distance) {
+    return ThrowNotImplemented("scrollUp");
+  }
+  async scrollDown(distance) {
+    return ThrowNotImplemented("scrollDown");
+  }
+  async scrollLeft(distance) {
+    return ThrowNotImplemented("scrollLeft");
   }
-  async scrollDownOneScreen() {
-    return ThrowNotImplemented("scrollDownOneScreen");
+  async scrollRight(distance) {
+    return ThrowNotImplemented("scrollRight");
   }
   async clearInput() {
     return ThrowNotImplemented("clearInput");

package/dist/es/playwright.js

@@ -721,26 +721,42 @@ var PageTaskExecutor = class {
           param: plan2.param,
           thought: plan2.thought,
           locate: plan2.locate,
-          executor: async (taskParam) => {
-            const scrollToEventName = taskParam.scrollType;
-            switch (scrollToEventName) {
-              case "scrollUntilTop":
-                await this.page.scrollUntilTop();
-                break;
-              case "scrollUntilBottom":
-                await this.page.scrollUntilBottom();
-                break;
-              case "scrollUpOneScreen":
-                await this.page.scrollUpOneScreen();
-                break;
-              case "scrollDownOneScreen":
-                await this.page.scrollDownOneScreen();
-                break;
-              default:
-                console.error(
-                  "Unknown scroll event type:",
-                  scrollToEventName
+          executor: async (taskParam, { element }) => {
+            if (element) {
+              await this.page.mouse.move(
+                element.center[0],
+                element.center[1]
+              );
+            }
+            const scrollToEventName = taskParam == null ? void 0 : taskParam.scrollType;
+            if (scrollToEventName === "untilTop") {
+              await this.page.scrollUntilTop();
+            } else if (scrollToEventName === "untilBottom") {
+              await this.page.scrollUntilBottom();
+            } else if (scrollToEventName === "untilRight") {
+              await this.page.scrollUntilRight();
+            } else if (scrollToEventName === "untilLeft") {
+              await this.page.scrollUntilLeft();
+            } else if (scrollToEventName === "once") {
+              if (taskParam.direction === "down") {
+                await this.page.scrollDown(taskParam.distance || void 0);
+              } else if (taskParam.direction === "up") {
+                await this.page.scrollUp(taskParam.distance || void 0);
+              } else if (taskParam.direction === "left") {
+                await this.page.scrollLeft(taskParam.distance || void 0);
+              } else if (taskParam.direction === "right") {
+                await this.page.scrollRight(taskParam.distance || void 0);
+              } else {
+                throw new Error(
+                  `Unknown scroll direction: ${taskParam.direction}`
                 );
+              }
+            } else {
+              throw new Error(
+                `Unknown scroll event type: ${scrollToEventName}, taskParam: ${JSON.stringify(
+                  taskParam
+                )}`
+              );
             }
           }
         };
@@ -769,6 +785,17 @@ var PageTaskExecutor = class {
           }
         };
         tasks.push(taskActionError);
+      } else if (plan2.type === "FalsyConditionStatement") {
+        const taskActionFalsyConditionStatement = {
+          type: "Action",
+          subType: "FalsyConditionStatement",
+          param: null,
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async () => {
+          }
+        };
+        tasks.push(taskActionFalsyConditionStatement);
       } else {
         throw new Error(`Unknown or unsupported task type: ${plan2.type}`);
       }
@@ -888,6 +915,9 @@ var PageTaskExecutor = class {
         const errorMsg = "Replanning too many times, please split the task into multiple steps";
         return this.appendErrorPlan(taskExecutor, errorMsg);
       }
+      if (replanCount > 0) {
+        await (0, import_utils3.sleep)(300);
+      }
       await taskExecutor.append(planningTask);
       const planResult = await taskExecutor.flush();
       if (taskExecutor.isInErrorState()) {
@@ -1573,15 +1603,31 @@ var Page = class {
   scrollUntilBottom() {
     return this.mouse.wheel(0, 9999999);
   }
-  async scrollUpOneScreen() {
+  scrollUntilLeft() {
+    return this.mouse.wheel(-9999999, 0);
+  }
+  scrollUntilRight() {
+    return this.mouse.wheel(9999999, 0);
+  }
+  async scrollUp(distance) {
     const innerHeight = await this.evaluate(() => window.innerHeight);
-    const distance = innerHeight * 0.7;
-    await this.mouse.wheel(0, -distance);
+    const scrollDistance = distance || innerHeight * 0.7;
+    await this.mouse.wheel(0, -scrollDistance);
   }
-  async scrollDownOneScreen() {
+  async scrollDown(distance) {
     const innerHeight = await this.evaluate(() => window.innerHeight);
-    const distance = innerHeight * 0.7;
-    await this.mouse.wheel(0, distance);
+    const scrollDistance = distance || innerHeight * 0.7;
+    await this.mouse.wheel(0, scrollDistance);
+  }
+  async scrollLeft(distance) {
+    const innerWidth = await this.evaluate(() => window.innerWidth);
+    const scrollDistance = distance || innerWidth * 0.7;
+    await this.mouse.wheel(-scrollDistance, 0);
+  }
+  async scrollRight(distance) {
+    const innerWidth = await this.evaluate(() => window.innerWidth);
+    const scrollDistance = distance || innerWidth * 0.7;
+    await this.mouse.wheel(scrollDistance, 0);
   }
   async destroy() {
   }

package/dist/es/puppeteer.js

@@ -717,26 +717,42 @@ var PageTaskExecutor = class {
           param: plan2.param,
           thought: plan2.thought,
           locate: plan2.locate,
-          executor: async (taskParam) => {
-            const scrollToEventName = taskParam.scrollType;
-            switch (scrollToEventName) {
-              case "scrollUntilTop":
-                await this.page.scrollUntilTop();
-                break;
-              case "scrollUntilBottom":
-                await this.page.scrollUntilBottom();
-                break;
-              case "scrollUpOneScreen":
-                await this.page.scrollUpOneScreen();
-                break;
-              case "scrollDownOneScreen":
-                await this.page.scrollDownOneScreen();
-                break;
-              default:
-                console.error(
-                  "Unknown scroll event type:",
-                  scrollToEventName
+          executor: async (taskParam, { element }) => {
+            if (element) {
+              await this.page.mouse.move(
+                element.center[0],
+                element.center[1]
+              );
+            }
+            const scrollToEventName = taskParam == null ? void 0 : taskParam.scrollType;
+            if (scrollToEventName === "untilTop") {
+              await this.page.scrollUntilTop();
+            } else if (scrollToEventName === "untilBottom") {
+              await this.page.scrollUntilBottom();
+            } else if (scrollToEventName === "untilRight") {
+              await this.page.scrollUntilRight();
+            } else if (scrollToEventName === "untilLeft") {
+              await this.page.scrollUntilLeft();
+            } else if (scrollToEventName === "once") {
+              if (taskParam.direction === "down") {
+                await this.page.scrollDown(taskParam.distance || void 0);
+              } else if (taskParam.direction === "up") {
+                await this.page.scrollUp(taskParam.distance || void 0);
+              } else if (taskParam.direction === "left") {
+                await this.page.scrollLeft(taskParam.distance || void 0);
+              } else if (taskParam.direction === "right") {
+                await this.page.scrollRight(taskParam.distance || void 0);
+              } else {
+                throw new Error(
+                  `Unknown scroll direction: ${taskParam.direction}`
                 );
+              }
+            } else {
+              throw new Error(
+                `Unknown scroll event type: ${scrollToEventName}, taskParam: ${JSON.stringify(
+                  taskParam
+                )}`
+              );
             }
           }
         };
@@ -765,6 +781,17 @@ var PageTaskExecutor = class {
           }
         };
         tasks.push(taskActionError);
+      } else if (plan2.type === "FalsyConditionStatement") {
+        const taskActionFalsyConditionStatement = {
+          type: "Action",
+          subType: "FalsyConditionStatement",
+          param: null,
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async () => {
+          }
+        };
+        tasks.push(taskActionFalsyConditionStatement);
       } else {
         throw new Error(`Unknown or unsupported task type: ${plan2.type}`);
       }
@@ -884,6 +911,9 @@ var PageTaskExecutor = class {
         const errorMsg = "Replanning too many times, please split the task into multiple steps";
         return this.appendErrorPlan(taskExecutor, errorMsg);
       }
+      if (replanCount > 0) {
+        await (0, import_utils3.sleep)(300);
+      }
       await taskExecutor.append(planningTask);
       const planResult = await taskExecutor.flush();
       if (taskExecutor.isInErrorState()) {
@@ -1417,15 +1447,31 @@ var Page = class {
   scrollUntilBottom() {
     return this.mouse.wheel(0, 9999999);
   }
-  async scrollUpOneScreen() {
+  scrollUntilLeft() {
+    return this.mouse.wheel(-9999999, 0);
+  }
+  scrollUntilRight() {
+    return this.mouse.wheel(9999999, 0);
+  }
+  async scrollUp(distance) {
     const innerHeight = await this.evaluate(() => window.innerHeight);
-    const distance = innerHeight * 0.7;
-    await this.mouse.wheel(0, -distance);
+    const scrollDistance = distance || innerHeight * 0.7;
+    await this.mouse.wheel(0, -scrollDistance);
   }
-  async scrollDownOneScreen() {
+  async scrollDown(distance) {
     const innerHeight = await this.evaluate(() => window.innerHeight);
-    const distance = innerHeight * 0.7;
-    await this.mouse.wheel(0, distance);
+    const scrollDistance = distance || innerHeight * 0.7;
+    await this.mouse.wheel(0, scrollDistance);
+  }
+  async scrollLeft(distance) {
+    const innerWidth = await this.evaluate(() => window.innerWidth);
+    const scrollDistance = distance || innerWidth * 0.7;
+    await this.mouse.wheel(-scrollDistance, 0);
+  }
+  async scrollRight(distance) {
+    const innerWidth = await this.evaluate(() => window.innerWidth);
+    const scrollDistance = distance || innerWidth * 0.7;
+    await this.mouse.wheel(scrollDistance, 0);
   }
   async destroy() {
   }

package/dist/es/ui-utils.js

@@ -28,7 +28,7 @@ function typeStr(task) {
   return task.subType ? `${task.type} / ${task.subType || ""}` : task.type;
 }
 function paramStr(task) {
-  var _a, _b, _c, _d, _e, _f, _g, _h;
+  var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k;
   let value;
   if (task.type === "Planning") {
     value = (_a = task == null ? void 0 : task.param) == null ? void 0 : _a.userPrompt;
@@ -38,10 +38,15 @@ function paramStr(task) {
   }
   if (task.type === "Action") {
     const sleepMs = (_f = task == null ? void 0 : task.param) == null ? void 0 : _f.timeMs;
+    const scrollType = (_g = task == null ? void 0 : task.param) == null ? void 0 : _g.scrollType;
     if (sleepMs) {
       value = `${sleepMs}ms`;
+    } else if (scrollType) {
+      const scrollDirection = (_h = task == null ? void 0 : task.param) == null ? void 0 : _h.direction;
+      const scrollDistance = (_i = task == null ? void 0 : task.param) == null ? void 0 : _i.distance;
+      value = `${scrollDirection}, ${scrollType}, ${scrollDistance || "distance-not-set"}`;
     } else {
-      value = ((_g = task == null ? void 0 : task.param) == null ? void 0 : _g.value) || ((_h = task == null ? void 0 : task.param) == null ? void 0 : _h.scrollType);
+      value = ((_j = task == null ? void 0 : task.param) == null ? void 0 : _j.value) || ((_k = task == null ? void 0 : task.param) == null ? void 0 : _k.scrollType);
     }
     if (!value) {
       value = task.thought;

package/dist/lib/appium.js

@@ -1235,26 +1235,42 @@ var PageTaskExecutor = class {
           param: plan2.param,
           thought: plan2.thought,
           locate: plan2.locate,
-          executor: async (taskParam) => {
-            const scrollToEventName = taskParam.scrollType;
-            switch (scrollToEventName) {
-              case "scrollUntilTop":
-                await this.page.scrollUntilTop();
-                break;
-              case "scrollUntilBottom":
-                await this.page.scrollUntilBottom();
-                break;
-              case "scrollUpOneScreen":
-                await this.page.scrollUpOneScreen();
-                break;
-              case "scrollDownOneScreen":
-                await this.page.scrollDownOneScreen();
-                break;
-              default:
-                console.error(
-                  "Unknown scroll event type:",
-                  scrollToEventName
+          executor: async (taskParam, { element }) => {
+            if (element) {
+              await this.page.mouse.move(
+                element.center[0],
+                element.center[1]
+              );
+            }
+            const scrollToEventName = taskParam == null ? void 0 : taskParam.scrollType;
+            if (scrollToEventName === "untilTop") {
+              await this.page.scrollUntilTop();
+            } else if (scrollToEventName === "untilBottom") {
+              await this.page.scrollUntilBottom();
+            } else if (scrollToEventName === "untilRight") {
+              await this.page.scrollUntilRight();
+            } else if (scrollToEventName === "untilLeft") {
+              await this.page.scrollUntilLeft();
+            } else if (scrollToEventName === "once") {
+              if (taskParam.direction === "down") {
+                await this.page.scrollDown(taskParam.distance || void 0);
+              } else if (taskParam.direction === "up") {
+                await this.page.scrollUp(taskParam.distance || void 0);
+              } else if (taskParam.direction === "left") {
+                await this.page.scrollLeft(taskParam.distance || void 0);
+              } else if (taskParam.direction === "right") {
+                await this.page.scrollRight(taskParam.distance || void 0);
+              } else {
+                throw new Error(
+                  `Unknown scroll direction: ${taskParam.direction}`
                 );
+              }
+            } else {
+              throw new Error(
+                `Unknown scroll event type: ${scrollToEventName}, taskParam: ${JSON.stringify(
+                  taskParam
+                )}`
+              );
             }
           }
         };
@@ -1283,6 +1299,17 @@ var PageTaskExecutor = class {
           }
         };
         tasks.push(taskActionError);
+      } else if (plan2.type === "FalsyConditionStatement") {
+        const taskActionFalsyConditionStatement = {
+          type: "Action",
+          subType: "FalsyConditionStatement",
+          param: null,
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async () => {
+          }
+        };
+        tasks.push(taskActionFalsyConditionStatement);
       } else {
         throw new Error(`Unknown or unsupported task type: ${plan2.type}`);
       }
@@ -1402,6 +1429,9 @@ var PageTaskExecutor = class {
         const errorMsg = "Replanning too many times, please split the task into multiple steps";
         return this.appendErrorPlan(taskExecutor, errorMsg);
       }
+      if (replanCount > 0) {
+        await (0, import_utils3.sleep)(300);
+      }
       await taskExecutor.append(planningTask);
       const planResult = await taskExecutor.flush();
       if (taskExecutor.isInErrorState()) {
@@ -2116,24 +2146,48 @@ var Page = class {
     return "";
   }
   // Scroll to top element
-  async scrollUntilTop() {
+  async scrollUntilTop(distance) {
     const { height } = await this.browser.getWindowSize();
-    await this.mouseWheel(0, height, 100);
+    const scrollDistance = distance || height * 0.7;
+    await this.mouseWheel(0, -scrollDistance, 100);
   }
   // Scroll to bottom element
-  async scrollUntilBottom() {
+  async scrollUntilBottom(distance) {
     const { height } = await this.browser.getWindowSize();
-    await this.mouseWheel(0, -height, 100);
+    const scrollDistance = distance || height * 0.7;
+    await this.mouseWheel(0, scrollDistance, 100);
+  }
+  async scrollUntilLeft(distance) {
+    const { width } = await this.browser.getWindowSize();
+    const scrollDistance = distance || width * 0.7;
+    await this.mouseWheel(-scrollDistance, 0, 100);
+  }
+  async scrollUntilRight(distance) {
+    const { width } = await this.browser.getWindowSize();
+    const scrollDistance = distance || width * 0.7;
+    await this.mouseWheel(scrollDistance, 0, 100);
   }
   // Scroll up one screen
-  async scrollUpOneScreen() {
+  async scrollUp(distance) {
     const { height } = await this.browser.getWindowSize();
-    await this.mouseWheel(0, height, 1e3);
+    const scrollDistance = distance || height * 0.7;
+    await this.mouseWheel(0, -scrollDistance, 1e3);
   }
   // Scroll down one screen
-  async scrollDownOneScreen() {
+  async scrollDown(distance) {
     const { height } = await this.browser.getWindowSize();
-    await this.mouseWheel(0, -height, 1e3);
+    const scrollDistance = distance || height * 0.7;
+    await this.mouseWheel(0, scrollDistance, 1e3);
+  }
+  async scrollLeft(distance) {
+    const { width } = await this.browser.getWindowSize();
+    const scrollDistance = distance || width * 0.7;
+    await this.mouseWheel(-scrollDistance, 0, 1e3);
+  }
+  async scrollRight(distance) {
+    const { width } = await this.browser.getWindowSize();
+    const scrollDistance = distance || width * 0.7;
+    await this.mouseWheel(scrollDistance, 0, 1e3);
   }
   async keyboardType(text) {
     const actions = [];