@midscene/web 0.3.0 → 0.3.1-beta-20240821105917.0

package/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2021-present Midscene.js
+Copyright (c) 2024-present Bytedance, Inc. and its affiliates.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

package/dist/es/index.js

@@ -483,17 +483,24 @@ async function alignElements(screenshotBuffer, elements, page) {
     return item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold;
   });
   for (const item of validElements) {
-    const { rect } = item;
+    const { rect, id, content, attributes, locator } = item;
     const aligned = await alignCoordByTrim(screenshotBuffer, rect);
+    if (aligned.width < 0)
+      continue;
     item.rect = aligned;
     item.center = [
       Math.round(aligned.left + aligned.width / 2),
       Math.round(aligned.top + aligned.height / 2)
     ];
     textsAligned.push(
-      new WebElementInfo(__spreadProps(__spreadValues({}, item), {
+      new WebElementInfo({
+        rect,
+        locator,
+        id,
+        content,
+        attributes,
         page
-      }))
+      })
     );
   }
   return textsAligned;
@@ -617,7 +624,7 @@ var PageTaskExecutor = class {
         };
         return taskFind;
       }
-      if (plan2.type === "Assert") {
+      if (plan2.type === "Assert" || plan2.type === "AssertWithoutThrow") {
         const assertPlan = plan2;
         const taskAssert = {
           type: "Insight",
@@ -634,13 +641,16 @@ var PageTaskExecutor = class {
               assertPlan.param.assertion
             );
             if (!assertion.pass) {
-              task.output = assertion;
-              task.log = {
-                dump: insightDump
-              };
-              throw new Error(
-                assertion.thought || "Assertion failed without reason"
-              );
+              if (plan2.type === "Assert") {
+                task.output = assertion;
+                task.log = {
+                  dump: insightDump
+                };
+                throw new Error(
+                  assertion.thought || "Assertion failed without reason"
+                );
+              }
+              task.error = assertion.thought;
             }
             return {
               output: assertion,
@@ -756,7 +766,19 @@ var PageTaskExecutor = class {
         return taskActionSleep;
       }
       if (plan2.type === "Error") {
-        throw new Error(`Got a task plan with type Error: ${plan2.thought}`);
+        const taskActionError = {
+          type: "Action",
+          subType: "Error",
+          param: plan2.param,
+          executor: async (taskParam) => {
+            assert2(
+              taskParam.thought,
+              "An error occurred, but no thought provided"
+            );
+            throw new Error(taskParam.thought);
+          }
+        };
+        return taskActionError;
       }
       throw new Error(`Unknown or Unsupported task type: ${plan2.type}`);
     }).map((task) => {
@@ -766,7 +788,6 @@ var PageTaskExecutor = class {
   }
   async action(userPrompt) {
     const taskExecutor = new Executor(userPrompt);
-    taskExecutor.description = userPrompt;
     let plans = [];
     const planningTask = {
       type: "Planning",
@@ -826,7 +847,6 @@ var PageTaskExecutor = class {
   async query(demand) {
     const description = typeof demand === "string" ? demand : JSON.stringify(demand);
     const taskExecutor = new Executor(description);
-    taskExecutor.description = description;
     const queryTask = {
       type: "Insight",
       subType: "Query",
@@ -854,9 +874,8 @@ var PageTaskExecutor = class {
     };
   }
   async assert(assertion) {
-    const description = assertion;
+    const description = `assert: ${assertion}`;
     const taskExecutor = new Executor(description);
-    taskExecutor.description = description;
     const assertionPlan = {
       type: "Assert",
       param: {
@@ -871,6 +890,64 @@ var PageTaskExecutor = class {
       executor: taskExecutor
     };
   }
+  async waitFor(assertion, opt) {
+    const description = `waitFor: ${assertion}`;
+    const taskExecutor = new Executor(description);
+    const { timeoutMs, checkIntervalMs } = opt;
+    assert2(assertion, "No assertion for waitFor");
+    assert2(timeoutMs, "No timeoutMs for waitFor");
+    assert2(checkIntervalMs, "No checkIntervalMs for waitFor");
+    const overallStartTime = Date.now();
+    let startTime = Date.now();
+    let errorThought = "";
+    while (Date.now() - overallStartTime < timeoutMs) {
+      startTime = Date.now();
+      const assertPlan = {
+        type: "AssertWithoutThrow",
+        param: {
+          assertion
+        }
+      };
+      const assertTask = await this.convertPlanToExecutable([assertPlan]);
+      await taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
+      const output = await taskExecutor.flush();
+      if (output.pass) {
+        return {
+          output: void 0,
+          executor: taskExecutor
+        };
+      }
+      errorThought = output.thought;
+      const now = Date.now();
+      if (now - startTime < checkIntervalMs) {
+        const timeRemaining = checkIntervalMs - (now - startTime);
+        const sleepPlan = {
+          type: "Sleep",
+          param: {
+            timeMs: timeRemaining
+          }
+        };
+        const sleepTask = await this.convertPlanToExecutable([sleepPlan]);
+        await taskExecutor.append(
+          this.wrapExecutorWithScreenshot(sleepTask[0])
+        );
+        await taskExecutor.flush();
+      }
+    }
+    const errorPlan = {
+      type: "Error",
+      param: {
+        thought: `waitFor timeout: ${errorThought}`
+      }
+    };
+    const errorTask = await this.convertPlanToExecutable([errorPlan]);
+    await taskExecutor.append(errorTask[0]);
+    await taskExecutor.flush();
+    return {
+      output: void 0,
+      executor: taskExecutor
+    };
+  }
 };
 
 // src/common/agent.ts
@@ -880,6 +957,7 @@ var PageAgent = class {
     this.opts = Object.assign(
       {
         generateReport: true,
+        autoPrintReportMsg: true,
         groupName: "Midscene Report",
         groupDescription: ""
       },
@@ -905,7 +983,7 @@ var PageAgent = class {
     return stringifyDumpData(this.dump);
   }
   writeOutActionDumps() {
-    const generateReport = this.opts.generateReport;
+    const { generateReport, autoPrintReportMsg } = this.opts;
     this.reportFile = writeLogFile({
       fileName: this.reportFileName,
       fileExt: groupedActionDumpFileExt,
@@ -913,7 +991,7 @@ var PageAgent = class {
       type: "dump",
       generateReport
     });
-    if (generateReport) {
+    if (generateReport && autoPrintReportMsg) {
       printReportMsg(this.reportFile);
     }
   }
@@ -944,11 +1022,25 @@ ${errorTask == null ? void 0 : errorTask.errorStack}`);
     this.writeOutActionDumps();
     if (!(output == null ? void 0 : output.pass)) {
       const errMsg = msg || `Assertion failed: ${assertion}`;
-      const reasonMsg = `Reason: ${output == null ? void 0 : output.thought} || (no_reason)`;
+      const reasonMsg = `Reason: ${(output == null ? void 0 : output.thought) || "(no_reason)"}`;
       throw new Error(`${errMsg}
 ${reasonMsg}`);
     }
   }
+  async aiWaitFor(assertion, opt) {
+    const { executor } = await this.taskExecutor.waitFor(assertion, {
+      timeoutMs: (opt == null ? void 0 : opt.timeoutMs) || 15 * 1e3,
+      checkIntervalMs: (opt == null ? void 0 : opt.checkIntervalMs) || 3 * 1e3,
+      assertion
+    });
+    this.appendExecutionDump(executor.dump());
+    this.writeOutActionDumps();
+    if (executor.isInErrorState()) {
+      const errorTask = executor.latestErrorTask();
+      throw new Error(`${errorTask == null ? void 0 : errorTask.error}
+${errorTask == null ? void 0 : errorTask.errorStack}`);
+    }
+  }
   async ai(taskPrompt, type = "action") {
     if (type === "action") {
       return this.aiAction(taskPrompt);
@@ -1124,6 +1216,13 @@ var PlaywrightAiFixture = () => {
         await agent.aiAssert(assertion, errorMsg);
       });
       updateDumpAnnotation(testInfo, agent.dumpDataString());
+    },
+    aiWaitFor: async ({ page }, use, testInfo) => {
+      const agent = agentForPage(page, testInfo);
+      await use(async (assertion, opt) => {
+        await agent.aiWaitFor(assertion, opt);
+      });
+      updateDumpAnnotation(testInfo, agent.dumpDataString());
     }
   };
 };

package/dist/lib/index.js

@@ -486,17 +486,24 @@ async function alignElements(screenshotBuffer, elements, page) {
     return item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold;
   });
   for (const item of validElements) {
-    const { rect } = item;
+    const { rect, id, content, attributes, locator } = item;
     const aligned = await (0, import_image.alignCoordByTrim)(screenshotBuffer, rect);
+    if (aligned.width < 0)
+      continue;
     item.rect = aligned;
     item.center = [
       Math.round(aligned.left + aligned.width / 2),
       Math.round(aligned.top + aligned.height / 2)
     ];
     textsAligned.push(
-      new WebElementInfo(__spreadProps(__spreadValues({}, item), {
+      new WebElementInfo({
+        rect,
+        locator,
+        id,
+        content,
+        attributes,
         page
-      }))
+      })
     );
   }
   return textsAligned;
@@ -620,7 +627,7 @@ var PageTaskExecutor = class {
         };
         return taskFind;
       }
-      if (plan2.type === "Assert") {
+      if (plan2.type === "Assert" || plan2.type === "AssertWithoutThrow") {
         const assertPlan = plan2;
         const taskAssert = {
           type: "Insight",
@@ -637,13 +644,16 @@ var PageTaskExecutor = class {
               assertPlan.param.assertion
             );
             if (!assertion.pass) {
-              task.output = assertion;
-              task.log = {
-                dump: insightDump
-              };
-              throw new Error(
-                assertion.thought || "Assertion failed without reason"
-              );
+              if (plan2.type === "Assert") {
+                task.output = assertion;
+                task.log = {
+                  dump: insightDump
+                };
+                throw new Error(
+                  assertion.thought || "Assertion failed without reason"
+                );
+              }
+              task.error = assertion.thought;
             }
             return {
               output: assertion,
@@ -759,7 +769,19 @@ var PageTaskExecutor = class {
         return taskActionSleep;
       }
       if (plan2.type === "Error") {
-        throw new Error(`Got a task plan with type Error: ${plan2.thought}`);
+        const taskActionError = {
+          type: "Action",
+          subType: "Error",
+          param: plan2.param,
+          executor: async (taskParam) => {
+            (0, import_node_assert2.default)(
+              taskParam.thought,
+              "An error occurred, but no thought provided"
+            );
+            throw new Error(taskParam.thought);
+          }
+        };
+        return taskActionError;
       }
       throw new Error(`Unknown or Unsupported task type: ${plan2.type}`);
     }).map((task) => {
@@ -769,7 +791,6 @@ var PageTaskExecutor = class {
   }
   async action(userPrompt) {
     const taskExecutor = new import_core.Executor(userPrompt);
-    taskExecutor.description = userPrompt;
     let plans = [];
     const planningTask = {
       type: "Planning",
@@ -829,7 +850,6 @@ var PageTaskExecutor = class {
   async query(demand) {
     const description = typeof demand === "string" ? demand : JSON.stringify(demand);
     const taskExecutor = new import_core.Executor(description);
-    taskExecutor.description = description;
     const queryTask = {
       type: "Insight",
       subType: "Query",
@@ -857,9 +877,8 @@ var PageTaskExecutor = class {
     };
   }
   async assert(assertion) {
-    const description = assertion;
+    const description = `assert: ${assertion}`;
     const taskExecutor = new import_core.Executor(description);
-    taskExecutor.description = description;
     const assertionPlan = {
       type: "Assert",
       param: {
@@ -874,6 +893,64 @@ var PageTaskExecutor = class {
       executor: taskExecutor
     };
   }
+  async waitFor(assertion, opt) {
+    const description = `waitFor: ${assertion}`;
+    const taskExecutor = new import_core.Executor(description);
+    const { timeoutMs, checkIntervalMs } = opt;
+    (0, import_node_assert2.default)(assertion, "No assertion for waitFor");
+    (0, import_node_assert2.default)(timeoutMs, "No timeoutMs for waitFor");
+    (0, import_node_assert2.default)(checkIntervalMs, "No checkIntervalMs for waitFor");
+    const overallStartTime = Date.now();
+    let startTime = Date.now();
+    let errorThought = "";
+    while (Date.now() - overallStartTime < timeoutMs) {
+      startTime = Date.now();
+      const assertPlan = {
+        type: "AssertWithoutThrow",
+        param: {
+          assertion
+        }
+      };
+      const assertTask = await this.convertPlanToExecutable([assertPlan]);
+      await taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
+      const output = await taskExecutor.flush();
+      if (output.pass) {
+        return {
+          output: void 0,
+          executor: taskExecutor
+        };
+      }
+      errorThought = output.thought;
+      const now = Date.now();
+      if (now - startTime < checkIntervalMs) {
+        const timeRemaining = checkIntervalMs - (now - startTime);
+        const sleepPlan = {
+          type: "Sleep",
+          param: {
+            timeMs: timeRemaining
+          }
+        };
+        const sleepTask = await this.convertPlanToExecutable([sleepPlan]);
+        await taskExecutor.append(
+          this.wrapExecutorWithScreenshot(sleepTask[0])
+        );
+        await taskExecutor.flush();
+      }
+    }
+    const errorPlan = {
+      type: "Error",
+      param: {
+        thought: `waitFor timeout: ${errorThought}`
+      }
+    };
+    const errorTask = await this.convertPlanToExecutable([errorPlan]);
+    await taskExecutor.append(errorTask[0]);
+    await taskExecutor.flush();
+    return {
+      output: void 0,
+      executor: taskExecutor
+    };
+  }
 };
 
 // src/common/agent.ts
@@ -883,6 +960,7 @@ var PageAgent = class {
     this.opts = Object.assign(
       {
         generateReport: true,
+        autoPrintReportMsg: true,
         groupName: "Midscene Report",
         groupDescription: ""
       },
@@ -908,7 +986,7 @@ var PageAgent = class {
     return (0, import_utils4.stringifyDumpData)(this.dump);
   }
   writeOutActionDumps() {
-    const generateReport = this.opts.generateReport;
+    const { generateReport, autoPrintReportMsg } = this.opts;
     this.reportFile = (0, import_utils4.writeLogFile)({
       fileName: this.reportFileName,
       fileExt: import_utils4.groupedActionDumpFileExt,
@@ -916,7 +994,7 @@ var PageAgent = class {
       type: "dump",
       generateReport
     });
-    if (generateReport) {
+    if (generateReport && autoPrintReportMsg) {
       printReportMsg(this.reportFile);
     }
   }
@@ -947,11 +1025,25 @@ ${errorTask == null ? void 0 : errorTask.errorStack}`);
     this.writeOutActionDumps();
     if (!(output == null ? void 0 : output.pass)) {
       const errMsg = msg || `Assertion failed: ${assertion}`;
-      const reasonMsg = `Reason: ${output == null ? void 0 : output.thought} || (no_reason)`;
+      const reasonMsg = `Reason: ${(output == null ? void 0 : output.thought) || "(no_reason)"}`;
       throw new Error(`${errMsg}
 ${reasonMsg}`);
     }
   }
+  async aiWaitFor(assertion, opt) {
+    const { executor } = await this.taskExecutor.waitFor(assertion, {
+      timeoutMs: (opt == null ? void 0 : opt.timeoutMs) || 15 * 1e3,
+      checkIntervalMs: (opt == null ? void 0 : opt.checkIntervalMs) || 3 * 1e3,
+      assertion
+    });
+    this.appendExecutionDump(executor.dump());
+    this.writeOutActionDumps();
+    if (executor.isInErrorState()) {
+      const errorTask = executor.latestErrorTask();
+      throw new Error(`${errorTask == null ? void 0 : errorTask.error}
+${errorTask == null ? void 0 : errorTask.errorStack}`);
+    }
+  }
   async ai(taskPrompt, type = "action") {
     if (type === "action") {
       return this.aiAction(taskPrompt);
@@ -1123,6 +1215,13 @@ var PlaywrightAiFixture = () => {
         await agent.aiAssert(assertion, errorMsg);
       });
       updateDumpAnnotation(testInfo, agent.dumpDataString());
+    },
+    aiWaitFor: async ({ page }, use, testInfo) => {
+      const agent = agentForPage(page, testInfo);
+      await use(async (assertion, opt) => {
+        await agent.aiWaitFor(assertion, opt);
+      });
+      updateDumpAnnotation(testInfo, agent.dumpDataString());
     }
   };
 };