@midscene/web 0.3.0 → 0.3.1

package/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2021-present Midscene.js
+Copyright (c) 2024-present Bytedance, Inc. and its affiliates.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

package/dist/es/index.js

@@ -21,7 +21,7 @@ var __spreadValues = (a, b) => {
   return a;
 };
 var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
-var __commonJS = (cb, mod) => function __require() {
+var __commonJS = (cb, mod) => function __require2() {
   return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
 };
 var __copyProps = (to, from, except, desc) => {
@@ -416,7 +416,6 @@ import assert from "assert";
 import fs, { readFileSync } from "fs";
 import path from "path";
 import {
-  alignCoordByTrim,
   base64Encoded,
   imageInfoOfBase64
 } from "@midscene/core/image";
@@ -483,17 +482,16 @@ async function alignElements(screenshotBuffer, elements, page) {
     return item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold;
   });
   for (const item of validElements) {
-    const { rect } = item;
-    const aligned = await alignCoordByTrim(screenshotBuffer, rect);
-    item.rect = aligned;
-    item.center = [
-      Math.round(aligned.left + aligned.width / 2),
-      Math.round(aligned.top + aligned.height / 2)
-    ];
+    const { rect, id, content, attributes, locator } = item;
     textsAligned.push(
-      new WebElementInfo(__spreadProps(__spreadValues({}, item), {
+      new WebElementInfo({
+        rect,
+        locator,
+        id,
+        content,
+        attributes,
         page
-      }))
+      })
     );
   }
   return textsAligned;
@@ -617,7 +615,7 @@ var PageTaskExecutor = class {
         };
         return taskFind;
       }
-      if (plan2.type === "Assert") {
+      if (plan2.type === "Assert" || plan2.type === "AssertWithoutThrow") {
         const assertPlan = plan2;
         const taskAssert = {
           type: "Insight",
@@ -634,13 +632,16 @@ var PageTaskExecutor = class {
               assertPlan.param.assertion
             );
             if (!assertion.pass) {
-              task.output = assertion;
-              task.log = {
-                dump: insightDump
-              };
-              throw new Error(
-                assertion.thought || "Assertion failed without reason"
-              );
+              if (plan2.type === "Assert") {
+                task.output = assertion;
+                task.log = {
+                  dump: insightDump
+                };
+                throw new Error(
+                  assertion.thought || "Assertion failed without reason"
+                );
+              }
+              task.error = assertion.thought;
             }
             return {
               output: assertion,
@@ -756,7 +757,19 @@ var PageTaskExecutor = class {
         return taskActionSleep;
       }
       if (plan2.type === "Error") {
-        throw new Error(`Got a task plan with type Error: ${plan2.thought}`);
+        const taskActionError = {
+          type: "Action",
+          subType: "Error",
+          param: plan2.param,
+          executor: async (taskParam) => {
+            assert2(
+              taskParam.thought,
+              "An error occurred, but no thought provided"
+            );
+            throw new Error(taskParam.thought);
+          }
+        };
+        return taskActionError;
       }
       throw new Error(`Unknown or Unsupported task type: ${plan2.type}`);
     }).map((task) => {
@@ -766,7 +779,6 @@ var PageTaskExecutor = class {
   }
   async action(userPrompt) {
     const taskExecutor = new Executor(userPrompt);
-    taskExecutor.description = userPrompt;
     let plans = [];
     const planningTask = {
       type: "Planning",
@@ -826,7 +838,6 @@ var PageTaskExecutor = class {
   async query(demand) {
     const description = typeof demand === "string" ? demand : JSON.stringify(demand);
     const taskExecutor = new Executor(description);
-    taskExecutor.description = description;
     const queryTask = {
       type: "Insight",
       subType: "Query",
@@ -854,9 +865,8 @@ var PageTaskExecutor = class {
     };
   }
   async assert(assertion) {
-    const description = assertion;
+    const description = `assert: ${assertion}`;
     const taskExecutor = new Executor(description);
-    taskExecutor.description = description;
     const assertionPlan = {
       type: "Assert",
       param: {
@@ -871,6 +881,64 @@ var PageTaskExecutor = class {
       executor: taskExecutor
     };
   }
+  async waitFor(assertion, opt) {
+    const description = `waitFor: ${assertion}`;
+    const taskExecutor = new Executor(description);
+    const { timeoutMs, checkIntervalMs } = opt;
+    assert2(assertion, "No assertion for waitFor");
+    assert2(timeoutMs, "No timeoutMs for waitFor");
+    assert2(checkIntervalMs, "No checkIntervalMs for waitFor");
+    const overallStartTime = Date.now();
+    let startTime = Date.now();
+    let errorThought = "";
+    while (Date.now() - overallStartTime < timeoutMs) {
+      startTime = Date.now();
+      const assertPlan = {
+        type: "AssertWithoutThrow",
+        param: {
+          assertion
+        }
+      };
+      const assertTask = await this.convertPlanToExecutable([assertPlan]);
+      await taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
+      const output = await taskExecutor.flush();
+      if (output.pass) {
+        return {
+          output: void 0,
+          executor: taskExecutor
+        };
+      }
+      errorThought = output.thought;
+      const now = Date.now();
+      if (now - startTime < checkIntervalMs) {
+        const timeRemaining = checkIntervalMs - (now - startTime);
+        const sleepPlan = {
+          type: "Sleep",
+          param: {
+            timeMs: timeRemaining
+          }
+        };
+        const sleepTask = await this.convertPlanToExecutable([sleepPlan]);
+        await taskExecutor.append(
+          this.wrapExecutorWithScreenshot(sleepTask[0])
+        );
+        await taskExecutor.flush();
+      }
+    }
+    const errorPlan = {
+      type: "Error",
+      param: {
+        thought: `waitFor timeout: ${errorThought}`
+      }
+    };
+    const errorTask = await this.convertPlanToExecutable([errorPlan]);
+    await taskExecutor.append(errorTask[0]);
+    await taskExecutor.flush();
+    return {
+      output: void 0,
+      executor: taskExecutor
+    };
+  }
 };
 
 // src/common/agent.ts
@@ -880,6 +948,7 @@ var PageAgent = class {
     this.opts = Object.assign(
       {
         generateReport: true,
+        autoPrintReportMsg: true,
         groupName: "Midscene Report",
         groupDescription: ""
       },
@@ -905,7 +974,7 @@ var PageAgent = class {
     return stringifyDumpData(this.dump);
   }
   writeOutActionDumps() {
-    const generateReport = this.opts.generateReport;
+    const { generateReport, autoPrintReportMsg } = this.opts;
     this.reportFile = writeLogFile({
       fileName: this.reportFileName,
       fileExt: groupedActionDumpFileExt,
@@ -913,7 +982,7 @@ var PageAgent = class {
       type: "dump",
       generateReport
     });
-    if (generateReport) {
+    if (generateReport && autoPrintReportMsg) {
       printReportMsg(this.reportFile);
     }
   }
@@ -949,6 +1018,20 @@ ${errorTask == null ? void 0 : errorTask.errorStack}`);
 ${reasonMsg}`);
     }
   }
+  async aiWaitFor(assertion, opt) {
+    const { executor } = await this.taskExecutor.waitFor(assertion, {
+      timeoutMs: (opt == null ? void 0 : opt.timeoutMs) || 15 * 1e3,
+      checkIntervalMs: (opt == null ? void 0 : opt.checkIntervalMs) || 3 * 1e3,
+      assertion
+    });
+    this.appendExecutionDump(executor.dump());
+    this.writeOutActionDumps();
+    if (executor.isInErrorState()) {
+      const errorTask = executor.latestErrorTask();
+      throw new Error(`${errorTask == null ? void 0 : errorTask.error}
+${errorTask == null ? void 0 : errorTask.errorStack}`);
+    }
+  }
   async ai(taskPrompt, type = "action") {
     if (type === "action") {
       return this.aiAction(taskPrompt);
@@ -965,6 +1048,9 @@ ${reasonMsg}`);
   }
 };
 
+// src/playwright/index.ts
+import { test } from "@playwright/test";
+
 // src/playwright/cache.ts
 import fs2 from "fs";
 import path2, { join } from "path";
@@ -1070,14 +1156,14 @@ var PlaywrightAiFixture = () => {
     }
     return pageAgentMap[idForPage];
   };
-  const updateDumpAnnotation = (test, dump) => {
-    const currentAnnotation = test.annotations.find((item) => {
+  const updateDumpAnnotation = (test2, dump) => {
+    const currentAnnotation = test2.annotations.find((item) => {
       return item.type === midsceneDumpAnnotationId;
     });
     if (currentAnnotation) {
       currentAnnotation.description = dump;
     } else {
-      test.annotations.push({
+      test2.annotations.push({
         type: midsceneDumpAnnotationId,
         description: dump
       });
@@ -1089,10 +1175,14 @@ var PlaywrightAiFixture = () => {
       const agent = agentForPage(page, testInfo);
       await use(
         async (taskPrompt, opts) => {
-          await page.waitForLoadState("networkidle");
-          const actionType = (opts == null ? void 0 : opts.type) || "action";
-          const result = await agent.ai(taskPrompt, actionType);
-          return result;
+          return new Promise((resolve, reject) => {
+            test.step(`ai - ${taskPrompt}`, async () => {
+              await page.waitForLoadState("networkidle");
+              const actionType = (opts == null ? void 0 : opts.type) || "action";
+              const result = await agent.ai(taskPrompt, actionType);
+              resolve(result);
+            });
+          });
         }
       );
       const taskCacheJson = agent.taskExecutor.taskCache.generateTaskCache();
@@ -1103,31 +1193,243 @@ var PlaywrightAiFixture = () => {
       const { taskFile, taskTitle } = groupAndCaseForTest(testInfo);
       const agent = agentForPage(page, testInfo);
       await use(async (taskPrompt) => {
-        await page.waitForLoadState("networkidle");
-        await agent.aiAction(taskPrompt);
+        test.step(`aiAction - ${taskPrompt}`, async () => {
+          await page.waitForLoadState("networkidle");
+          await agent.aiAction(taskPrompt);
+        });
       });
       updateDumpAnnotation(testInfo, agent.dumpDataString());
     },
     aiQuery: async ({ page }, use, testInfo) => {
       const agent = agentForPage(page, testInfo);
       await use(async (demand) => {
-        await page.waitForLoadState("networkidle");
-        const result = await agent.aiQuery(demand);
-        return result;
+        return new Promise((resolve, reject) => {
+          test.step(`aiQuery - ${JSON.stringify(demand)}`, async () => {
+            await page.waitForLoadState("networkidle");
+            const result = await agent.aiQuery(demand);
+            resolve(result);
+          });
+        });
       });
       updateDumpAnnotation(testInfo, agent.dumpDataString());
     },
     aiAssert: async ({ page }, use, testInfo) => {
       const agent = agentForPage(page, testInfo);
       await use(async (assertion, errorMsg) => {
-        await page.waitForLoadState("networkidle");
-        await agent.aiAssert(assertion, errorMsg);
+        return new Promise((resolve, reject) => {
+          test.step(`aiAssert - ${assertion}`, async () => {
+            await page.waitForLoadState("networkidle");
+            await agent.aiAssert(assertion, errorMsg);
+            resolve(null);
+          });
+        });
+      });
+      updateDumpAnnotation(testInfo, agent.dumpDataString());
+    },
+    aiWaitFor: async ({ page }, use, testInfo) => {
+      const agent = agentForPage(page, testInfo);
+      await use(async (assertion, opt) => {
+        return new Promise((resolve, reject) => {
+          test.step(`aiWaitFor - ${assertion}`, async () => {
+            await agent.aiWaitFor(assertion, opt);
+            resolve(null);
+          });
+        });
       });
       updateDumpAnnotation(testInfo, agent.dumpDataString());
     }
   };
 };
+
+// src/debug/index.ts
+import { existsSync, mkdirSync, writeFileSync } from "fs";
+import path3 from "path";
+
+// src/img/img.ts
+import assert3 from "assert";
+import { Buffer as Buffer2 } from "buffer";
+import sharp from "sharp";
+var createSvgOverlay = (elements, imageWidth, imageHeight) => {
+  let svgContent = `<svg width="${imageWidth}" height="${imageHeight}" xmlns="http://www.w3.org/2000/svg">`;
+  const colors = [
+    { rect: "blue", text: "white" },
+    { rect: "green", text: "white" }
+  ];
+  svgContent += "<defs>";
+  elements.forEach((element, index) => {
+    svgContent += `
+      <clipPath id="clip${index}">
+        <rect x="${element.x}" y="${element.y}" width="${element.width}" height="${element.height}" />
+      </clipPath>
+    `;
+  });
+  svgContent += "</defs>";
+  elements.forEach((element, index) => {
+    const textWidth = element.label.length * 8;
+    const textHeight = 12;
+    const rectWidth = textWidth + 5;
+    const rectHeight = textHeight + 4;
+    let rectX = element.x - rectWidth;
+    let rectY = element.y + element.height / 2 - textHeight / 2 - 2;
+    let textX = rectX + rectWidth / 2;
+    let textY = rectY + rectHeight / 2 + 6;
+    if (rectX < 0) {
+      rectX = element.x;
+      rectY = element.y - rectHeight;
+      textX = rectX + rectWidth / 2;
+      textY = rectY + rectHeight / 2 + 6;
+    }
+    const color = colors[index % colors.length];
+    svgContent += `
+      <rect x="${element.x}" y="${element.y}" width="${element.width}" height="${element.height}" 
+            style="fill:none;stroke:${color.rect};stroke-width:4" clip-path="url(#clip${index})" />
+      <rect x="${rectX}" y="${rectY}" width="${rectWidth}" height="${rectHeight}" style="fill:${color.rect};" />
+      <text x="${textX}" y="${textY}" 
+            text-anchor="middle" dominant-baseline="middle" style="fill:${color.text};font-size:12px;font-weight:bold;">
+        ${element.label}
+      </text>
+    `;
+  });
+  svgContent += "</svg>";
+  return Buffer2.from(svgContent);
+};
+var processImageElementInfo = async (options) => {
+  const base64Image = options.inputImgBase64.split(";base64,").pop();
+  assert3(base64Image, "base64Image is undefined");
+  const imageBuffer = Buffer2.from(base64Image, "base64");
+  const metadata = await sharp(imageBuffer).metadata();
+  const { width, height } = metadata;
+  if (width && height) {
+    const svgOverlay = createSvgOverlay(
+      options.elementsPositionInfo,
+      width,
+      height
+    );
+    const svgOverlayWithoutText = createSvgOverlay(
+      options.elementsPositionInfoWithoutText,
+      width,
+      height
+    );
+    const compositeElementInfoImgBase64 = await sharp(imageBuffer).composite([{ input: svgOverlay, blend: "over" }]).toBuffer().then((data) => {
+      return data.toString("base64");
+    }).catch((err) => {
+      throw err;
+    });
+    const compositeElementInfoImgWithoutTextBase64 = await sharp(imageBuffer).composite([{ input: svgOverlayWithoutText, blend: "over" }]).toBuffer().then((data) => {
+      return data.toString("base64");
+    }).catch((err) => {
+      throw err;
+    });
+    return {
+      compositeElementInfoImgBase64,
+      compositeElementInfoImgWithoutTextBase64
+    };
+  }
+  throw Error("Image processing failed because width or height is undefined");
+};
+
+// src/img/util.ts
+async function getElementInfos(page) {
+  const captureElementSnapshot = await getElementInfosFromPage(page);
+  const elementsPositionInfo = captureElementSnapshot.map((elementInfo) => {
+    return {
+      label: elementInfo.indexId.toString(),
+      x: elementInfo.rect.left,
+      y: elementInfo.rect.top,
+      width: elementInfo.rect.width,
+      height: elementInfo.rect.height,
+      attributes: elementInfo.attributes
+    };
+  });
+  const elementsPositionInfoWithoutText = elementsPositionInfo.filter(
+    (elementInfo) => {
+      if (elementInfo.attributes.nodeType === "TEXT Node" /* TEXT */) {
+        return false;
+      }
+      return true;
+    }
+  );
+  return {
+    elementsPositionInfo,
+    captureElementSnapshot,
+    elementsPositionInfoWithoutText
+  };
+}
+
+// src/debug/index.ts
+import { resizeImg, saveBase64Image } from "@midscene/core/image";
+async function generateExtractData(page, targetDir, saveImgType) {
+  const buffer = await page.screenshot({
+    encoding: "base64"
+  });
+  const inputImgBase64 = buffer.toString("base64");
+  const {
+    elementsPositionInfo,
+    captureElementSnapshot,
+    elementsPositionInfoWithoutText
+  } = await getElementInfos(page);
+  const inputImagePath = path3.join(targetDir, "input.png");
+  const outputImagePath = path3.join(targetDir, "output.png");
+  const outputWithoutTextImgPath = path3.join(
+    targetDir,
+    "output_without_text.png"
+  );
+  const resizeOutputImgPath = path3.join(targetDir, "resize-output.png");
+  const snapshotJsonPath = path3.join(targetDir, "element-snapshot.json");
+  const {
+    compositeElementInfoImgBase64,
+    compositeElementInfoImgWithoutTextBase64
+  } = await processImageElementInfo({
+    elementsPositionInfo,
+    elementsPositionInfoWithoutText,
+    inputImgBase64
+  });
+  const resizeImgBase64 = await resizeImg(inputImgBase64);
+  if (!(saveImgType == null ? void 0 : saveImgType.disableSnapshot)) {
+    writeFileSyncWithDir(
+      snapshotJsonPath,
+      JSON.stringify(captureElementSnapshot, null, 2)
+    );
+  }
+  if (!(saveImgType == null ? void 0 : saveImgType.disableInputImage)) {
+    await saveBase64Image({
+      base64Data: inputImgBase64,
+      outputPath: inputImagePath
+    });
+  }
+  if (!(saveImgType == null ? void 0 : saveImgType.disableOutputImage)) {
+    await saveBase64Image({
+      base64Data: compositeElementInfoImgBase64,
+      outputPath: outputImagePath
+    });
+  }
+  if (!(saveImgType == null ? void 0 : saveImgType.disableOutputWithoutTextImg)) {
+    await saveBase64Image({
+      base64Data: compositeElementInfoImgWithoutTextBase64,
+      outputPath: outputWithoutTextImgPath
+    });
+  }
+  if (!(saveImgType == null ? void 0 : saveImgType.disableResizeOutputImg)) {
+    await saveBase64Image({
+      base64Data: resizeImgBase64,
+      outputPath: resizeOutputImgPath
+    });
+  }
+}
+function ensureDirectoryExistence(filePath) {
+  const dirname2 = path3.dirname(filePath);
+  if (existsSync(dirname2)) {
+    return;
+  }
+  ensureDirectoryExistence(dirname2);
+  mkdirSync(dirname2);
+}
+function writeFileSyncWithDir(filePath, content, options = {}) {
+  ensureDirectoryExistence(filePath);
+  writeFileSync(filePath, content, options);
+}
 export {
   PlaywrightAiFixture,
-  PageAgent as PuppeteerAgent
+  PageAgent as PuppeteerAgent,
+  generateExtractData
 };

package/dist/es/playwright-report.js

@@ -325,7 +325,6 @@ import assert from "assert";
 import fs, { readFileSync } from "fs";
 import path from "path";
 import {
-  alignCoordByTrim,
   base64Encoded,
   imageInfoOfBase64
 } from "@midscene/core/image";