@midscene/web 0.3.1-beta-20240821105917.0 → 0.3.2

package/dist/es/index.js

@@ -21,7 +21,7 @@ var __spreadValues = (a, b) => {
   return a;
 };
 var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
-var __commonJS = (cb, mod) => function __require() {
+var __commonJS = (cb, mod) => function __require2() {
   return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
 };
 var __copyProps = (to, from, except, desc) => {
@@ -416,7 +416,6 @@ import assert from "assert";
 import fs, { readFileSync } from "fs";
 import path from "path";
 import {
-  alignCoordByTrim,
   base64Encoded,
   imageInfoOfBase64
 } from "@midscene/core/image";
@@ -484,14 +483,6 @@ async function alignElements(screenshotBuffer, elements, page) {
   });
   for (const item of validElements) {
     const { rect, id, content, attributes, locator } = item;
-    const aligned = await alignCoordByTrim(screenshotBuffer, rect);
-    if (aligned.width < 0)
-      continue;
-    item.rect = aligned;
-    item.center = [
-      Math.round(aligned.left + aligned.width / 2),
-      Math.round(aligned.top + aligned.height / 2)
-    ];
     textsAligned.push(
       new WebElementInfo({
         rect,
@@ -1022,7 +1013,7 @@ ${errorTask == null ? void 0 : errorTask.errorStack}`);
     this.writeOutActionDumps();
     if (!(output == null ? void 0 : output.pass)) {
       const errMsg = msg || `Assertion failed: ${assertion}`;
-      const reasonMsg = `Reason: ${(output == null ? void 0 : output.thought) || "(no_reason)"}`;
+      const reasonMsg = `Reason: ${output == null ? void 0 : output.thought} || (no_reason)`;
       throw new Error(`${errMsg}
 ${reasonMsg}`);
     }
@@ -1057,6 +1048,9 @@ ${errorTask == null ? void 0 : errorTask.errorStack}`);
   }
 };
 
+// src/playwright/index.ts
+import { test } from "@playwright/test";
+
 // src/playwright/cache.ts
 import fs2 from "fs";
 import path2, { join } from "path";
@@ -1162,14 +1156,14 @@ var PlaywrightAiFixture = () => {
     }
     return pageAgentMap[idForPage];
   };
-  const updateDumpAnnotation = (test, dump) => {
-    const currentAnnotation = test.annotations.find((item) => {
+  const updateDumpAnnotation = (test2, dump) => {
+    const currentAnnotation = test2.annotations.find((item) => {
       return item.type === midsceneDumpAnnotationId;
     });
     if (currentAnnotation) {
       currentAnnotation.description = dump;
     } else {
-      test.annotations.push({
+      test2.annotations.push({
         type: midsceneDumpAnnotationId,
         description: dump
       });
@@ -1181,10 +1175,14 @@ var PlaywrightAiFixture = () => {
       const agent = agentForPage(page, testInfo);
       await use(
         async (taskPrompt, opts) => {
-          await page.waitForLoadState("networkidle");
-          const actionType = (opts == null ? void 0 : opts.type) || "action";
-          const result = await agent.ai(taskPrompt, actionType);
-          return result;
+          return new Promise((resolve, reject) => {
+            test.step(`ai - ${taskPrompt}`, async () => {
+              await page.waitForLoadState("networkidle");
+              const actionType = (opts == null ? void 0 : opts.type) || "action";
+              const result = await agent.ai(taskPrompt, actionType);
+              resolve(result);
+            });
+          });
         }
       );
       const taskCacheJson = agent.taskExecutor.taskCache.generateTaskCache();
@@ -1195,38 +1193,243 @@ var PlaywrightAiFixture = () => {
       const { taskFile, taskTitle } = groupAndCaseForTest(testInfo);
       const agent = agentForPage(page, testInfo);
       await use(async (taskPrompt) => {
-        await page.waitForLoadState("networkidle");
-        await agent.aiAction(taskPrompt);
+        test.step(`aiAction - ${taskPrompt}`, async () => {
+          await page.waitForLoadState("networkidle");
+          await agent.aiAction(taskPrompt);
+        });
       });
       updateDumpAnnotation(testInfo, agent.dumpDataString());
     },
     aiQuery: async ({ page }, use, testInfo) => {
       const agent = agentForPage(page, testInfo);
       await use(async (demand) => {
-        await page.waitForLoadState("networkidle");
-        const result = await agent.aiQuery(demand);
-        return result;
+        return new Promise((resolve, reject) => {
+          test.step(`aiQuery - ${JSON.stringify(demand)}`, async () => {
+            await page.waitForLoadState("networkidle");
+            const result = await agent.aiQuery(demand);
+            resolve(result);
+          });
+        });
       });
       updateDumpAnnotation(testInfo, agent.dumpDataString());
     },
     aiAssert: async ({ page }, use, testInfo) => {
       const agent = agentForPage(page, testInfo);
       await use(async (assertion, errorMsg) => {
-        await page.waitForLoadState("networkidle");
-        await agent.aiAssert(assertion, errorMsg);
+        return new Promise((resolve, reject) => {
+          test.step(`aiAssert - ${assertion}`, async () => {
+            await page.waitForLoadState("networkidle");
+            await agent.aiAssert(assertion, errorMsg);
+            resolve(null);
+          });
+        });
       });
       updateDumpAnnotation(testInfo, agent.dumpDataString());
     },
     aiWaitFor: async ({ page }, use, testInfo) => {
       const agent = agentForPage(page, testInfo);
       await use(async (assertion, opt) => {
-        await agent.aiWaitFor(assertion, opt);
+        return new Promise((resolve, reject) => {
+          test.step(`aiWaitFor - ${assertion}`, async () => {
+            await agent.aiWaitFor(assertion, opt);
+            resolve(null);
+          });
+        });
       });
       updateDumpAnnotation(testInfo, agent.dumpDataString());
     }
   };
 };
+
+// src/debug/index.ts
+import { existsSync, mkdirSync, writeFileSync } from "fs";
+import path3 from "path";
+
+// src/img/img.ts
+import assert3 from "assert";
+import { Buffer as Buffer2 } from "buffer";
+import sharp from "sharp";
+var createSvgOverlay = (elements, imageWidth, imageHeight) => {
+  let svgContent = `<svg width="${imageWidth}" height="${imageHeight}" xmlns="http://www.w3.org/2000/svg">`;
+  const colors = [
+    { rect: "blue", text: "white" },
+    { rect: "green", text: "white" }
+  ];
+  svgContent += "<defs>";
+  elements.forEach((element, index) => {
+    svgContent += `
+      <clipPath id="clip${index}">
+        <rect x="${element.x}" y="${element.y}" width="${element.width}" height="${element.height}" />
+      </clipPath>
+    `;
+  });
+  svgContent += "</defs>";
+  elements.forEach((element, index) => {
+    const textWidth = element.label.length * 8;
+    const textHeight = 12;
+    const rectWidth = textWidth + 5;
+    const rectHeight = textHeight + 4;
+    let rectX = element.x - rectWidth;
+    let rectY = element.y + element.height / 2 - textHeight / 2 - 2;
+    let textX = rectX + rectWidth / 2;
+    let textY = rectY + rectHeight / 2 + 6;
+    if (rectX < 0) {
+      rectX = element.x;
+      rectY = element.y - rectHeight;
+      textX = rectX + rectWidth / 2;
+      textY = rectY + rectHeight / 2 + 6;
+    }
+    const color = colors[index % colors.length];
+    svgContent += `
+      <rect x="${element.x}" y="${element.y}" width="${element.width}" height="${element.height}" 
+            style="fill:none;stroke:${color.rect};stroke-width:4" clip-path="url(#clip${index})" />
+      <rect x="${rectX}" y="${rectY}" width="${rectWidth}" height="${rectHeight}" style="fill:${color.rect};" />
+      <text x="${textX}" y="${textY}" 
+            text-anchor="middle" dominant-baseline="middle" style="fill:${color.text};font-size:12px;font-weight:bold;">
+        ${element.label}
+      </text>
+    `;
+  });
+  svgContent += "</svg>";
+  return Buffer2.from(svgContent);
+};
+var processImageElementInfo = async (options) => {
+  const base64Image = options.inputImgBase64.split(";base64,").pop();
+  assert3(base64Image, "base64Image is undefined");
+  const imageBuffer = Buffer2.from(base64Image, "base64");
+  const metadata = await sharp(imageBuffer).metadata();
+  const { width, height } = metadata;
+  if (width && height) {
+    const svgOverlay = createSvgOverlay(
+      options.elementsPositionInfo,
+      width,
+      height
+    );
+    const svgOverlayWithoutText = createSvgOverlay(
+      options.elementsPositionInfoWithoutText,
+      width,
+      height
+    );
+    const compositeElementInfoImgBase64 = await sharp(imageBuffer).composite([{ input: svgOverlay, blend: "over" }]).toBuffer().then((data) => {
+      return data.toString("base64");
+    }).catch((err) => {
+      throw err;
+    });
+    const compositeElementInfoImgWithoutTextBase64 = await sharp(imageBuffer).composite([{ input: svgOverlayWithoutText, blend: "over" }]).toBuffer().then((data) => {
+      return data.toString("base64");
+    }).catch((err) => {
+      throw err;
+    });
+    return {
+      compositeElementInfoImgBase64,
+      compositeElementInfoImgWithoutTextBase64
+    };
+  }
+  throw Error("Image processing failed because width or height is undefined");
+};
+
+// src/img/util.ts
+async function getElementInfos(page) {
+  const captureElementSnapshot = await getElementInfosFromPage(page);
+  const elementsPositionInfo = captureElementSnapshot.map((elementInfo) => {
+    return {
+      label: elementInfo.indexId.toString(),
+      x: elementInfo.rect.left,
+      y: elementInfo.rect.top,
+      width: elementInfo.rect.width,
+      height: elementInfo.rect.height,
+      attributes: elementInfo.attributes
+    };
+  });
+  const elementsPositionInfoWithoutText = elementsPositionInfo.filter(
+    (elementInfo) => {
+      if (elementInfo.attributes.nodeType === "TEXT Node" /* TEXT */) {
+        return false;
+      }
+      return true;
+    }
+  );
+  return {
+    elementsPositionInfo,
+    captureElementSnapshot,
+    elementsPositionInfoWithoutText
+  };
+}
+
+// src/debug/index.ts
+import { resizeImg, saveBase64Image } from "@midscene/core/image";
+async function generateExtractData(page, targetDir, saveImgType) {
+  const buffer = await page.screenshot({
+    encoding: "base64"
+  });
+  const inputImgBase64 = buffer.toString("base64");
+  const {
+    elementsPositionInfo,
+    captureElementSnapshot,
+    elementsPositionInfoWithoutText
+  } = await getElementInfos(page);
+  const inputImagePath = path3.join(targetDir, "input.png");
+  const outputImagePath = path3.join(targetDir, "output.png");
+  const outputWithoutTextImgPath = path3.join(
+    targetDir,
+    "output_without_text.png"
+  );
+  const resizeOutputImgPath = path3.join(targetDir, "resize-output.png");
+  const snapshotJsonPath = path3.join(targetDir, "element-snapshot.json");
+  const {
+    compositeElementInfoImgBase64,
+    compositeElementInfoImgWithoutTextBase64
+  } = await processImageElementInfo({
+    elementsPositionInfo,
+    elementsPositionInfoWithoutText,
+    inputImgBase64
+  });
+  const resizeImgBase64 = await resizeImg(inputImgBase64);
+  if (!(saveImgType == null ? void 0 : saveImgType.disableSnapshot)) {
+    writeFileSyncWithDir(
+      snapshotJsonPath,
+      JSON.stringify(captureElementSnapshot, null, 2)
+    );
+  }
+  if (!(saveImgType == null ? void 0 : saveImgType.disableInputImage)) {
+    await saveBase64Image({
+      base64Data: inputImgBase64,
+      outputPath: inputImagePath
+    });
+  }
+  if (!(saveImgType == null ? void 0 : saveImgType.disableOutputImage)) {
+    await saveBase64Image({
+      base64Data: compositeElementInfoImgBase64,
+      outputPath: outputImagePath
+    });
+  }
+  if (!(saveImgType == null ? void 0 : saveImgType.disableOutputWithoutTextImg)) {
+    await saveBase64Image({
+      base64Data: compositeElementInfoImgWithoutTextBase64,
+      outputPath: outputWithoutTextImgPath
+    });
+  }
+  if (!(saveImgType == null ? void 0 : saveImgType.disableResizeOutputImg)) {
+    await saveBase64Image({
+      base64Data: resizeImgBase64,
+      outputPath: resizeOutputImgPath
+    });
+  }
+}
+function ensureDirectoryExistence(filePath) {
+  const dirname2 = path3.dirname(filePath);
+  if (existsSync(dirname2)) {
+    return;
+  }
+  ensureDirectoryExistence(dirname2);
+  mkdirSync(dirname2);
+}
+function writeFileSyncWithDir(filePath, content, options = {}) {
+  ensureDirectoryExistence(filePath);
+  writeFileSync(filePath, content, options);
+}
 export {
   PlaywrightAiFixture,
-  PageAgent as PuppeteerAgent
+  PageAgent as PuppeteerAgent,
+  generateExtractData
 };

package/dist/es/playwright-report.js

@@ -325,7 +325,6 @@ import assert from "assert";
 import fs, { readFileSync } from "fs";
 import path from "path";
 import {
-  alignCoordByTrim,
   base64Encoded,
   imageInfoOfBase64
 } from "@midscene/core/image";

package/dist/lib/index.js

@@ -346,7 +346,8 @@ var require_dayjs_min = __commonJS({
 var src_exports = {};
 __export(src_exports, {
   PlaywrightAiFixture: () => PlaywrightAiFixture,
-  PuppeteerAgent: () => PageAgent
+  PuppeteerAgent: () => PageAgent,
+  generateExtractData: () => generateExtractData
 });
 module.exports = __toCommonJS(src_exports);
 
@@ -487,14 +488,6 @@ async function alignElements(screenshotBuffer, elements, page) {
   });
   for (const item of validElements) {
     const { rect, id, content, attributes, locator } = item;
-    const aligned = await (0, import_image.alignCoordByTrim)(screenshotBuffer, rect);
-    if (aligned.width < 0)
-      continue;
-    item.rect = aligned;
-    item.center = [
-      Math.round(aligned.left + aligned.width / 2),
-      Math.round(aligned.top + aligned.height / 2)
-    ];
     textsAligned.push(
       new WebElementInfo({
         rect,
@@ -1025,7 +1018,7 @@ ${errorTask == null ? void 0 : errorTask.errorStack}`);
     this.writeOutActionDumps();
     if (!(output == null ? void 0 : output.pass)) {
       const errMsg = msg || `Assertion failed: ${assertion}`;
-      const reasonMsg = `Reason: ${(output == null ? void 0 : output.thought) || "(no_reason)"}`;
+      const reasonMsg = `Reason: ${output == null ? void 0 : output.thought} || (no_reason)`;
       throw new Error(`${errMsg}
 ${reasonMsg}`);
     }
@@ -1060,6 +1053,9 @@ ${errorTask == null ? void 0 : errorTask.errorStack}`);
   }
 };
 
+// src/playwright/index.ts
+var import_test = require("@playwright/test");
+
 // src/playwright/cache.ts
 var import_node_fs2 = __toESM(require("fs"));
 var import_node_path2 = __toESM(require("path"));
@@ -1161,14 +1157,14 @@ var PlaywrightAiFixture = () => {
     }
     return pageAgentMap[idForPage];
   };
-  const updateDumpAnnotation = (test, dump) => {
-    const currentAnnotation = test.annotations.find((item) => {
+  const updateDumpAnnotation = (test2, dump) => {
+    const currentAnnotation = test2.annotations.find((item) => {
       return item.type === midsceneDumpAnnotationId;
     });
     if (currentAnnotation) {
       currentAnnotation.description = dump;
     } else {
-      test.annotations.push({
+      test2.annotations.push({
         type: midsceneDumpAnnotationId,
         description: dump
       });
@@ -1180,10 +1176,14 @@ var PlaywrightAiFixture = () => {
       const agent = agentForPage(page, testInfo);
       await use(
         async (taskPrompt, opts) => {
-          await page.waitForLoadState("networkidle");
-          const actionType = (opts == null ? void 0 : opts.type) || "action";
-          const result = await agent.ai(taskPrompt, actionType);
-          return result;
+          return new Promise((resolve, reject) => {
+            import_test.test.step(`ai - ${taskPrompt}`, async () => {
+              await page.waitForLoadState("networkidle");
+              const actionType = (opts == null ? void 0 : opts.type) || "action";
+              const result = await agent.ai(taskPrompt, actionType);
+              resolve(result);
+            });
+          });
         }
       );
       const taskCacheJson = agent.taskExecutor.taskCache.generateTaskCache();
@@ -1194,39 +1194,244 @@ var PlaywrightAiFixture = () => {
       const { taskFile, taskTitle } = groupAndCaseForTest(testInfo);
       const agent = agentForPage(page, testInfo);
       await use(async (taskPrompt) => {
-        await page.waitForLoadState("networkidle");
-        await agent.aiAction(taskPrompt);
+        import_test.test.step(`aiAction - ${taskPrompt}`, async () => {
+          await page.waitForLoadState("networkidle");
+          await agent.aiAction(taskPrompt);
+        });
       });
       updateDumpAnnotation(testInfo, agent.dumpDataString());
     },
     aiQuery: async ({ page }, use, testInfo) => {
       const agent = agentForPage(page, testInfo);
       await use(async (demand) => {
-        await page.waitForLoadState("networkidle");
-        const result = await agent.aiQuery(demand);
-        return result;
+        return new Promise((resolve, reject) => {
+          import_test.test.step(`aiQuery - ${JSON.stringify(demand)}`, async () => {
+            await page.waitForLoadState("networkidle");
+            const result = await agent.aiQuery(demand);
+            resolve(result);
+          });
+        });
       });
       updateDumpAnnotation(testInfo, agent.dumpDataString());
     },
     aiAssert: async ({ page }, use, testInfo) => {
       const agent = agentForPage(page, testInfo);
       await use(async (assertion, errorMsg) => {
-        await page.waitForLoadState("networkidle");
-        await agent.aiAssert(assertion, errorMsg);
+        return new Promise((resolve, reject) => {
+          import_test.test.step(`aiAssert - ${assertion}`, async () => {
+            await page.waitForLoadState("networkidle");
+            await agent.aiAssert(assertion, errorMsg);
+            resolve(null);
+          });
+        });
       });
       updateDumpAnnotation(testInfo, agent.dumpDataString());
     },
     aiWaitFor: async ({ page }, use, testInfo) => {
       const agent = agentForPage(page, testInfo);
       await use(async (assertion, opt) => {
-        await agent.aiWaitFor(assertion, opt);
+        return new Promise((resolve, reject) => {
+          import_test.test.step(`aiWaitFor - ${assertion}`, async () => {
+            await agent.aiWaitFor(assertion, opt);
+            resolve(null);
+          });
+        });
       });
       updateDumpAnnotation(testInfo, agent.dumpDataString());
     }
   };
 };
+
+// src/debug/index.ts
+var import_node_fs3 = require("fs");
+var import_node_path3 = __toESM(require("path"));
+
+// src/img/img.ts
+var import_node_assert3 = __toESM(require("assert"));
+var import_node_buffer = require("buffer");
+var import_sharp = __toESM(require("sharp"));
+var createSvgOverlay = (elements, imageWidth, imageHeight) => {
+  let svgContent = `<svg width="${imageWidth}" height="${imageHeight}" xmlns="http://www.w3.org/2000/svg">`;
+  const colors = [
+    { rect: "blue", text: "white" },
+    { rect: "green", text: "white" }
+  ];
+  svgContent += "<defs>";
+  elements.forEach((element, index) => {
+    svgContent += `
+      <clipPath id="clip${index}">
+        <rect x="${element.x}" y="${element.y}" width="${element.width}" height="${element.height}" />
+      </clipPath>
+    `;
+  });
+  svgContent += "</defs>";
+  elements.forEach((element, index) => {
+    const textWidth = element.label.length * 8;
+    const textHeight = 12;
+    const rectWidth = textWidth + 5;
+    const rectHeight = textHeight + 4;
+    let rectX = element.x - rectWidth;
+    let rectY = element.y + element.height / 2 - textHeight / 2 - 2;
+    let textX = rectX + rectWidth / 2;
+    let textY = rectY + rectHeight / 2 + 6;
+    if (rectX < 0) {
+      rectX = element.x;
+      rectY = element.y - rectHeight;
+      textX = rectX + rectWidth / 2;
+      textY = rectY + rectHeight / 2 + 6;
+    }
+    const color = colors[index % colors.length];
+    svgContent += `
+      <rect x="${element.x}" y="${element.y}" width="${element.width}" height="${element.height}" 
+            style="fill:none;stroke:${color.rect};stroke-width:4" clip-path="url(#clip${index})" />
+      <rect x="${rectX}" y="${rectY}" width="${rectWidth}" height="${rectHeight}" style="fill:${color.rect};" />
+      <text x="${textX}" y="${textY}" 
+            text-anchor="middle" dominant-baseline="middle" style="fill:${color.text};font-size:12px;font-weight:bold;">
+        ${element.label}
+      </text>
+    `;
+  });
+  svgContent += "</svg>";
+  return import_node_buffer.Buffer.from(svgContent);
+};
+var processImageElementInfo = async (options) => {
+  const base64Image = options.inputImgBase64.split(";base64,").pop();
+  (0, import_node_assert3.default)(base64Image, "base64Image is undefined");
+  const imageBuffer = import_node_buffer.Buffer.from(base64Image, "base64");
+  const metadata = await (0, import_sharp.default)(imageBuffer).metadata();
+  const { width, height } = metadata;
+  if (width && height) {
+    const svgOverlay = createSvgOverlay(
+      options.elementsPositionInfo,
+      width,
+      height
+    );
+    const svgOverlayWithoutText = createSvgOverlay(
+      options.elementsPositionInfoWithoutText,
+      width,
+      height
+    );
+    const compositeElementInfoImgBase64 = await (0, import_sharp.default)(imageBuffer).composite([{ input: svgOverlay, blend: "over" }]).toBuffer().then((data) => {
+      return data.toString("base64");
+    }).catch((err) => {
+      throw err;
+    });
+    const compositeElementInfoImgWithoutTextBase64 = await (0, import_sharp.default)(imageBuffer).composite([{ input: svgOverlayWithoutText, blend: "over" }]).toBuffer().then((data) => {
+      return data.toString("base64");
+    }).catch((err) => {
+      throw err;
+    });
+    return {
+      compositeElementInfoImgBase64,
+      compositeElementInfoImgWithoutTextBase64
+    };
+  }
+  throw Error("Image processing failed because width or height is undefined");
+};
+
+// src/img/util.ts
+async function getElementInfos(page) {
+  const captureElementSnapshot = await getElementInfosFromPage(page);
+  const elementsPositionInfo = captureElementSnapshot.map((elementInfo) => {
+    return {
+      label: elementInfo.indexId.toString(),
+      x: elementInfo.rect.left,
+      y: elementInfo.rect.top,
+      width: elementInfo.rect.width,
+      height: elementInfo.rect.height,
+      attributes: elementInfo.attributes
+    };
+  });
+  const elementsPositionInfoWithoutText = elementsPositionInfo.filter(
+    (elementInfo) => {
+      if (elementInfo.attributes.nodeType === "TEXT Node" /* TEXT */) {
+        return false;
+      }
+      return true;
+    }
+  );
+  return {
+    elementsPositionInfo,
+    captureElementSnapshot,
+    elementsPositionInfoWithoutText
+  };
+}
+
+// src/debug/index.ts
+var import_image3 = require("@midscene/core/image");
+async function generateExtractData(page, targetDir, saveImgType) {
+  const buffer = await page.screenshot({
+    encoding: "base64"
+  });
+  const inputImgBase64 = buffer.toString("base64");
+  const {
+    elementsPositionInfo,
+    captureElementSnapshot,
+    elementsPositionInfoWithoutText
+  } = await getElementInfos(page);
+  const inputImagePath = import_node_path3.default.join(targetDir, "input.png");
+  const outputImagePath = import_node_path3.default.join(targetDir, "output.png");
+  const outputWithoutTextImgPath = import_node_path3.default.join(
+    targetDir,
+    "output_without_text.png"
+  );
+  const resizeOutputImgPath = import_node_path3.default.join(targetDir, "resize-output.png");
+  const snapshotJsonPath = import_node_path3.default.join(targetDir, "element-snapshot.json");
+  const {
+    compositeElementInfoImgBase64,
+    compositeElementInfoImgWithoutTextBase64
+  } = await processImageElementInfo({
+    elementsPositionInfo,
+    elementsPositionInfoWithoutText,
+    inputImgBase64
+  });
+  const resizeImgBase64 = await (0, import_image3.resizeImg)(inputImgBase64);
+  if (!(saveImgType == null ? void 0 : saveImgType.disableSnapshot)) {
+    writeFileSyncWithDir(
+      snapshotJsonPath,
+      JSON.stringify(captureElementSnapshot, null, 2)
+    );
+  }
+  if (!(saveImgType == null ? void 0 : saveImgType.disableInputImage)) {
+    await (0, import_image3.saveBase64Image)({
+      base64Data: inputImgBase64,
+      outputPath: inputImagePath
+    });
+  }
+  if (!(saveImgType == null ? void 0 : saveImgType.disableOutputImage)) {
+    await (0, import_image3.saveBase64Image)({
+      base64Data: compositeElementInfoImgBase64,
+      outputPath: outputImagePath
+    });
+  }
+  if (!(saveImgType == null ? void 0 : saveImgType.disableOutputWithoutTextImg)) {
+    await (0, import_image3.saveBase64Image)({
+      base64Data: compositeElementInfoImgWithoutTextBase64,
+      outputPath: outputWithoutTextImgPath
+    });
+  }
+  if (!(saveImgType == null ? void 0 : saveImgType.disableResizeOutputImg)) {
+    await (0, import_image3.saveBase64Image)({
+      base64Data: resizeImgBase64,
+      outputPath: resizeOutputImgPath
+    });
+  }
+}
+function ensureDirectoryExistence(filePath) {
+  const dirname2 = import_node_path3.default.dirname(filePath);
+  if ((0, import_node_fs3.existsSync)(dirname2)) {
+    return;
+  }
+  ensureDirectoryExistence(dirname2);
+  (0, import_node_fs3.mkdirSync)(dirname2);
+}
+function writeFileSyncWithDir(filePath, content, options = {}) {
+  ensureDirectoryExistence(filePath);
+  (0, import_node_fs3.writeFileSync)(filePath, content, options);
+}
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   PlaywrightAiFixture,
-  PuppeteerAgent
+  PuppeteerAgent,
+  generateExtractData
 });

package/dist/script/htmlElement.js

@@ -581,8 +581,8 @@ var midscene_element_inspector = (() => {
   });
 
   // src/extractor/dom-util.ts
-  function isFormElement(node) {
-    return node instanceof HTMLElement && (node.tagName.toLowerCase() === "input" || node.tagName.toLowerCase() === "textarea" || node.tagName.toLowerCase() === "label" || node.tagName.toLowerCase() === "select" || node.tagName.toLowerCase() === "option");
+  function isInputElement(node) {
+    return node instanceof HTMLElement && (node.tagName.toLowerCase() === "input" || node.tagName.toLowerCase() === "textarea");
   }
   function isButtonElement(node) {
     return node instanceof HTMLElement && node.tagName.toLowerCase() === "button";
@@ -644,23 +644,23 @@ var midscene_element_inspector = (() => {
   }
   function visibleRect(el) {
     if (!el) {
-      logger("Element is not in the DOM hierarchy");
+      logger(el, "Element is not in the DOM hierarchy");
       return false;
     }
     if (!(el instanceof HTMLElement) && el.nodeType !== Node.TEXT_NODE) {
-      logger("Element is not in the DOM hierarchy");
+      logger(el, "Element is not in the DOM hierarchy");
       return false;
     }
     if (el instanceof HTMLElement) {
       const style = window.getComputedStyle(el);
       if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0" && el.tagName !== "INPUT") {
-        logger("Element is hidden");
+        logger(el, "Element is hidden");
         return false;
       }
     }
     const rect = getRect(el);
     if (rect.width === 0 && rect.height === 0) {
-      logger("Element has no size");
+      logger(el, "Element has no size");
       return false;
     }
     const scrollLeft = window.pageXOffset || document.documentElement.scrollLeft;
@@ -669,8 +669,13 @@ var midscene_element_inspector = (() => {
     const viewportHeight = window.innerHeight || document.documentElement.clientHeight;
     const isPartiallyInViewport = rect.right > 0 && rect.bottom > 0 && rect.left < viewportWidth && rect.top < viewportHeight;
     if (!isPartiallyInViewport) {
-      logger("Element is completely outside the viewport");
-      logger(rect, viewportHeight, viewportWidth, scrollTop, scrollLeft);
+      logger(el, "Element is completely outside the viewport", {
+        rect,
+        viewportHeight,
+        viewportWidth,
+        scrollTop,
+        scrollLeft
+      });
       return false;
     }
     let parent = el;
@@ -739,7 +744,6 @@ var midscene_element_inspector = (() => {
       }
       const shouldContinue = collectElementInfo(node);
       if (!shouldContinue) {
-        logger("should NOT continue for node", node);
         return;
       }
       for (let i = 0; i < node.childNodes.length; i++) {
@@ -753,29 +757,22 @@ var midscene_element_inspector = (() => {
       logger("collectElementInfo", node, node.nodeName, rect);
       if (!rect) {
         logger("Element is not visible", node);
-        return;
+        return true;
       }
-      if (isFormElement(node)) {
+      if (isInputElement(node)) {
         const attributes = getNodeAttributes(node);
         const nodeHashId = generateHash(attributes.placeholder, rect);
         const selector = setDataForNode(node, nodeHashId);
-        let valueContent = attributes.value || attributes.placeholder || node.textContent || "";
-        const tagName = node.tagName.toLowerCase();
-        if (node.tagName.toLowerCase() === "select") {
-          const selectedOption = node.options[node.selectedIndex];
-          valueContent = selectedOption.textContent || "";
-        }
         elementInfoArray.push({
           id: nodeHashId,
           indexId: generateId(nodeIndex++),
           nodeHashId,
           locator: selector,
-          nodeType: "FORM_ITEM Node" /* FORM_ITEM */,
+          nodeType: "INPUT Node" /* INPUT */,
           attributes: __spreadProps(__spreadValues({}, attributes), {
-            htmlTagName: `<${tagName}>`,
-            nodeType: "FORM_ITEM Node" /* FORM_ITEM */
+            nodeType: "INPUT Node" /* INPUT */
           }),
-          content: valueContent.trim(),
+          content: attributes.placeholder || "",
           rect,
           center: [
             Math.round(rect.left + rect.width / 2),
@@ -783,8 +780,6 @@ var midscene_element_inspector = (() => {
           ],
           htmlNode: debugMode2 ? node : null
         });
-        if (tagName === "label")
-          return true;
         return;
       }
       if (isButtonElement(node)) {
@@ -841,10 +836,6 @@ var midscene_element_inspector = (() => {
           return;
         }
         const attributes = getNodeAttributes(node);
-        const attributeKeys = Object.keys(attributes);
-        if (!text.trim() && attributeKeys.length === 0) {
-          return;
-        }
         const nodeHashId = generateHash(text, rect);
         const selector = setDataForNode(node, nodeHashId);
         elementInfoArray.push({

package/dist/script/htmlElementDebug.js

@@ -570,8 +570,8 @@ var midscene_element_inspector = (() => {
   });
 
   // src/extractor/dom-util.ts
-  function isFormElement(node) {
-    return node instanceof HTMLElement && (node.tagName.toLowerCase() === "input" || node.tagName.toLowerCase() === "textarea" || node.tagName.toLowerCase() === "label" || node.tagName.toLowerCase() === "select" || node.tagName.toLowerCase() === "option");
+  function isInputElement(node) {
+    return node instanceof HTMLElement && (node.tagName.toLowerCase() === "input" || node.tagName.toLowerCase() === "textarea");
   }
   function isButtonElement(node) {
     return node instanceof HTMLElement && node.tagName.toLowerCase() === "button";
@@ -633,23 +633,23 @@ var midscene_element_inspector = (() => {
   }
   function visibleRect(el) {
     if (!el) {
-      logger("Element is not in the DOM hierarchy");
+      logger(el, "Element is not in the DOM hierarchy");
       return false;
     }
     if (!(el instanceof HTMLElement) && el.nodeType !== Node.TEXT_NODE) {
-      logger("Element is not in the DOM hierarchy");
+      logger(el, "Element is not in the DOM hierarchy");
       return false;
     }
     if (el instanceof HTMLElement) {
       const style = window.getComputedStyle(el);
       if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0" && el.tagName !== "INPUT") {
-        logger("Element is hidden");
+        logger(el, "Element is hidden");
         return false;
       }
     }
     const rect = getRect(el);
     if (rect.width === 0 && rect.height === 0) {
-      logger("Element has no size");
+      logger(el, "Element has no size");
       return false;
     }
     const scrollLeft = window.pageXOffset || document.documentElement.scrollLeft;
@@ -658,8 +658,13 @@ var midscene_element_inspector = (() => {
     const viewportHeight = window.innerHeight || document.documentElement.clientHeight;
     const isPartiallyInViewport = rect.right > 0 && rect.bottom > 0 && rect.left < viewportWidth && rect.top < viewportHeight;
     if (!isPartiallyInViewport) {
-      logger("Element is completely outside the viewport");
-      logger(rect, viewportHeight, viewportWidth, scrollTop, scrollLeft);
+      logger(el, "Element is completely outside the viewport", {
+        rect,
+        viewportHeight,
+        viewportWidth,
+        scrollTop,
+        scrollLeft
+      });
       return false;
     }
     let parent = el;
@@ -728,7 +733,6 @@ var midscene_element_inspector = (() => {
       }
       const shouldContinue = collectElementInfo(node);
       if (!shouldContinue) {
-        logger("should NOT continue for node", node);
         return;
       }
       for (let i = 0; i < node.childNodes.length; i++) {
@@ -742,29 +746,22 @@ var midscene_element_inspector = (() => {
       logger("collectElementInfo", node, node.nodeName, rect);
       if (!rect) {
         logger("Element is not visible", node);
-        return;
+        return true;
       }
-      if (isFormElement(node)) {
+      if (isInputElement(node)) {
         const attributes = getNodeAttributes(node);
         const nodeHashId = generateHash(attributes.placeholder, rect);
         const selector = setDataForNode(node, nodeHashId);
-        let valueContent = attributes.value || attributes.placeholder || node.textContent || "";
-        const tagName = node.tagName.toLowerCase();
-        if (node.tagName.toLowerCase() === "select") {
-          const selectedOption = node.options[node.selectedIndex];
-          valueContent = selectedOption.textContent || "";
-        }
         elementInfoArray.push({
           id: nodeHashId,
           indexId: generateId(nodeIndex++),
           nodeHashId,
           locator: selector,
-          nodeType: "FORM_ITEM Node" /* FORM_ITEM */,
+          nodeType: "INPUT Node" /* INPUT */,
           attributes: __spreadProps(__spreadValues({}, attributes), {
-            htmlTagName: `<${tagName}>`,
-            nodeType: "FORM_ITEM Node" /* FORM_ITEM */
+            nodeType: "INPUT Node" /* INPUT */
           }),
-          content: valueContent.trim(),
+          content: attributes.placeholder || "",
           rect,
           center: [
             Math.round(rect.left + rect.width / 2),
@@ -772,8 +769,6 @@ var midscene_element_inspector = (() => {
           ],
           htmlNode: debugMode2 ? node : null
         });
-        if (tagName === "label")
-          return true;
         return;
       }
       if (isButtonElement(node)) {
@@ -830,10 +825,6 @@ var midscene_element_inspector = (() => {
           return;
         }
         const attributes = getNodeAttributes(node);
-        const attributeKeys = Object.keys(attributes);
-        if (!text.trim() && attributeKeys.length === 0) {
-          return;
-        }
         const nodeHashId = generateHash(text, rect);
         const selector = setDataForNode(node, nodeHashId);
         elementInfoArray.push({

package/dist/script/types/htmlElement.d.ts

@@ -1,5 +1,5 @@
 declare enum NodeType {
-    FORM_ITEM = "FORM_ITEM Node",
+    INPUT = "INPUT Node",
     BUTTON = "BUTTON Node",
     IMG = "IMG Node",
     TEXT = "TEXT Node"

package/dist/types/index.d.ts

@@ -7,7 +7,7 @@ import Insight, { BaseElement, Rect, UIContext, PlanningAction, AIElementParseRe
 type WebPage = Page | Page$1;
 
 declare enum NodeType {
-    FORM_ITEM = "FORM_ITEM Node",
+    INPUT = "INPUT Node",
     BUTTON = "BUTTON Node",
     IMG = "IMG Node",
     TEXT = "TEXT Node"
@@ -180,4 +180,12 @@ declare class PageAgent {
     ai(taskPrompt: string, type?: string): Promise<any>;
 }
 
-export { type PlayWrightAiFixtureType, PlaywrightAiFixture, PageAgent as PuppeteerAgent };
+declare function generateExtractData(page: WebPage, targetDir: string, saveImgType?: {
+    disableInputImage: boolean;
+    disableOutputImage: boolean;
+    disableOutputWithoutTextImg: boolean;
+    disableResizeOutputImg: boolean;
+    disableSnapshot: boolean;
+}): Promise<void>;
+
+export { type PlayWrightAiFixtureType, PlaywrightAiFixture, PageAgent as PuppeteerAgent, generateExtractData };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/web",
   "description": "Web integration for Midscene.js",
-  "version": "0.3.1-beta-20240821105917.0",
+  "version": "0.3.2",
   "jsnext:source": "./src/index.ts",
   "main": "./dist/lib/index.js",
   "module": "./dist/es/index.js",
@@ -52,7 +52,7 @@
     "openai": "4.47.1",
     "sharp": "0.33.3",
     "inquirer": "10.1.5",
-    "@midscene/core": "0.3.1-beta-20240821105917.0"
+    "@midscene/core": "0.3.2"
   },
   "devDependencies": {
     "@modern-js/module-tools": "^2.56.1",
@@ -70,7 +70,7 @@
   "peerDependencies": {
     "@playwright/test": "^1.44.1",
     "playwright": "^1.44.1",
-    "puppeteer": "^23.0.2"
+    "puppeteer": ">=20.0.0"
   },
   "peerDependenciesMeta": {
     "@playwright/test": {
@@ -100,8 +100,9 @@
     "new": "modern new",
     "upgrade": "modern upgrade",
     "e2e": "playwright test --config=playwright.config.ts",
+    "e2e:report": "MIDSCENE_REPORT=true playwright test --config=playwright.config.ts",
     "e2e:cache": "MIDSCENE_CACHE=true playwright test --config=playwright.config.ts",
     "e2e:ui": "playwright test --config=playwright.config.ts --ui",
-    "e2e:ui-cache": "MIDSCENE_CACHE=true playwright test --config=playwright.config.ts  --ui"
+    "e2e:ui:cache": "MIDSCENE_CACHE=true playwright test --config=playwright.config.ts  --ui"
   }
 }