@midscene/web 0.3.4 → 0.4.0

package/dist/es/debug.js

@@ -329,104 +329,31 @@ var require_dayjs_min = __commonJS({
 // src/debug/index.ts
 import { existsSync, mkdirSync, writeFileSync } from "fs";
 import path2 from "path";
-import { resizeImg, saveBase64Image } from "@midscene/core/image";
 
-// src/debug/img/index.ts
-import assert from "assert";
-import { Buffer as Buffer2 } from "buffer";
-import sharp from "sharp";
-var createSvgOverlay = (elements, imageWidth, imageHeight) => {
-  let svgContent = `<svg width="${imageWidth}" height="${imageHeight}" xmlns="http://www.w3.org/2000/svg">`;
-  const colors = [
-    { rect: "blue", text: "white" },
-    { rect: "green", text: "white" }
-  ];
-  svgContent += "<defs>";
-  elements.forEach((element, index) => {
-    svgContent += `
-      <clipPath id="clip${index}">
-        <rect x="${element.x}" y="${element.y}" width="${element.width}" height="${element.height}" />
-      </clipPath>
-    `;
-  });
-  svgContent += "</defs>";
-  elements.forEach((element, index) => {
-    const textWidth = element.label.length * 8;
-    const textHeight = 12;
-    const rectWidth = textWidth + 5;
-    const rectHeight = textHeight + 4;
-    let rectX = element.x - rectWidth;
-    let rectY = element.y + element.height / 2 - textHeight / 2 - 2;
-    let textX = rectX + rectWidth / 2;
-    let textY = rectY + rectHeight / 2 + 6;
-    if (rectX < 0) {
-      rectX = element.x;
-      rectY = element.y - rectHeight;
-      textX = rectX + rectWidth / 2;
-      textY = rectY + rectHeight / 2 + 6;
-    }
-    const color = colors[index % colors.length];
-    svgContent += `
-      <rect x="${element.x}" y="${element.y}" width="${element.width}" height="${element.height}" 
-            style="fill:none;stroke:${color.rect};stroke-width:4" clip-path="url(#clip${index})" />
-      <rect x="${rectX}" y="${rectY}" width="${rectWidth}" height="${rectHeight}" style="fill:${color.rect};" />
-      <text x="${textX}" y="${textY}" 
-            text-anchor="middle" dominant-baseline="middle" style="fill:${color.text};font-size:12px;font-weight:bold;">
-        ${element.label}
-      </text>
-    `;
-  });
-  svgContent += "</svg>";
-  return Buffer2.from(svgContent);
-};
-var processImageElementInfo = async (options) => {
-  const base64Image = options.inputImgBase64.split(";base64,").pop();
-  assert(base64Image, "base64Image is undefined");
-  const imageBuffer = Buffer2.from(base64Image, "base64");
-  const metadata = await sharp(imageBuffer).metadata();
-  const { width, height } = metadata;
-  if (width && height) {
-    const svgOverlay = createSvgOverlay(
-      options.elementsPositionInfo,
-      width,
-      height
-    );
-    const svgOverlayWithoutText = createSvgOverlay(
-      options.elementsPositionInfoWithoutText,
-      width,
-      height
-    );
-    const compositeElementInfoImgBase64 = await sharp(imageBuffer).composite([{ input: svgOverlay, blend: "over" }]).toBuffer().then((data) => {
-      return data.toString("base64");
-    }).catch((err) => {
-      throw err;
-    });
-    const compositeElementInfoImgWithoutTextBase64 = await sharp(imageBuffer).composite([{ input: svgOverlayWithoutText, blend: "over" }]).toBuffer().then((data) => {
-      return data.toString("base64");
-    }).catch((err) => {
-      throw err;
-    });
-    return {
-      compositeElementInfoImgBase64,
-      compositeElementInfoImgWithoutTextBase64
-    };
-  }
-  throw Error("Image processing failed because width or height is undefined");
-};
+// src/extractor/constants.ts
+import {
+  NodeType,
+  TEXT_MAX_SIZE,
+  TEXT_SIZE_THRESHOLD
+} from "@midscene/shared/constants";
+
+// src/debug/index.ts
+import {
+  processImageElementInfo,
+  resizeImg,
+  saveBase64Image
+} from "@midscene/shared/img";
 
 // src/common/utils.ts
 var import_dayjs = __toESM(require_dayjs_min());
-import assert2 from "assert";
+import assert from "assert";
 import fs, { readFileSync } from "fs";
 import path from "path";
-import {
-  base64Encoded,
-  imageInfoOfBase64
-} from "@midscene/core/image";
 import { getTmpFile } from "@midscene/core/utils";
+import { base64Encoded, imageInfoOfBase64 } from "@midscene/shared/img";
 async function getElementInfosFromPage(page) {
   const pathDir = findNearestPackageJson(__dirname);
-  assert2(pathDir, `can't find pathDir, with ${__dirname}`);
+  assert(pathDir, `can't find pathDir, with ${__dirname}`);
   const scriptPath = path.join(pathDir, "./dist/script/htmlElement.js");
   const elementInfosScriptContent = readFileSync(scriptPath, "utf-8");
   const extraReturnLogic = `${elementInfosScriptContent}midscene_element_inspector.extractTextWithPosition()`;
@@ -445,34 +372,6 @@ function findNearestPackageJson(dir) {
   return findNearestPackageJson(parentDir);
 }
 
-// src/debug/img/util.ts
-async function getElementInfos(page) {
-  const captureElementSnapshot = await getElementInfosFromPage(page);
-  const elementsPositionInfo = captureElementSnapshot.map((elementInfo) => {
-    return {
-      label: elementInfo.indexId.toString(),
-      x: elementInfo.rect.left,
-      y: elementInfo.rect.top,
-      width: elementInfo.rect.width,
-      height: elementInfo.rect.height,
-      attributes: elementInfo.attributes
-    };
-  });
-  const elementsPositionInfoWithoutText = elementsPositionInfo.filter(
-    (elementInfo) => {
-      if (elementInfo.attributes.nodeType === "TEXT Node" /* TEXT */) {
-        return false;
-      }
-      return true;
-    }
-  );
-  return {
-    elementsPositionInfo,
-    captureElementSnapshot,
-    elementsPositionInfoWithoutText
-  };
-}
-
 // src/debug/index.ts
 async function generateExtractData(page, targetDir, saveImgType) {
   const buffer = await page.screenshot({
@@ -492,6 +391,7 @@ async function generateExtractData(page, targetDir, saveImgType) {
   );
   const resizeOutputImgPath = path2.join(targetDir, "resize-output.png");
   const snapshotJsonPath = path2.join(targetDir, "element-snapshot.json");
+  const startTime = Date.now();
   const {
     compositeElementInfoImgBase64,
     compositeElementInfoImgWithoutTextBase64
@@ -500,6 +400,9 @@ async function generateExtractData(page, targetDir, saveImgType) {
     elementsPositionInfoWithoutText,
     inputImgBase64
   });
+  const endTime = Date.now();
+  const executionTime = (endTime - startTime) / 1e3;
+  console.log(`Execution time: ${executionTime.toFixed(2)}s`);
   const resizeImgBase64 = await resizeImg(inputImgBase64);
   if (!(saveImgType == null ? void 0 : saveImgType.disableSnapshot)) {
     writeFileSyncWithDir(
@@ -552,8 +455,35 @@ function writeFileSyncWithDir(filePath, content, options = {}) {
   ensureDirectoryExistence(filePath);
   writeFileSync(filePath, content, options);
 }
+async function getElementInfos(page) {
+  const captureElementSnapshot = await getElementInfosFromPage(page);
+  const elementsPositionInfo = captureElementSnapshot.map((elementInfo) => {
+    return {
+      label: elementInfo.indexId.toString(),
+      x: elementInfo.rect.left,
+      y: elementInfo.rect.top,
+      width: elementInfo.rect.width,
+      height: elementInfo.rect.height,
+      attributes: elementInfo.attributes
+    };
+  });
+  const elementsPositionInfoWithoutText = elementsPositionInfo.filter(
+    (elementInfo) => {
+      if (elementInfo.attributes.nodeType === NodeType.TEXT) {
+        return false;
+      }
+      return true;
+    }
+  );
+  return {
+    elementsPositionInfo,
+    captureElementSnapshot,
+    elementsPositionInfoWithoutText
+  };
+}
 export {
   generateExtractData,
   generateTestDataPath,
+  getElementInfos,
   writeFileSyncWithDir
 };

package/dist/es/index.js

@@ -352,8 +352,8 @@ import Insight, {
   Executor,
   plan
 } from "@midscene/core";
-import { base64Encoded as base64Encoded2 } from "@midscene/core/image";
 import { commonScreenshotParam, getTmpFile as getTmpFile2, sleep } from "@midscene/core/utils";
+import { base64Encoded as base64Encoded2 } from "@midscene/shared/img";
 
 // src/common/task-cache.ts
 var TaskCache = class {
@@ -415,11 +415,8 @@ var import_dayjs = __toESM(require_dayjs_min());
 import assert from "assert";
 import fs, { readFileSync } from "fs";
 import path from "path";
-import {
-  base64Encoded,
-  imageInfoOfBase64
-} from "@midscene/core/image";
 import { getTmpFile } from "@midscene/core/utils";
+import { base64Encoded, imageInfoOfBase64 } from "@midscene/shared/img";
 
 // src/web-element.ts
 var WebElementInfo = class {
@@ -477,10 +474,10 @@ async function getElementInfosFromPage(page) {
 }
 var sizeThreshold = 3;
 async function alignElements(screenshotBuffer, elements, page) {
-  const textsAligned = [];
   const validElements = elements.filter((item) => {
     return item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold;
   });
+  const textsAligned = [];
   for (const item of validElements) {
     const { rect, id, content, attributes, locator } = item;
     textsAligned.push(
@@ -1013,7 +1010,7 @@ ${errorTask == null ? void 0 : errorTask.errorStack}`);
     this.writeOutActionDumps();
     if (!(output == null ? void 0 : output.pass)) {
       const errMsg = msg || `Assertion failed: ${assertion}`;
-      const reasonMsg = `Reason: ${output == null ? void 0 : output.thought} || (no_reason)`;
+      const reasonMsg = `Reason: ${(output == null ? void 0 : output.thought) || "(no_reason)"}`;
       throw new Error(`${errMsg}
 ${reasonMsg}`);
     }
@@ -1244,120 +1241,20 @@ var PlaywrightAiFixture = () => {
 // src/debug/index.ts
 import { existsSync, mkdirSync, writeFileSync } from "fs";
 import path3 from "path";
-import { resizeImg, saveBase64Image } from "@midscene/core/image";
 
-// src/debug/img/index.ts
-import assert3 from "assert";
-import { Buffer as Buffer2 } from "buffer";
-import sharp from "sharp";
-var createSvgOverlay = (elements, imageWidth, imageHeight) => {
-  let svgContent = `<svg width="${imageWidth}" height="${imageHeight}" xmlns="http://www.w3.org/2000/svg">`;
-  const colors = [
-    { rect: "blue", text: "white" },
-    { rect: "green", text: "white" }
-  ];
-  svgContent += "<defs>";
-  elements.forEach((element, index) => {
-    svgContent += `
-      <clipPath id="clip${index}">
-        <rect x="${element.x}" y="${element.y}" width="${element.width}" height="${element.height}" />
-      </clipPath>
-    `;
-  });
-  svgContent += "</defs>";
-  elements.forEach((element, index) => {
-    const textWidth = element.label.length * 8;
-    const textHeight = 12;
-    const rectWidth = textWidth + 5;
-    const rectHeight = textHeight + 4;
-    let rectX = element.x - rectWidth;
-    let rectY = element.y + element.height / 2 - textHeight / 2 - 2;
-    let textX = rectX + rectWidth / 2;
-    let textY = rectY + rectHeight / 2 + 6;
-    if (rectX < 0) {
-      rectX = element.x;
-      rectY = element.y - rectHeight;
-      textX = rectX + rectWidth / 2;
-      textY = rectY + rectHeight / 2 + 6;
-    }
-    const color = colors[index % colors.length];
-    svgContent += `
-      <rect x="${element.x}" y="${element.y}" width="${element.width}" height="${element.height}" 
-            style="fill:none;stroke:${color.rect};stroke-width:4" clip-path="url(#clip${index})" />
-      <rect x="${rectX}" y="${rectY}" width="${rectWidth}" height="${rectHeight}" style="fill:${color.rect};" />
-      <text x="${textX}" y="${textY}" 
-            text-anchor="middle" dominant-baseline="middle" style="fill:${color.text};font-size:12px;font-weight:bold;">
-        ${element.label}
-      </text>
-    `;
-  });
-  svgContent += "</svg>";
-  return Buffer2.from(svgContent);
-};
-var processImageElementInfo = async (options) => {
-  const base64Image = options.inputImgBase64.split(";base64,").pop();
-  assert3(base64Image, "base64Image is undefined");
-  const imageBuffer = Buffer2.from(base64Image, "base64");
-  const metadata = await sharp(imageBuffer).metadata();
-  const { width, height } = metadata;
-  if (width && height) {
-    const svgOverlay = createSvgOverlay(
-      options.elementsPositionInfo,
-      width,
-      height
-    );
-    const svgOverlayWithoutText = createSvgOverlay(
-      options.elementsPositionInfoWithoutText,
-      width,
-      height
-    );
-    const compositeElementInfoImgBase64 = await sharp(imageBuffer).composite([{ input: svgOverlay, blend: "over" }]).toBuffer().then((data) => {
-      return data.toString("base64");
-    }).catch((err) => {
-      throw err;
-    });
-    const compositeElementInfoImgWithoutTextBase64 = await sharp(imageBuffer).composite([{ input: svgOverlayWithoutText, blend: "over" }]).toBuffer().then((data) => {
-      return data.toString("base64");
-    }).catch((err) => {
-      throw err;
-    });
-    return {
-      compositeElementInfoImgBase64,
-      compositeElementInfoImgWithoutTextBase64
-    };
-  }
-  throw Error("Image processing failed because width or height is undefined");
-};
-
-// src/debug/img/util.ts
-async function getElementInfos(page) {
-  const captureElementSnapshot = await getElementInfosFromPage(page);
-  const elementsPositionInfo = captureElementSnapshot.map((elementInfo) => {
-    return {
-      label: elementInfo.indexId.toString(),
-      x: elementInfo.rect.left,
-      y: elementInfo.rect.top,
-      width: elementInfo.rect.width,
-      height: elementInfo.rect.height,
-      attributes: elementInfo.attributes
-    };
-  });
-  const elementsPositionInfoWithoutText = elementsPositionInfo.filter(
-    (elementInfo) => {
-      if (elementInfo.attributes.nodeType === "TEXT Node" /* TEXT */) {
-        return false;
-      }
-      return true;
-    }
-  );
-  return {
-    elementsPositionInfo,
-    captureElementSnapshot,
-    elementsPositionInfoWithoutText
-  };
-}
+// src/extractor/constants.ts
+import {
+  NodeType,
+  TEXT_MAX_SIZE,
+  TEXT_SIZE_THRESHOLD
+} from "@midscene/shared/constants";
 
 // src/debug/index.ts
+import {
+  processImageElementInfo,
+  resizeImg,
+  saveBase64Image
+} from "@midscene/shared/img";
 async function generateExtractData(page, targetDir, saveImgType) {
   const buffer = await page.screenshot({
     encoding: "base64"
@@ -1376,6 +1273,7 @@ async function generateExtractData(page, targetDir, saveImgType) {
   );
   const resizeOutputImgPath = path3.join(targetDir, "resize-output.png");
   const snapshotJsonPath = path3.join(targetDir, "element-snapshot.json");
+  const startTime = Date.now();
   const {
     compositeElementInfoImgBase64,
     compositeElementInfoImgWithoutTextBase64
@@ -1384,6 +1282,9 @@ async function generateExtractData(page, targetDir, saveImgType) {
     elementsPositionInfoWithoutText,
     inputImgBase64
   });
+  const endTime = Date.now();
+  const executionTime = (endTime - startTime) / 1e3;
+  console.log(`Execution time: ${executionTime.toFixed(2)}s`);
   const resizeImgBase64 = await resizeImg(inputImgBase64);
   if (!(saveImgType == null ? void 0 : saveImgType.disableSnapshot)) {
     writeFileSyncWithDir(
@@ -1428,6 +1329,32 @@ function writeFileSyncWithDir(filePath, content, options = {}) {
   ensureDirectoryExistence(filePath);
   writeFileSync(filePath, content, options);
 }
+async function getElementInfos(page) {
+  const captureElementSnapshot = await getElementInfosFromPage(page);
+  const elementsPositionInfo = captureElementSnapshot.map((elementInfo) => {
+    return {
+      label: elementInfo.indexId.toString(),
+      x: elementInfo.rect.left,
+      y: elementInfo.rect.top,
+      width: elementInfo.rect.width,
+      height: elementInfo.rect.height,
+      attributes: elementInfo.attributes
+    };
+  });
+  const elementsPositionInfoWithoutText = elementsPositionInfo.filter(
+    (elementInfo) => {
+      if (elementInfo.attributes.nodeType === NodeType.TEXT) {
+        return false;
+      }
+      return true;
+    }
+  );
+  return {
+    elementsPositionInfo,
+    captureElementSnapshot,
+    elementsPositionInfoWithoutText
+  };
+}
 export {
   PlaywrightAiFixture,
   PageAgent as PuppeteerAgent,

package/dist/es/playwright-report.js

@@ -324,11 +324,8 @@ var import_dayjs = __toESM(require_dayjs_min());
 import assert from "assert";
 import fs, { readFileSync } from "fs";
 import path from "path";
-import {
-  base64Encoded,
-  imageInfoOfBase64
-} from "@midscene/core/image";
 import { getTmpFile } from "@midscene/core/utils";
+import { base64Encoded, imageInfoOfBase64 } from "@midscene/shared/img";
 function reportFileName(tag = "web") {
   const dateTimeInFileName = (0, import_dayjs.default)().format("YYYY-MM-DD_HH-mm-ss-SSS");
   return `${tag}-${dateTimeInFileName}`;

package/dist/es/playwright.js

@@ -352,8 +352,8 @@ import Insight, {
   Executor,
   plan
 } from "@midscene/core";
-import { base64Encoded as base64Encoded2 } from "@midscene/core/image";
 import { commonScreenshotParam, getTmpFile as getTmpFile2, sleep } from "@midscene/core/utils";
+import { base64Encoded as base64Encoded2 } from "@midscene/shared/img";
 
 // src/common/task-cache.ts
 var TaskCache = class {
@@ -415,11 +415,8 @@ var import_dayjs = __toESM(require_dayjs_min());
 import assert from "assert";
 import fs, { readFileSync } from "fs";
 import path from "path";
-import {
-  base64Encoded,
-  imageInfoOfBase64
-} from "@midscene/core/image";
 import { getTmpFile } from "@midscene/core/utils";
+import { base64Encoded, imageInfoOfBase64 } from "@midscene/shared/img";
 
 // src/web-element.ts
 var WebElementInfo = class {
@@ -477,10 +474,10 @@ async function getElementInfosFromPage(page) {
 }
 var sizeThreshold = 3;
 async function alignElements(screenshotBuffer, elements, page) {
-  const textsAligned = [];
   const validElements = elements.filter((item) => {
     return item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold;
   });
+  const textsAligned = [];
   for (const item of validElements) {
     const { rect, id, content, attributes, locator } = item;
     textsAligned.push(
@@ -1013,7 +1010,7 @@ ${errorTask == null ? void 0 : errorTask.errorStack}`);
     this.writeOutActionDumps();
     if (!(output == null ? void 0 : output.pass)) {
       const errMsg = msg || `Assertion failed: ${assertion}`;
-      const reasonMsg = `Reason: ${output == null ? void 0 : output.thought} || (no_reason)`;
+      const reasonMsg = `Reason: ${(output == null ? void 0 : output.thought) || "(no_reason)"}`;
       throw new Error(`${errMsg}
 ${reasonMsg}`);
     }

package/dist/es/puppeteer.js

@@ -349,8 +349,8 @@ import Insight, {
   Executor,
   plan
 } from "@midscene/core";
-import { base64Encoded as base64Encoded2 } from "@midscene/core/image";
 import { commonScreenshotParam, getTmpFile as getTmpFile2, sleep } from "@midscene/core/utils";
+import { base64Encoded as base64Encoded2 } from "@midscene/shared/img";
 
 // src/common/task-cache.ts
 var TaskCache = class {
@@ -412,11 +412,8 @@ var import_dayjs = __toESM(require_dayjs_min());
 import assert from "assert";
 import fs, { readFileSync } from "fs";
 import path from "path";
-import {
-  base64Encoded,
-  imageInfoOfBase64
-} from "@midscene/core/image";
 import { getTmpFile } from "@midscene/core/utils";
+import { base64Encoded, imageInfoOfBase64 } from "@midscene/shared/img";
 
 // src/web-element.ts
 var WebElementInfo = class {
@@ -474,10 +471,10 @@ async function getElementInfosFromPage(page) {
 }
 var sizeThreshold = 3;
 async function alignElements(screenshotBuffer, elements, page) {
-  const textsAligned = [];
   const validElements = elements.filter((item) => {
     return item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold;
   });
+  const textsAligned = [];
   for (const item of validElements) {
     const { rect, id, content, attributes, locator } = item;
     textsAligned.push(
@@ -1010,7 +1007,7 @@ ${errorTask == null ? void 0 : errorTask.errorStack}`);
     this.writeOutActionDumps();
     if (!(output == null ? void 0 : output.pass)) {
       const errMsg = msg || `Assertion failed: ${assertion}`;
-      const reasonMsg = `Reason: ${output == null ? void 0 : output.thought} || (no_reason)`;
+      const reasonMsg = `Reason: ${(output == null ? void 0 : output.thought) || "(no_reason)"}`;
       throw new Error(`${errMsg}
 ${reasonMsg}`);
     }