@midscene/web 0.5.2-beta-20241010035503.0 → 0.5.2

package/dist/es/appium.js

@@ -894,8 +894,13 @@ import assert from "assert";
 import { randomUUID } from "crypto";
 import { readFileSync } from "fs";
 import path from "path";
+import { NodeType } from "@midscene/shared/constants";
 import { findNearestPackageJson } from "@midscene/shared/fs";
-import { base64Encoded, imageInfoOfBase64 } from "@midscene/shared/img";
+import {
+  base64Encoded,
+  imageInfoOfBase64
+} from "@midscene/shared/img";
+import { compositeElementInfoImg } from "@midscene/shared/img";
 
 // src/web-element.ts
 var WebElementInfo = class {
@@ -905,7 +910,8 @@ var WebElementInfo = class {
     page,
     locator,
     id,
-    attributes
+    attributes,
+    indexId
   }) {
     this.content = content;
     this.rect = rect;
@@ -917,6 +923,7 @@ var WebElementInfo = class {
     this.locator = locator;
     this.id = id;
     this.attributes = attributes;
+    this.indexId = indexId;
   }
 };
 
@@ -925,30 +932,35 @@ async function parseContextFromWebPage(page, _opt) {
   assert(page, "page is required");
   const url = page.url();
   const file = await page.screenshot();
-  const screenshotBuffer = readFileSync(file);
   const screenshotBase64 = base64Encoded(file);
   const captureElementSnapshot = await page.getElementInfos();
-  const elementsInfo = await alignElements(
-    screenshotBuffer,
-    captureElementSnapshot,
-    page
-  );
+  const elementsInfo = await alignElements(captureElementSnapshot, page);
+  const elementsPositionInfoWithoutText = elementsInfo.filter((elementInfo) => {
+    if (elementInfo.attributes.nodeType === NodeType.TEXT) {
+      return false;
+    }
+    return true;
+  });
   const size = await imageInfoOfBase64(screenshotBase64);
+  const screenshotBase64WithElementInfos = await compositeElementInfoImg({
+    inputImgBase64: screenshotBase64.split(";base64,").pop(),
+    elementsPositionInfo: elementsPositionInfoWithoutText
+  });
   return {
     content: elementsInfo,
     size,
-    screenshotBase64,
+    screenshotBase64: `data:image/png;base64,${screenshotBase64WithElementInfos}`,
     url
   };
 }
 var sizeThreshold = 3;
-async function alignElements(screenshotBuffer, elements, page) {
+async function alignElements(elements, page) {
   const validElements = elements.filter((item) => {
     return item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold;
   });
   const textsAligned = [];
   for (const item of validElements) {
-    const { rect, id, content, attributes, locator } = item;
+    const { rect, id, content, attributes, locator, indexId } = item;
     textsAligned.push(
       new WebElementInfo({
         rect,
@@ -956,7 +968,8 @@ async function alignElements(screenshotBuffer, elements, page) {
         id,
         content,
         attributes,
-        page
+        page,
+        indexId
       })
     );
   }
@@ -1510,13 +1523,13 @@ var PageTaskExecutor = class {
       const assertTask = await this.convertPlanToExecutable([assertPlan]);
       await taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
       const output = await taskExecutor.flush();
-      if (output.pass) {
+      if (output == null ? void 0 : output.pass) {
         return {
           output: void 0,
           executor: taskExecutor
         };
       }
-      errorThought = output.thought;
+      errorThought = (output == null ? void 0 : output.thought) || "unknown error";
       const now = Date.now();
       if (now - startTime < checkIntervalMs) {
         const timeRemaining = checkIntervalMs - (now - startTime);
@@ -1666,7 +1679,7 @@ import { DOMParser } from "@xmldom/xmldom";
 import {
   CONTAINER_MINI_HEIGHT,
   CONTAINER_MINI_WIDTH,
-  NodeType
+  NodeType as NodeType2
 } from "@midscene/shared/constants";
 
 // src/extractor/util.ts
@@ -1684,13 +1697,10 @@ function midsceneGenerateHash(content, rect) {
   const hashHex = (0, import_js_sha256.default)(combined);
   return hashHex.slice(0, 10);
 }
-function generateId(numberId) {
-  return `${numberId}`;
-}
 
 // src/extractor/constants.ts
 import {
-  NodeType as NodeType2,
+  NodeType as NodeType3,
   TEXT_MAX_SIZE,
   TEXT_SIZE_THRESHOLD
 } from "@midscene/shared/constants";
@@ -1802,35 +1812,35 @@ function extractTextWithPosition2(initNode) {
     let nodeType;
     switch (node.nodeName.toUpperCase()) {
       case "TEXT":
-        nodeType = NodeType2.TEXT;
+        nodeType = NodeType3.TEXT;
         break;
       case "IMAGE":
-        nodeType = NodeType2.IMG;
+        nodeType = NodeType3.IMG;
         break;
       case "BUTTON":
-        nodeType = NodeType2.BUTTON;
+        nodeType = NodeType3.BUTTON;
         break;
       case "SEARCHINPUT":
       case "INPUT":
-        nodeType = NodeType2.FORM_ITEM;
+        nodeType = NodeType3.FORM_ITEM;
         break;
       case "NAV":
       case "LIST":
       case "CELL":
-        nodeType = NodeType2.CONTAINER;
+        nodeType = NodeType3.CONTAINER;
         break;
       default:
         if (attributes.id === "android:id/input" || attributes.id === "android:id/inputArea") {
-          nodeType = NodeType2.FORM_ITEM;
+          nodeType = NodeType3.FORM_ITEM;
         } else {
-          nodeType = NodeType2.CONTAINER;
+          nodeType = NodeType3.CONTAINER;
         }
         break;
     }
     const xpath = getXPathForElement(node);
     const elementInfo = {
       id: nodeHashId,
-      indexId: generateId(nodeIndex++),
+      indexId: nodeIndex++,
       nodeHashId,
       locator: xpath,
       attributes: __spreadValues({
@@ -1845,7 +1855,7 @@ function extractTextWithPosition2(initNode) {
       nodeType,
       nodePath: ""
     };
-    if (elementInfo.nodeType !== NodeType2.CONTAINER) {
+    if (elementInfo.nodeType !== NodeType3.CONTAINER) {
       elementInfoArray.push(elementInfo);
     }
   }

package/dist/es/debug.js

@@ -107,20 +107,7 @@ function writeFileSyncWithDir(filePath, content, options = {}) {
 }
 async function getElementInfos(page) {
   const captureElementSnapshot = await page.getElementInfos();
-  const elementsPositionInfo = captureElementSnapshot.map(
-    (elementInfo, index) => {
-      var _a;
-      return {
-        label: ((_a = elementInfo.indexId) == null ? void 0 : _a.toString()) || index.toString(),
-        x: elementInfo.rect.left,
-        y: elementInfo.rect.top,
-        width: elementInfo.rect.width,
-        height: elementInfo.rect.height,
-        attributes: elementInfo.attributes
-      };
-    }
-  );
-  const elementsPositionInfoWithoutText = elementsPositionInfo.filter(
+  const elementsPositionInfoWithoutText = captureElementSnapshot.filter(
     (elementInfo) => {
       if (elementInfo.attributes.nodeType === NodeType.TEXT) {
         return false;
@@ -129,7 +116,7 @@ async function getElementInfos(page) {
     }
   );
   return {
-    elementsPositionInfo,
+    elementsPositionInfo: captureElementSnapshot,
     captureElementSnapshot,
     elementsPositionInfoWithoutText
   };

package/dist/es/index.js

@@ -897,8 +897,13 @@ import assert from "assert";
 import { randomUUID } from "crypto";
 import { readFileSync } from "fs";
 import path from "path";
+import { NodeType } from "@midscene/shared/constants";
 import { findNearestPackageJson } from "@midscene/shared/fs";
-import { base64Encoded, imageInfoOfBase64 } from "@midscene/shared/img";
+import {
+  base64Encoded,
+  imageInfoOfBase64
+} from "@midscene/shared/img";
+import { compositeElementInfoImg } from "@midscene/shared/img";
 
 // src/web-element.ts
 var WebElementInfo = class {
@@ -908,7 +913,8 @@ var WebElementInfo = class {
     page,
     locator,
     id,
-    attributes
+    attributes,
+    indexId
   }) {
     this.content = content;
     this.rect = rect;
@@ -920,6 +926,7 @@ var WebElementInfo = class {
     this.locator = locator;
     this.id = id;
     this.attributes = attributes;
+    this.indexId = indexId;
   }
 };
 
@@ -928,19 +935,24 @@ async function parseContextFromWebPage(page, _opt) {
   assert(page, "page is required");
   const url = page.url();
   const file = await page.screenshot();
-  const screenshotBuffer = readFileSync(file);
   const screenshotBase64 = base64Encoded(file);
   const captureElementSnapshot = await page.getElementInfos();
-  const elementsInfo = await alignElements(
-    screenshotBuffer,
-    captureElementSnapshot,
-    page
-  );
+  const elementsInfo = await alignElements(captureElementSnapshot, page);
+  const elementsPositionInfoWithoutText = elementsInfo.filter((elementInfo) => {
+    if (elementInfo.attributes.nodeType === NodeType.TEXT) {
+      return false;
+    }
+    return true;
+  });
   const size = await imageInfoOfBase64(screenshotBase64);
+  const screenshotBase64WithElementInfos = await compositeElementInfoImg({
+    inputImgBase64: screenshotBase64.split(";base64,").pop(),
+    elementsPositionInfo: elementsPositionInfoWithoutText
+  });
   return {
     content: elementsInfo,
     size,
-    screenshotBase64,
+    screenshotBase64: `data:image/png;base64,${screenshotBase64WithElementInfos}`,
     url
   };
 }
@@ -952,13 +964,13 @@ async function getExtraReturnLogic() {
   return `${elementInfosScriptContent}midscene_element_inspector.webExtractTextWithPosition()`;
 }
 var sizeThreshold = 3;
-async function alignElements(screenshotBuffer, elements, page) {
+async function alignElements(elements, page) {
   const validElements = elements.filter((item) => {
     return item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold;
   });
   const textsAligned = [];
   for (const item of validElements) {
-    const { rect, id, content, attributes, locator } = item;
+    const { rect, id, content, attributes, locator, indexId } = item;
     textsAligned.push(
       new WebElementInfo({
         rect,
@@ -966,7 +978,8 @@ async function alignElements(screenshotBuffer, elements, page) {
         id,
         content,
         attributes,
-        page
+        page,
+        indexId
       })
     );
   }
@@ -1520,13 +1533,13 @@ var PageTaskExecutor = class {
       const assertTask = await this.convertPlanToExecutable([assertPlan]);
       await taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
       const output = await taskExecutor.flush();
-      if (output.pass) {
+      if (output == null ? void 0 : output.pass) {
         return {
           output: void 0,
           executor: taskExecutor
         };
       }
-      errorThought = output.thought;
+      errorThought = (output == null ? void 0 : output.thought) || "unknown error";
       const now = Date.now();
       if (now - startTime < checkIntervalMs) {
         const timeRemaining = checkIntervalMs - (now - startTime);
@@ -1822,7 +1835,7 @@ var Page = class {
         deviceScaleFactor: window.devicePixelRatio
       };
     });
-    const path3 = getTmpFile("jpeg");
+    const path3 = getTmpFile("png");
     await this.page.screenshot({
       path: path3,
       type: "png"
@@ -1928,7 +1941,7 @@ import { DOMParser } from "@xmldom/xmldom";
 import {
   CONTAINER_MINI_HEIGHT,
   CONTAINER_MINI_WIDTH,
-  NodeType
+  NodeType as NodeType2
 } from "@midscene/shared/constants";
 
 // src/extractor/util.ts
@@ -1946,13 +1959,10 @@ function midsceneGenerateHash(content, rect) {
   const hashHex = (0, import_js_sha256.default)(combined);
   return hashHex.slice(0, 10);
 }
-function generateId(numberId) {
-  return `${numberId}`;
-}
 
 // src/extractor/constants.ts
 import {
-  NodeType as NodeType2,
+  NodeType as NodeType3,
   TEXT_MAX_SIZE,
   TEXT_SIZE_THRESHOLD
 } from "@midscene/shared/constants";
@@ -2064,35 +2074,35 @@ function extractTextWithPosition2(initNode) {
     let nodeType;
     switch (node.nodeName.toUpperCase()) {
       case "TEXT":
-        nodeType = NodeType2.TEXT;
+        nodeType = NodeType3.TEXT;
         break;
       case "IMAGE":
-        nodeType = NodeType2.IMG;
+        nodeType = NodeType3.IMG;
         break;
       case "BUTTON":
-        nodeType = NodeType2.BUTTON;
+        nodeType = NodeType3.BUTTON;
         break;
       case "SEARCHINPUT":
       case "INPUT":
-        nodeType = NodeType2.FORM_ITEM;
+        nodeType = NodeType3.FORM_ITEM;
         break;
       case "NAV":
       case "LIST":
       case "CELL":
-        nodeType = NodeType2.CONTAINER;
+        nodeType = NodeType3.CONTAINER;
         break;
       default:
         if (attributes.id === "android:id/input" || attributes.id === "android:id/inputArea") {
-          nodeType = NodeType2.FORM_ITEM;
+          nodeType = NodeType3.FORM_ITEM;
         } else {
-          nodeType = NodeType2.CONTAINER;
+          nodeType = NodeType3.CONTAINER;
         }
         break;
     }
     const xpath = getXPathForElement(node);
     const elementInfo = {
       id: nodeHashId,
-      indexId: generateId(nodeIndex++),
+      indexId: nodeIndex++,
       nodeHashId,
       locator: xpath,
       attributes: __spreadValues({
@@ -2107,7 +2117,7 @@ function extractTextWithPosition2(initNode) {
       nodeType,
       nodePath: ""
     };
-    if (elementInfo.nodeType !== NodeType2.CONTAINER) {
+    if (elementInfo.nodeType !== NodeType3.CONTAINER) {
       elementInfoArray.push(elementInfo);
     }
   }
@@ -2379,29 +2389,16 @@ function writeFileSyncWithDir(filePath, content, options = {}) {
 }
 async function getElementInfos(page) {
   const captureElementSnapshot = await page.getElementInfos();
-  const elementsPositionInfo = captureElementSnapshot.map(
-    (elementInfo, index) => {
-      var _a;
-      return {
-        label: ((_a = elementInfo.indexId) == null ? void 0 : _a.toString()) || index.toString(),
-        x: elementInfo.rect.left,
-        y: elementInfo.rect.top,
-        width: elementInfo.rect.width,
-        height: elementInfo.rect.height,
-        attributes: elementInfo.attributes
-      };
-    }
-  );
-  const elementsPositionInfoWithoutText = elementsPositionInfo.filter(
+  const elementsPositionInfoWithoutText = captureElementSnapshot.filter(
     (elementInfo) => {
-      if (elementInfo.attributes.nodeType === NodeType2.TEXT) {
+      if (elementInfo.attributes.nodeType === NodeType3.TEXT) {
         return false;
       }
       return true;
     }
   );
   return {
-    elementsPositionInfo,
+    elementsPositionInfo: captureElementSnapshot,
     captureElementSnapshot,
     elementsPositionInfoWithoutText
   };

package/dist/es/playwright-report.js

@@ -325,8 +325,13 @@ import assert from "assert";
 import { randomUUID } from "crypto";
 import { readFileSync } from "fs";
 import path from "path";
+import { NodeType } from "@midscene/shared/constants";
 import { findNearestPackageJson } from "@midscene/shared/fs";
-import { base64Encoded, imageInfoOfBase64 } from "@midscene/shared/img";
+import {
+  base64Encoded,
+  imageInfoOfBase64
+} from "@midscene/shared/img";
+import { compositeElementInfoImg } from "@midscene/shared/img";
 function reportFileName(tag = "web") {
   const dateTimeInFileName = (0, import_dayjs.default)().format("YYYY-MM-DD_HH-mm-ss-SSS");
   return `${tag}-${dateTimeInFileName}`;

package/dist/es/playwright.js

@@ -371,8 +371,13 @@ import assert from "assert";
 import { randomUUID } from "crypto";
 import { readFileSync } from "fs";
 import path from "path";
+import { NodeType } from "@midscene/shared/constants";
 import { findNearestPackageJson } from "@midscene/shared/fs";
-import { base64Encoded, imageInfoOfBase64 } from "@midscene/shared/img";
+import {
+  base64Encoded,
+  imageInfoOfBase64
+} from "@midscene/shared/img";
+import { compositeElementInfoImg } from "@midscene/shared/img";
 
 // src/web-element.ts
 var WebElementInfo = class {
@@ -382,7 +387,8 @@ var WebElementInfo = class {
     page,
     locator,
     id,
-    attributes
+    attributes,
+    indexId
   }) {
     this.content = content;
     this.rect = rect;
@@ -394,6 +400,7 @@ var WebElementInfo = class {
     this.locator = locator;
     this.id = id;
     this.attributes = attributes;
+    this.indexId = indexId;
   }
 };
 
@@ -402,19 +409,24 @@ async function parseContextFromWebPage(page, _opt) {
   assert(page, "page is required");
   const url = page.url();
   const file = await page.screenshot();
-  const screenshotBuffer = readFileSync(file);
   const screenshotBase64 = base64Encoded(file);
   const captureElementSnapshot = await page.getElementInfos();
-  const elementsInfo = await alignElements(
-    screenshotBuffer,
-    captureElementSnapshot,
-    page
-  );
+  const elementsInfo = await alignElements(captureElementSnapshot, page);
+  const elementsPositionInfoWithoutText = elementsInfo.filter((elementInfo) => {
+    if (elementInfo.attributes.nodeType === NodeType.TEXT) {
+      return false;
+    }
+    return true;
+  });
   const size = await imageInfoOfBase64(screenshotBase64);
+  const screenshotBase64WithElementInfos = await compositeElementInfoImg({
+    inputImgBase64: screenshotBase64.split(";base64,").pop(),
+    elementsPositionInfo: elementsPositionInfoWithoutText
+  });
   return {
     content: elementsInfo,
     size,
-    screenshotBase64,
+    screenshotBase64: `data:image/png;base64,${screenshotBase64WithElementInfos}`,
     url
   };
 }
@@ -426,13 +438,13 @@ async function getExtraReturnLogic() {
   return `${elementInfosScriptContent}midscene_element_inspector.webExtractTextWithPosition()`;
 }
 var sizeThreshold = 3;
-async function alignElements(screenshotBuffer, elements, page) {
+async function alignElements(elements, page) {
   const validElements = elements.filter((item) => {
     return item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold;
   });
   const textsAligned = [];
   for (const item of validElements) {
-    const { rect, id, content, attributes, locator } = item;
+    const { rect, id, content, attributes, locator, indexId } = item;
     textsAligned.push(
       new WebElementInfo({
         rect,
@@ -440,7 +452,8 @@ async function alignElements(screenshotBuffer, elements, page) {
         id,
         content,
         attributes,
-        page
+        page,
+        indexId
       })
     );
   }
@@ -994,13 +1007,13 @@ var PageTaskExecutor = class {
       const assertTask = await this.convertPlanToExecutable([assertPlan]);
       await taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
       const output = await taskExecutor.flush();
-      if (output.pass) {
+      if (output == null ? void 0 : output.pass) {
         return {
           output: void 0,
           executor: taskExecutor
         };
       }
-      errorThought = output.thought;
+      errorThought = (output == null ? void 0 : output.thought) || "unknown error";
       const now = Date.now();
       if (now - startTime < checkIntervalMs) {
         const timeRemaining = checkIntervalMs - (now - startTime);
@@ -1296,7 +1309,7 @@ var Page = class {
         deviceScaleFactor: window.devicePixelRatio
       };
     });
-    const path2 = getTmpFile("jpeg");
+    const path2 = getTmpFile("png");
     await this.page.screenshot({
       path: path2,
       type: "png"

package/dist/es/puppeteer.js

@@ -368,8 +368,13 @@ import assert from "assert";
 import { randomUUID } from "crypto";
 import { readFileSync } from "fs";
 import path from "path";
+import { NodeType } from "@midscene/shared/constants";
 import { findNearestPackageJson } from "@midscene/shared/fs";
-import { base64Encoded, imageInfoOfBase64 } from "@midscene/shared/img";
+import {
+  base64Encoded,
+  imageInfoOfBase64
+} from "@midscene/shared/img";
+import { compositeElementInfoImg } from "@midscene/shared/img";
 
 // src/web-element.ts
 var WebElementInfo = class {
@@ -379,7 +384,8 @@ var WebElementInfo = class {
     page,
     locator,
     id,
-    attributes
+    attributes,
+    indexId
   }) {
     this.content = content;
     this.rect = rect;
@@ -391,6 +397,7 @@ var WebElementInfo = class {
     this.locator = locator;
     this.id = id;
     this.attributes = attributes;
+    this.indexId = indexId;
   }
 };
 
@@ -399,19 +406,24 @@ async function parseContextFromWebPage(page, _opt) {
   assert(page, "page is required");
   const url = page.url();
   const file = await page.screenshot();
-  const screenshotBuffer = readFileSync(file);
   const screenshotBase64 = base64Encoded(file);
   const captureElementSnapshot = await page.getElementInfos();
-  const elementsInfo = await alignElements(
-    screenshotBuffer,
-    captureElementSnapshot,
-    page
-  );
+  const elementsInfo = await alignElements(captureElementSnapshot, page);
+  const elementsPositionInfoWithoutText = elementsInfo.filter((elementInfo) => {
+    if (elementInfo.attributes.nodeType === NodeType.TEXT) {
+      return false;
+    }
+    return true;
+  });
   const size = await imageInfoOfBase64(screenshotBase64);
+  const screenshotBase64WithElementInfos = await compositeElementInfoImg({
+    inputImgBase64: screenshotBase64.split(";base64,").pop(),
+    elementsPositionInfo: elementsPositionInfoWithoutText
+  });
   return {
     content: elementsInfo,
     size,
-    screenshotBase64,
+    screenshotBase64: `data:image/png;base64,${screenshotBase64WithElementInfos}`,
     url
   };
 }
@@ -423,13 +435,13 @@ async function getExtraReturnLogic() {
   return `${elementInfosScriptContent}midscene_element_inspector.webExtractTextWithPosition()`;
 }
 var sizeThreshold = 3;
-async function alignElements(screenshotBuffer, elements, page) {
+async function alignElements(elements, page) {
   const validElements = elements.filter((item) => {
     return item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold;
   });
   const textsAligned = [];
   for (const item of validElements) {
-    const { rect, id, content, attributes, locator } = item;
+    const { rect, id, content, attributes, locator, indexId } = item;
     textsAligned.push(
       new WebElementInfo({
         rect,
@@ -437,7 +449,8 @@ async function alignElements(screenshotBuffer, elements, page) {
         id,
         content,
         attributes,
-        page
+        page,
+        indexId
       })
     );
   }
@@ -991,13 +1004,13 @@ var PageTaskExecutor = class {
       const assertTask = await this.convertPlanToExecutable([assertPlan]);
       await taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
       const output = await taskExecutor.flush();
-      if (output.pass) {
+      if (output == null ? void 0 : output.pass) {
         return {
           output: void 0,
           executor: taskExecutor
         };
       }
-      errorThought = output.thought;
+      errorThought = (output == null ? void 0 : output.thought) || "unknown error";
       const now = Date.now();
       if (now - startTime < checkIntervalMs) {
         const timeRemaining = checkIntervalMs - (now - startTime);
@@ -1165,7 +1178,7 @@ var Page = class {
         deviceScaleFactor: window.devicePixelRatio
       };
     });
-    const path2 = getTmpFile("jpeg");
+    const path2 = getTmpFile("png");
     await this.page.screenshot({
       path: path2,
       type: "png"