@midscene/web 0.5.2-beta-20241010035503.0 → 0.5.2

package/dist/lib/appium.js

@@ -876,7 +876,7 @@ var import_utils5 = require("@midscene/core/utils");
 var import_node_assert2 = __toESM(require("assert"));
 var import_core = __toESM(require("@midscene/core"));
 var import_utils3 = require("@midscene/core/utils");
-var import_img2 = require("@midscene/shared/img");
+var import_img3 = require("@midscene/shared/img");
 
 // src/common/task-cache.ts
 var import_node_fs2 = require("fs");
@@ -889,8 +889,10 @@ var import_node_assert = __toESM(require("assert"));
 var import_node_crypto = require("crypto");
 var import_node_fs = require("fs");
 var import_node_path = __toESM(require("path"));
+var import_constants = require("@midscene/shared/constants");
 var import_fs = require("@midscene/shared/fs");
 var import_img = require("@midscene/shared/img");
+var import_img2 = require("@midscene/shared/img");
 var import_dayjs = __toESM(require_dayjs_min());
 
 // src/web-element.ts
@@ -901,7 +903,8 @@ var WebElementInfo = class {
     page,
     locator,
     id,
-    attributes
+    attributes,
+    indexId
   }) {
     this.content = content;
     this.rect = rect;
@@ -913,6 +916,7 @@ var WebElementInfo = class {
     this.locator = locator;
     this.id = id;
     this.attributes = attributes;
+    this.indexId = indexId;
   }
 };
 
@@ -921,30 +925,35 @@ async function parseContextFromWebPage(page, _opt) {
   (0, import_node_assert.default)(page, "page is required");
   const url = page.url();
   const file = await page.screenshot();
-  const screenshotBuffer = (0, import_node_fs.readFileSync)(file);
   const screenshotBase64 = (0, import_img.base64Encoded)(file);
   const captureElementSnapshot = await page.getElementInfos();
-  const elementsInfo = await alignElements(
-    screenshotBuffer,
-    captureElementSnapshot,
-    page
-  );
+  const elementsInfo = await alignElements(captureElementSnapshot, page);
+  const elementsPositionInfoWithoutText = elementsInfo.filter((elementInfo) => {
+    if (elementInfo.attributes.nodeType === import_constants.NodeType.TEXT) {
+      return false;
+    }
+    return true;
+  });
   const size = await (0, import_img.imageInfoOfBase64)(screenshotBase64);
+  const screenshotBase64WithElementInfos = await (0, import_img2.compositeElementInfoImg)({
+    inputImgBase64: screenshotBase64.split(";base64,").pop(),
+    elementsPositionInfo: elementsPositionInfoWithoutText
+  });
   return {
     content: elementsInfo,
     size,
-    screenshotBase64,
+    screenshotBase64: `data:image/png;base64,${screenshotBase64WithElementInfos}`,
     url
   };
 }
 var sizeThreshold = 3;
-async function alignElements(screenshotBuffer, elements, page) {
+async function alignElements(elements, page) {
   const validElements = elements.filter((item) => {
     return item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold;
   });
   const textsAligned = [];
   for (const item of validElements) {
-    const { rect, id, content, attributes, locator } = item;
+    const { rect, id, content, attributes, locator, indexId } = item;
     textsAligned.push(
       new WebElementInfo({
         rect,
@@ -952,7 +961,8 @@ async function alignElements(screenshotBuffer, elements, page) {
         id,
         content,
         attributes,
-        page
+        page,
+        indexId
       })
     );
   }
@@ -1137,7 +1147,7 @@ var PageTaskExecutor = class {
     const item = {
       type: "screenshot",
       ts: Date.now(),
-      screenshot: (0, import_img2.base64Encoded)(file),
+      screenshot: (0, import_img3.base64Encoded)(file),
       timing
     };
     return item;
@@ -1506,13 +1516,13 @@ var PageTaskExecutor = class {
       const assertTask = await this.convertPlanToExecutable([assertPlan]);
       await taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
       const output = await taskExecutor.flush();
-      if (output.pass) {
+      if (output == null ? void 0 : output.pass) {
         return {
           output: void 0,
           executor: taskExecutor
         };
       }
-      errorThought = output.thought;
+      errorThought = (output == null ? void 0 : output.thought) || "unknown error";
       const now = Date.now();
       if (now - startTime < checkIntervalMs) {
         const timeRemaining = checkIntervalMs - (now - startTime);
@@ -1655,11 +1665,11 @@ ${errorTask == null ? void 0 : errorTask.errorStack}`);
 // src/appium/page.ts
 var import_node_fs3 = __toESM(require("fs"));
 var import_utils7 = require("@midscene/core/utils");
-var import_img3 = require("@midscene/shared/img");
+var import_img4 = require("@midscene/shared/img");
 var import_xmldom = require("@xmldom/xmldom");
 
 // src/extractor/web-extractor.ts
-var import_constants = require("@midscene/shared/constants");
+var import_constants2 = require("@midscene/shared/constants");
 
 // src/extractor/util.ts
 var import_js_sha256 = __toESM(require_sha256());
@@ -1676,12 +1686,9 @@ function midsceneGenerateHash(content, rect) {
   const hashHex = (0, import_js_sha256.default)(combined);
   return hashHex.slice(0, 10);
 }
-function generateId(numberId) {
-  return `${numberId}`;
-}
 
 // src/extractor/constants.ts
-var import_constants2 = require("@midscene/shared/constants");
+var import_constants3 = require("@midscene/shared/constants");
 
 // src/extractor/client-extractor.ts
 function getNodeAttributes2(node) {
@@ -1790,35 +1797,35 @@ function extractTextWithPosition2(initNode) {
     let nodeType;
     switch (node.nodeName.toUpperCase()) {
       case "TEXT":
-        nodeType = import_constants2.NodeType.TEXT;
+        nodeType = import_constants3.NodeType.TEXT;
         break;
       case "IMAGE":
-        nodeType = import_constants2.NodeType.IMG;
+        nodeType = import_constants3.NodeType.IMG;
         break;
       case "BUTTON":
-        nodeType = import_constants2.NodeType.BUTTON;
+        nodeType = import_constants3.NodeType.BUTTON;
         break;
       case "SEARCHINPUT":
       case "INPUT":
-        nodeType = import_constants2.NodeType.FORM_ITEM;
+        nodeType = import_constants3.NodeType.FORM_ITEM;
         break;
       case "NAV":
       case "LIST":
       case "CELL":
-        nodeType = import_constants2.NodeType.CONTAINER;
+        nodeType = import_constants3.NodeType.CONTAINER;
         break;
       default:
         if (attributes.id === "android:id/input" || attributes.id === "android:id/inputArea") {
-          nodeType = import_constants2.NodeType.FORM_ITEM;
+          nodeType = import_constants3.NodeType.FORM_ITEM;
         } else {
-          nodeType = import_constants2.NodeType.CONTAINER;
+          nodeType = import_constants3.NodeType.CONTAINER;
         }
         break;
     }
     const xpath = getXPathForElement(node);
     const elementInfo = {
       id: nodeHashId,
-      indexId: generateId(nodeIndex++),
+      indexId: nodeIndex++,
       nodeHashId,
       locator: xpath,
       attributes: __spreadValues({
@@ -1833,7 +1840,7 @@ function extractTextWithPosition2(initNode) {
       nodeType,
       nodePath: ""
     };
-    if (elementInfo.nodeType !== import_constants2.NodeType.CONTAINER) {
+    if (elementInfo.nodeType !== import_constants3.NodeType.CONTAINER) {
       elementInfoArray.push(elementInfo);
     }
   }
@@ -1866,7 +1873,7 @@ var Page = class {
     const { width, height } = await this.browser.getWindowSize();
     const path2 = (0, import_utils7.getTmpFile)("png");
     const screenshotBuffer = await this.browser.saveScreenshot(path2);
-    const resizedScreenshotBuffer = await (0, import_img3.resizeImg)(screenshotBuffer, {
+    const resizedScreenshotBuffer = await (0, import_img4.resizeImg)(screenshotBuffer, {
       width,
       height
     });

package/dist/lib/debug.js

@@ -128,20 +128,7 @@ function writeFileSyncWithDir(filePath, content, options = {}) {
 }
 async function getElementInfos(page) {
   const captureElementSnapshot = await page.getElementInfos();
-  const elementsPositionInfo = captureElementSnapshot.map(
-    (elementInfo, index) => {
-      var _a;
-      return {
-        label: ((_a = elementInfo.indexId) == null ? void 0 : _a.toString()) || index.toString(),
-        x: elementInfo.rect.left,
-        y: elementInfo.rect.top,
-        width: elementInfo.rect.width,
-        height: elementInfo.rect.height,
-        attributes: elementInfo.attributes
-      };
-    }
-  );
-  const elementsPositionInfoWithoutText = elementsPositionInfo.filter(
+  const elementsPositionInfoWithoutText = captureElementSnapshot.filter(
     (elementInfo) => {
       if (elementInfo.attributes.nodeType === import_constants.NodeType.TEXT) {
         return false;
@@ -150,7 +137,7 @@ async function getElementInfos(page) {
     }
   );
   return {
-    elementsPositionInfo,
+    elementsPositionInfo: captureElementSnapshot,
     captureElementSnapshot,
     elementsPositionInfoWithoutText
   };

package/dist/lib/index.js

@@ -883,7 +883,7 @@ var import_utils5 = require("@midscene/core/utils");
 var import_node_assert2 = __toESM(require("assert"));
 var import_core = __toESM(require("@midscene/core"));
 var import_utils3 = require("@midscene/core/utils");
-var import_img2 = require("@midscene/shared/img");
+var import_img3 = require("@midscene/shared/img");
 
 // src/common/task-cache.ts
 var import_node_fs2 = require("fs");
@@ -896,8 +896,10 @@ var import_node_assert = __toESM(require("assert"));
 var import_node_crypto = require("crypto");
 var import_node_fs = require("fs");
 var import_node_path = __toESM(require("path"));
+var import_constants = require("@midscene/shared/constants");
 var import_fs = require("@midscene/shared/fs");
 var import_img = require("@midscene/shared/img");
+var import_img2 = require("@midscene/shared/img");
 var import_dayjs = __toESM(require_dayjs_min());
 
 // src/web-element.ts
@@ -908,7 +910,8 @@ var WebElementInfo = class {
     page,
     locator,
     id,
-    attributes
+    attributes,
+    indexId
   }) {
     this.content = content;
     this.rect = rect;
@@ -920,6 +923,7 @@ var WebElementInfo = class {
     this.locator = locator;
     this.id = id;
     this.attributes = attributes;
+    this.indexId = indexId;
   }
 };
 
@@ -928,19 +932,24 @@ async function parseContextFromWebPage(page, _opt) {
   (0, import_node_assert.default)(page, "page is required");
   const url = page.url();
   const file = await page.screenshot();
-  const screenshotBuffer = (0, import_node_fs.readFileSync)(file);
   const screenshotBase64 = (0, import_img.base64Encoded)(file);
   const captureElementSnapshot = await page.getElementInfos();
-  const elementsInfo = await alignElements(
-    screenshotBuffer,
-    captureElementSnapshot,
-    page
-  );
+  const elementsInfo = await alignElements(captureElementSnapshot, page);
+  const elementsPositionInfoWithoutText = elementsInfo.filter((elementInfo) => {
+    if (elementInfo.attributes.nodeType === import_constants.NodeType.TEXT) {
+      return false;
+    }
+    return true;
+  });
   const size = await (0, import_img.imageInfoOfBase64)(screenshotBase64);
+  const screenshotBase64WithElementInfos = await (0, import_img2.compositeElementInfoImg)({
+    inputImgBase64: screenshotBase64.split(";base64,").pop(),
+    elementsPositionInfo: elementsPositionInfoWithoutText
+  });
   return {
     content: elementsInfo,
     size,
-    screenshotBase64,
+    screenshotBase64: `data:image/png;base64,${screenshotBase64WithElementInfos}`,
     url
   };
 }
@@ -952,13 +961,13 @@ async function getExtraReturnLogic() {
   return `${elementInfosScriptContent}midscene_element_inspector.webExtractTextWithPosition()`;
 }
 var sizeThreshold = 3;
-async function alignElements(screenshotBuffer, elements, page) {
+async function alignElements(elements, page) {
   const validElements = elements.filter((item) => {
     return item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold;
   });
   const textsAligned = [];
   for (const item of validElements) {
-    const { rect, id, content, attributes, locator } = item;
+    const { rect, id, content, attributes, locator, indexId } = item;
     textsAligned.push(
       new WebElementInfo({
         rect,
@@ -966,7 +975,8 @@ async function alignElements(screenshotBuffer, elements, page) {
         id,
         content,
         attributes,
-        page
+        page,
+        indexId
       })
     );
   }
@@ -1151,7 +1161,7 @@ var PageTaskExecutor = class {
     const item = {
       type: "screenshot",
       ts: Date.now(),
-      screenshot: (0, import_img2.base64Encoded)(file),
+      screenshot: (0, import_img3.base64Encoded)(file),
       timing
     };
     return item;
@@ -1520,13 +1530,13 @@ var PageTaskExecutor = class {
       const assertTask = await this.convertPlanToExecutable([assertPlan]);
       await taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
       const output = await taskExecutor.flush();
-      if (output.pass) {
+      if (output == null ? void 0 : output.pass) {
         return {
           output: void 0,
           executor: taskExecutor
         };
       }
-      errorThought = output.thought;
+      errorThought = (output == null ? void 0 : output.thought) || "unknown error";
       const now = Date.now();
       if (now - startTime < checkIntervalMs) {
         const timeRemaining = checkIntervalMs - (now - startTime);
@@ -1797,7 +1807,7 @@ function waitForNetworkIdle(page) {
 // src/puppeteer/base-page.ts
 var import_node_fs3 = require("fs");
 var import_utils7 = require("@midscene/core/utils");
-var import_img3 = require("@midscene/shared/img");
+var import_img4 = require("@midscene/shared/img");
 var Page = class {
   evaluate(pageFunction, arg) {
     if (this.pageType === "puppeteer") {
@@ -1822,14 +1832,14 @@ var Page = class {
         deviceScaleFactor: window.devicePixelRatio
       };
     });
-    const path3 = (0, import_utils7.getTmpFile)("jpeg");
+    const path3 = (0, import_utils7.getTmpFile)("png");
     await this.page.screenshot({
       path: path3,
       type: "png"
     });
     let buf;
     if (viewportSize.deviceScaleFactor > 1) {
-      buf = await (0, import_img3.resizeImg)((0, import_node_fs3.readFileSync)(path3), {
+      buf = await (0, import_img4.resizeImg)((0, import_node_fs3.readFileSync)(path3), {
         width: viewportSize.width,
         height: viewportSize.height
       });
@@ -1921,11 +1931,11 @@ var PuppeteerAgent = class extends PageAgent {
 // src/appium/page.ts
 var import_node_fs4 = __toESM(require("fs"));
 var import_utils9 = require("@midscene/core/utils");
-var import_img4 = require("@midscene/shared/img");
+var import_img5 = require("@midscene/shared/img");
 var import_xmldom = require("@xmldom/xmldom");
 
 // src/extractor/web-extractor.ts
-var import_constants = require("@midscene/shared/constants");
+var import_constants2 = require("@midscene/shared/constants");
 
 // src/extractor/util.ts
 var import_js_sha256 = __toESM(require_sha256());
@@ -1942,12 +1952,9 @@ function midsceneGenerateHash(content, rect) {
   const hashHex = (0, import_js_sha256.default)(combined);
   return hashHex.slice(0, 10);
 }
-function generateId(numberId) {
-  return `${numberId}`;
-}
 
 // src/extractor/constants.ts
-var import_constants2 = require("@midscene/shared/constants");
+var import_constants3 = require("@midscene/shared/constants");
 
 // src/extractor/client-extractor.ts
 function getNodeAttributes2(node) {
@@ -2056,35 +2063,35 @@ function extractTextWithPosition2(initNode) {
     let nodeType;
     switch (node.nodeName.toUpperCase()) {
       case "TEXT":
-        nodeType = import_constants2.NodeType.TEXT;
+        nodeType = import_constants3.NodeType.TEXT;
         break;
       case "IMAGE":
-        nodeType = import_constants2.NodeType.IMG;
+        nodeType = import_constants3.NodeType.IMG;
         break;
       case "BUTTON":
-        nodeType = import_constants2.NodeType.BUTTON;
+        nodeType = import_constants3.NodeType.BUTTON;
         break;
       case "SEARCHINPUT":
       case "INPUT":
-        nodeType = import_constants2.NodeType.FORM_ITEM;
+        nodeType = import_constants3.NodeType.FORM_ITEM;
         break;
       case "NAV":
       case "LIST":
       case "CELL":
-        nodeType = import_constants2.NodeType.CONTAINER;
+        nodeType = import_constants3.NodeType.CONTAINER;
         break;
       default:
         if (attributes.id === "android:id/input" || attributes.id === "android:id/inputArea") {
-          nodeType = import_constants2.NodeType.FORM_ITEM;
+          nodeType = import_constants3.NodeType.FORM_ITEM;
         } else {
-          nodeType = import_constants2.NodeType.CONTAINER;
+          nodeType = import_constants3.NodeType.CONTAINER;
         }
         break;
     }
     const xpath = getXPathForElement(node);
     const elementInfo = {
       id: nodeHashId,
-      indexId: generateId(nodeIndex++),
+      indexId: nodeIndex++,
       nodeHashId,
       locator: xpath,
       attributes: __spreadValues({
@@ -2099,7 +2106,7 @@ function extractTextWithPosition2(initNode) {
       nodeType,
       nodePath: ""
     };
-    if (elementInfo.nodeType !== import_constants2.NodeType.CONTAINER) {
+    if (elementInfo.nodeType !== import_constants3.NodeType.CONTAINER) {
       elementInfoArray.push(elementInfo);
     }
   }
@@ -2132,7 +2139,7 @@ var Page2 = class {
     const { width, height } = await this.browser.getWindowSize();
     const path3 = (0, import_utils9.getTmpFile)("png");
     const screenshotBuffer = await this.browser.saveScreenshot(path3);
-    const resizedScreenshotBuffer = await (0, import_img4.resizeImg)(screenshotBuffer, {
+    const resizedScreenshotBuffer = await (0, import_img5.resizeImg)(screenshotBuffer, {
       width,
       height
     });
@@ -2290,7 +2297,7 @@ var Page2 = class {
 // src/debug/index.ts
 var import_node_fs5 = require("fs");
 var import_node_path3 = __toESM(require("path"));
-var import_img5 = require("@midscene/shared/img");
+var import_img6 = require("@midscene/shared/img");
 async function generateExtractData(page, targetDir, saveImgType) {
   const file = await page.screenshot();
   const screenshotBuffer = (0, import_node_fs5.readFileSync)(file);
@@ -2311,12 +2318,12 @@ async function generateExtractData(page, targetDir, saveImgType) {
   const {
     compositeElementInfoImgBase64,
     compositeElementInfoImgWithoutTextBase64
-  } = await (0, import_img5.processImageElementInfo)({
+  } = await (0, import_img6.processImageElementInfo)({
     elementsPositionInfo,
     elementsPositionInfoWithoutText,
     inputImgBase64
   });
-  const resizeImgBase64 = await (0, import_img5.resizeImg)(inputImgBase64);
+  const resizeImgBase64 = await (0, import_img6.resizeImg)(inputImgBase64);
   const existingSnapshot = (0, import_node_fs5.existsSync)(snapshotJsonPath) ? JSON.parse((0, import_node_fs5.readFileSync)(snapshotJsonPath, "utf-8")) : null;
   if (existingSnapshot && JSON.stringify(existingSnapshot) === JSON.stringify(captureElementSnapshot)) {
     console.log("skip save snapshot for ", targetDir);
@@ -2329,25 +2336,25 @@ async function generateExtractData(page, targetDir, saveImgType) {
     );
   }
   if (!(saveImgType == null ? void 0 : saveImgType.disableInputImage)) {
-    await (0, import_img5.saveBase64Image)({
+    await (0, import_img6.saveBase64Image)({
       base64Data: inputImgBase64,
       outputPath: inputImagePath
     });
   }
   if (!(saveImgType == null ? void 0 : saveImgType.disableOutputImage)) {
-    await (0, import_img5.saveBase64Image)({
+    await (0, import_img6.saveBase64Image)({
       base64Data: compositeElementInfoImgBase64,
       outputPath: outputImagePath
     });
   }
   if (!(saveImgType == null ? void 0 : saveImgType.disableOutputWithoutTextImg)) {
-    await (0, import_img5.saveBase64Image)({
+    await (0, import_img6.saveBase64Image)({
       base64Data: compositeElementInfoImgWithoutTextBase64,
       outputPath: outputWithoutTextImgPath
     });
   }
   if (!(saveImgType == null ? void 0 : saveImgType.disableResizeOutputImg)) {
-    await (0, import_img5.saveBase64Image)({
+    await (0, import_img6.saveBase64Image)({
       base64Data: resizeImgBase64,
       outputPath: resizeOutputImgPath
     });
@@ -2367,29 +2374,16 @@ function writeFileSyncWithDir(filePath, content, options = {}) {
 }
 async function getElementInfos(page) {
   const captureElementSnapshot = await page.getElementInfos();
-  const elementsPositionInfo = captureElementSnapshot.map(
-    (elementInfo, index) => {
-      var _a;
-      return {
-        label: ((_a = elementInfo.indexId) == null ? void 0 : _a.toString()) || index.toString(),
-        x: elementInfo.rect.left,
-        y: elementInfo.rect.top,
-        width: elementInfo.rect.width,
-        height: elementInfo.rect.height,
-        attributes: elementInfo.attributes
-      };
-    }
-  );
-  const elementsPositionInfoWithoutText = elementsPositionInfo.filter(
+  const elementsPositionInfoWithoutText = captureElementSnapshot.filter(
     (elementInfo) => {
-      if (elementInfo.attributes.nodeType === import_constants2.NodeType.TEXT) {
+      if (elementInfo.attributes.nodeType === import_constants3.NodeType.TEXT) {
         return false;
       }
       return true;
     }
   );
   return {
-    elementsPositionInfo,
+    elementsPositionInfo: captureElementSnapshot,
     captureElementSnapshot,
     elementsPositionInfoWithoutText
   };

package/dist/lib/playwright-report.js

@@ -337,8 +337,10 @@ var import_node_assert = __toESM(require("assert"));
 var import_node_crypto = require("crypto");
 var import_node_fs = require("fs");
 var import_node_path = __toESM(require("path"));
+var import_constants = require("@midscene/shared/constants");
 var import_fs = require("@midscene/shared/fs");
 var import_img = require("@midscene/shared/img");
+var import_img2 = require("@midscene/shared/img");
 var import_dayjs = __toESM(require_dayjs_min());
 function reportFileName(tag = "web") {
   const dateTimeInFileName = (0, import_dayjs.default)().format("YYYY-MM-DD_HH-mm-ss-SSS");