@midscene/web 0.5.1 → 0.5.2-beta-20240929094445.0

package/dist/script/htmlElement.js

@@ -595,17 +595,60 @@ var midscene_element_inspector = (() => {
 
   // src/extractor/dom-util.ts
   function isFormElement(node) {
-    return node instanceof HTMLElement && (node.tagName.toLowerCase() === "input" || node.tagName.toLowerCase() === "textarea" || node.tagName.toLowerCase() === "label" || node.tagName.toLowerCase() === "select" || node.tagName.toLowerCase() === "option");
+    return node instanceof HTMLElement && (node.tagName.toLowerCase() === "input" || node.tagName.toLowerCase() === "textarea" || node.tagName.toLowerCase() === "select" || node.tagName.toLowerCase() === "option");
   }
   function isButtonElement(node) {
     return node instanceof HTMLElement && node.tagName.toLowerCase() === "button";
   }
   function isImgElement(node) {
+    if (!includeBaseElement(node) && node instanceof Element) {
+      const computedStyle = window.getComputedStyle(node);
+      const backgroundImage = computedStyle.getPropertyValue("background-image");
+      if (backgroundImage !== "none") {
+        return true;
+      }
+    }
     return node instanceof HTMLElement && node.tagName.toLowerCase() === "img" || node instanceof SVGElement && node.tagName.toLowerCase() === "svg";
   }
   function isTextElement(node) {
     return node.nodeName.toLowerCase() === "#text";
   }
+  function isContainerElement(node) {
+    if (!(node instanceof HTMLElement))
+      return false;
+    if (includeBaseElement(node)) {
+      return false;
+    }
+    const computedStyle = window.getComputedStyle(node);
+    const backgroundColor = computedStyle.getPropertyValue("background-color");
+    if (backgroundColor) {
+      return true;
+    }
+    return false;
+  }
+  function includeBaseElement(node) {
+    if (!(node instanceof HTMLElement))
+      return false;
+    if (node.innerText) {
+      return true;
+    }
+    const includeList = [
+      "svg",
+      "button",
+      "input",
+      "textarea",
+      "select",
+      "option",
+      "img"
+    ];
+    for (const tagName of includeList) {
+      const element = node.querySelectorAll(tagName);
+      if (element.length > 0) {
+        return true;
+      }
+    }
+    return false;
+  }
 
   // src/extractor/util.ts
   var import_js_sha256 = __toESM(require_sha256());
@@ -633,9 +676,9 @@ var midscene_element_inspector = (() => {
   function selectorForValue(val) {
     return `[${taskIdKey}='${val}']`;
   }
-  function setDataForNode(node, nodeHash) {
+  function setDataForNode(node, nodeHash, setToParentNode = false) {
     const taskId = taskIdKey;
-    if (!(node instanceof HTMLElement)) {
+    if (!(node instanceof Element)) {
       return "";
     }
     if (!taskId) {
@@ -644,7 +687,13 @@ var midscene_element_inspector = (() => {
     }
     const selector = selectorForValue(nodeHash);
     if (getDebugMode()) {
-      node.setAttribute(taskIdKey, nodeHash.toString());
+      if (setToParentNode) {
+        if (node.parentNode instanceof HTMLElement) {
+          node.parentNode.setAttribute(taskIdKey, nodeHash.toString());
+        }
+      } else {
+        node.setAttribute(taskIdKey, nodeHash.toString());
+      }
     }
     return selector;
   }
@@ -702,6 +751,28 @@ var midscene_element_inspector = (() => {
       zoom
     };
   }
+  var isElementCovered = (el, rect) => {
+    const x = rect.left + rect.width / 2;
+    const y = rect.top + rect.height / 2;
+    const topElement = document.elementFromPoint(x, y);
+    if (topElement === el) {
+      return false;
+    }
+    if (el == null ? void 0 : el.contains(topElement)) {
+      return false;
+    }
+    if (topElement == null ? void 0 : topElement.contains(el)) {
+      return false;
+    }
+    logger(el, "Element is covered by another element", {
+      topElement,
+      el,
+      rect,
+      x,
+      y
+    });
+    return true;
+  };
   function visibleRect(el, baseZoom = 1) {
     if (!el) {
       logger(el, "Element is not in the DOM hierarchy");
@@ -723,6 +794,9 @@ var midscene_element_inspector = (() => {
       logger(el, "Element has no size");
       return false;
     }
+    if (baseZoom === 1 && isElementCovered(el, rect)) {
+      return false;
+    }
     const scrollLeft = window.pageXOffset || document.documentElement.scrollLeft;
     const scrollTop = window.pageYOffset || document.documentElement.scrollTop;
     const viewportWidth = window.innerWidth || document.documentElement.clientWidth;
@@ -815,12 +889,11 @@ var midscene_element_inspector = (() => {
       logger("Element is not visible", node);
       return null;
     }
-    const debugMode2 = getDebugMode();
     if (isFormElement(node)) {
-      const attributes2 = getNodeAttributes(node);
-      const nodeHashId2 = midsceneGenerateHash(attributes2.placeholder, rect);
-      const selector2 = setDataForNode(node, nodeHashId2);
-      let valueContent = attributes2.value || attributes2.placeholder || node.textContent || "";
+      const attributes = getNodeAttributes(node);
+      const nodeHashId = midsceneGenerateHash(attributes.placeholder, rect);
+      const selector = setDataForNode(node, nodeHashId);
+      let valueContent = attributes.value || attributes.placeholder || node.textContent || "";
       const tagName = node.tagName.toLowerCase();
       if (node.tagName.toLowerCase() === "select") {
         const selectedOption = node.options[node.selectedIndex];
@@ -829,13 +902,13 @@ var midscene_element_inspector = (() => {
       if ((node.tagName.toLowerCase() === "input" || node.tagName.toLowerCase() === "textarea") && node.value) {
         valueContent = node.value;
       }
-      const elementInfo2 = {
-        id: nodeHashId2,
+      const elementInfo = {
+        id: nodeHashId,
         nodePath,
-        nodeHashId: nodeHashId2,
-        locator: selector2,
+        nodeHashId,
+        locator: selector,
         nodeType: NodeType.FORM_ITEM,
-        attributes: __spreadProps(__spreadValues({}, attributes2), {
+        attributes: __spreadProps(__spreadValues({}, attributes), {
           htmlTagName: `<${tagName}>`,
           nodeType: NodeType.FORM_ITEM
         }),
@@ -845,24 +918,25 @@ var midscene_element_inspector = (() => {
           Math.round(rect.left + rect.width / 2),
           Math.round(rect.top + rect.height / 2)
         ],
-        htmlNode: debugMode2 ? node : null,
-        zoom: rect.zoom
+        zoom: rect.zoom,
+        screenWidth: window.innerWidth,
+        screenHeight: window.innerHeight
       };
-      return elementInfo2;
+      return elementInfo;
     }
     if (isButtonElement(node)) {
-      const attributes2 = getNodeAttributes(node);
+      const attributes = getNodeAttributes(node);
       const pseudo = getPseudoElementContent(node);
       const content = node.innerText || pseudo.before || pseudo.after || "";
-      const nodeHashId2 = midsceneGenerateHash(content, rect);
-      const selector2 = setDataForNode(node, nodeHashId2);
-      const elementInfo2 = {
-        id: nodeHashId2,
+      const nodeHashId = midsceneGenerateHash(content, rect);
+      const selector = setDataForNode(node, nodeHashId);
+      const elementInfo = {
+        id: nodeHashId,
         nodePath,
-        nodeHashId: nodeHashId2,
+        nodeHashId,
         nodeType: NodeType.BUTTON,
-        locator: selector2,
-        attributes: __spreadProps(__spreadValues({}, attributes2), {
+        locator: selector,
+        attributes: __spreadProps(__spreadValues({}, attributes), {
           nodeType: NodeType.BUTTON
         }),
         content,
@@ -871,21 +945,22 @@ var midscene_element_inspector = (() => {
           Math.round(rect.left + rect.width / 2),
           Math.round(rect.top + rect.height / 2)
         ],
-        htmlNode: debugMode2 ? node : null,
-        zoom: rect.zoom
+        zoom: rect.zoom,
+        screenWidth: window.innerWidth,
+        screenHeight: window.innerHeight
       };
-      return elementInfo2;
+      return elementInfo;
     }
     if (isImgElement(node)) {
-      const attributes2 = getNodeAttributes(node);
-      const nodeHashId2 = midsceneGenerateHash("", rect);
-      const selector2 = setDataForNode(node, nodeHashId2);
-      const elementInfo2 = {
-        id: nodeHashId2,
+      const attributes = getNodeAttributes(node);
+      const nodeHashId = midsceneGenerateHash("", rect);
+      const selector = setDataForNode(node, nodeHashId);
+      const elementInfo = {
+        id: nodeHashId,
         nodePath,
-        nodeHashId: nodeHashId2,
-        locator: selector2,
-        attributes: __spreadProps(__spreadValues(__spreadValues({}, attributes2), node.nodeName.toLowerCase() === "svg" ? {
+        nodeHashId,
+        locator: selector,
+        attributes: __spreadProps(__spreadValues(__spreadValues({}, attributes), node.nodeName.toLowerCase() === "svg" ? {
           svgContent: "true"
         } : {}), {
           nodeType: NodeType.IMG
@@ -897,30 +972,31 @@ var midscene_element_inspector = (() => {
           Math.round(rect.left + rect.width / 2),
           Math.round(rect.top + rect.height / 2)
         ],
-        htmlNode: debugMode2 ? node : null,
-        zoom: rect.zoom
+        zoom: rect.zoom,
+        screenWidth: window.innerWidth,
+        screenHeight: window.innerHeight
       };
-      return elementInfo2;
+      return elementInfo;
     }
     if (isTextElement(node)) {
       const text = (_a = node.textContent) == null ? void 0 : _a.trim().replace(/\n+/g, " ");
       if (!text) {
         return null;
       }
-      const attributes2 = getNodeAttributes(node);
-      const attributeKeys = Object.keys(attributes2);
+      const attributes = getNodeAttributes(node);
+      const attributeKeys = Object.keys(attributes);
       if (!text.trim() && attributeKeys.length === 0) {
         return null;
       }
-      const nodeHashId2 = midsceneGenerateHash(text, rect);
-      const selector2 = setDataForNode(node, nodeHashId2);
-      const elementInfo2 = {
-        id: nodeHashId2,
+      const nodeHashId = midsceneGenerateHash(text, rect);
+      const selector = setDataForNode(node, nodeHashId, true);
+      const elementInfo = {
+        id: nodeHashId,
         nodePath,
-        nodeHashId: nodeHashId2,
+        nodeHashId,
         nodeType: NodeType.TEXT,
-        locator: selector2,
-        attributes: __spreadProps(__spreadValues({}, attributes2), {
+        locator: selector,
+        attributes: __spreadProps(__spreadValues({}, attributes), {
           nodeType: NodeType.TEXT
         }),
         center: [
@@ -930,33 +1006,38 @@ var midscene_element_inspector = (() => {
         // attributes,
         content: text,
         rect,
-        htmlNode: debugMode2 ? node : null,
-        zoom: rect.zoom
+        zoom: rect.zoom,
+        screenWidth: window.innerWidth,
+        screenHeight: window.innerHeight
       };
-      return elementInfo2;
+      return elementInfo;
     }
-    const attributes = getNodeAttributes(node);
-    const nodeHashId = midsceneGenerateHash("", rect);
-    const selector = setDataForNode(node, nodeHashId);
-    const elementInfo = {
-      id: nodeHashId,
-      nodePath,
-      nodeHashId,
-      nodeType: NodeType.CONTAINER,
-      locator: selector,
-      attributes: __spreadProps(__spreadValues({}, attributes), {
-        nodeType: NodeType.CONTAINER
-      }),
-      content: "",
-      rect,
-      center: [
-        Math.round(rect.left + rect.width / 2),
-        Math.round(rect.top + rect.height / 2)
-      ],
-      htmlNode: debugMode2 ? node : null,
-      zoom: rect.zoom
-    };
-    return elementInfo;
+    if (isContainerElement(node)) {
+      const attributes = getNodeAttributes(node);
+      const nodeHashId = midsceneGenerateHash("", rect);
+      const selector = setDataForNode(node, nodeHashId);
+      const elementInfo = {
+        id: nodeHashId,
+        nodePath,
+        nodeHashId,
+        nodeType: NodeType.CONTAINER,
+        locator: selector,
+        attributes: __spreadProps(__spreadValues({}, attributes), {
+          nodeType: NodeType.CONTAINER
+        }),
+        content: "",
+        rect,
+        center: [
+          Math.round(rect.left + rect.width / 2),
+          Math.round(rect.top + rect.height / 2)
+        ],
+        zoom: rect.zoom,
+        screenWidth: window.innerWidth,
+        screenHeight: window.innerHeight
+      };
+      return elementInfo;
+    }
+    return null;
   }
   function extractTextWithPosition(initNode, debugMode2 = false, currentFrame = { id: 0, left: 0, top: 0 }) {
     setDebugMode(debugMode2);
@@ -967,34 +1048,14 @@ var midscene_element_inspector = (() => {
         return null;
       }
       const elementInfo = collectElementInfo(node, nodePath, baseZoom);
-      if ((elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.BUTTON || (elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.IMG) {
+      if ((elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.BUTTON || (elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.IMG || (elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.TEXT || (elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.FORM_ITEM || (elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.CONTAINER) {
+        elementInfoArray.push(elementInfo);
         return elementInfo;
       }
-      let hasNonContainerChildren = false;
-      const childrenPureContainers = [];
+      const rect = getRect(node, baseZoom);
       for (let i = 0; i < node.childNodes.length; i++) {
         logger("will dfs", node.childNodes[i]);
-        const resultLengthBeforeDfs = elementInfoArray.length;
-        const result = dfs(
-          node.childNodes[i],
-          `${nodePath}-${i}`,
-          elementInfo == null ? void 0 : elementInfo.zoom
-        );
-        if (!result)
-          continue;
-        if ((result == null ? void 0 : result.nodeType) === NodeType.CONTAINER && elementInfoArray.length > resultLengthBeforeDfs) {
-          hasNonContainerChildren = true;
-          continue;
-        }
-        if ((result == null ? void 0 : result.nodeType) === NodeType.CONTAINER) {
-          childrenPureContainers.push(result);
-        } else {
-          hasNonContainerChildren = true;
-          elementInfoArray.push(result);
-        }
-      }
-      if (hasNonContainerChildren) {
-        elementInfoArray.push(...childrenPureContainers);
+        dfs(node.childNodes[i], `${nodePath}-${i}`, rect.zoom);
       }
       if (!elementInfo) {
         logger("should NOT continue for node", node);
@@ -1002,10 +1063,7 @@ var midscene_element_inspector = (() => {
       }
       return elementInfo;
     }
-    const outerMostElementInfo = dfs(initNode || getDocument(), "0");
-    if (outerMostElementInfo && !elementInfoArray.length) {
-      elementInfoArray.push(outerMostElementInfo);
-    }
+    dfs(initNode || getDocument(), "0");
     for (let i = 0; i < elementInfoArray.length; i++) {
       elementInfoArray[i].indexId = (i + 1).toString();
     }
@@ -1169,7 +1227,6 @@ var midscene_element_inspector = (() => {
           Math.round(rect.top + rect.height / 2)
         ],
         nodeType,
-        htmlNode: null,
         nodePath: ""
       };
       if (elementInfo.nodeType !== NodeType.CONTAINER) {