@midscene/shared 0.17.2 → 0.17.3-beta-20250526031130.0

package/dist/es/constants.d.ts

@@ -6,6 +6,7 @@ declare enum NodeType {
     CONTAINER = "CONTAINER Node",
     FORM_ITEM = "FORM_ITEM Node",
     BUTTON = "BUTTON Node",
+    A = "Anchor Node",
     IMG = "IMG Node",
     TEXT = "TEXT Node",
     POSITION = "POSITION Node"

package/dist/es/constants.js

@@ -7,6 +7,7 @@ var NodeType = /* @__PURE__ */ ((NodeType2) => {
   NodeType2["CONTAINER"] = "CONTAINER Node";
   NodeType2["FORM_ITEM"] = "FORM_ITEM Node";
   NodeType2["BUTTON"] = "BUTTON Node";
+  NodeType2["A"] = "Anchor Node";
   NodeType2["IMG"] = "IMG Node";
   NodeType2["TEXT"] = "TEXT Node";
   NodeType2["POSITION"] = "POSITION Node";

package/dist/es/extractor-debug.js

@@ -39,6 +39,9 @@ function isFormElement(node) {
 function isButtonElement(node) {
   return node instanceof HTMLElement && node.tagName.toLowerCase() === "button";
 }
+function isAElement(node) {
+  return node instanceof HTMLElement && node.tagName.toLowerCase() === "a";
+}
 function isImgElement(node) {
   if (!includeBaseElement(node) && node instanceof Element) {
     const computedStyle = window.getComputedStyle(node);
@@ -89,7 +92,8 @@ function includeBaseElement(node) {
     "textarea",
     "select",
     "option",
-    "img"
+    "img",
+    "a"
   ];
   for (const tagName of includeList) {
     const element = node.querySelectorAll(tagName);
@@ -471,10 +475,20 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
     return elementInfo;
   }
   if (isButtonElement(node)) {
+    const rect2 = mergeElementAndChildrenRects(
+      node,
+      currentWindow,
+      currentDocument,
+      baseZoom,
+      visibleOnly
+    );
+    if (!rect2) {
+      return null;
+    }
     const attributes = getNodeAttributes(node, currentWindow);
     const pseudo = getPseudoElementContent(node, currentWindow);
     const content = node.innerText || pseudo.before || pseudo.after || "";
-    const nodeHashId = midsceneGenerateHash(node, content, rect);
+    const nodeHashId = midsceneGenerateHash(node, content, rect2);
     const selector = setDataForNode(node, nodeHashId, false, currentWindow);
     const elementInfo = {
       id: nodeHashId,
@@ -488,12 +502,12 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
         nodeType: "BUTTON Node" /* BUTTON */
       },
       content,
-      rect,
+      rect: rect2,
       center: [
-        Math.round(rect.left + rect.width / 2),
-        Math.round(rect.top + rect.height / 2)
+        Math.round(rect2.left + rect2.width / 2),
+        Math.round(rect2.top + rect2.height / 2)
       ],
-      zoom: rect.zoom
+      zoom: rect2.zoom
     };
     return elementInfo;
   }
@@ -558,6 +572,33 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
     };
     return elementInfo;
   }
+  if (isAElement(node)) {
+    const attributes = getNodeAttributes(node, currentWindow);
+    const pseudo = getPseudoElementContent(node, currentWindow);
+    const content = node.innerText || pseudo.before || pseudo.after || "";
+    const nodeHashId = midsceneGenerateHash(node, content, rect);
+    const selector = setDataForNode(node, nodeHashId, false, currentWindow);
+    const elementInfo = {
+      id: nodeHashId,
+      indexId: indexId++,
+      nodeHashId,
+      nodeType: "Anchor Node" /* A */,
+      locator: selector,
+      attributes: {
+        ...attributes,
+        htmlTagName: tagNameOfNode(node),
+        nodeType: "Anchor Node" /* A */
+      },
+      content,
+      rect,
+      center: [
+        Math.round(rect.left + rect.width / 2),
+        Math.round(rect.top + rect.height / 2)
+      ],
+      zoom: rect.zoom
+    };
+    return elementInfo;
+  }
   if (isContainerElement(node)) {
     const attributes = getNodeAttributes(node, currentWindow);
     const nodeHashId = midsceneGenerateHash(node, "", rect);
@@ -628,7 +669,7 @@ function extractTreeNode(initNode, debugMode2 = false) {
       node: elementInfo,
       children: []
     };
-    if (elementInfo?.nodeType === "BUTTON Node" /* BUTTON */ || elementInfo?.nodeType === "IMG Node" /* IMG */ || elementInfo?.nodeType === "TEXT Node" /* TEXT */ || elementInfo?.nodeType === "FORM_ITEM Node" /* FORM_ITEM */ || elementInfo?.nodeType === "CONTAINER Node" /* CONTAINER */) {
+    if (elementInfo?.nodeType === "BUTTON Node" /* BUTTON */ || elementInfo?.nodeType === "IMG Node" /* IMG */ || elementInfo?.nodeType === "TEXT Node" /* TEXT */ || elementInfo?.nodeType === "Anchor Node" /* A */ || elementInfo?.nodeType === "FORM_ITEM Node" /* FORM_ITEM */ || elementInfo?.nodeType === "CONTAINER Node" /* CONTAINER */) {
       return nodeInfo;
     }
     const rect = getRect(node, baseZoom, currentWindow);
@@ -683,6 +724,50 @@ function extractTreeNode(initNode, debugMode2 = false) {
     children: topChildren
   };
 }
+function mergeElementAndChildrenRects(node, currentWindow, currentDocument, baseZoom = 1, visibleOnly = true) {
+  const selfRect = elementRect(
+    node,
+    currentWindow,
+    currentDocument,
+    baseZoom,
+    visibleOnly
+  );
+  if (!selfRect)
+    return null;
+  let minLeft = selfRect.left;
+  let minTop = selfRect.top;
+  let maxRight = selfRect.left + selfRect.width;
+  let maxBottom = selfRect.top + selfRect.height;
+  function traverse(child) {
+    for (let i = 0; i < child.childNodes.length; i++) {
+      const sub = child.childNodes[i];
+      if (sub.nodeType === 1) {
+        const rect = elementRect(
+          sub,
+          currentWindow,
+          currentDocument,
+          baseZoom,
+          visibleOnly
+        );
+        if (rect) {
+          minLeft = Math.min(minLeft, rect.left);
+          minTop = Math.min(minTop, rect.top);
+          maxRight = Math.max(maxRight, rect.left + rect.width);
+          maxBottom = Math.max(maxBottom, rect.top + rect.height);
+        }
+        traverse(sub);
+      }
+    }
+  }
+  traverse(node);
+  return {
+    ...selfRect,
+    left: minLeft,
+    top: minTop,
+    width: maxRight - minLeft,
+    height: maxBottom - minTop
+  };
+}
 
 // src/extractor/debug.ts
 console.log(extractTextWithPosition(document.body, true));

package/dist/es/extractor.js

@@ -160,6 +160,9 @@ function isFormElement(node) {
 function isButtonElement(node) {
   return node instanceof HTMLElement && node.tagName.toLowerCase() === "button";
 }
+function isAElement(node) {
+  return node instanceof HTMLElement && node.tagName.toLowerCase() === "a";
+}
 function isImgElement(node) {
   if (!includeBaseElement(node) && node instanceof Element) {
     const computedStyle = window.getComputedStyle(node);
@@ -210,7 +213,8 @@ function includeBaseElement(node) {
     "textarea",
     "select",
     "option",
-    "img"
+    "img",
+    "a"
   ];
   for (const tagName of includeList) {
     const element = node.querySelectorAll(tagName);
@@ -599,10 +603,20 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
     return elementInfo;
   }
   if (isButtonElement(node)) {
+    const rect2 = mergeElementAndChildrenRects(
+      node,
+      currentWindow,
+      currentDocument,
+      baseZoom,
+      visibleOnly
+    );
+    if (!rect2) {
+      return null;
+    }
     const attributes = getNodeAttributes(node, currentWindow);
     const pseudo = getPseudoElementContent(node, currentWindow);
     const content = node.innerText || pseudo.before || pseudo.after || "";
-    const nodeHashId = midsceneGenerateHash(node, content, rect);
+    const nodeHashId = midsceneGenerateHash(node, content, rect2);
     const selector = setDataForNode(node, nodeHashId, false, currentWindow);
     const elementInfo = {
       id: nodeHashId,
@@ -616,12 +630,12 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
         nodeType: "BUTTON Node" /* BUTTON */
       },
       content,
-      rect,
+      rect: rect2,
       center: [
-        Math.round(rect.left + rect.width / 2),
-        Math.round(rect.top + rect.height / 2)
+        Math.round(rect2.left + rect2.width / 2),
+        Math.round(rect2.top + rect2.height / 2)
       ],
-      zoom: rect.zoom
+      zoom: rect2.zoom
     };
     return elementInfo;
   }
@@ -686,6 +700,33 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
     };
     return elementInfo;
   }
+  if (isAElement(node)) {
+    const attributes = getNodeAttributes(node, currentWindow);
+    const pseudo = getPseudoElementContent(node, currentWindow);
+    const content = node.innerText || pseudo.before || pseudo.after || "";
+    const nodeHashId = midsceneGenerateHash(node, content, rect);
+    const selector = setDataForNode(node, nodeHashId, false, currentWindow);
+    const elementInfo = {
+      id: nodeHashId,
+      indexId: indexId++,
+      nodeHashId,
+      nodeType: "Anchor Node" /* A */,
+      locator: selector,
+      attributes: {
+        ...attributes,
+        htmlTagName: tagNameOfNode(node),
+        nodeType: "Anchor Node" /* A */
+      },
+      content,
+      rect,
+      center: [
+        Math.round(rect.left + rect.width / 2),
+        Math.round(rect.top + rect.height / 2)
+      ],
+      zoom: rect.zoom
+    };
+    return elementInfo;
+  }
   if (isContainerElement(node)) {
     const attributes = getNodeAttributes(node, currentWindow);
     const nodeHashId = midsceneGenerateHash(node, "", rect);
@@ -760,7 +801,7 @@ function extractTreeNode(initNode, debugMode2 = false) {
       node: elementInfo,
       children: []
     };
-    if (elementInfo?.nodeType === "BUTTON Node" /* BUTTON */ || elementInfo?.nodeType === "IMG Node" /* IMG */ || elementInfo?.nodeType === "TEXT Node" /* TEXT */ || elementInfo?.nodeType === "FORM_ITEM Node" /* FORM_ITEM */ || elementInfo?.nodeType === "CONTAINER Node" /* CONTAINER */) {
+    if (elementInfo?.nodeType === "BUTTON Node" /* BUTTON */ || elementInfo?.nodeType === "IMG Node" /* IMG */ || elementInfo?.nodeType === "TEXT Node" /* TEXT */ || elementInfo?.nodeType === "Anchor Node" /* A */ || elementInfo?.nodeType === "FORM_ITEM Node" /* FORM_ITEM */ || elementInfo?.nodeType === "CONTAINER Node" /* CONTAINER */) {
       return nodeInfo;
     }
     const rect = getRect(node, baseZoom, currentWindow);
@@ -815,6 +856,50 @@ function extractTreeNode(initNode, debugMode2 = false) {
     children: topChildren
   };
 }
+function mergeElementAndChildrenRects(node, currentWindow, currentDocument, baseZoom = 1, visibleOnly = true) {
+  const selfRect = elementRect(
+    node,
+    currentWindow,
+    currentDocument,
+    baseZoom,
+    visibleOnly
+  );
+  if (!selfRect)
+    return null;
+  let minLeft = selfRect.left;
+  let minTop = selfRect.top;
+  let maxRight = selfRect.left + selfRect.width;
+  let maxBottom = selfRect.top + selfRect.height;
+  function traverse(child) {
+    for (let i = 0; i < child.childNodes.length; i++) {
+      const sub = child.childNodes[i];
+      if (sub.nodeType === 1) {
+        const rect = elementRect(
+          sub,
+          currentWindow,
+          currentDocument,
+          baseZoom,
+          visibleOnly
+        );
+        if (rect) {
+          minLeft = Math.min(minLeft, rect.left);
+          minTop = Math.min(minTop, rect.top);
+          maxRight = Math.max(maxRight, rect.left + rect.width);
+          maxBottom = Math.max(maxBottom, rect.top + rect.height);
+        }
+        traverse(sub);
+      }
+    }
+  }
+  traverse(node);
+  return {
+    ...selfRect,
+    left: minLeft,
+    top: minTop,
+    width: maxRight - minLeft,
+    height: maxBottom - minTop
+  };
+}
 
 // src/extractor/locator.ts
 var getElementIndex = (element) => {
@@ -828,16 +913,6 @@ var getElementIndex = (element) => {
   }
   return index;
 };
-var findFirstAncestorWithId = (element) => {
-  let current = element;
-  while (current?.parentElement) {
-    if (current.id) {
-      return current;
-    }
-    current = current.parentElement;
-  }
-  return null;
-};
 var getTextNodeIndex = (textNode) => {
   let index = 1;
   let current = textNode.previousSibling;
@@ -855,6 +930,10 @@ var getElementXPath = (element) => {
     if (parentNode && parentNode.nodeType === Node.ELEMENT_NODE) {
       const parentXPath = getElementXPath(parentNode);
       const textIndex = getTextNodeIndex(element);
+      const textContent = element.textContent?.trim();
+      if (textContent) {
+        return `${parentXPath}/text()[${textIndex}][normalize-space()="${textContent}"]`;
+      }
       return `${parentXPath}/text()[${textIndex}]`;
     }
     return "";
@@ -868,22 +947,6 @@ var getElementXPath = (element) => {
   if (el === document.body) {
     return "/html/body";
   }
-  if (el.id) {
-    return `//*[@id="${el.id}"]`;
-  }
-  const ancestorWithId = findFirstAncestorWithId(el);
-  if (ancestorWithId) {
-    const ancestorPath = `//*[@id="${ancestorWithId.id}"]`;
-    let current = el;
-    const pathParts = [];
-    while (current && current !== ancestorWithId) {
-      const index2 = getElementIndex(current);
-      const tagName2 = current.nodeName.toLowerCase();
-      pathParts.unshift(`${tagName2}[${index2}]`);
-      current = current.parentElement;
-    }
-    return pathParts.length > 0 ? `${ancestorPath}/${pathParts.join("/")}` : ancestorPath;
-  }
   if (!el.parentNode) {
     return `/${el.nodeName.toLowerCase()}`;
   }
@@ -898,8 +961,8 @@ var getElementXPath = (element) => {
 function generateXPaths(node) {
   if (!node)
     return [];
-  const xPath = getElementXPath(node);
-  return [xPath];
+  const fullXPath = getElementXPath(node);
+  return [fullXPath];
 }
 function getXpathsById(id) {
   const node = getNodeFromCacheList(id);

package/dist/lib/constants.d.ts

@@ -6,6 +6,7 @@ declare enum NodeType {
     CONTAINER = "CONTAINER Node",
     FORM_ITEM = "FORM_ITEM Node",
     BUTTON = "BUTTON Node",
+    A = "Anchor Node",
     IMG = "IMG Node",
     TEXT = "TEXT Node",
     POSITION = "POSITION Node"

package/dist/lib/constants.js

@@ -41,6 +41,7 @@ var NodeType = /* @__PURE__ */ ((NodeType2) => {
   NodeType2["CONTAINER"] = "CONTAINER Node";
   NodeType2["FORM_ITEM"] = "FORM_ITEM Node";
   NodeType2["BUTTON"] = "BUTTON Node";
+  NodeType2["A"] = "Anchor Node";
   NodeType2["IMG"] = "IMG Node";
   NodeType2["TEXT"] = "TEXT Node";
   NodeType2["POSITION"] = "POSITION Node";

package/dist/lib/extractor-debug.js

@@ -41,6 +41,9 @@ function isFormElement(node) {
 function isButtonElement(node) {
   return node instanceof HTMLElement && node.tagName.toLowerCase() === "button";
 }
+function isAElement(node) {
+  return node instanceof HTMLElement && node.tagName.toLowerCase() === "a";
+}
 function isImgElement(node) {
   if (!includeBaseElement(node) && node instanceof Element) {
     const computedStyle = window.getComputedStyle(node);
@@ -91,7 +94,8 @@ function includeBaseElement(node) {
     "textarea",
     "select",
     "option",
-    "img"
+    "img",
+    "a"
   ];
   for (const tagName of includeList) {
     const element = node.querySelectorAll(tagName);
@@ -473,10 +477,20 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
     return elementInfo;
   }
   if (isButtonElement(node)) {
+    const rect2 = mergeElementAndChildrenRects(
+      node,
+      currentWindow,
+      currentDocument,
+      baseZoom,
+      visibleOnly
+    );
+    if (!rect2) {
+      return null;
+    }
     const attributes = getNodeAttributes(node, currentWindow);
     const pseudo = getPseudoElementContent(node, currentWindow);
     const content = node.innerText || pseudo.before || pseudo.after || "";
-    const nodeHashId = midsceneGenerateHash(node, content, rect);
+    const nodeHashId = midsceneGenerateHash(node, content, rect2);
     const selector = setDataForNode(node, nodeHashId, false, currentWindow);
     const elementInfo = {
       id: nodeHashId,
@@ -490,12 +504,12 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
         nodeType: "BUTTON Node" /* BUTTON */
       },
       content,
-      rect,
+      rect: rect2,
       center: [
-        Math.round(rect.left + rect.width / 2),
-        Math.round(rect.top + rect.height / 2)
+        Math.round(rect2.left + rect2.width / 2),
+        Math.round(rect2.top + rect2.height / 2)
       ],
-      zoom: rect.zoom
+      zoom: rect2.zoom
     };
     return elementInfo;
   }
@@ -560,6 +574,33 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
     };
     return elementInfo;
   }
+  if (isAElement(node)) {
+    const attributes = getNodeAttributes(node, currentWindow);
+    const pseudo = getPseudoElementContent(node, currentWindow);
+    const content = node.innerText || pseudo.before || pseudo.after || "";
+    const nodeHashId = midsceneGenerateHash(node, content, rect);
+    const selector = setDataForNode(node, nodeHashId, false, currentWindow);
+    const elementInfo = {
+      id: nodeHashId,
+      indexId: indexId++,
+      nodeHashId,
+      nodeType: "Anchor Node" /* A */,
+      locator: selector,
+      attributes: {
+        ...attributes,
+        htmlTagName: tagNameOfNode(node),
+        nodeType: "Anchor Node" /* A */
+      },
+      content,
+      rect,
+      center: [
+        Math.round(rect.left + rect.width / 2),
+        Math.round(rect.top + rect.height / 2)
+      ],
+      zoom: rect.zoom
+    };
+    return elementInfo;
+  }
   if (isContainerElement(node)) {
     const attributes = getNodeAttributes(node, currentWindow);
     const nodeHashId = midsceneGenerateHash(node, "", rect);
@@ -630,7 +671,7 @@ function extractTreeNode(initNode, debugMode2 = false) {
       node: elementInfo,
       children: []
     };
-    if (elementInfo?.nodeType === "BUTTON Node" /* BUTTON */ || elementInfo?.nodeType === "IMG Node" /* IMG */ || elementInfo?.nodeType === "TEXT Node" /* TEXT */ || elementInfo?.nodeType === "FORM_ITEM Node" /* FORM_ITEM */ || elementInfo?.nodeType === "CONTAINER Node" /* CONTAINER */) {
+    if (elementInfo?.nodeType === "BUTTON Node" /* BUTTON */ || elementInfo?.nodeType === "IMG Node" /* IMG */ || elementInfo?.nodeType === "TEXT Node" /* TEXT */ || elementInfo?.nodeType === "Anchor Node" /* A */ || elementInfo?.nodeType === "FORM_ITEM Node" /* FORM_ITEM */ || elementInfo?.nodeType === "CONTAINER Node" /* CONTAINER */) {
       return nodeInfo;
     }
     const rect = getRect(node, baseZoom, currentWindow);
@@ -685,6 +726,50 @@ function extractTreeNode(initNode, debugMode2 = false) {
     children: topChildren
   };
 }
+function mergeElementAndChildrenRects(node, currentWindow, currentDocument, baseZoom = 1, visibleOnly = true) {
+  const selfRect = elementRect(
+    node,
+    currentWindow,
+    currentDocument,
+    baseZoom,
+    visibleOnly
+  );
+  if (!selfRect)
+    return null;
+  let minLeft = selfRect.left;
+  let minTop = selfRect.top;
+  let maxRight = selfRect.left + selfRect.width;
+  let maxBottom = selfRect.top + selfRect.height;
+  function traverse(child) {
+    for (let i = 0; i < child.childNodes.length; i++) {
+      const sub = child.childNodes[i];
+      if (sub.nodeType === 1) {
+        const rect = elementRect(
+          sub,
+          currentWindow,
+          currentDocument,
+          baseZoom,
+          visibleOnly
+        );
+        if (rect) {
+          minLeft = Math.min(minLeft, rect.left);
+          minTop = Math.min(minTop, rect.top);
+          maxRight = Math.max(maxRight, rect.left + rect.width);
+          maxBottom = Math.max(maxBottom, rect.top + rect.height);
+        }
+        traverse(sub);
+      }
+    }
+  }
+  traverse(node);
+  return {
+    ...selfRect,
+    left: minLeft,
+    top: minTop,
+    width: maxRight - minLeft,
+    height: maxBottom - minTop
+  };
+}
 
 // src/extractor/debug.ts
 console.log(extractTextWithPosition(document.body, true));