misoai-web 1.0.2 → 1.0.4

package/iife-script/htmlElement.js

@@ -48,9 +48,9 @@ var midscene_element_inspector = (() => {
   ));
   var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 
-  // resolve-false:/empty-stub
+  // resolve-false:\empty-stub
   var require_empty_stub = __commonJS({
-    "resolve-false:/empty-stub"(exports, module) {
+    "resolve-false:\\empty-stub"(exports, module) {
       "use strict";
       module.exports = {};
     }
@@ -755,6 +755,9 @@ ${indentStr}${after}`;
   function isButtonElement(node) {
     return node instanceof HTMLElement && node.tagName.toLowerCase() === "button";
   }
+  function isAElement(node) {
+    return node instanceof HTMLElement && node.tagName.toLowerCase() === "a";
+  }
   function isImgElement(node) {
     if (!includeBaseElement(node) && node instanceof Element) {
       const computedStyle = window.getComputedStyle(node);
@@ -805,7 +808,8 @@ ${indentStr}${after}`;
       "textarea",
       "select",
       "option",
-      "img"
+      "img",
+      "a"
     ];
     for (const tagName of includeList) {
       const element = node.querySelectorAll(tagName);
@@ -1196,10 +1200,20 @@ ${indentStr}${after}`;
       return elementInfo;
     }
     if (isButtonElement(node)) {
+      const rect2 = mergeElementAndChildrenRects(
+        node,
+        currentWindow,
+        currentDocument,
+        baseZoom,
+        visibleOnly
+      );
+      if (!rect2) {
+        return null;
+      }
       const attributes = getNodeAttributes(node, currentWindow);
       const pseudo = getPseudoElementContent(node, currentWindow);
       const content = node.innerText || pseudo.before || pseudo.after || "";
-      const nodeHashId = midsceneGenerateHash(node, content, rect);
+      const nodeHashId = midsceneGenerateHash(node, content, rect2);
       const selector = setDataForNode(node, nodeHashId, false, currentWindow);
       const elementInfo = {
         id: nodeHashId,
@@ -1212,12 +1226,12 @@ ${indentStr}${after}`;
           nodeType: "BUTTON Node" /* BUTTON */
         }),
         content,
-        rect,
+        rect: rect2,
         center: [
-          Math.round(rect.left + rect.width / 2),
-          Math.round(rect.top + rect.height / 2)
+          Math.round(rect2.left + rect2.width / 2),
+          Math.round(rect2.top + rect2.height / 2)
         ],
-        zoom: rect.zoom
+        zoom: rect2.zoom
       };
       return elementInfo;
     }
@@ -1279,6 +1293,32 @@ ${indentStr}${after}`;
       };
       return elementInfo;
     }
+    if (isAElement(node)) {
+      const attributes = getNodeAttributes(node, currentWindow);
+      const pseudo = getPseudoElementContent(node, currentWindow);
+      const content = node.innerText || pseudo.before || pseudo.after || "";
+      const nodeHashId = midsceneGenerateHash(node, content, rect);
+      const selector = setDataForNode(node, nodeHashId, false, currentWindow);
+      const elementInfo = {
+        id: nodeHashId,
+        indexId: indexId++,
+        nodeHashId,
+        nodeType: "Anchor Node" /* A */,
+        locator: selector,
+        attributes: __spreadProps(__spreadValues({}, attributes), {
+          htmlTagName: tagNameOfNode(node),
+          nodeType: "Anchor Node" /* A */
+        }),
+        content,
+        rect,
+        center: [
+          Math.round(rect.left + rect.width / 2),
+          Math.round(rect.top + rect.height / 2)
+        ],
+        zoom: rect.zoom
+      };
+      return elementInfo;
+    }
     if (isContainerElement(node)) {
       const attributes = getNodeAttributes(node, currentWindow);
       const nodeHashId = midsceneGenerateHash(node, "", rect);
@@ -1352,7 +1392,7 @@ ${indentStr}${after}`;
         node: elementInfo,
         children: []
       };
-      if ((elementInfo == null ? void 0 : elementInfo.nodeType) === "BUTTON Node" /* BUTTON */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "IMG Node" /* IMG */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "TEXT Node" /* TEXT */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "FORM_ITEM Node" /* FORM_ITEM */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "CONTAINER Node" /* CONTAINER */) {
+      if ((elementInfo == null ? void 0 : elementInfo.nodeType) === "BUTTON Node" /* BUTTON */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "IMG Node" /* IMG */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "TEXT Node" /* TEXT */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "Anchor Node" /* A */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "FORM_ITEM Node" /* FORM_ITEM */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "CONTAINER Node" /* CONTAINER */) {
         return nodeInfo;
       }
       const rect = getRect(node, baseZoom, currentWindow);
@@ -1407,6 +1447,49 @@ ${indentStr}${after}`;
       children: topChildren
     };
   }
+  function mergeElementAndChildrenRects(node, currentWindow, currentDocument, baseZoom = 1, visibleOnly = true) {
+    const selfRect = elementRect(
+      node,
+      currentWindow,
+      currentDocument,
+      baseZoom,
+      visibleOnly
+    );
+    if (!selfRect)
+      return null;
+    let minLeft = selfRect.left;
+    let minTop = selfRect.top;
+    let maxRight = selfRect.left + selfRect.width;
+    let maxBottom = selfRect.top + selfRect.height;
+    function traverse(child) {
+      for (let i = 0; i < child.childNodes.length; i++) {
+        const sub = child.childNodes[i];
+        if (sub.nodeType === 1) {
+          const rect = elementRect(
+            sub,
+            currentWindow,
+            currentDocument,
+            baseZoom,
+            visibleOnly
+          );
+          if (rect) {
+            minLeft = Math.min(minLeft, rect.left);
+            minTop = Math.min(minTop, rect.top);
+            maxRight = Math.max(maxRight, rect.left + rect.width);
+            maxBottom = Math.max(maxBottom, rect.top + rect.height);
+          }
+          traverse(sub);
+        }
+      }
+    }
+    traverse(node);
+    return __spreadProps(__spreadValues({}, selfRect), {
+      left: minLeft,
+      top: minTop,
+      width: maxRight - minLeft,
+      height: maxBottom - minTop
+    });
+  }
 
   // src/extractor/locator.ts
   var getElementIndex = (element) => {
@@ -1420,16 +1503,6 @@ ${indentStr}${after}`;
     }
     return index;
   };
-  var findFirstAncestorWithId = (element) => {
-    let current = element;
-    while (current == null ? void 0 : current.parentElement) {
-      if (current.id) {
-        return current;
-      }
-      current = current.parentElement;
-    }
-    return null;
-  };
   var getTextNodeIndex = (textNode) => {
     let index = 1;
     let current = textNode.previousSibling;
@@ -1442,11 +1515,16 @@ ${indentStr}${after}`;
     return index;
   };
   var getElementXPath = (element) => {
+    var _a;
     if (element.nodeType === Node.TEXT_NODE) {
       const parentNode = element.parentNode;
       if (parentNode && parentNode.nodeType === Node.ELEMENT_NODE) {
         const parentXPath = getElementXPath(parentNode);
         const textIndex = getTextNodeIndex(element);
+        const textContent = (_a = element.textContent) == null ? void 0 : _a.trim();
+        if (textContent) {
+          return `${parentXPath}/text()[${textIndex}][normalize-space()="${textContent}"]`;
+        }
         return `${parentXPath}/text()[${textIndex}]`;
       }
       return "";
@@ -1460,22 +1538,6 @@ ${indentStr}${after}`;
     if (el === document.body) {
       return "/html/body";
     }
-    if (el.id) {
-      return `//*[@id="${el.id}"]`;
-    }
-    const ancestorWithId = findFirstAncestorWithId(el);
-    if (ancestorWithId) {
-      const ancestorPath = `//*[@id="${ancestorWithId.id}"]`;
-      let current = el;
-      const pathParts = [];
-      while (current && current !== ancestorWithId) {
-        const index2 = getElementIndex(current);
-        const tagName2 = current.nodeName.toLowerCase();
-        pathParts.unshift(`${tagName2}[${index2}]`);
-        current = current.parentElement;
-      }
-      return pathParts.length > 0 ? `${ancestorPath}/${pathParts.join("/")}` : ancestorPath;
-    }
     if (!el.parentNode) {
       return `/${el.nodeName.toLowerCase()}`;
     }
@@ -1490,8 +1552,8 @@ ${indentStr}${after}`;
   function generateXPaths(node) {
     if (!node)
       return [];
-    const xPath = getElementXPath(node);
-    return [xPath];
+    const fullXPath = getElementXPath(node);
+    return [fullXPath];
   }
   function getXpathsById(id) {
     const node = getNodeFromCacheList(id);

package/iife-script/htmlElementDebug.js

@@ -43,9 +43,9 @@ var midscene_element_inspector = (() => {
     mod
   ));
 
-  // resolve-false:/empty-stub
+  // resolve-false:\empty-stub
   var require_empty_stub = __commonJS({
-    "resolve-false:/empty-stub"(exports, module) {
+    "resolve-false:\\empty-stub"(exports, module) {
       "use strict";
       module.exports = {};
     }
@@ -611,6 +611,9 @@ var midscene_element_inspector = (() => {
   function isButtonElement(node) {
     return node instanceof HTMLElement && node.tagName.toLowerCase() === "button";
   }
+  function isAElement(node) {
+    return node instanceof HTMLElement && node.tagName.toLowerCase() === "a";
+  }
   function isImgElement(node) {
     if (!includeBaseElement(node) && node instanceof Element) {
       const computedStyle = window.getComputedStyle(node);
@@ -661,7 +664,8 @@ var midscene_element_inspector = (() => {
       "textarea",
       "select",
       "option",
-      "img"
+      "img",
+      "a"
     ];
     for (const tagName of includeList) {
       const element = node.querySelectorAll(tagName);
@@ -1045,10 +1049,20 @@ var midscene_element_inspector = (() => {
       return elementInfo;
     }
     if (isButtonElement(node)) {
+      const rect2 = mergeElementAndChildrenRects(
+        node,
+        currentWindow,
+        currentDocument,
+        baseZoom,
+        visibleOnly
+      );
+      if (!rect2) {
+        return null;
+      }
       const attributes = getNodeAttributes(node, currentWindow);
       const pseudo = getPseudoElementContent(node, currentWindow);
       const content = node.innerText || pseudo.before || pseudo.after || "";
-      const nodeHashId = midsceneGenerateHash(node, content, rect);
+      const nodeHashId = midsceneGenerateHash(node, content, rect2);
       const selector = setDataForNode(node, nodeHashId, false, currentWindow);
       const elementInfo = {
         id: nodeHashId,
@@ -1061,12 +1075,12 @@ var midscene_element_inspector = (() => {
           nodeType: "BUTTON Node" /* BUTTON */
         }),
         content,
-        rect,
+        rect: rect2,
         center: [
-          Math.round(rect.left + rect.width / 2),
-          Math.round(rect.top + rect.height / 2)
+          Math.round(rect2.left + rect2.width / 2),
+          Math.round(rect2.top + rect2.height / 2)
         ],
-        zoom: rect.zoom
+        zoom: rect2.zoom
       };
       return elementInfo;
     }
@@ -1128,6 +1142,32 @@ var midscene_element_inspector = (() => {
       };
       return elementInfo;
     }
+    if (isAElement(node)) {
+      const attributes = getNodeAttributes(node, currentWindow);
+      const pseudo = getPseudoElementContent(node, currentWindow);
+      const content = node.innerText || pseudo.before || pseudo.after || "";
+      const nodeHashId = midsceneGenerateHash(node, content, rect);
+      const selector = setDataForNode(node, nodeHashId, false, currentWindow);
+      const elementInfo = {
+        id: nodeHashId,
+        indexId: indexId++,
+        nodeHashId,
+        nodeType: "Anchor Node" /* A */,
+        locator: selector,
+        attributes: __spreadProps(__spreadValues({}, attributes), {
+          htmlTagName: tagNameOfNode(node),
+          nodeType: "Anchor Node" /* A */
+        }),
+        content,
+        rect,
+        center: [
+          Math.round(rect.left + rect.width / 2),
+          Math.round(rect.top + rect.height / 2)
+        ],
+        zoom: rect.zoom
+      };
+      return elementInfo;
+    }
     if (isContainerElement(node)) {
       const attributes = getNodeAttributes(node, currentWindow);
       const nodeHashId = midsceneGenerateHash(node, "", rect);
@@ -1197,7 +1237,7 @@ var midscene_element_inspector = (() => {
         node: elementInfo,
         children: []
       };
-      if ((elementInfo == null ? void 0 : elementInfo.nodeType) === "BUTTON Node" /* BUTTON */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "IMG Node" /* IMG */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "TEXT Node" /* TEXT */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "FORM_ITEM Node" /* FORM_ITEM */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "CONTAINER Node" /* CONTAINER */) {
+      if ((elementInfo == null ? void 0 : elementInfo.nodeType) === "BUTTON Node" /* BUTTON */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "IMG Node" /* IMG */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "TEXT Node" /* TEXT */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "Anchor Node" /* A */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "FORM_ITEM Node" /* FORM_ITEM */ || (elementInfo == null ? void 0 : elementInfo.nodeType) === "CONTAINER Node" /* CONTAINER */) {
         return nodeInfo;
       }
       const rect = getRect(node, baseZoom, currentWindow);
@@ -1252,6 +1292,49 @@ var midscene_element_inspector = (() => {
       children: topChildren
     };
   }
+  function mergeElementAndChildrenRects(node, currentWindow, currentDocument, baseZoom = 1, visibleOnly = true) {
+    const selfRect = elementRect(
+      node,
+      currentWindow,
+      currentDocument,
+      baseZoom,
+      visibleOnly
+    );
+    if (!selfRect)
+      return null;
+    let minLeft = selfRect.left;
+    let minTop = selfRect.top;
+    let maxRight = selfRect.left + selfRect.width;
+    let maxBottom = selfRect.top + selfRect.height;
+    function traverse(child) {
+      for (let i = 0; i < child.childNodes.length; i++) {
+        const sub = child.childNodes[i];
+        if (sub.nodeType === 1) {
+          const rect = elementRect(
+            sub,
+            currentWindow,
+            currentDocument,
+            baseZoom,
+            visibleOnly
+          );
+          if (rect) {
+            minLeft = Math.min(minLeft, rect.left);
+            minTop = Math.min(minTop, rect.top);
+            maxRight = Math.max(maxRight, rect.left + rect.width);
+            maxBottom = Math.max(maxBottom, rect.top + rect.height);
+          }
+          traverse(sub);
+        }
+      }
+    }
+    traverse(node);
+    return __spreadProps(__spreadValues({}, selfRect), {
+      left: minLeft,
+      top: minTop,
+      width: maxRight - minLeft,
+      height: maxBottom - minTop
+    });
+  }
 
   // src/extractor/debug.ts
   console.log(extractTextWithPosition(document.body, true));

package/package.json

@@ -8,7 +8,7 @@
     "Browser use",
     "Android use"
   ],
-  "version": "1.0.2",
+  "version": "1.0.4",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "jsnext:source": "./src/index.ts",
@@ -137,25 +137,6 @@
       "quiet": false
     }
   },
-  "scripts": {
-    "dev": "npm run build && npx npm-watch",
-    "dev:server": "npm run build && ./bin/midscene-playground",
-    "build": "modern build -c ./modern.config.ts",
-    "postbuild": "node scripts/check-exports.js",
-    "build:watch": "modern build -w -c ./modern.config.ts",
-    "test": "vitest --run",
-    "test:u": "vitest --run -u",
-    "test:ai": "AI_TEST_TYPE=web npm run test",
-    "test:ai:temp": "MIDSCENE_CACHE=true BRIDGE_MODE=true vitest --run tests/ai/bridge/open-new-tab.test.ts",
-    "test:ai:bridge": "MIDSCENE_CACHE=true BRIDGE_MODE=true npm run test --inspect tests/ai/bridge/temp.test.ts",
-    "test:ai:cache": "MIDSCENE_CACHE=true npm run test",
-    "upgrade": "modern upgrade",
-    "e2e": "playwright test --config=tests/playwright.config.ts",
-    "e2e:report": "MIDSCENE_REPORT=true playwright test --config=tests/playwright.config.ts",
-    "e2e:cache": "MIDSCENE_CACHE=true playwright test --config=tests/playwright.config.ts",
-    "e2e:ui": "playwright test --config=tests/playwright.config.ts --ui",
-    "e2e:ui:cache": "MIDSCENE_CACHE=true playwright test --config=tests/playwright.config.ts --ui"
-  },
   "files": [
     "static",
     "dist",
@@ -164,8 +145,8 @@
     "bin"
   ],
   "dependencies": {
-    "misoai-core": "1.0.1",
-    "misoai-shared": "1.0.0",
+    "misoai-core": "1.0.4",
+    "misoai-shared": "1.0.1",
     "@xmldom/xmldom": "0.8.10",
     "cors": "2.8.5",
     "dayjs": "1.11.11",
@@ -217,5 +198,23 @@
     "access": "public",
     "registry": "https://registry.npmjs.org"
   },
-  "license": "MIT"
-}
+  "license": "MIT",
+  "scripts": {
+    "dev": "npm run build && npx npm-watch",
+    "dev:server": "npm run build && ./bin/midscene-playground",
+    "build": "modern build -c ./modern.config.ts",
+    "postbuild": "node scripts/check-exports.js",
+    "build:watch": "modern build -w -c ./modern.config.ts",
+    "test": "vitest --run",
+    "test:u": "vitest --run -u",
+    "test:ai": "AI_TEST_TYPE=web npm run test",
+    "test:ai:bridge": "MIDSCENE_CACHE=true BRIDGE_MODE=true vitest --run tests/ai/bridge/open-new-tab.test.ts",
+    "test:ai:cache": "MIDSCENE_CACHE=true npm run test",
+    "upgrade": "modern upgrade",
+    "e2e": "playwright test --config=tests/playwright.config.ts",
+    "e2e:report": "MIDSCENE_REPORT=true playwright test --config=tests/playwright.config.ts",
+    "e2e:cache": "MIDSCENE_CACHE=true playwright test --config=tests/playwright.config.ts",
+    "e2e:ui": "playwright test --config=tests/playwright.config.ts --ui",
+    "e2e:ui:cache": "MIDSCENE_CACHE=true playwright test --config=tests/playwright.config.ts --ui"
+  }
+}