@midscene/shared 0.16.10 → 0.16.11-beta-20250512134434.0

package/dist/es/extractor-debug.js

@@ -362,8 +362,24 @@ function getNodeAttributes(node, currentWindow) {
 }
 function midsceneGenerateHash(node, content, rect) {
   const slicedHash = generateHashId(rect, content);
+  if (node) {
+    if (!window.midsceneNodeHashCacheList) {
+      setNodeHashCacheListOnWindow();
+    }
+    setNodeToCacheList(node, slicedHash);
+  }
   return slicedHash;
 }
+function setNodeHashCacheListOnWindow() {
+  if (typeof window !== "undefined") {
+    window.midsceneNodeHashCacheList = [];
+  }
+}
+function setNodeToCacheList(node, id) {
+  if (typeof window !== "undefined") {
+    window.midsceneNodeHashCacheList.push({ node, id });
+  }
+}
 function setMidsceneVisibleRectOnWindow() {
   if (typeof window !== "undefined") {
     window.midsceneVisibleRect = visibleRect;
@@ -434,9 +450,7 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
         Math.round(rect.left + rect.width / 2),
         Math.round(rect.top + rect.height / 2)
       ],
-      zoom: rect.zoom,
-      screenWidth: currentWindow.innerWidth,
-      screenHeight: currentWindow.innerHeight
+      zoom: rect.zoom
     };
     return elementInfo;
   }
@@ -463,9 +477,7 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
         Math.round(rect.left + rect.width / 2),
         Math.round(rect.top + rect.height / 2)
       ],
-      zoom: rect.zoom,
-      screenWidth: currentWindow.innerWidth,
-      screenHeight: currentWindow.innerHeight
+      zoom: rect.zoom
     };
     return elementInfo;
   }
@@ -493,9 +505,7 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
         Math.round(rect.left + rect.width / 2),
         Math.round(rect.top + rect.height / 2)
       ],
-      zoom: rect.zoom,
-      screenWidth: currentWindow.innerWidth,
-      screenHeight: currentWindow.innerHeight
+      zoom: rect.zoom
     };
     return elementInfo;
   }
@@ -526,12 +536,9 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
         Math.round(rect.left + rect.width / 2),
         Math.round(rect.top + rect.height / 2)
       ],
-      // attributes,
       content: text,
       rect,
-      zoom: rect.zoom,
-      screenWidth: currentWindow.innerWidth,
-      screenHeight: currentWindow.innerHeight
+      zoom: rect.zoom
     };
     return elementInfo;
   }
@@ -556,9 +563,7 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
         Math.round(rect.left + rect.width / 2),
         Math.round(rect.top + rect.height / 2)
       ],
-      zoom: rect.zoom,
-      screenWidth: currentWindow.innerWidth,
-      screenHeight: currentWindow.innerHeight
+      zoom: rect.zoom
     };
     return elementInfo;
   }

package/dist/es/extractor.d.ts

@@ -1,47 +1,2 @@
-import { NodeType } from './constants.js';
-import { B as BaseElement, E as ElementTreeNode } from './index-305e7a7e.js';
-
-declare function descriptionOfTree<ElementType extends BaseElement = BaseElement>(tree: ElementTreeNode<ElementType>, truncateTextLength?: number, filterNonTextContent?: boolean): string;
-declare function treeToList<T extends BaseElement>(tree: ElementTreeNode<T>): T[];
-declare function traverseTree<T extends BaseElement, ReturnNodeType extends BaseElement>(tree: ElementTreeNode<T>, onNode: (node: T) => ReturnNodeType): ElementTreeNode<ReturnNodeType>;
-
-interface WebElementInfo extends ElementInfo {
-    zoom: number;
-    screenWidth?: number;
-    screenHeight?: number;
-}
-interface WebElementNode {
-    node: WebElementInfo | null;
-    children: WebElementNode[];
-}
-declare function extractTextWithPosition$1(initNode: globalThis.Node, debugMode?: boolean): WebElementInfo[];
-declare function extractTreeNodeAsString(initNode: globalThis.Node, debugMode?: boolean): string;
-declare function extractTreeNode(initNode: globalThis.Node, debugMode?: boolean): WebElementNode;
-
-declare function extractTextWithPosition(initNode: globalThis.Document): ElementInfo[];
-
-interface ElementInfo {
-    id: string;
-    indexId: number;
-    nodeHashId: string;
-    locator: string;
-    attributes: {
-        nodeType: NodeType;
-        [key: string]: string;
-    };
-    nodeType: NodeType;
-    content: string;
-    rect: {
-        left: number;
-        top: number;
-        width: number;
-        height: number;
-    };
-    center: [number, number];
-}
-interface ElementNode {
-    node: ElementInfo | null;
-    children: ElementNode[];
-}
-
-export { type ElementInfo, type ElementNode, extractTextWithPosition as clientExtractTextWithPosition, descriptionOfTree, traverseTree, treeToList, extractTreeNode as webExtractNodeTree, extractTreeNodeAsString as webExtractNodeTreeAsString, extractTextWithPosition$1 as webExtractTextWithPosition };
+import './constants.js';
+export { E as ElementInfo, a as ElementNode, d as descriptionOfTree, i as getElementInfoByXpath, h as getNodeInfoByXpath, g as getXpathsById, s as setNodeHashCacheListOnWindow, t as traverseTree, b as treeToList, c as webExtractNodeTree, f as webExtractNodeTreeAsString, e as webExtractTextWithPosition } from './index-2cb7865f.js';

package/dist/es/extractor.js

@@ -483,8 +483,32 @@ function getNodeAttributes(node, currentWindow) {
 }
 function midsceneGenerateHash(node, content, rect) {
   const slicedHash = generateHashId(rect, content);
+  if (node) {
+    if (!window.midsceneNodeHashCacheList) {
+      setNodeHashCacheListOnWindow();
+    }
+    setNodeToCacheList(node, slicedHash);
+  }
   return slicedHash;
 }
+function setNodeHashCacheListOnWindow() {
+  if (typeof window !== "undefined") {
+    window.midsceneNodeHashCacheList = [];
+  }
+}
+function setNodeToCacheList(node, id) {
+  if (typeof window !== "undefined") {
+    window.midsceneNodeHashCacheList.push({ node, id });
+  }
+}
+function getNodeFromCacheList(id) {
+  if (typeof window !== "undefined") {
+    return window.midsceneNodeHashCacheList.find(
+      (item) => item.id === id
+    )?.node;
+  }
+  return null;
+}
 function getTopDocument() {
   const container = document.body || document;
   return container;
@@ -545,9 +569,7 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
         Math.round(rect.left + rect.width / 2),
         Math.round(rect.top + rect.height / 2)
       ],
-      zoom: rect.zoom,
-      screenWidth: currentWindow.innerWidth,
-      screenHeight: currentWindow.innerHeight
+      zoom: rect.zoom
     };
     return elementInfo;
   }
@@ -574,9 +596,7 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
         Math.round(rect.left + rect.width / 2),
         Math.round(rect.top + rect.height / 2)
       ],
-      zoom: rect.zoom,
-      screenWidth: currentWindow.innerWidth,
-      screenHeight: currentWindow.innerHeight
+      zoom: rect.zoom
     };
     return elementInfo;
   }
@@ -604,9 +624,7 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
         Math.round(rect.left + rect.width / 2),
         Math.round(rect.top + rect.height / 2)
       ],
-      zoom: rect.zoom,
-      screenWidth: currentWindow.innerWidth,
-      screenHeight: currentWindow.innerHeight
+      zoom: rect.zoom
     };
     return elementInfo;
   }
@@ -637,12 +655,9 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
         Math.round(rect.left + rect.width / 2),
         Math.round(rect.top + rect.height / 2)
       ],
-      // attributes,
       content: text,
       rect,
-      zoom: rect.zoom,
-      screenWidth: currentWindow.innerWidth,
-      screenHeight: currentWindow.innerHeight
+      zoom: rect.zoom
     };
     return elementInfo;
   }
@@ -667,9 +682,7 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
         Math.round(rect.left + rect.width / 2),
         Math.round(rect.top + rect.height / 2)
       ],
-      zoom: rect.zoom,
-      screenWidth: currentWindow.innerWidth,
-      screenHeight: currentWindow.innerHeight
+      zoom: rect.zoom
     };
     return elementInfo;
   }
@@ -778,166 +791,113 @@ function extractTreeNode(initNode, debugMode2 = false) {
   };
 }
 
-// src/extractor/client-extractor.ts
-function getNodeAttributes2(node) {
-  const attrs = {};
-  if (node && node.nodeType === 1) {
-    const element = node;
-    for (let i = 0; i < element.attributes.length; i++) {
-      const attr = element.attributes[i];
-      attrs[attr.nodeName] = attr.nodeValue ?? "";
-    }
-  }
-  return attrs;
-}
-function getRect2(attributes) {
-  const x = Math.round(Number.parseFloat(attributes.x ?? "0"));
-  const y = Math.round(Number.parseFloat(attributes.y ?? "0"));
-  const width = Math.round(Number.parseFloat(attributes.width ?? "0"));
-  const height = Math.round(Number.parseFloat(attributes.height ?? "0"));
-  return {
-    left: Math.max(0, Math.floor(x)),
-    top: Math.max(0, Math.floor(y)),
-    width: Math.max(0, width),
-    height: Math.max(0, height)
-  };
-}
-function validTextNodeContent(node) {
-  if (node.nodeType === 3) {
-    return node.nodeValue?.trim() || "";
-  }
-  return "";
-}
-function getXPathForElement(element) {
-  if (element.nodeType !== 1) {
-    return "";
+// src/extractor/locator.ts
+var getElementIndex = (element) => {
+  let index = 1;
+  let prev = element.previousElementSibling;
+  while (prev) {
+    if (prev.nodeName.toLowerCase() === element.nodeName.toLowerCase()) {
+      index++;
+    }
+    prev = prev.previousElementSibling;
   }
-  const getIndex = (sib, name) => {
-    let count = 1;
-    for (let cur = sib.previousSibling; cur; cur = cur.previousSibling) {
-      if (cur.nodeType === 1 && cur.nodeName === name) {
-        count++;
-      }
+  return index;
+};
+var findFirstAncestorWithId = (element) => {
+  let current = element;
+  while (current?.parentElement) {
+    if (current.id) {
+      return current;
     }
-    return count;
-  };
-  const buildAttributePart = (elem) => {
-    const attributes = ["id", "resource-id", "content-desc", "class"];
-    for (const attr of attributes) {
-      if (elem.hasAttribute(attr)) {
-        const value = elem.getAttribute(attr);
-        if (value && value.trim() !== "") {
-          return `[@${attr}="${value}"]`;
-        }
-      }
+    current = current.parentElement;
+  }
+  return null;
+};
+var getElementXPath = (element) => {
+  if (element.nodeType === Node.TEXT_NODE) {
+    const parentNode = element.parentNode;
+    if (parentNode && parentNode.nodeType === Node.ELEMENT_NODE) {
+      const parentXPath = getElementXPath(parentNode);
+      return `${parentXPath}/text()`;
     }
     return "";
-  };
-  const getPath = (node, path = "") => {
-    if (node.parentNode) {
-      path = getPath(node.parentNode, path);
-    }
-    if (node.nodeType === 1) {
-      const elem = node;
-      const tagName = elem.nodeName.toLowerCase();
-      let part = `/${tagName}`;
-      const attributePart = buildAttributePart(elem);
-      if (attributePart) {
-        part += attributePart;
-      } else {
-        const index = getIndex(node, node.nodeName);
-        if (index > 1) {
-          part += `[${index}]`;
-        }
-      }
-      path += part;
-    }
-    return path;
-  };
-  return getPath(element);
+  }
+  if (element.nodeType !== Node.ELEMENT_NODE)
+    return "";
+  const el = element;
+  if (el === document.documentElement) {
+    return "/html";
+  }
+  if (el === document.body) {
+    return "/html/body";
+  }
+  if (el.id) {
+    return `//*[@id="${el.id}"]`;
+  }
+  const ancestorWithId = findFirstAncestorWithId(el);
+  if (ancestorWithId) {
+    const ancestorPath = `//*[@id="${ancestorWithId.id}"]`;
+    let current = el;
+    const pathParts = [];
+    while (current && current !== ancestorWithId) {
+      const index2 = getElementIndex(current);
+      const tagName2 = current.nodeName.toLowerCase();
+      pathParts.unshift(`${tagName2}[${index2}]`);
+      current = current.parentElement;
+    }
+    return pathParts.length > 0 ? `${ancestorPath}/${pathParts.join("/")}` : ancestorPath;
+  }
+  if (!el.parentNode) {
+    return `/${el.nodeName.toLowerCase()}`;
+  }
+  const index = getElementIndex(el);
+  const tagName = el.nodeName.toLowerCase();
+  if (el.parentNode) {
+    const parentXPath = getElementXPath(el.parentNode);
+    return `${parentXPath}/${tagName}[${index}]`;
+  }
+  return `/${tagName}[${index}]`;
+};
+function generateXPaths(node) {
+  if (!node)
+    return [];
+  const xPath = getElementXPath(node);
+  return [xPath];
+}
+function getXpathsById(id) {
+  const node = getNodeFromCacheList(id);
+  if (!node) {
+    return null;
+  }
+  return generateXPaths(node);
 }
-function extractTextWithPosition2(initNode) {
-  const elementInfoArray = [];
-  let nodeIndex = 1;
-  function dfs(node, parentNode = null) {
-    if (!node) {
-      return;
-    }
-    const currentNodeDes = { node, children: [] };
-    if (parentNode) {
-      parentNode.children.push(currentNodeDes);
-    }
-    collectElementInfo2(node);
-    if (node.childNodes && node.childNodes.length > 0) {
-      for (let i = 0; i < node.childNodes.length; i++) {
-        dfs(node.childNodes[i], currentNodeDes);
-      }
-    }
+function getNodeInfoByXpath(xpath) {
+  const xpathResult = document.evaluate(
+    xpath,
+    document,
+    null,
+    XPathResult.ORDERED_NODE_SNAPSHOT_TYPE,
+    null
+  );
+  if (xpathResult.snapshotLength !== 1) {
+    return null;
   }
-  function collectElementInfo2(node) {
-    const attributes = getNodeAttributes2(node);
-    const rect = getRect2(attributes);
-    const nodeHashId = midsceneGenerateHash(null, attributes.placeholder, rect);
-    const text = validTextNodeContent(node);
-    let nodeType;
-    switch (node.nodeName.toUpperCase()) {
-      case "TEXT":
-        nodeType = "TEXT Node" /* TEXT */;
-        break;
-      case "IMAGE":
-        nodeType = "IMG Node" /* IMG */;
-        break;
-      case "BUTTON":
-        nodeType = "BUTTON Node" /* BUTTON */;
-        break;
-      case "SEARCHINPUT":
-      case "TEXTINPUT":
-      case "INPUT":
-        nodeType = "FORM_ITEM Node" /* FORM_ITEM */;
-        break;
-      case "NAV":
-      case "LIST":
-      case "CELL":
-        nodeType = "CONTAINER Node" /* CONTAINER */;
-        break;
-      default:
-        if (attributes.id === "android:id/input" || attributes.id === "android:id/inputArea") {
-          nodeType = "FORM_ITEM Node" /* FORM_ITEM */;
-        } else {
-          nodeType = "CONTAINER Node" /* CONTAINER */;
-        }
-        break;
-    }
-    const xpath = getXPathForElement(node);
-    const elementInfo = {
-      id: nodeHashId,
-      indexId: nodeIndex++,
-      nodeHashId,
-      locator: xpath,
-      attributes: {
-        nodeType,
-        ...attributes
-      },
-      content: text,
-      rect,
-      center: [
-        Math.round(rect.left + rect.width / 2),
-        Math.round(rect.top + rect.height / 2)
-      ],
-      nodeType
-    };
-    if (elementInfo.nodeType !== "CONTAINER Node" /* CONTAINER */) {
-      elementInfoArray.push(elementInfo);
-    }
+  const node = xpathResult.snapshotItem(0);
+  return node;
+}
+function getElementInfoByXpath(xpath) {
+  const node = getNodeInfoByXpath(xpath);
+  if (!node) {
+    return null;
   }
-  const rootNode = initNode;
-  const rootDescriptor = { node: rootNode, children: [] };
-  dfs(rootNode, rootDescriptor);
-  return elementInfoArray;
+  return collectElementInfo(node, window, document);
 }
 export {
-  extractTextWithPosition2 as clientExtractTextWithPosition,
   descriptionOfTree,
+  getElementInfoByXpath,
+  getNodeInfoByXpath,
+  getXpathsById,
+  setNodeHashCacheListOnWindow,
   traverseTree,
   treeToList,
   extractTreeNode as webExtractNodeTree,

package/dist/es/fs.d.ts

@@ -10,6 +10,7 @@ declare function getRunningPkgInfo(dir?: string): PkgInfo | null;
  * @returns {string|null} - The most recent package.json file path or null
  */
 declare function findNearestPackageJson(dir: string): string | null;
+declare function getElementInfosScriptContent(): string;
 declare function getExtraReturnLogic(tree?: boolean): Promise<string | null>;
 
-export { findNearestPackageJson, getExtraReturnLogic, getRunningPkgInfo };
+export { findNearestPackageJson, getElementInfosScriptContent, getExtraReturnLogic, getRunningPkgInfo };

package/dist/es/fs.js

@@ -50,15 +50,19 @@ function findNearestPackageJson(dir) {
   }
   return findNearestPackageJson(parentDir);
 }
-async function getExtraReturnLogic(tree = false) {
-  if (ifInBrowser) {
-    return null;
-  }
+function getElementInfosScriptContent() {
   const currentFilePath = __filename;
   const pathDir = findNearestPackageJson(dirname(currentFilePath));
   assert(pathDir, `can't find pathDir, with ${dirname}`);
   const scriptPath = path.join(pathDir, "./dist/script/htmlElement.js");
   const elementInfosScriptContent = readFileSync(scriptPath, "utf-8");
+  return elementInfosScriptContent;
+}
+async function getExtraReturnLogic(tree = false) {
+  if (ifInBrowser) {
+    return null;
+  }
+  const elementInfosScriptContent = `${getElementInfosScriptContent()}midscene_element_inspector.setNodeHashCacheListOnWindow();`;
   if (tree) {
     return `${elementInfosScriptContent}midscene_element_inspector.webExtractNodeTree()`;
   }
@@ -66,6 +70,7 @@ async function getExtraReturnLogic(tree = false) {
 }
 export {
   findNearestPackageJson,
+  getElementInfosScriptContent,
   getExtraReturnLogic,
   getRunningPkgInfo
 };

package/dist/es/img.d.ts

@@ -1,6 +1,6 @@
 import { Buffer } from 'node:buffer';
 import Jimp from 'jimp';
-import { R as Rect, B as BaseElement } from './index-305e7a7e.js';
+import { R as Rect, B as BaseElement } from './index-2cb7865f.js';
 import './constants.js';
 
 interface Size {

package/dist/es/index-2cb7865f.d.ts

@@ -0,0 +1,78 @@
+import { NodeType } from './constants.js';
+
+declare function descriptionOfTree<ElementType extends BaseElement = BaseElement>(tree: ElementTreeNode<ElementType>, truncateTextLength?: number, filterNonTextContent?: boolean): string;
+declare function treeToList<T extends BaseElement>(tree: ElementTreeNode<T>): T[];
+declare function traverseTree<T extends BaseElement, ReturnNodeType extends BaseElement>(tree: ElementTreeNode<T>, onNode: (node: T) => ReturnNodeType): ElementTreeNode<ReturnNodeType>;
+
+interface WebElementNode {
+    node: WebElementInfo | null;
+    children: WebElementNode[];
+}
+declare function extractTextWithPosition(initNode: globalThis.Node, debugMode?: boolean): WebElementInfo[];
+declare function extractTreeNodeAsString(initNode: globalThis.Node, debugMode?: boolean): string;
+declare function extractTreeNode(initNode: globalThis.Node, debugMode?: boolean): WebElementNode;
+
+declare function setNodeHashCacheListOnWindow(): void;
+
+declare function getXpathsById(id: string): string[] | null;
+declare function getNodeInfoByXpath(xpath: string): Node | null;
+declare function getElementInfoByXpath(xpath: string): ElementInfo | null;
+
+interface ElementInfo {
+    id: string;
+    indexId: number;
+    nodeHashId: string;
+    locator: string;
+    xpaths?: string[];
+    attributes: {
+        nodeType: NodeType;
+        [key: string]: string;
+    };
+    nodeType: NodeType;
+    content: string;
+    rect: {
+        left: number;
+        top: number;
+        width: number;
+        height: number;
+    };
+    center: [number, number];
+}
+interface ElementNode {
+    node: ElementInfo | null;
+    children: ElementNode[];
+}
+
+interface Point {
+    left: number;
+    top: number;
+}
+interface Size {
+    width: number;
+    height: number;
+    dpr?: number;
+}
+type Rect = Point & Size & {
+    zoom?: number;
+};
+declare abstract class BaseElement {
+    abstract id: string;
+    abstract indexId?: number;
+    abstract attributes: {
+        nodeType: NodeType;
+        [key: string]: string;
+    };
+    abstract content: string;
+    abstract rect: Rect;
+    abstract center: [number, number];
+    abstract locator?: string;
+}
+interface ElementTreeNode<ElementType extends BaseElement = BaseElement> {
+    node: ElementType | null;
+    children: ElementTreeNode<ElementType>[];
+}
+interface WebElementInfo extends ElementInfo {
+    zoom: number;
+}
+
+export { BaseElement as B, type ElementInfo as E, type Point as P, type Rect as R, type Size as S, type WebElementInfo as W, type ElementNode as a, treeToList as b, extractTreeNode as c, descriptionOfTree as d, extractTextWithPosition as e, extractTreeNodeAsString as f, getXpathsById as g, getNodeInfoByXpath as h, getElementInfoByXpath as i, type ElementTreeNode as j, setNodeHashCacheListOnWindow as s, traverseTree as t };

package/dist/es/types.d.ts

@@ -0,0 +1,2 @@
+import './constants.js';
+export { B as BaseElement, j as ElementTreeNode, P as Point, R as Rect, S as Size, W as WebElementInfo } from './index-2cb7865f.js';

package/dist/es/types.js

@@ -0,0 +1,6 @@
+// src/types/index.ts
+var BaseElement = class {
+};
+export {
+  BaseElement
+};