@midscene/shared 0.16.9-beta-20250508125518.0 → 0.16.9

package/src/extractor/client-extractor.ts

@@ -0,0 +1,211 @@
+import type { ElementInfo } from './';
+import { NodeType } from './constants';
+import { midsceneGenerateHash } from './util';
+
+// Definition of NodeDescriptor interface
+interface NodeDescriptor {
+  node: globalThis.Node;
+  children: NodeDescriptor[];
+}
+
+// Retrieve attributes from a node
+function getNodeAttributes(node: globalThis.Node): { [key: string]: string } {
+  const attrs: { [key: string]: string } = {};
+
+  // Check if node exists and its type is ELEMENT_NODE
+  if (node && node.nodeType === 1) {
+    const element = node as globalThis.Element;
+
+    for (let i = 0; i < element.attributes.length; i++) {
+      const attr = element.attributes[i];
+      attrs[attr.nodeName] = attr.nodeValue ?? '';
+    }
+  }
+
+  return attrs;
+}
+
+// Retrieve rectangle information
+function getRect(attributes: { [key: string]: string }): {
+  left: number;
+  top: number;
+  width: number;
+  height: number;
+} {
+  const x = Math.round(Number.parseFloat(attributes.x ?? '0'));
+  const y = Math.round(Number.parseFloat(attributes.y ?? '0'));
+  const width = Math.round(Number.parseFloat(attributes.width ?? '0'));
+  const height = Math.round(Number.parseFloat(attributes.height ?? '0'));
+
+  return {
+    left: Math.max(0, Math.floor(x)),
+    top: Math.max(0, Math.floor(y)),
+    width: Math.max(0, width),
+    height: Math.max(0, height),
+  };
+}
+
+// Validate if the node can provide text content
+function validTextNodeContent(node: Node): string {
+  if (node.nodeType === 3) {
+    return node.nodeValue?.trim() || '';
+  }
+  return '';
+}
+
+function getXPathForElement(element: Node): string {
+  if (element.nodeType !== 1) {
+    return '';
+  }
+
+  const getIndex = (sib: Node, name: string) => {
+    let count = 1;
+    for (let cur = sib.previousSibling; cur; cur = cur.previousSibling) {
+      if (cur.nodeType === 1 && cur.nodeName === name) {
+        count++;
+      }
+    }
+    return count;
+  };
+
+  const buildAttributePart = (elem: Element): string => {
+    const attributes = ['id', 'resource-id', 'content-desc', 'class'];
+    for (const attr of attributes) {
+      if (elem.hasAttribute(attr)) {
+        const value = elem.getAttribute(attr);
+        if (value && value.trim() !== '') {
+          return `[@${attr}="${value}"]`;
+        }
+      }
+    }
+    return '';
+  };
+
+  const getPath = (node: Node, path = ''): string => {
+    if (node.parentNode) {
+      path = getPath(node.parentNode, path);
+    }
+
+    if (node.nodeType === 1) {
+      const elem = node as Element;
+      const tagName = elem.nodeName.toLowerCase();
+      let part = `/${tagName}`;
+
+      const attributePart = buildAttributePart(elem);
+
+      // 如果找到有意义的属性，则添加属性部分
+      if (attributePart) {
+        part += attributePart;
+      } else {
+        // 如果没有有意义的属性，则添加索引
+        const index = getIndex(node, node.nodeName);
+        if (index > 1) {
+          part += `[${index}]`;
+        }
+      }
+
+      path += part;
+    }
+
+    return path;
+  };
+
+  return getPath(element);
+}
+
+// Perform DFS traversal and collect element information
+export function extractTextWithPosition(
+  initNode: globalThis.Document,
+): ElementInfo[] {
+  const elementInfoArray: ElementInfo[] = [];
+  let nodeIndex = 1;
+
+  function dfs(node: Node, parentNode: NodeDescriptor | null = null): void {
+    if (!node) {
+      return;
+    }
+
+    const currentNodeDes: NodeDescriptor = { node, children: [] };
+    if (parentNode) {
+      parentNode.children.push(currentNodeDes);
+    }
+
+    collectElementInfo(node);
+
+    if (node.childNodes && node.childNodes.length > 0) {
+      for (let i = 0; i < node.childNodes.length; i++) {
+        dfs(node.childNodes[i], currentNodeDes);
+      }
+    }
+  }
+
+  function collectElementInfo(node: Node) {
+    const attributes = getNodeAttributes(node);
+    const rect = getRect(attributes);
+    const nodeHashId = midsceneGenerateHash(null, attributes.placeholder, rect);
+    const text = validTextNodeContent(node);
+
+    let nodeType;
+
+    switch (node.nodeName.toUpperCase()) {
+      case 'TEXT':
+        nodeType = NodeType.TEXT;
+        break;
+      case 'IMAGE':
+        nodeType = NodeType.IMG;
+        break;
+      case 'BUTTON':
+        nodeType = NodeType.BUTTON;
+        break;
+      case 'SEARCHINPUT':
+      case 'TEXTINPUT':
+      case 'INPUT':
+        nodeType = NodeType.FORM_ITEM;
+        break;
+      case 'NAV':
+      case 'LIST':
+      case 'CELL':
+        nodeType = NodeType.CONTAINER;
+        break;
+      default:
+        if (
+          attributes.id === 'android:id/input' ||
+          attributes.id === 'android:id/inputArea'
+        ) {
+          nodeType = NodeType.FORM_ITEM;
+        } else {
+          nodeType = NodeType.CONTAINER;
+        }
+        break;
+    }
+
+    const xpath = getXPathForElement(node);
+    const elementInfo: ElementInfo = {
+      id: nodeHashId,
+      indexId: nodeIndex++,
+      nodeHashId,
+      locator: xpath,
+      attributes: {
+        nodeType,
+        ...attributes,
+      },
+      content: text,
+      rect,
+      center: [
+        Math.round(rect.left + rect.width / 2),
+        Math.round(rect.top + rect.height / 2),
+      ],
+      nodeType,
+    };
+
+    if (elementInfo.nodeType !== NodeType.CONTAINER) {
+      elementInfoArray.push(elementInfo);
+    }
+  }
+
+  const rootNode = initNode;
+  const rootDescriptor: NodeDescriptor = { node: rootNode, children: [] };
+  dfs(rootNode, rootDescriptor);
+
+  return elementInfoArray;
+}

package/src/extractor/index.ts

@@ -1,10 +1,10 @@
 import type { NodeType } from '../constants/index';
+
 export interface ElementInfo {
   id: string;
   indexId: number;
   nodeHashId: string;
   locator: string;
-  xpaths?: string[];
   attributes: {
     nodeType: NodeType;
     [key: string]: string;
@@ -24,10 +24,8 @@ export { descriptionOfTree, traverseTree, treeToList } from './tree';
 
 export { extractTextWithPosition as webExtractTextWithPosition } from './web-extractor';
 
+export { extractTextWithPosition as clientExtractTextWithPosition } from './client-extractor';
+
 export { extractTreeNode as webExtractNodeTree } from './web-extractor';
 
 export { extractTreeNodeAsString as webExtractNodeTreeAsString } from './web-extractor';
-
-export { setNodeHashCacheListOnWindow } from './util';
-
-export { getXpathsById, getElementInfoByNode } from './locator';

package/src/extractor/util.ts

@@ -1,7 +1,7 @@
-import type { Rect } from '../types';
 import { generateHashId } from '../utils';
 import { extractTextWithPosition } from './web-extractor';
 
+// import { TEXT_MAX_SIZE } from './constants';
 const MAX_VALUE_LENGTH = 300;
 let debugMode = false;
 
@@ -444,43 +444,14 @@ export function getNodeAttributes(
 export function midsceneGenerateHash(
   node: globalThis.Node | null,
   content: string,
-  rect: Rect,
+  rect: any,
 ): string {
   const slicedHash = generateHashId(rect, content);
 
-  if (node) {
-    if (!(window as any).midsceneNodeHashCacheList) {
-      setNodeHashCacheListOnWindow();
-    }
-
-    setNodeToCacheList(node, slicedHash);
-  }
-
   // Returns the first 10 characters as a short hash
   return slicedHash;
 }
 
-export function setNodeHashCacheListOnWindow() {
-  if (typeof window !== 'undefined') {
-    (window as any).midsceneNodeHashCacheList = [];
-  }
-}
-
-export function setNodeToCacheList(node: globalThis.Node, id: string) {
-  if (typeof window !== 'undefined') {
-    (window as any).midsceneNodeHashCacheList.push({ node, id });
-  }
-}
-
-export function getNodeFromCacheList(id: string) {
-  if (typeof window !== 'undefined') {
-    return (window as any).midsceneNodeHashCacheList.find(
-      (item: { node: Node; id: string }) => item.id === id,
-    )?.node;
-  }
-  return null;
-}
-
 export function generateId(numberId: number) {
   //   const letters = 'ABCDEFGHIJKLMNPRSTUVXYZ';
   //   const numbers = '0123456789';

package/src/extractor/web-extractor.ts

@@ -1,9 +1,9 @@
+import type { ElementInfo } from '.';
 import {
   CONTAINER_MINI_HEIGHT,
   CONTAINER_MINI_WIDTH,
   NodeType,
 } from '../constants/index';
-import type { WebElementInfo } from '../types';
 import type { Point } from '../types';
 import {
   isButtonElement,
@@ -25,6 +25,12 @@ import {
   visibleRect,
 } from './util';
 
+interface WebElementInfo extends ElementInfo {
+  zoom: number;
+  screenWidth?: number;
+  screenHeight?: number;
+}
+
 let indexId = 0;
 
 function tagNameOfNode(node: globalThis.Node): string {
@@ -41,7 +47,7 @@ function tagNameOfNode(node: globalThis.Node): string {
   return tagName ? `<${tagName}>` : '';
 }
 
-export function collectElementInfo(
+function collectElementInfo(
   node: Node,
   currentWindow: typeof window,
   currentDocument: typeof document,
@@ -109,6 +115,8 @@ export function collectElementInfo(
         Math.round(rect.top + rect.height / 2),
       ],
       zoom: rect.zoom,
+      screenWidth: currentWindow.innerWidth,
+      screenHeight: currentWindow.innerHeight,
     };
     return elementInfo;
   }
@@ -137,6 +145,8 @@ export function collectElementInfo(
         Math.round(rect.top + rect.height / 2),
       ],
       zoom: rect.zoom,
+      screenWidth: currentWindow.innerWidth,
+      screenHeight: currentWindow.innerHeight,
     };
     return elementInfo;
   }
@@ -168,6 +178,8 @@ export function collectElementInfo(
         Math.round(rect.top + rect.height / 2),
       ],
       zoom: rect.zoom,
+      screenWidth: currentWindow.innerWidth,
+      screenHeight: currentWindow.innerHeight,
     };
     return elementInfo;
   }
@@ -199,9 +211,12 @@ export function collectElementInfo(
         Math.round(rect.left + rect.width / 2),
         Math.round(rect.top + rect.height / 2),
       ],
+      // attributes,
       content: text,
       rect,
       zoom: rect.zoom,
+      screenWidth: currentWindow.innerWidth,
+      screenHeight: currentWindow.innerHeight,
     };
     return elementInfo;
   }
@@ -229,6 +244,8 @@ export function collectElementInfo(
         Math.round(rect.top + rect.height / 2),
       ],
       zoom: rect.zoom,
+      screenWidth: currentWindow.innerWidth,
+      screenHeight: currentWindow.innerHeight,
     };
     return elementInfo;
   }

package/src/node/fs.ts

@@ -60,24 +60,16 @@ export function findNearestPackageJson(dir: string): string | null {
   return findNearestPackageJson(parentDir);
 }
 
-export function getElementInfosScriptContent() {
+export async function getExtraReturnLogic(tree = false) {
+  if (ifInBrowser) {
+    return null;
+  }
   // Get __dirname equivalent in Node.js environment
   const currentFilePath = __filename;
   const pathDir = findNearestPackageJson(dirname(currentFilePath));
   assert(pathDir, `can't find pathDir, with ${dirname}`);
   const scriptPath = path.join(pathDir, './dist/script/htmlElement.js');
   const elementInfosScriptContent = readFileSync(scriptPath, 'utf-8');
-
-  return elementInfosScriptContent;
-}
-
-export async function getExtraReturnLogic(tree = false) {
-  if (ifInBrowser) {
-    return null;
-  }
-
-  const elementInfosScriptContent = `${getElementInfosScriptContent()}midscene_element_inspector.setNodeHashCacheListOnWindow();`;
-
   if (tree) {
     return `${elementInfosScriptContent}midscene_element_inspector.webExtractNodeTree()`;
   }

package/src/types/index.ts

@@ -1,5 +1,4 @@
 import type { NodeType } from '../constants';
-import type { ElementInfo } from '../extractor';
 
 export interface Point {
   left: number;
@@ -39,7 +38,3 @@ export interface ElementTreeNode<
   node: ElementType | null;
   children: ElementTreeNode<ElementType>[];
 }
-
-export interface WebElementInfo extends ElementInfo {
-  zoom: number;
-}

package/dist/es/index-df4e781b.d.ts

@@ -1,77 +0,0 @@
-import { NodeType } from './constants.js';
-
-declare function descriptionOfTree<ElementType extends BaseElement = BaseElement>(tree: ElementTreeNode<ElementType>, truncateTextLength?: number, filterNonTextContent?: boolean): string;
-declare function treeToList<T extends BaseElement>(tree: ElementTreeNode<T>): T[];
-declare function traverseTree<T extends BaseElement, ReturnNodeType extends BaseElement>(tree: ElementTreeNode<T>, onNode: (node: T) => ReturnNodeType): ElementTreeNode<ReturnNodeType>;
-
-interface WebElementNode {
-    node: WebElementInfo | null;
-    children: WebElementNode[];
-}
-declare function extractTextWithPosition(initNode: globalThis.Node, debugMode?: boolean): WebElementInfo[];
-declare function extractTreeNodeAsString(initNode: globalThis.Node, debugMode?: boolean): string;
-declare function extractTreeNode(initNode: globalThis.Node, debugMode?: boolean): WebElementNode;
-
-declare function setNodeHashCacheListOnWindow(): void;
-
-declare function getXpathsById(id: string): string[] | null;
-declare function getElementInfoByNode(node: Node): ElementInfo | null;
-
-interface ElementInfo {
-    id: string;
-    indexId: number;
-    nodeHashId: string;
-    locator: string;
-    xpaths?: string[];
-    attributes: {
-        nodeType: NodeType;
-        [key: string]: string;
-    };
-    nodeType: NodeType;
-    content: string;
-    rect: {
-        left: number;
-        top: number;
-        width: number;
-        height: number;
-    };
-    center: [number, number];
-}
-interface ElementNode {
-    node: ElementInfo | null;
-    children: ElementNode[];
-}
-
-interface Point {
-    left: number;
-    top: number;
-}
-interface Size {
-    width: number;
-    height: number;
-    dpr?: number;
-}
-type Rect = Point & Size & {
-    zoom?: number;
-};
-declare abstract class BaseElement {
-    abstract id: string;
-    abstract indexId?: number;
-    abstract attributes: {
-        nodeType: NodeType;
-        [key: string]: string;
-    };
-    abstract content: string;
-    abstract rect: Rect;
-    abstract center: [number, number];
-    abstract locator?: string;
-}
-interface ElementTreeNode<ElementType extends BaseElement = BaseElement> {
-    node: ElementType | null;
-    children: ElementTreeNode<ElementType>[];
-}
-interface WebElementInfo extends ElementInfo {
-    zoom: number;
-}
-
-export { BaseElement as B, type ElementInfo as E, type Point as P, type Rect as R, type Size as S, type WebElementInfo as W, type ElementNode as a, treeToList as b, extractTreeNode as c, descriptionOfTree as d, extractTextWithPosition as e, extractTreeNodeAsString as f, getXpathsById as g, getElementInfoByNode as h, type ElementTreeNode as i, setNodeHashCacheListOnWindow as s, traverseTree as t };

package/dist/es/types.d.ts

@@ -1,2 +0,0 @@
-import './constants.js';
-export { B as BaseElement, i as ElementTreeNode, P as Point, R as Rect, S as Size, W as WebElementInfo } from './index-df4e781b.js';

package/dist/es/types.js

@@ -1,6 +0,0 @@
-// src/types/index.ts
-var BaseElement = class {
-};
-export {
-  BaseElement
-};

package/dist/lib/index-df4e781b.d.ts

@@ -1,77 +0,0 @@
-import { NodeType } from './constants.js';
-
-declare function descriptionOfTree<ElementType extends BaseElement = BaseElement>(tree: ElementTreeNode<ElementType>, truncateTextLength?: number, filterNonTextContent?: boolean): string;
-declare function treeToList<T extends BaseElement>(tree: ElementTreeNode<T>): T[];
-declare function traverseTree<T extends BaseElement, ReturnNodeType extends BaseElement>(tree: ElementTreeNode<T>, onNode: (node: T) => ReturnNodeType): ElementTreeNode<ReturnNodeType>;
-
-interface WebElementNode {
-    node: WebElementInfo | null;
-    children: WebElementNode[];
-}
-declare function extractTextWithPosition(initNode: globalThis.Node, debugMode?: boolean): WebElementInfo[];
-declare function extractTreeNodeAsString(initNode: globalThis.Node, debugMode?: boolean): string;
-declare function extractTreeNode(initNode: globalThis.Node, debugMode?: boolean): WebElementNode;
-
-declare function setNodeHashCacheListOnWindow(): void;
-
-declare function getXpathsById(id: string): string[] | null;
-declare function getElementInfoByNode(node: Node): ElementInfo | null;
-
-interface ElementInfo {
-    id: string;
-    indexId: number;
-    nodeHashId: string;
-    locator: string;
-    xpaths?: string[];
-    attributes: {
-        nodeType: NodeType;
-        [key: string]: string;
-    };
-    nodeType: NodeType;
-    content: string;
-    rect: {
-        left: number;
-        top: number;
-        width: number;
-        height: number;
-    };
-    center: [number, number];
-}
-interface ElementNode {
-    node: ElementInfo | null;
-    children: ElementNode[];
-}
-
-interface Point {
-    left: number;
-    top: number;
-}
-interface Size {
-    width: number;
-    height: number;
-    dpr?: number;
-}
-type Rect = Point & Size & {
-    zoom?: number;
-};
-declare abstract class BaseElement {
-    abstract id: string;
-    abstract indexId?: number;
-    abstract attributes: {
-        nodeType: NodeType;
-        [key: string]: string;
-    };
-    abstract content: string;
-    abstract rect: Rect;
-    abstract center: [number, number];
-    abstract locator?: string;
-}
-interface ElementTreeNode<ElementType extends BaseElement = BaseElement> {
-    node: ElementType | null;
-    children: ElementTreeNode<ElementType>[];
-}
-interface WebElementInfo extends ElementInfo {
-    zoom: number;
-}
-
-export { BaseElement as B, type ElementInfo as E, type Point as P, type Rect as R, type Size as S, type WebElementInfo as W, type ElementNode as a, treeToList as b, extractTreeNode as c, descriptionOfTree as d, extractTextWithPosition as e, extractTreeNodeAsString as f, getXpathsById as g, getElementInfoByNode as h, type ElementTreeNode as i, setNodeHashCacheListOnWindow as s, traverseTree as t };

package/dist/lib/types.d.ts

@@ -1,2 +0,0 @@
-import './constants.js';
-export { B as BaseElement, i as ElementTreeNode, P as Point, R as Rect, S as Size, W as WebElementInfo } from './index-df4e781b.js';

package/dist/lib/types.js

@@ -1,31 +0,0 @@
-"use strict";
-var __defProp = Object.defineProperty;
-var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
-var __getOwnPropNames = Object.getOwnPropertyNames;
-var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __export = (target, all) => {
-  for (var name in all)
-    __defProp(target, name, { get: all[name], enumerable: true });
-};
-var __copyProps = (to, from, except, desc) => {
-  if (from && typeof from === "object" || typeof from === "function") {
-    for (let key of __getOwnPropNames(from))
-      if (!__hasOwnProp.call(to, key) && key !== except)
-        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
-  }
-  return to;
-};
-var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
-
-// src/types/index.ts
-var types_exports = {};
-__export(types_exports, {
-  BaseElement: () => BaseElement
-});
-module.exports = __toCommonJS(types_exports);
-var BaseElement = class {
-};
-// Annotate the CommonJS export names for ESM import in node:
-0 && (module.exports = {
-  BaseElement
-});

package/dist/types/index-df4e781b.d.ts

@@ -1,77 +0,0 @@
-import { NodeType } from './constants.js';
-
-declare function descriptionOfTree<ElementType extends BaseElement = BaseElement>(tree: ElementTreeNode<ElementType>, truncateTextLength?: number, filterNonTextContent?: boolean): string;
-declare function treeToList<T extends BaseElement>(tree: ElementTreeNode<T>): T[];
-declare function traverseTree<T extends BaseElement, ReturnNodeType extends BaseElement>(tree: ElementTreeNode<T>, onNode: (node: T) => ReturnNodeType): ElementTreeNode<ReturnNodeType>;
-
-interface WebElementNode {
-    node: WebElementInfo | null;
-    children: WebElementNode[];
-}
-declare function extractTextWithPosition(initNode: globalThis.Node, debugMode?: boolean): WebElementInfo[];
-declare function extractTreeNodeAsString(initNode: globalThis.Node, debugMode?: boolean): string;
-declare function extractTreeNode(initNode: globalThis.Node, debugMode?: boolean): WebElementNode;
-
-declare function setNodeHashCacheListOnWindow(): void;
-
-declare function getXpathsById(id: string): string[] | null;
-declare function getElementInfoByNode(node: Node): ElementInfo | null;
-
-interface ElementInfo {
-    id: string;
-    indexId: number;
-    nodeHashId: string;
-    locator: string;
-    xpaths?: string[];
-    attributes: {
-        nodeType: NodeType;
-        [key: string]: string;
-    };
-    nodeType: NodeType;
-    content: string;
-    rect: {
-        left: number;
-        top: number;
-        width: number;
-        height: number;
-    };
-    center: [number, number];
-}
-interface ElementNode {
-    node: ElementInfo | null;
-    children: ElementNode[];
-}
-
-interface Point {
-    left: number;
-    top: number;
-}
-interface Size {
-    width: number;
-    height: number;
-    dpr?: number;
-}
-type Rect = Point & Size & {
-    zoom?: number;
-};
-declare abstract class BaseElement {
-    abstract id: string;
-    abstract indexId?: number;
-    abstract attributes: {
-        nodeType: NodeType;
-        [key: string]: string;
-    };
-    abstract content: string;
-    abstract rect: Rect;
-    abstract center: [number, number];
-    abstract locator?: string;
-}
-interface ElementTreeNode<ElementType extends BaseElement = BaseElement> {
-    node: ElementType | null;
-    children: ElementTreeNode<ElementType>[];
-}
-interface WebElementInfo extends ElementInfo {
-    zoom: number;
-}
-
-export { BaseElement as B, type ElementInfo as E, type Point as P, type Rect as R, type Size as S, type WebElementInfo as W, type ElementNode as a, treeToList as b, extractTreeNode as c, descriptionOfTree as d, extractTextWithPosition as e, extractTreeNodeAsString as f, getXpathsById as g, getElementInfoByNode as h, type ElementTreeNode as i, setNodeHashCacheListOnWindow as s, traverseTree as t };

package/dist/types/types.d.ts

@@ -1,2 +0,0 @@
-import './constants.js';
-export { B as BaseElement, i as ElementTreeNode, P as Point, R as Rect, S as Size, W as WebElementInfo } from './index-df4e781b.js';