@rpascene/shared 0.30.8

package/src/extractor/web-extractor.ts

@@ -0,0 +1,559 @@
+import {
+  CONTAINER_MINI_HEIGHT,
+  CONTAINER_MINI_WIDTH,
+  NodeType,
+} from '../constants/index';
+import type { WebElementInfo } from '../types';
+import type { Point } from '../types';
+import {
+  isAElement,
+  isButtonElement,
+  isContainerElement,
+  isFormElement,
+  isImgElement,
+  isTextElement,
+} from './dom-util';
+import { descriptionOfTree } from './tree';
+import {
+  elementRect,
+  getNodeAttributes,
+  getPseudoElementContent,
+  getRect,
+  getTopDocument,
+  logger,
+  rpasceneGenerateHash,
+  setDebugMode,
+} from './util';
+import { postWindowMessage, onWindowMessage } from './cs_postmessage';
+
+import {
+  getContainerPath,
+  getLocators
+} from './customLocator';
+
+let indexId = 0;
+
+function tagNameOfNode(node: globalThis.Node, currentWindow = globalThis): string {
+  let tagName = '';
+  if (node instanceof currentWindow.HTMLElement) {
+    tagName = node.tagName?.toLowerCase();
+  } else {
+    const parentElement = node.parentElement;
+    if (parentElement && parentElement instanceof currentWindow.HTMLElement) {
+      tagName = parentElement.tagName?.toLowerCase();
+    }
+  }
+
+  return tagName ? `<${tagName}>` : '';
+}
+
+export function collectElementInfo(
+  node: Node,
+  currentWindow: typeof window,
+  currentDocument: typeof document,
+  baseZoom = 1,
+  basePoint: Point = { left: 0, top: 0 },
+  isContainer = false, // if true, the element will be considered as a container
+): WebElementInfo | null | any {
+  const rect = elementRect(node, currentWindow, currentDocument, baseZoom);
+
+  if (!rect) {
+    return null;
+  }
+
+  if (
+    rect.width < CONTAINER_MINI_WIDTH ||
+    rect.height < CONTAINER_MINI_HEIGHT
+  ) {
+    return null;
+  }
+
+  if (basePoint.left !== 0 || basePoint.top !== 0) {
+    rect.left += basePoint.left;
+    rect.top += basePoint.top;
+  }
+  // Skip elements that cover the entire viewport, as they are likely background containers
+  // rather than meaningful interactive elements
+  if (rect.height >= window.innerHeight && rect.width >= window.innerWidth) {
+    return null;
+  }
+
+
+  if (isFormElement(node, currentWindow)) {
+    const attributes = getNodeAttributes(node, currentWindow);
+    let valueContent =
+      attributes.value || attributes.placeholder || node.textContent || '';
+    const nodeHashId = rpasceneGenerateHash(node, valueContent, rect);
+    const tagName = (node as HTMLElement).tagName.toLowerCase();
+    if ((node as HTMLElement).tagName.toLowerCase() === 'select') {
+      // Get the selected option using the selectedIndex property
+      const selectedOption = (node as HTMLSelectElement).options[
+        (node as HTMLSelectElement).selectedIndex
+      ];
+
+      // Retrieve the text content of the selected option
+      valueContent = selectedOption?.textContent || '';
+    }
+
+    if (
+      ((node as HTMLElement).tagName.toLowerCase() === 'input' ||
+        (node as HTMLElement).tagName.toLowerCase() === 'textarea') &&
+      (node as HTMLInputElement).value
+    ) {
+      valueContent = (node as HTMLInputElement).value;
+    }
+
+    const elementInfo: WebElementInfo = {
+      id: nodeHashId,
+      nodeHashId,
+      nodeType: NodeType.FORM_ITEM,
+      indexId: indexId++,
+      attributes: {
+        ...attributes,
+        htmlTagName: `<${tagName}>`,
+        nodeType: NodeType.FORM_ITEM,
+      },
+      content: valueContent.trim(),
+      rect,
+      center: [
+        Math.round(rect.left + rect.width / 2),
+        Math.round(rect.top + rect.height / 2),
+      ],
+      zoom: rect.zoom,
+      isVisible: rect.isVisible
+    };
+    return elementInfo;
+  }
+
+  if (isButtonElement(node, currentWindow)) {
+    const rect = mergeElementAndChildrenRects(
+      node,
+      currentWindow,
+      currentDocument,
+      baseZoom,
+    );
+    if (!rect) {
+      return null;
+    }
+    const attributes = getNodeAttributes(node, currentWindow);
+    const pseudo = getPseudoElementContent(node, currentWindow);
+    const content = node.innerText || pseudo.before || pseudo.after || '';
+    const nodeHashId = rpasceneGenerateHash(node, content, rect);
+    const elementInfo: WebElementInfo = {
+      id: nodeHashId,
+      indexId: indexId++,
+      nodeHashId,
+      nodeType: NodeType.BUTTON,
+      attributes: {
+        ...attributes,
+        htmlTagName: tagNameOfNode(node, currentWindow),
+        nodeType: NodeType.BUTTON,
+      },
+      content,
+      rect,
+      center: [
+        Math.round(rect.left + rect.width / 2),
+        Math.round(rect.top + rect.height / 2),
+      ],
+      zoom: rect.zoom,
+      isVisible: rect.isVisible
+    };
+    return elementInfo;
+  }
+
+  if (isImgElement(node, currentWindow)) {
+    const attributes = getNodeAttributes(node, currentWindow);
+    const nodeHashId = rpasceneGenerateHash(node, '', rect);
+    const elementInfo: WebElementInfo = {
+      id: nodeHashId,
+      indexId: indexId++,
+      nodeHashId,
+      attributes: {
+        ...attributes,
+        ...(node.nodeName?.toLowerCase() === 'svg'
+          ? {
+            svgContent: 'true',
+          }
+          : {}),
+        nodeType: NodeType.IMG,
+        htmlTagName: tagNameOfNode(node, currentWindow),
+      },
+      nodeType: NodeType.IMG,
+      content: '',
+      rect,
+      center: [
+        Math.round(rect.left + rect.width / 2),
+        Math.round(rect.top + rect.height / 2),
+      ],
+      zoom: rect.zoom,
+      isVisible: rect.isVisible
+    };
+    return elementInfo;
+  }
+
+  if (isTextElement(node, currentWindow)) {
+    const text = node.textContent?.trim().replace(/\n+/g, ' ');
+    if (!text) {
+      return null;
+    }
+    const attributes = getNodeAttributes(node, currentWindow);
+    const attributeKeys = Object.keys(attributes);
+    if (!text.trim() && attributeKeys.length === 0) {
+      return null;
+    }
+    const nodeHashId = rpasceneGenerateHash(node, text, rect);
+    const elementInfo: WebElementInfo = {
+      id: nodeHashId,
+      indexId: indexId++,
+      nodeHashId,
+      nodeType: NodeType.TEXT,
+      attributes: {
+        ...attributes,
+        nodeType: NodeType.TEXT,
+        htmlTagName: tagNameOfNode(node, currentWindow),
+      },
+      center: [
+        Math.round(rect.left + rect.width / 2),
+        Math.round(rect.top + rect.height / 2),
+      ],
+      content: text,
+      rect,
+      zoom: rect.zoom,
+      isVisible: rect.isVisible
+    };
+    return elementInfo;
+  }
+
+  if (isAElement(node, currentWindow)) {
+    const attributes = getNodeAttributes(node, currentWindow);
+    const pseudo = getPseudoElementContent(node, currentWindow);
+    const content = node.innerText || pseudo.before || pseudo.after || '';
+    const nodeHashId = rpasceneGenerateHash(node, content, rect);
+    const elementInfo: WebElementInfo = {
+      id: nodeHashId,
+      indexId: indexId++,
+      nodeHashId,
+      nodeType: NodeType.A,
+      attributes: {
+        ...attributes,
+        htmlTagName: tagNameOfNode(node, currentWindow),
+        nodeType: NodeType.A,
+      },
+      content,
+      rect,
+      center: [
+        Math.round(rect.left + rect.width / 2),
+        Math.round(rect.top + rect.height / 2),
+      ],
+      zoom: rect.zoom,
+      isVisible: rect.isVisible
+    };
+    return elementInfo;
+  }
+
+  // else, consider as a container
+  // 视为容器
+  if (isContainerElement(node, currentWindow) || isContainer) {
+    const attributes = getNodeAttributes(node, currentWindow);
+    const nodeHashId = rpasceneGenerateHash(node, '', rect);
+    const elementInfo: WebElementInfo = {
+      id: nodeHashId,
+      nodeHashId,
+      indexId: indexId++,
+      nodeType: NodeType.CONTAINER,
+      attributes: {
+        ...attributes,
+        nodeType: NodeType.CONTAINER,
+        htmlTagName: tagNameOfNode(node, currentWindow),
+      },
+      content: '',
+      rect,
+      center: [
+        Math.round(rect.left + rect.width / 2),
+        Math.round(rect.top + rect.height / 2),
+      ],
+      zoom: rect.zoom,
+      isVisible: rect.isVisible
+    };
+    return elementInfo;
+  }
+  return null;
+}
+
+interface WebElementNode {
+  node: WebElementInfo | null;
+  children: WebElementNode[];
+}
+
+// 提取带位置的文本
+// @deprecated
+export async function extractTextWithPosition(
+  initNode: globalThis.Node,
+  debugMode = false,
+): Promise<WebElementInfo[]> {
+  const elementNode = await extractTreeNode(initNode, debugMode);
+
+  // dfs topChildren
+  const elementInfoArray: WebElementInfo[] = [];
+  function dfsTopChildren(node: WebElementNode) {
+    if (node.node) {
+      elementInfoArray.push(node.node);
+    }
+    for (let i = 0; i < node.children.length; i++) {
+      dfsTopChildren(node.children[i]);
+    }
+  }
+  dfsTopChildren({ children: elementNode.children, node: elementNode.node });
+
+  console.log(elementInfoArray, 'elementInfoArray')
+  return elementInfoArray;
+}
+// 提取树节点为字符串
+export async function extractTreeNodeAsString(
+  initNode: globalThis.Node,
+  visibleOnly = false,
+  debugMode = false,
+): Promise<string> {
+  const elementNode = await extractTreeNode(initNode, debugMode);
+  return descriptionOfTree(elementNode, undefined, false, visibleOnly);
+}
+// 提取树节点
+export async function extractTreeNode(
+  initNode: globalThis.Node,
+  debugMode: boolean = false,
+  basePoint: Point = { left: 0, top: 0 },
+  baseZoom: number = 1,
+  containerPaths: any[] = []
+): Promise<WebElementNode> {
+  setDebugMode(debugMode);
+  indexId = 0;
+  let containerPathClone = [...containerPaths]
+  const topDocument = getTopDocument();
+  const startNode = initNode || topDocument;
+  const topChildren: WebElementNode[] = [];
+
+  // 递归函数dfs
+  function dfs(
+    node: globalThis.Node,
+    currentWindow: typeof globalThis.window,
+    currentDocument: typeof globalThis.document,
+    baseZoom = 1,
+    basePoint: Point = { left: 0, top: 0 },
+  ): WebElementNode | WebElementNode[] | null {
+    if (!node) {
+      return null;
+    }
+
+    if (node.nodeType && node.nodeType === 10) {
+      // Doctype node
+      return null;
+    }
+
+    const elementInfo = collectElementInfo(
+      node,
+      currentWindow,
+      currentDocument,
+      baseZoom,
+      basePoint,
+      false
+    );
+
+    if (elementInfo) {
+      const allPaths = getLocators(node);
+      elementInfo.allPaths = allPaths;
+      elementInfo.containerPaths = containerPathClone;
+    }
+    if (node instanceof currentWindow.HTMLIFrameElement) {
+      if (
+        (node as HTMLIFrameElement).contentWindow &&
+        (node as HTMLIFrameElement).contentWindow
+      ) {
+        return null;
+      }
+    }
+
+    const nodeInfo: WebElementNode = {
+      node: elementInfo,
+      children: [],
+    };
+    // stop collecting if the node is a Button/Image/Text/FormItem/Container
+    if (
+      elementInfo?.nodeType === NodeType.BUTTON ||
+      elementInfo?.nodeType === NodeType.IMG ||
+      elementInfo?.nodeType === NodeType.TEXT ||
+      elementInfo?.nodeType === NodeType.FORM_ITEM ||
+      elementInfo?.nodeType === NodeType.CONTAINER // TODO: need return the container node?
+    ) {
+      return nodeInfo;
+    }
+
+    const rect = getRect(node, baseZoom, currentWindow);
+    for (let i = 0; i < node.childNodes.length; i++) {
+      logger('will dfs', node.childNodes[i]);
+      const childNodeInfo = dfs(
+        node.childNodes[i],
+        currentWindow,
+        currentDocument,
+        rect.zoom,
+        basePoint,
+      );
+      if (Array.isArray(childNodeInfo)) {
+        // if the recursive return is an array, expand and merge it into children
+        nodeInfo.children.push(...childNodeInfo);
+      } else if (childNodeInfo) {
+        nodeInfo.children.push(childNodeInfo);
+      }
+    }
+
+    // if nodeInfo.node is null
+    if (nodeInfo.node === null) {
+      if (nodeInfo.children.length === 0) {
+        return null;
+      }
+      // promote children to the upper layer
+      return nodeInfo.children;
+    }
+
+    return nodeInfo;
+  }
+
+  /**
+ * 获取css样式中元素的transform: scale(.8);
+ * @param {*} ele
+ * @returns
+ */
+  function getCssScale(ele: any) {
+    try {
+      const { width, height } = ele.getBoundingClientRect();
+      return {
+        cssScaleX: Number((width / ele.offsetWidth).toFixed(1)),
+        cssScaleY: Number((height / ele.offsetHeight).toFixed(1)),
+      };
+    } catch (error) {
+      return {
+        cssScaleX: 1,
+        cssScaleY: 1,
+      };
+    }
+  }
+
+  const rootNodeInfo = dfs(startNode, window, document, baseZoom, basePoint);
+  if (Array.isArray(rootNodeInfo)) {
+    topChildren.push(...rootNodeInfo);
+  } else if (rootNodeInfo) {
+    topChildren.push(rootNodeInfo);
+  }
+  if (startNode === topDocument) {
+    // find all the same-origin iframes
+    const iframes = document.querySelectorAll('iframe');
+    for (let i = 0; i < iframes.length; i++) {
+      const iframe = iframes[i];
+      const iframeInfo = collectElementInfo(iframe, window, document, 1);
+      const baseZoom = getCssScale(iframe).cssScaleX;
+      const containerPath = getContainerPath(iframe);
+      containerPathClone = containerPathClone.concat(containerPath)
+      if (iframeInfo) {
+        if (iframe.contentDocument && iframe.contentWindow) {
+          console.log('iframe同域')
+          // when the iframe is in the viewport, we need to collect its children
+          const iframeChildren = dfs(
+            iframe.contentDocument.body,
+            iframe.contentWindow as any,
+            iframe.contentDocument,
+            baseZoom,
+            {
+              left: iframeInfo.rect.left,
+              top: iframeInfo.rect.top,
+            },
+          );
+          if (Array.isArray(iframeChildren)) {
+            topChildren.push(...iframeChildren);
+          } else if (iframeChildren) {
+            topChildren.push(iframeChildren);
+          }
+        } else {
+          console.log(iframeInfo, 'iframe跨域了')
+
+          // 跨域了,请求获取tree
+          const iframeChildren = await postWindowMessage(iframe.contentWindow, window, {
+            action: 'extractTreeNode',
+            data: {
+              basePoint: {
+                left: iframeInfo.rect.left,
+                top: iframeInfo.rect.top,
+              },
+              baseZoom,
+              containerPaths: containerPathClone
+            },
+          })
+          if (Array.isArray(iframeChildren)) {
+            topChildren.push(...iframeChildren);
+          } else if (iframeChildren) {
+            topChildren.push(iframeChildren);
+          }
+        }
+      }
+
+    }
+  }
+
+  return {
+    node: null,
+    children: topChildren,
+  };
+}
+// 合并元素及其子元素矩形
+export function mergeElementAndChildrenRects(
+  node: Node,
+  currentWindow: typeof window,
+  currentDocument: typeof document,
+  baseZoom = 1,
+) {
+  const selfRect = elementRect(node, currentWindow, currentDocument, baseZoom);
+  if (!selfRect) return null;
+
+  let minLeft = selfRect.left;
+  let minTop = selfRect.top;
+  let maxRight = selfRect.left + selfRect.width;
+  let maxBottom = selfRect.top + selfRect.height;
+
+  function traverse(child: Node) {
+    for (let i = 0; i < child.childNodes.length; i++) {
+      const sub = child.childNodes[i];
+      if (sub.nodeType === 1) {
+        const rect = elementRect(sub, currentWindow, currentDocument, baseZoom);
+        if (rect) {
+          minLeft = Math.min(minLeft, rect.left);
+          minTop = Math.min(minTop, rect.top);
+          maxRight = Math.max(maxRight, rect.left + rect.width);
+          maxBottom = Math.max(maxBottom, rect.top + rect.height);
+        }
+        traverse(sub);
+      }
+    }
+  }
+  traverse(node);
+
+  return {
+    ...selfRect,
+    left: minLeft,
+    top: minTop,
+    width: maxRight - minLeft,
+    height: maxBottom - minTop,
+  };
+}
+
+if (typeof window !== 'undefined') {
+  onWindowMessage(window, async ({ action, data }: any, { source }: any) => {
+    if (action === 'extractTreeNode') {
+      // @ts-ignore
+      try {
+        return await extractTreeNode(document.body, false, data.basePoint, data.baseZoom, data.containerPaths)
+      } catch (error) {
+        return {
+          error: 'error message'
+        }
+      }
+    }
+  })
+}