@godscene/shared 1.7.11

package/src/extractor/web-extractor.ts

@@ -0,0 +1,617 @@
+import {
+  CONTAINER_MINI_HEIGHT,
+  CONTAINER_MINI_WIDTH,
+  NodeType,
+} from '../constants/index';
+import type { WebElementInfo } from '../types';
+import type { Point } from '../types';
+import {
+  isAElement,
+  isButtonElement,
+  isContainerElement,
+  isFormElement,
+  isImgElement,
+  isTextElement,
+} from './dom-util';
+import { descriptionOfTree } from './tree';
+import {
+  elementRect,
+  getNodeAttributes,
+  getPseudoElementContent,
+  getRect,
+  getTopDocument,
+  logger,
+  midsceneGenerateHash,
+  setDebugMode,
+} from './util';
+import { onWindowMessage, postWindowMessage } from './cs_postmessage';
+import { getContainerPath, getLocators } from './customLocator';
+
+let indexId = 0;
+
+function tagNameOfNode(node: globalThis.Node): string {
+  let tagName = '';
+  if (node instanceof HTMLElement) {
+    tagName = node.tagName?.toLowerCase();
+  } else {
+    const parentElement = node.parentElement;
+    if (parentElement && parentElement instanceof HTMLElement) {
+      tagName = parentElement.tagName?.toLowerCase();
+    }
+  }
+
+  return tagName ? `<${tagName}>` : '';
+}
+
+export function collectElementInfo(
+  node: Node,
+  currentWindow: typeof window,
+  currentDocument: typeof document,
+  baseZoom = 1,
+  basePoint: Point = { left: 0, top: 0 },
+  isContainer = false, // if true, the element will be considered as a container
+): WebElementInfo | null | any {
+  const rect = elementRect(node, currentWindow, currentDocument, baseZoom);
+
+  if (!rect) {
+    return null;
+  }
+
+  if (
+    rect.width < CONTAINER_MINI_WIDTH ||
+    rect.height < CONTAINER_MINI_HEIGHT
+  ) {
+    return null;
+  }
+
+  if (basePoint.left !== 0 || basePoint.top !== 0) {
+    rect.left += basePoint.left;
+    rect.top += basePoint.top;
+  }
+  // Skip elements that cover the entire viewport, as they are likely background containers
+  // rather than meaningful interactive elements
+  if (rect.height >= window.innerHeight && rect.width >= window.innerWidth) {
+    return null;
+  }
+
+  if (isFormElement(node)) {
+    const attributes = getNodeAttributes(node, currentWindow);
+    let valueContent =
+      attributes.value || attributes.placeholder || node.textContent || '';
+    const nodeHashId = midsceneGenerateHash(node, valueContent, rect);
+    const tagName = (node as HTMLElement).tagName.toLowerCase();
+    if ((node as HTMLElement).tagName.toLowerCase() === 'select') {
+      // Get the selected option using the selectedIndex property
+      const selectedOption = (node as HTMLSelectElement).options[
+        (node as HTMLSelectElement).selectedIndex
+      ];
+
+      // Retrieve the text content of the selected option
+      valueContent = selectedOption?.textContent || '';
+    }
+
+    if (
+      ((node as HTMLElement).tagName.toLowerCase() === 'input' ||
+        (node as HTMLElement).tagName.toLowerCase() === 'textarea') &&
+      (node as HTMLInputElement).value
+    ) {
+      valueContent = (node as HTMLInputElement).value;
+    }
+
+    const elementInfo: WebElementInfo = {
+      id: nodeHashId,
+      nodeHashId,
+      nodeType: NodeType.FORM_ITEM,
+      indexId: indexId++,
+      attributes: {
+        ...attributes,
+        htmlTagName: `<${tagName}>`,
+        nodeType: NodeType.FORM_ITEM,
+      },
+      content: valueContent.trim(),
+      rect,
+      center: [
+        Math.round(rect.left + rect.width / 2),
+        Math.round(rect.top + rect.height / 2),
+      ],
+      zoom: rect.zoom,
+      isVisible: rect.isVisible,
+    };
+    return elementInfo;
+  }
+
+  if (isButtonElement(node)) {
+    const rect = mergeElementAndChildrenRects(
+      node,
+      currentWindow,
+      currentDocument,
+      baseZoom,
+    );
+    if (!rect) {
+      return null;
+    }
+    const attributes = getNodeAttributes(node, currentWindow);
+    const pseudo = getPseudoElementContent(node, currentWindow);
+    const content = node.innerText || pseudo.before || pseudo.after || '';
+    const nodeHashId = midsceneGenerateHash(node, content, rect);
+    const elementInfo: WebElementInfo = {
+      id: nodeHashId,
+      indexId: indexId++,
+      nodeHashId,
+      nodeType: NodeType.BUTTON,
+      attributes: {
+        ...attributes,
+        htmlTagName: tagNameOfNode(node),
+        nodeType: NodeType.BUTTON,
+      },
+      content,
+      rect,
+      center: [
+        Math.round(rect.left + rect.width / 2),
+        Math.round(rect.top + rect.height / 2),
+      ],
+      zoom: rect.zoom,
+      isVisible: rect.isVisible,
+    };
+    return elementInfo;
+  }
+
+  if (isImgElement(node)) {
+    const attributes = getNodeAttributes(node, currentWindow);
+    const nodeHashId = midsceneGenerateHash(node, '', rect);
+    const elementInfo: WebElementInfo = {
+      id: nodeHashId,
+      indexId: indexId++,
+      nodeHashId,
+      attributes: {
+        ...attributes,
+        ...(node.nodeName?.toLowerCase() === 'svg'
+          ? {
+              svgContent: 'true',
+            }
+          : {}),
+        nodeType: NodeType.IMG,
+        htmlTagName: tagNameOfNode(node),
+      },
+      nodeType: NodeType.IMG,
+      content: '',
+      rect,
+      center: [
+        Math.round(rect.left + rect.width / 2),
+        Math.round(rect.top + rect.height / 2),
+      ],
+      zoom: rect.zoom,
+      isVisible: rect.isVisible,
+    };
+    return elementInfo;
+  }
+
+  if (isTextElement(node)) {
+    const text = node.textContent?.trim().replace(/\n+/g, ' ');
+    if (!text) {
+      return null;
+    }
+    const attributes = getNodeAttributes(node, currentWindow);
+    const attributeKeys = Object.keys(attributes);
+    if (!text.trim() && attributeKeys.length === 0) {
+      return null;
+    }
+    const nodeHashId = midsceneGenerateHash(node, text, rect);
+    const elementInfo: WebElementInfo = {
+      id: nodeHashId,
+      indexId: indexId++,
+      nodeHashId,
+      nodeType: NodeType.TEXT,
+      attributes: {
+        ...attributes,
+        nodeType: NodeType.TEXT,
+        htmlTagName: tagNameOfNode(node),
+      },
+      center: [
+        Math.round(rect.left + rect.width / 2),
+        Math.round(rect.top + rect.height / 2),
+      ],
+      content: text,
+      rect,
+      zoom: rect.zoom,
+      isVisible: rect.isVisible,
+    };
+    return elementInfo;
+  }
+
+  if (isAElement(node)) {
+    const attributes = getNodeAttributes(node, currentWindow);
+    const pseudo = getPseudoElementContent(node, currentWindow);
+    const content = node.innerText || pseudo.before || pseudo.after || '';
+    const nodeHashId = midsceneGenerateHash(node, content, rect);
+    const elementInfo: WebElementInfo = {
+      id: nodeHashId,
+      indexId: indexId++,
+      nodeHashId,
+      nodeType: NodeType.A,
+      attributes: {
+        ...attributes,
+        htmlTagName: tagNameOfNode(node),
+        nodeType: NodeType.A,
+      },
+      content,
+      rect,
+      center: [
+        Math.round(rect.left + rect.width / 2),
+        Math.round(rect.top + rect.height / 2),
+      ],
+      zoom: rect.zoom,
+      isVisible: rect.isVisible,
+    };
+    return elementInfo;
+  }
+
+  // else, consider as a container
+  if (isContainerElement(node) || isContainer) {
+    const attributes = getNodeAttributes(node, currentWindow);
+    const nodeHashId = midsceneGenerateHash(node, '', rect);
+    const elementInfo: WebElementInfo = {
+      id: nodeHashId,
+      nodeHashId,
+      indexId: indexId++,
+      nodeType: NodeType.CONTAINER,
+      attributes: {
+        ...attributes,
+        nodeType: NodeType.CONTAINER,
+        htmlTagName: tagNameOfNode(node),
+      },
+      content: '',
+      rect,
+      center: [
+        Math.round(rect.left + rect.width / 2),
+        Math.round(rect.top + rect.height / 2),
+      ],
+      zoom: rect.zoom,
+      isVisible: rect.isVisible,
+    };
+    return elementInfo;
+  }
+  return null;
+}
+
+interface WebElementNode {
+  node: WebElementInfo | null;
+  children: WebElementNode[];
+}
+
+// @deprecated
+export async function extractTextWithPosition(
+  initNode: globalThis.Node,
+  debugMode = false,
+): Promise<WebElementInfo[]>  {
+  const elementNode = await extractTreeNode(initNode, debugMode);
+
+  // dfs topChildren
+  const elementInfoArray: WebElementInfo[] = [];
+  function dfsTopChildren(node: WebElementNode) {
+    if (node.node) {
+      elementInfoArray.push(node.node);
+    }
+    for (let i = 0; i < node.children.length; i++) {
+      dfsTopChildren(node.children[i]);
+    }
+  }
+  dfsTopChildren({ children: elementNode.children, node: elementNode.node });
+  return elementInfoArray;
+}
+
+export async function extractTreeNodeAsString(
+  initNode: globalThis.Node,
+  visibleOnly = false,
+  debugMode = false,
+): Promise<string>  {
+  const elementNode = await extractTreeNode(initNode, debugMode);
+  return descriptionOfTree(elementNode, undefined, false, visibleOnly);
+}
+
+// 提取树节点
+export async function extractTreeNode(
+  initNode: globalThis.Node,
+  debugMode = false,
+  basePoint: Point = { left: 0, top: 0 },
+  baseZoom = 1,
+): Promise<WebElementNode> {
+  setDebugMode(debugMode);
+  indexId = 0;
+  const topDocument = getTopDocument();
+  const startNode = initNode || topDocument;
+  const topChildren: WebElementNode[] = [];
+
+  // 递归函数dfs
+  function dfs(
+    node: globalThis.Node,
+    currentWindow: typeof globalThis.window,
+    currentDocument: typeof globalThis.document,
+    baseZoom = 1,
+    basePoint: Point = { left: 0, top: 0 },
+  ): WebElementNode | WebElementNode[] | null {
+    if (!node) {
+      return null;
+    }
+
+    if (node.nodeType && node.nodeType === 10) {
+      // Doctype node
+      return null;
+    }
+
+    const elementInfo = collectElementInfo(
+      node,
+      currentWindow,
+      currentDocument,
+      baseZoom,
+      basePoint,
+      false,
+    );
+
+    if (node instanceof currentWindow.HTMLIFrameElement) {
+      if (
+        (node as HTMLIFrameElement).contentWindow &&
+        (node as HTMLIFrameElement).contentWindow
+      ) {
+        return null;
+      }
+    }
+
+    const nodeInfo: WebElementNode = {
+      node: elementInfo,
+      children: [],
+    };
+    // stop collecting if the node is a Button/Image/Text/FormItem/Container
+    if (
+      elementInfo?.nodeType === NodeType.BUTTON ||
+      elementInfo?.nodeType === NodeType.IMG ||
+      elementInfo?.nodeType === NodeType.TEXT ||
+      elementInfo?.nodeType === NodeType.FORM_ITEM ||
+      elementInfo?.nodeType === NodeType.CONTAINER // TODO: need return the container node?
+    ) {
+      return nodeInfo;
+    }
+
+    const rect = getRect(node, baseZoom, currentWindow);
+    for (let i = 0; i < node.childNodes.length; i++) {
+      logger('will dfs', node.childNodes[i]);
+      const childNodeInfo = dfs(
+        node.childNodes[i],
+        currentWindow,
+        currentDocument,
+        rect.zoom,
+        basePoint,
+      );
+      if (Array.isArray(childNodeInfo)) {
+        // if the recursive return is an array, expand and merge it into children
+        nodeInfo.children.push(...childNodeInfo);
+      } else if (childNodeInfo) {
+        nodeInfo.children.push(childNodeInfo);
+      }
+    }
+
+    // if nodeInfo.node is null
+    if (nodeInfo.node === null) {
+      if (nodeInfo.children.length === 0) {
+        return null;
+      }
+      // promote children to the upper layer
+      return nodeInfo.children;
+    }
+
+    return nodeInfo;
+  }
+
+  /**
+   * 获取css样式中元素的transform: scale(.8);
+   * @param {*} ele
+   * @returns
+   */
+  function getCssScale(ele: any) {
+    try {
+      const { width, height } = ele.getBoundingClientRect();
+      return {
+        cssScaleX: Number((width / ele.offsetWidth).toFixed(1)),
+        cssScaleY: Number((height / ele.offsetHeight).toFixed(1)),
+      };
+    } catch (error) {
+      return {
+        cssScaleX: 1,
+        cssScaleY: 1,
+      };
+    }
+  }
+
+  const rootNodeInfo = dfs(startNode, window, document, baseZoom, basePoint);
+  if (Array.isArray(rootNodeInfo)) {
+    topChildren.push(...rootNodeInfo);
+  } else if (rootNodeInfo) {
+    topChildren.push(rootNodeInfo);
+  }
+  if (startNode === topDocument) {
+    // find all the same-origin iframes
+    const iframes = document.querySelectorAll('iframe');
+    for (let i = 0; i < iframes.length; i++) {
+      const iframe = iframes[i];
+      const iframeInfo = collectElementInfo(iframe, window, document, 1);
+      // 这里不再生成containerPath
+      const baseZoom = getCssScale(iframe).cssScaleX;
+      if (iframeInfo) {
+        const iframeChildren = await postWindowMessage(
+          iframe.contentWindow,
+          window,
+          {
+            action: 'extractTreeNode',
+            data: {
+              basePoint: {
+                left: iframeInfo.rect.left,
+                top: iframeInfo.rect.top,
+              },
+              baseZoom,
+            },
+          },
+        );
+        if (Array.isArray(iframeChildren)) {
+          topChildren.push(...iframeChildren);
+        } else if (iframeChildren) {
+          topChildren.push(iframeChildren);
+        }
+      }
+    }
+  }
+
+  return {
+    node: null,
+    children: topChildren,
+  };
+}
+
+export function mergeElementAndChildrenRects(
+  node: Node,
+  currentWindow: typeof window,
+  currentDocument: typeof document,
+  baseZoom = 1,
+) {
+  const selfRect = elementRect(node, currentWindow, currentDocument, baseZoom);
+  if (!selfRect) return null;
+
+  let minLeft = selfRect.left;
+  let minTop = selfRect.top;
+  let maxRight = selfRect.left + selfRect.width;
+  let maxBottom = selfRect.top + selfRect.height;
+
+  function traverse(child: Node) {
+    for (let i = 0; i < child.childNodes.length; i++) {
+      const sub = child.childNodes[i];
+      if (sub.nodeType === 1) {
+        const rect = elementRect(sub, currentWindow, currentDocument, baseZoom);
+        if (rect) {
+          minLeft = Math.min(minLeft, rect.left);
+          minTop = Math.min(minTop, rect.top);
+          maxRight = Math.max(maxRight, rect.left + rect.width);
+          maxBottom = Math.max(maxBottom, rect.top + rect.height);
+        }
+        traverse(sub);
+      }
+    }
+  }
+  traverse(node);
+
+  return {
+    ...selfRect,
+    left: minLeft,
+    top: minTop,
+    width: maxRight - minLeft,
+    height: maxBottom - minTop,
+  };
+}
+
+if (typeof window !== 'undefined') {
+  onWindowMessage(window, async ({ action, data }: any, { source }: any) => {
+    if (action === 'extractTreeNode') {
+      // @ts-ignore
+      try {
+        return await extractTreeNode(
+          document.body,
+          false,
+          data.basePoint,
+          data.baseZoom,
+        );
+      } catch (error) {
+        return {
+          error: 'error message',
+        };
+      }
+    } else if (action === 'getElementFromPoint') {
+      // @ts-ignore
+      try {
+        return await getElementFromPoint({
+          x: data.x,
+          y: data.y,
+          containerPaths: data.containerPaths,
+        });
+      } catch (error) {
+        return {
+          error: 'error message',
+        };
+      }
+    }
+  });
+}
+
+const getElementOffset = (element: any) => {
+  const pLeft =
+    Number.parseInt(
+      window.getComputedStyle(element, null).getPropertyValue('padding-left'),
+      10,
+    ) || 0;
+  const pTop =
+    Number.parseInt(
+      window.getComputedStyle(element, null).getPropertyValue('padding-top'),
+      10,
+    ) || 0;
+  const bLeft =
+    Number.parseInt(
+      window
+        .getComputedStyle(element, null)
+        .getPropertyValue('border-left-width'),
+      10,
+    ) || 0;
+  const bTop =
+    Number.parseInt(
+      window
+        .getComputedStyle(element, null)
+        .getPropertyValue('border-top-width'),
+      10,
+    ) || 0;
+  return {
+    x: pLeft + bLeft,
+    y: pTop + bTop,
+  };
+};
+
+const getCssScale = (element: any) => {
+  try {
+    const { width, height } = element.getBoundingClientRect();
+    return {
+      cssScaleX: Number((width / element.offsetWidth).toFixed(1)),
+      cssScaleY: Number((height / element.offsetHeight).toFixed(1)),
+    };
+  } catch (error) {
+    return {
+      cssScaleX: 1,
+      cssScaleY: 1,
+    };
+  }
+};
+
+export const getElementFromPoint = async (args: {
+  x: number;
+  y: number;
+  containerPaths: string[];
+}) => {
+  if (!args.containerPaths) {
+    args.containerPaths = [];
+  }
+  const dom: any = document.elementFromPoint(args.x, args.y);
+  if (dom?.nodeName === 'IFRAME') {
+    const containerPath: any = getContainerPath(dom);
+    const bounding = dom.getBoundingClientRect();
+    const offset = getElementOffset(dom);
+    const { cssScaleX, cssScaleY } = getCssScale(dom);
+    return await postWindowMessage(dom.contentWindow, window, {
+      action: 'getElementFromPoint',
+      data: {
+        x: (args.x - bounding.x - offset.x) / cssScaleX,
+        y: (args.y - bounding.y - offset.y) / cssScaleY,
+        containerPaths: args.containerPaths.concat(containerPath),
+      },
+    });
+  } else {
+    const elementInfo = {
+      allPaths: getLocators(dom),
+      containerPaths: args.containerPaths,
+    };
+    return elementInfo;
+  }
+};