@midscene/shared 0.12.4 → 0.12.5-beta-20250306111616.0

package/dist/es/extractor/util.js

@@ -0,0 +1,344 @@
+import { generateHashId } from "../utils";
+import { extractTextWithPosition } from "./web-extractor";
+const MAX_VALUE_LENGTH = 300;
+let debugMode = false;
+function setDebugMode(mode) {
+  debugMode = mode;
+}
+function getDebugMode() {
+  return debugMode;
+}
+function logger(..._msg) {
+  if (!debugMode) {
+    return;
+  }
+  console.log(..._msg);
+}
+const taskIdKey = "_midscene_retrieve_task_id";
+function selectorForValue(val) {
+  return `[${taskIdKey}='${val}']`;
+}
+function setDataForNode(node, nodeHash, setToParentNode, currentWindow) {
+  const taskId = taskIdKey;
+  if (!(node instanceof currentWindow.HTMLElement)) {
+    return "";
+  }
+  if (!taskId) {
+    console.error("No task id found");
+    return "";
+  }
+  const selector = selectorForValue(nodeHash);
+  if (getDebugMode()) {
+    if (setToParentNode) {
+      if (node.parentNode instanceof currentWindow.HTMLElement) {
+        node.parentNode.setAttribute(taskIdKey, nodeHash.toString());
+      }
+    } else {
+      node.setAttribute(taskIdKey, nodeHash.toString());
+    }
+  }
+  return selector;
+}
+function isElementPartiallyInViewport(rect, currentWindow, currentDocument) {
+  const elementHeight = rect.height;
+  const elementWidth = rect.width;
+  const viewportRect = {
+    left: 0,
+    top: 0,
+    width: currentWindow.innerWidth || currentDocument.documentElement.clientWidth,
+    height: currentWindow.innerHeight || currentDocument.documentElement.clientHeight,
+    right: currentWindow.innerWidth || currentDocument.documentElement.clientWidth,
+    bottom: currentWindow.innerHeight || currentDocument.documentElement.clientHeight,
+    x: 0,
+    y: 0,
+    zoom: 1
+  };
+  const overlapRect = overlappedRect(rect, viewportRect);
+  if (!overlapRect) {
+    return false;
+  }
+  const visibleArea = overlapRect.width * overlapRect.height;
+  const totalArea = elementHeight * elementWidth;
+  return visibleArea / totalArea >= 2 / 3;
+}
+function getPseudoElementContent(element, currentWindow) {
+  if (!(element instanceof currentWindow.HTMLElement)) {
+    return { before: "", after: "" };
+  }
+  const beforeContent = currentWindow.getComputedStyle(element, "::before").getPropertyValue("content");
+  const afterContent = currentWindow.getComputedStyle(element, "::after").getPropertyValue("content");
+  return {
+    before: beforeContent === "none" ? "" : beforeContent.replace(/"/g, ""),
+    after: afterContent === "none" ? "" : afterContent.replace(/"/g, "")
+  };
+}
+function hasOverflowY(element, currentWindow) {
+  const style = currentWindow.getComputedStyle(element);
+  return style.overflowY === "scroll" || style.overflowY === "auto" || style.overflowY === "hidden";
+}
+function overlappedRect(rect1, rect2) {
+  const left = Math.max(rect1.left, rect2.left);
+  const top = Math.max(rect1.top, rect2.top);
+  const right = Math.min(rect1.right, rect2.right);
+  const bottom = Math.min(rect1.bottom, rect2.bottom);
+  if (left < right && top < bottom) {
+    return {
+      left,
+      top,
+      right,
+      bottom,
+      width: right - left,
+      height: bottom - top,
+      x: left,
+      y: top,
+      zoom: 1
+    };
+  }
+  return null;
+}
+function getRect(el, baseZoom, currentWindow) {
+  let originalRect;
+  let newZoom = 1;
+  if (!(el instanceof currentWindow.HTMLElement)) {
+    const range = currentWindow.document.createRange();
+    range.selectNodeContents(el);
+    originalRect = range.getBoundingClientRect();
+  } else {
+    originalRect = el.getBoundingClientRect();
+    if (!("currentCSSZoom" in el)) {
+      newZoom = Number.parseFloat(currentWindow.getComputedStyle(el).zoom) || 1;
+    }
+  }
+  const zoom = newZoom * baseZoom;
+  return {
+    width: originalRect.width * zoom,
+    height: originalRect.height * zoom,
+    left: originalRect.left * zoom,
+    top: originalRect.top * zoom,
+    right: originalRect.right * zoom,
+    bottom: originalRect.bottom * zoom,
+    x: originalRect.x * zoom,
+    y: originalRect.y * zoom,
+    zoom
+  };
+}
+const isElementCovered = (el, rect, currentWindow) => {
+  const x = rect.left + rect.width / 2;
+  const y = rect.top + rect.height / 2;
+  const topElement = currentWindow.document.elementFromPoint(x, y);
+  if (!topElement) {
+    return false;
+  }
+  if (topElement === el) {
+    return false;
+  }
+  if (el?.contains(topElement)) {
+    return false;
+  }
+  if (topElement?.contains(el)) {
+    return false;
+  }
+  const rectOfTopElement = getRect(topElement, 1, currentWindow);
+  const overlapRect = overlappedRect(rect, rectOfTopElement);
+  if (!overlapRect) {
+    return false;
+  }
+  logger(el, "Element is covered by another element", {
+    topElement,
+    el,
+    rect,
+    x,
+    y
+  });
+  return true;
+};
+function visibleRect(el, currentWindow, currentDocument, baseZoom = 1) {
+  if (!el) {
+    logger(el, "Element is not in the DOM hierarchy");
+    return false;
+  }
+  if (!(el instanceof currentWindow.HTMLElement) && el.nodeType !== Node.TEXT_NODE && el.nodeName.toLowerCase() !== "svg") {
+    logger(el, "Element is not in the DOM hierarchy");
+    return false;
+  }
+  if (el instanceof currentWindow.HTMLElement) {
+    const style = currentWindow.getComputedStyle(el);
+    if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0" && el.tagName !== "INPUT") {
+      logger(el, "Element is hidden");
+      return false;
+    }
+  }
+  const rect = getRect(el, baseZoom, currentWindow);
+  if (rect.width === 0 && rect.height === 0) {
+    logger(el, "Element has no size");
+    return false;
+  }
+  if (baseZoom === 1 && isElementCovered(el, rect, currentWindow)) {
+    return false;
+  }
+  const scrollLeft = currentWindow.pageXOffset || currentDocument.documentElement.scrollLeft;
+  const scrollTop = currentWindow.pageYOffset || currentDocument.documentElement.scrollTop;
+  const viewportWidth = currentWindow.innerWidth || currentDocument.documentElement.clientWidth;
+  const viewportHeight = currentWindow.innerHeight || currentDocument.documentElement.clientHeight;
+  const isPartiallyInViewport = isElementPartiallyInViewport(
+    rect,
+    currentWindow,
+    currentDocument
+  );
+  if (!isPartiallyInViewport) {
+    logger(el, "Element is completely outside the viewport", {
+      rect,
+      viewportHeight,
+      viewportWidth,
+      scrollTop,
+      scrollLeft
+    });
+    return false;
+  }
+  let parent = el;
+  const parentUntilNonStatic = (currentNode) => {
+    let parent2 = currentNode?.parentElement;
+    while (parent2) {
+      const style = currentWindow.getComputedStyle(parent2);
+      if (style.position !== "static") {
+        return parent2;
+      }
+      parent2 = parent2.parentElement;
+    }
+    return null;
+  };
+  while (parent && parent !== currentDocument.body) {
+    if (!(parent instanceof currentWindow.HTMLElement)) {
+      parent = parent.parentElement;
+      continue;
+    }
+    const parentStyle = currentWindow.getComputedStyle(parent);
+    if (parentStyle.overflow === "hidden") {
+      const parentRect = getRect(parent, 1, currentWindow);
+      const tolerance = 10;
+      if (rect.right < parentRect.left - tolerance || rect.left > parentRect.right + tolerance || rect.bottom < parentRect.top - tolerance || rect.top > parentRect.bottom + tolerance) {
+        logger(el, "element is partially or totally hidden by an ancestor", {
+          rect,
+          parentRect
+        });
+        return false;
+      }
+    }
+    if (parentStyle.position === "fixed" || parentStyle.position === "sticky") {
+      break;
+    }
+    if (parentStyle.position === "absolute") {
+      parent = parentUntilNonStatic(parent);
+    } else {
+      parent = parent.parentElement;
+    }
+  }
+  return {
+    left: Math.round(rect.left),
+    top: Math.round(rect.top),
+    width: Math.round(rect.width),
+    height: Math.round(rect.height),
+    zoom: rect.zoom
+  };
+}
+function validTextNodeContent(node) {
+  if (!node) {
+    return false;
+  }
+  if (node.nodeType !== Node.ELEMENT_NODE && node.nodeType !== Node.TEXT_NODE && node.nodeName !== "#text") {
+    return false;
+  }
+  const content = node.textContent || node.innerText;
+  if (content && !/^\s*$/.test(content)) {
+    return content.trim();
+  }
+  return false;
+}
+function getNodeAttributes(node, currentWindow) {
+  if (!node || !(node instanceof currentWindow.HTMLElement) || !node.attributes) {
+    return {};
+  }
+  const attributesList = Array.from(node.attributes).map((attr) => {
+    if (attr.name === "class") {
+      return [attr.name, `.${attr.value.split(" ").join(".")}`];
+    }
+    if (!attr.value) {
+      return [];
+    }
+    let value = attr.value;
+    if (value.startsWith("data:image")) {
+      value = "image";
+    }
+    if (value.length > MAX_VALUE_LENGTH) {
+      value = `${value.slice(0, MAX_VALUE_LENGTH)}...`;
+    }
+    return [attr.name, value];
+  });
+  return Object.fromEntries(attributesList);
+}
+let nodeHashCacheList = [];
+if (typeof window !== "undefined") {
+  window.midsceneNodeHashCacheList = window.midsceneNodeHashCacheList || [];
+  nodeHashCacheList = window.midsceneNodeHashCacheList;
+}
+const hashMap = {};
+function resetNodeHashCacheList() {
+  if (typeof window !== "undefined") {
+    nodeHashCacheList = window.midsceneNodeHashCacheList || [];
+    window.midsceneNodeHashCacheList = [];
+  } else {
+    nodeHashCacheList = [];
+  }
+}
+function midsceneGenerateHash(node, content, rect) {
+  if (node && nodeHashCacheList.find((item) => item.node === node)) {
+    return nodeHashCacheList.find((item) => item.node === node)?.id || "";
+  }
+  const slicedHash = generateHashId(rect, content);
+  if (node && typeof window !== "undefined") {
+    window.midsceneNodeHashCacheList.push({ node, id: slicedHash });
+  }
+  return slicedHash;
+}
+function generateId(numberId) {
+  return `${numberId}`;
+}
+function setGenerateHashOnWindow() {
+  if (typeof window !== "undefined") {
+    window.midsceneGenerateHash = midsceneGenerateHash;
+  }
+}
+function setMidsceneVisibleRectOnWindow() {
+  if (typeof window !== "undefined") {
+    window.midsceneVisibleRect = visibleRect;
+  }
+}
+function setExtractTextWithPositionOnWindow() {
+  if (typeof window !== "undefined") {
+    window.extractTextWithPosition = extractTextWithPosition;
+  }
+}
+function getTopDocument() {
+  const container = document.body || document;
+  return container;
+}
+export {
+  generateId,
+  getDebugMode,
+  getNodeAttributes,
+  getPseudoElementContent,
+  getRect,
+  getTopDocument,
+  hasOverflowY,
+  logger,
+  midsceneGenerateHash,
+  overlappedRect,
+  resetNodeHashCacheList,
+  setDataForNode,
+  setDebugMode,
+  setExtractTextWithPositionOnWindow,
+  setGenerateHashOnWindow,
+  setMidsceneVisibleRectOnWindow,
+  validTextNodeContent,
+  visibleRect
+};

package/dist/es/extractor/web-extractor.js

@@ -0,0 +1,317 @@
+import {
+  CONTAINER_MINI_HEIGHT,
+  CONTAINER_MINI_WIDTH,
+  NodeType
+} from "../constants";
+import {
+  isButtonElement,
+  isContainerElement,
+  isFormElement,
+  isImgElement,
+  isTextElement
+} from "./dom-util";
+import { descriptionOfTree } from "./tree";
+import {
+  getNodeAttributes,
+  getPseudoElementContent,
+  getRect,
+  getTopDocument,
+  logger,
+  midsceneGenerateHash,
+  resetNodeHashCacheList,
+  setDataForNode,
+  setDebugMode,
+  visibleRect
+} from "./util";
+let indexId = 0;
+function tagNameOfNode(node) {
+  let tagName = "";
+  if (node instanceof HTMLElement) {
+    tagName = node.tagName.toLowerCase();
+  }
+  const parentElement = node.parentElement;
+  if (parentElement && parentElement instanceof HTMLElement) {
+    tagName = parentElement.tagName.toLowerCase();
+  }
+  return tagName ? `<${tagName}>` : "";
+}
+function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1, basePoint = { left: 0, top: 0 }) {
+  const rect = visibleRect(node, currentWindow, currentDocument, baseZoom);
+  if (!rect || rect.width < CONTAINER_MINI_WIDTH || rect.height < CONTAINER_MINI_HEIGHT) {
+    return null;
+  }
+  if (basePoint.left !== 0 || basePoint.top !== 0) {
+    rect.left += basePoint.left;
+    rect.top += basePoint.top;
+  }
+  if (rect.height >= window.innerHeight && rect.width >= window.innerWidth) {
+    return null;
+  }
+  if (isFormElement(node)) {
+    const attributes = getNodeAttributes(node, currentWindow);
+    let valueContent = attributes.value || attributes.placeholder || node.textContent || "";
+    const nodeHashId = midsceneGenerateHash(node, valueContent, rect);
+    const selector = setDataForNode(node, nodeHashId, false, currentWindow);
+    const tagName = node.tagName.toLowerCase();
+    if (node.tagName.toLowerCase() === "select") {
+      const selectedOption = node.options[node.selectedIndex];
+      valueContent = selectedOption.textContent || "";
+    }
+    if ((node.tagName.toLowerCase() === "input" || node.tagName.toLowerCase() === "textarea") && node.value) {
+      valueContent = node.value;
+    }
+    const elementInfo = {
+      id: nodeHashId,
+      nodeHashId,
+      locator: selector,
+      nodeType: NodeType.FORM_ITEM,
+      indexId: indexId++,
+      attributes: {
+        ...attributes,
+        htmlTagName: `<${tagName}>`,
+        nodeType: NodeType.FORM_ITEM
+      },
+      content: valueContent.trim(),
+      rect,
+      center: [
+        Math.round(rect.left + rect.width / 2),
+        Math.round(rect.top + rect.height / 2)
+      ],
+      zoom: rect.zoom,
+      screenWidth: currentWindow.innerWidth,
+      screenHeight: currentWindow.innerHeight
+    };
+    return elementInfo;
+  }
+  if (isButtonElement(node)) {
+    const attributes = getNodeAttributes(node, currentWindow);
+    const pseudo = getPseudoElementContent(node, currentWindow);
+    const content = node.innerText || pseudo.before || pseudo.after || "";
+    const nodeHashId = midsceneGenerateHash(node, content, rect);
+    const selector = setDataForNode(node, nodeHashId, false, currentWindow);
+    const elementInfo = {
+      id: nodeHashId,
+      indexId: indexId++,
+      nodeHashId,
+      nodeType: NodeType.BUTTON,
+      locator: selector,
+      attributes: {
+        ...attributes,
+        htmlTagName: tagNameOfNode(node),
+        nodeType: NodeType.BUTTON
+      },
+      content,
+      rect,
+      center: [
+        Math.round(rect.left + rect.width / 2),
+        Math.round(rect.top + rect.height / 2)
+      ],
+      zoom: rect.zoom,
+      screenWidth: currentWindow.innerWidth,
+      screenHeight: currentWindow.innerHeight
+    };
+    return elementInfo;
+  }
+  if (isImgElement(node)) {
+    const attributes = getNodeAttributes(node, currentWindow);
+    const nodeHashId = midsceneGenerateHash(node, "", rect);
+    const selector = setDataForNode(node, nodeHashId, false, currentWindow);
+    const elementInfo = {
+      id: nodeHashId,
+      indexId: indexId++,
+      nodeHashId,
+      locator: selector,
+      attributes: {
+        ...attributes,
+        ...node.nodeName.toLowerCase() === "svg" ? {
+          svgContent: "true"
+        } : {},
+        nodeType: NodeType.IMG,
+        htmlTagName: tagNameOfNode(node)
+      },
+      nodeType: NodeType.IMG,
+      content: "",
+      rect,
+      center: [
+        Math.round(rect.left + rect.width / 2),
+        Math.round(rect.top + rect.height / 2)
+      ],
+      zoom: rect.zoom,
+      screenWidth: currentWindow.innerWidth,
+      screenHeight: currentWindow.innerHeight
+    };
+    return elementInfo;
+  }
+  if (isTextElement(node)) {
+    const text = node.textContent?.trim().replace(/\n+/g, " ");
+    if (!text) {
+      return null;
+    }
+    const attributes = getNodeAttributes(node, currentWindow);
+    const attributeKeys = Object.keys(attributes);
+    if (!text.trim() && attributeKeys.length === 0) {
+      return null;
+    }
+    const nodeHashId = midsceneGenerateHash(node, text, rect);
+    const selector = setDataForNode(node, nodeHashId, true, currentWindow);
+    const elementInfo = {
+      id: nodeHashId,
+      indexId: indexId++,
+      nodeHashId,
+      nodeType: NodeType.TEXT,
+      locator: selector,
+      attributes: {
+        ...attributes,
+        nodeType: NodeType.TEXT,
+        htmlTagName: tagNameOfNode(node)
+      },
+      center: [
+        Math.round(rect.left + rect.width / 2),
+        Math.round(rect.top + rect.height / 2)
+      ],
+      // attributes,
+      content: text,
+      rect,
+      zoom: rect.zoom,
+      screenWidth: currentWindow.innerWidth,
+      screenHeight: currentWindow.innerHeight
+    };
+    return elementInfo;
+  }
+  if (isContainerElement(node)) {
+    const attributes = getNodeAttributes(node, currentWindow);
+    const nodeHashId = midsceneGenerateHash(node, "", rect);
+    const selector = setDataForNode(node, nodeHashId, false, currentWindow);
+    const elementInfo = {
+      id: nodeHashId,
+      nodeHashId,
+      indexId: indexId++,
+      nodeType: NodeType.CONTAINER,
+      locator: selector,
+      attributes: {
+        ...attributes,
+        nodeType: NodeType.CONTAINER,
+        htmlTagName: tagNameOfNode(node)
+      },
+      content: "",
+      rect,
+      center: [
+        Math.round(rect.left + rect.width / 2),
+        Math.round(rect.top + rect.height / 2)
+      ],
+      zoom: rect.zoom,
+      screenWidth: currentWindow.innerWidth,
+      screenHeight: currentWindow.innerHeight
+    };
+    return elementInfo;
+  }
+  return null;
+}
+function extractTextWithPosition(initNode, debugMode = false) {
+  const elementNode = extractTreeNode(initNode, debugMode);
+  const elementInfoArray = [];
+  function dfsTopChildren(node) {
+    if (node.node) {
+      elementInfoArray.push(node.node);
+    }
+    for (let i = 0; i < node.children.length; i++) {
+      dfsTopChildren(node.children[i]);
+    }
+  }
+  dfsTopChildren({ children: elementNode.children, node: elementNode.node });
+  return elementInfoArray;
+}
+function extractTreeNodeAsString(initNode, debugMode = false) {
+  const elementNode = extractTreeNode(initNode, debugMode);
+  return descriptionOfTree(elementNode);
+}
+function extractTreeNode(initNode, debugMode = false) {
+  setDebugMode(debugMode);
+  resetNodeHashCacheList();
+  indexId = 0;
+  const topDocument = getTopDocument();
+  const startNode = initNode || topDocument;
+  const topChildren = [];
+  function dfs(node, currentWindow, currentDocument, baseZoom = 1, basePoint = { left: 0, top: 0 }) {
+    if (!node) {
+      return null;
+    }
+    if (node.nodeType && node.nodeType === 10) {
+      return null;
+    }
+    const elementInfo = collectElementInfo(
+      node,
+      currentWindow,
+      currentDocument,
+      baseZoom,
+      basePoint
+    );
+    if (node instanceof currentWindow.HTMLIFrameElement) {
+      if (node.contentWindow && node.contentWindow) {
+        return null;
+      }
+    }
+    const nodeInfo = {
+      node: elementInfo,
+      children: []
+    };
+    if (elementInfo?.nodeType === NodeType.BUTTON || elementInfo?.nodeType === NodeType.IMG || elementInfo?.nodeType === NodeType.TEXT || elementInfo?.nodeType === NodeType.FORM_ITEM || elementInfo?.nodeType === NodeType.CONTAINER) {
+      return nodeInfo;
+    }
+    const rect = getRect(node, baseZoom, currentWindow);
+    for (let i = 0; i < node.childNodes.length; i++) {
+      logger("will dfs", node.childNodes[i]);
+      const childNodeInfo = dfs(
+        node.childNodes[i],
+        currentWindow,
+        currentDocument,
+        rect.zoom,
+        basePoint
+      );
+      if (childNodeInfo) {
+        nodeInfo.children.push(childNodeInfo);
+      }
+    }
+    return nodeInfo;
+  }
+  const rootNodeInfo = dfs(startNode, window, document, 1, {
+    left: 0,
+    top: 0
+  });
+  if (rootNodeInfo) {
+    topChildren.push(rootNodeInfo);
+  }
+  if (startNode === topDocument) {
+    const iframes = document.querySelectorAll("iframe");
+    for (let i = 0; i < iframes.length; i++) {
+      const iframe = iframes[i];
+      if (iframe.contentDocument && iframe.contentWindow) {
+        const iframeInfo = collectElementInfo(iframe, window, document, 1);
+        if (iframeInfo) {
+          const iframeChildren = dfs(
+            iframe.contentDocument.body,
+            iframe.contentWindow,
+            iframe.contentDocument,
+            1,
+            {
+              left: iframeInfo.rect.left,
+              top: iframeInfo.rect.top
+            }
+          );
+          if (iframeChildren) {
+            topChildren.push(iframeChildren);
+          }
+        }
+      }
+    }
+  }
+  return {
+    node: null,
+    children: topChildren
+  };
+}
+export {
+  extractTextWithPosition,
+  extractTreeNode,
+  extractTreeNodeAsString
+};