@midscene/web 0.0.1

package/src/html-element/extractInfo.ts

@@ -0,0 +1,168 @@
+import {
+  getNodeAttributes,
+  getPseudoElementContent,
+  logger,
+  setDataForNode,
+  validTextNodeContent,
+  visibleRect,
+} from './util';
+import { NodeType, TEXT_SIZE_THRESHOLD } from './constants';
+import { isButtonElement, isImgElement, isInputElement } from './dom-util';
+
+interface NodeDescriptor {
+  node: Node;
+  childrens: NodeDescriptor[];
+}
+
+export interface ElementInfo {
+  id: string;
+  locator: string | void;
+  attributes: {
+    nodeType: NodeType;
+    [key: string]: string;
+  };
+  content: string;
+  rect: { left: number; top: number; width: number; height: number };
+  center: [number, number];
+}
+
+const container: HTMLElement = document.body;
+
+function generateId(numberId: number) {
+  //   const letters = 'ABCDEFGHIJKLMNPRSTUVXYZ';
+  //   const numbers = '0123456789';
+  //   const randomLetter = letters.charAt(Math.floor(Math.random() * letters.length)).toUpperCase();
+  // const randomNumber = numbers.charAt(Math.floor(Math.random() * numbers.length));
+  // return randomLetter + numberId;
+  return `${numberId}`;
+}
+
+export function extractTextWithPositionDFS(initNode: Node = container): ElementInfo[] {
+  const elementInfoArray: ElementInfo[] = [];
+  const nodeMapTree: NodeDescriptor = { node: initNode, childrens: [] };
+  let nodeIndex = 1;
+
+  function dfs(node: Node, parentNode: NodeDescriptor | null = null): void {
+    if (!node) {
+      return;
+    }
+
+    const currentNodeDes: NodeDescriptor = { node, childrens: [] };
+    if (parentNode?.childrens) {
+      parentNode.childrens.push(currentNodeDes);
+    }
+
+    collectElementInfo(node);
+
+    // eslint-disable-next-line @typescript-eslint/prefer-for-of
+    for (let i = 0; i < node.childNodes.length; i++) {
+      logger('will dfs', node.childNodes[i]);
+      dfs(node.childNodes[i], currentNodeDes);
+    }
+  }
+
+  function collectElementInfo(node: Node) {
+    const rect = visibleRect(node);
+    if (!rect) {
+      logger('Element is not visible', node);
+      return;
+    }
+
+    if (isInputElement(node)) {
+      const attributes = getNodeAttributes(node);
+      const selector = setDataForNode(node, nodeIndex);
+      elementInfoArray.push({
+        id: generateId(nodeIndex++),
+        locator: selector,
+        attributes: {
+          ...attributes,
+          nodeType: NodeType.INPUT,
+        },
+        content: attributes.placeholder || '',
+        rect,
+        center: [Math.round(rect.left + rect.width / 2), Math.round(rect.top + rect.height / 2)],
+      });
+      return;
+    }
+
+    if (isButtonElement(node)) {
+      const attributes = getNodeAttributes(node);
+      const pseudo = getPseudoElementContent(node);
+      const selector = setDataForNode(node, nodeIndex);
+      elementInfoArray.push({
+        id: generateId(nodeIndex++),
+        locator: selector,
+        attributes: {
+          ...attributes,
+          nodeType: NodeType.BUTTON,
+        },
+        content: node.innerText || pseudo.before || pseudo.after || '',
+        rect,
+        center: [Math.round(rect.left + rect.width / 2), Math.round(rect.top + rect.height / 2)],
+      });
+      return;
+    }
+
+    if (isImgElement(node)) {
+      const attributes = getNodeAttributes(node);
+      const selector = setDataForNode(node, nodeIndex);
+      elementInfoArray.push({
+        id: generateId(nodeIndex++),
+        locator: selector,
+        attributes: {
+          ...attributes,
+          nodeType: NodeType.IMG,
+        },
+        content: '',
+        rect,
+        center: [Math.round(rect.left + rect.width / 2), Math.round(rect.top + rect.height / 2)],
+      });
+      return;
+    }
+
+    //   if (node instanceof HTMLElement && hasOverflowY(node)) {
+    //       const rect = visibleRect(node);
+    //       if (!rect || rect.height < 100) {
+    //           logger('Element is not visible', node);
+    //       } else {
+    //           const attributes = getNodeAttributes(node);
+    //           const selector = setDataForNode(node, nodeIndex);
+    //           elementInfoArray.push({
+    //               id: nodeIndex++,
+    //               nodeType: 'ScrollContainer Node',
+    //               locator: selector!,
+    //               parentIndex,
+    //               attributes,
+    //               content: "",
+    //               rect,
+    //               center: [Math.round(rect.left + rect.width / 2), Math.round(rect.top + rect.height / 2)],
+    //           });
+    //       }
+    //   }
+
+    const text = validTextNodeContent(node);
+    if (text) {
+      if (rect.width < TEXT_SIZE_THRESHOLD || rect.height < TEXT_SIZE_THRESHOLD) {
+        logger('Element is too small', text);
+        return;
+      }
+      const attributes = getNodeAttributes(node);
+      const selector = setDataForNode(node, nodeIndex);
+      elementInfoArray.push({
+        id: generateId(nodeIndex++),
+        attributes: {
+          ...attributes,
+          nodeType: NodeType.TEXT,
+        },
+        locator: selector,
+        center: [Math.round(rect.left + rect.width / 2), Math.round(rect.top + rect.height / 2)],
+        // attributes,
+        content: text,
+        rect,
+      });
+    }
+  }
+
+  dfs(initNode, nodeMapTree);
+  return elementInfoArray;
+}

package/src/html-element/index.ts

@@ -0,0 +1 @@
+export { extractTextWithPositionDFS } from './extractInfo';

package/src/html-element/util.ts

@@ -0,0 +1,160 @@
+// import { TEXT_MAX_SIZE } from './constants';
+
+export function logger(...msg: any[]): void {
+  // console.log(...msg);
+}
+
+// const nodeIndexCounter = 0;
+
+const taskIdKey = '_midscene_retrieve_task_id';
+// const nodeDataIdKey = 'data-midscene-task-';
+// const nodeIndexKey = '_midscene_retrieve_node_index';
+
+function selectorForValue(val: number): string {
+  return `[${taskIdKey}='${val}']`;
+}
+
+export function setDataForNode(node: HTMLElement | Node, nodeIndex: number): string {
+  const taskId = taskIdKey;
+  if (!(node instanceof HTMLElement)) {
+    return '';
+  }
+  if (!taskId) {
+    console.error('No task id found');
+    return '';
+  }
+
+  const selector = selectorForValue(nodeIndex);
+  node.setAttribute(taskIdKey, nodeIndex.toString());
+  return selector;
+}
+
+export function getPseudoElementContent(element: Node): { before: string; after: string } {
+  if (!(element instanceof HTMLElement)) {
+    return { before: '', after: '' };
+  }
+  const beforeContent = window.getComputedStyle(element, '::before').getPropertyValue('content');
+  const afterContent = window.getComputedStyle(element, '::after').getPropertyValue('content');
+  return {
+    before: beforeContent === 'none' ? '' : beforeContent.replace(/"/g, ''),
+    after: afterContent === 'none' ? '' : afterContent.replace(/"/g, ''),
+  };
+}
+
+export function hasOverflowY(element: HTMLElement): boolean {
+  const style = window.getComputedStyle(element);
+  return style.overflowY === 'scroll' || style.overflowY === 'auto' || style.overflowY === 'hidden';
+}
+
+export function visibleRect(
+  el: HTMLElement | Node | null,
+): { left: number; top: number; width: number; height: number } | false {
+  if (!el) {
+    logger('Element is not in the DOM hierarchy');
+    return false;
+  }
+
+  if (!(el instanceof HTMLElement)) {
+    logger('Element is not in the DOM hierarchy');
+    return false;
+  }
+
+  const style = window.getComputedStyle(el);
+  if (
+    style.display === 'none' ||
+    style.visibility === 'hidden' ||
+    (style.opacity === '0' && el.tagName !== 'INPUT')
+  ) {
+    logger('Element is hidden');
+    return false;
+  }
+
+  const rect = el.getBoundingClientRect();
+  if (rect.width === 0 && rect.height === 0) {
+    logger('Element has no size');
+    return false;
+  }
+
+  const scrollLeft = window.pageXOffset || document.documentElement.scrollLeft;
+  const scrollTop = window.pageYOffset || document.documentElement.scrollTop;
+  const isInViewport =
+    rect.top >= 0 + scrollTop &&
+    rect.left >= 0 + scrollLeft &&
+    rect.bottom <= (window.innerHeight || document.documentElement.clientHeight) + scrollTop &&
+    rect.right <= (window.innerWidth || document.documentElement.clientWidth) + scrollLeft;
+
+  if (!isInViewport) {
+    logger('Element is not in the viewport');
+    logger(rect, window.innerHeight, window.innerWidth, scrollTop, scrollLeft);
+    return false;
+  }
+
+  let parent: HTMLElement | null = el;
+  while (parent && parent !== document.body) {
+    const parentStyle = window.getComputedStyle(parent);
+    if (parentStyle.overflow === 'hidden') {
+      const parentRect = parent.getBoundingClientRect();
+      const tolerance = 10;
+      if (
+        rect.top < parentRect.top - tolerance ||
+        rect.left < parentRect.left - tolerance ||
+        rect.bottom > parentRect.bottom + tolerance ||
+        rect.right > parentRect.right + tolerance
+      ) {
+        logger('Element is clipped by an ancestor', parent, rect, parentRect);
+        return false;
+      }
+    }
+    parent = parent.parentElement;
+  }
+
+  return {
+    left: Math.round(rect.left - scrollLeft),
+    top: Math.round(rect.top - scrollTop),
+    width: Math.round(rect.width),
+    height: Math.round(rect.height),
+  };
+}
+
+export function validTextNodeContent(node: Node): string | false {
+  if (!node) {
+    return false;
+  }
+  console.log('node', node);
+  if (node.nodeType === Node.COMMENT_NODE) {
+    return false;
+  }
+
+  const everyChildNodeIsText = Array.from(node.childNodes).findIndex(
+    (child) => child.nodeType === Node.TEXT_NODE,
+  );
+
+  if (everyChildNodeIsText === -1) {
+    return false;
+  }
+
+  const content = node.textContent || (node as HTMLElement).innerText;
+  if (content && !/^\s*$/.test(content)) {
+    return content.trim();
+  }
+
+  return false;
+}
+
+export function getNodeAttributes(node: HTMLElement | Node): Record<string, string> {
+  if (!node || !(node instanceof HTMLElement) || !node.attributes) {
+    return {};
+  }
+
+  const attributesList = Array.from(node.attributes).map((attr) => {
+    if (attr.name === 'class') {
+      return [attr.name, `.${attr.value.split(' ').join('.')}`];
+    }
+    if (!attr.value) {
+      return [];
+    }
+    return [attr.name, attr.value];
+  });
+
+  return Object.fromEntries(attributesList);
+}

package/src/img/img.ts

@@ -0,0 +1,132 @@
+import assert from 'assert';
+import { Buffer } from 'node:buffer';
+import sharp from 'sharp';
+import { NodeType } from '@/html-element/constants';
+
+// Define picture path
+type ElementType = {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+  label: string;
+  attributes: {
+    [key: string]: string;
+    nodeType: NodeType;
+  };
+};
+
+const createSvgOverlay = (elements: Array<ElementType>, imageWidth: number, imageHeight: number) => {
+  let svgContent = `<svg width="${imageWidth}" height="${imageHeight}" xmlns="http://www.w3.org/2000/svg">`;
+
+  // Define color array
+  const colors = [
+    { rect: 'blue', text: 'white' },
+    { rect: 'green', text: 'white' },
+  ];
+
+  // Define clipping path
+  svgContent += `<defs>`;
+  elements.forEach((element, index) => {
+    svgContent += `
+      <clipPath id="clip${index}">
+        <rect x="${element.x}" y="${element.y}" width="${element.width}" height="${element.height}" />
+      </clipPath>
+    `;
+  });
+  svgContent += `</defs>`;
+
+  elements.forEach((element, index) => {
+    // Calculate the width and height of the text
+    const textWidth = element.label.length * 8; // Assume that each character is 8px wide
+    const textHeight = 12; // Assume that the text height is 20px
+
+    // Calculates the position of the initial color block so that it wraps and centers the text
+    const rectWidth = textWidth + 5;
+    const rectHeight = textHeight + 4;
+    let rectX = element.x - rectWidth;
+    let rectY = element.y + element.height / 2 - textHeight / 2 - 2;
+
+    // Initial text position
+    let textX = rectX + rectWidth / 2;
+    let textY = rectY + rectHeight / 2 + 6;
+
+    // Check to see if it's obscured by the left
+    if (rectX < 0) {
+      rectX = element.x;
+      rectY = element.y - rectHeight;
+      textX = rectX + rectWidth / 2;
+      textY = rectY + rectHeight / 2 + 6;
+    }
+
+    // Choose color
+    const color = colors[index % colors.length];
+
+    // Draw boxes and text
+    svgContent += `
+      <rect x="${element.x}" y="${element.y}" width="${element.width}" height="${element.height}" 
+            style="fill:none;stroke:${color.rect};stroke-width:4" clip-path="url(#clip${index})" />
+      <rect x="${rectX}" y="${rectY}" width="${rectWidth}" height="${rectHeight}" style="fill:${color.rect};" />
+      <text x="${textX}" y="${textY}" 
+            text-anchor="middle" dominant-baseline="middle" style="fill:${color.text};font-size:12px;font-weight:bold;">
+        ${element.label}
+      </text>
+    `;
+  });
+
+  svgContent += `</svg>`;
+  return Buffer.from(svgContent);
+};
+
+export const processImageElementInfo = async (options: {
+  inputImgBase64: string;
+  elementsPostionInfo: Array<ElementType>;
+  elementsPostionInfoWithoutText: Array<ElementType>;
+}) => {
+  // Get the size of the original image
+  const base64Image = options.inputImgBase64.split(';base64,').pop();
+  assert(base64Image, 'base64Image is undefined');
+
+  const imageBuffer = Buffer.from(base64Image, 'base64');
+  const metadata = await sharp(imageBuffer).metadata();
+  const { width, height } = metadata;
+
+  if (width && height) {
+    // Create svg overlay
+    const svgOverlay = createSvgOverlay(options.elementsPostionInfo, width, height);
+    const svgOverlayWithoutText = createSvgOverlay(options.elementsPostionInfoWithoutText, width, height);
+
+    // Composite picture
+    const compositeElementInfoImgBase64 = await sharp(imageBuffer)
+      // .resize(newDimensions.width, newDimensions.height)
+      .composite([{ input: svgOverlay, blend: 'over' }])
+      .toBuffer()
+      .then((data) => {
+        // Convert image data to base64 encoding
+        return data.toString('base64');
+      })
+      .catch((err) => {
+        throw err;
+      });
+
+    // Composite picture withtoutText
+    const compositeElementInfoImgWithoutTextBase64 = await sharp(imageBuffer)
+      // .resize(newDimensions.width, newDimensions.height)
+      .composite([{ input: svgOverlayWithoutText, blend: 'over' }])
+      .toBuffer()
+      .then((data) => {
+        // Convert image data to base64 encoding
+        return data.toString('base64');
+      })
+      .catch((err) => {
+        throw err;
+      });
+
+    return {
+      compositeElementInfoImgBase64,
+      compositeElementInfoImgWithoutTextBase64,
+    };
+  } else {
+    throw Error('Image processing failed because width or height is undefined');
+  }
+};

package/src/img/util.ts

@@ -0,0 +1,28 @@
+import { getElementInfosFromPage } from '../playwright/utils';
+import { NodeType } from '@/html-element/constants';
+import { ElementInfo } from '@/html-element/extractInfo';
+
+export async function getElementInfos(page: any) {
+  const captureElementSnapshot: Array<ElementInfo> = await getElementInfosFromPage(page);
+  const elementsPostionInfo = captureElementSnapshot.map((elementInfo) => {
+    return {
+      label: elementInfo.id.toString(),
+      x: elementInfo.rect.left,
+      y: elementInfo.rect.top,
+      width: elementInfo.rect.width,
+      height: elementInfo.rect.height,
+      attributes: elementInfo.attributes,
+    };
+  });
+  const elementsPostionInfoWithoutText = elementsPostionInfo.filter((elementInfo) => {
+    if (elementInfo.attributes.nodeType === NodeType.TEXT) {
+      return false;
+    }
+    return true;
+  });
+  return {
+    elementsPostionInfo,
+    captureElementSnapshot,
+    elementsPostionInfoWithoutText,
+  };
+}

package/src/index.ts

@@ -0,0 +1,2 @@
+export { PlaywrightAiFixture } from './playwright';
+export type { PlayWrightAiFixtureType } from './playwright';