@midscene/web 0.0.1 → 0.1.0

package/src/html-element/util.ts

@@ -1,160 +0,0 @@
-// import { TEXT_MAX_SIZE } from './constants';
-
-export function logger(...msg: any[]): void {
-  // console.log(...msg);
-}
-
-// const nodeIndexCounter = 0;
-
-const taskIdKey = '_midscene_retrieve_task_id';
-// const nodeDataIdKey = 'data-midscene-task-';
-// const nodeIndexKey = '_midscene_retrieve_node_index';
-
-function selectorForValue(val: number): string {
-  return `[${taskIdKey}='${val}']`;
-}
-
-export function setDataForNode(node: HTMLElement | Node, nodeIndex: number): string {
-  const taskId = taskIdKey;
-  if (!(node instanceof HTMLElement)) {
-    return '';
-  }
-  if (!taskId) {
-    console.error('No task id found');
-    return '';
-  }
-
-  const selector = selectorForValue(nodeIndex);
-  node.setAttribute(taskIdKey, nodeIndex.toString());
-  return selector;
-}
-
-export function getPseudoElementContent(element: Node): { before: string; after: string } {
-  if (!(element instanceof HTMLElement)) {
-    return { before: '', after: '' };
-  }
-  const beforeContent = window.getComputedStyle(element, '::before').getPropertyValue('content');
-  const afterContent = window.getComputedStyle(element, '::after').getPropertyValue('content');
-  return {
-    before: beforeContent === 'none' ? '' : beforeContent.replace(/"/g, ''),
-    after: afterContent === 'none' ? '' : afterContent.replace(/"/g, ''),
-  };
-}
-
-export function hasOverflowY(element: HTMLElement): boolean {
-  const style = window.getComputedStyle(element);
-  return style.overflowY === 'scroll' || style.overflowY === 'auto' || style.overflowY === 'hidden';
-}
-
-export function visibleRect(
-  el: HTMLElement | Node | null,
-): { left: number; top: number; width: number; height: number } | false {
-  if (!el) {
-    logger('Element is not in the DOM hierarchy');
-    return false;
-  }
-
-  if (!(el instanceof HTMLElement)) {
-    logger('Element is not in the DOM hierarchy');
-    return false;
-  }
-
-  const style = window.getComputedStyle(el);
-  if (
-    style.display === 'none' ||
-    style.visibility === 'hidden' ||
-    (style.opacity === '0' && el.tagName !== 'INPUT')
-  ) {
-    logger('Element is hidden');
-    return false;
-  }
-
-  const rect = el.getBoundingClientRect();
-  if (rect.width === 0 && rect.height === 0) {
-    logger('Element has no size');
-    return false;
-  }
-
-  const scrollLeft = window.pageXOffset || document.documentElement.scrollLeft;
-  const scrollTop = window.pageYOffset || document.documentElement.scrollTop;
-  const isInViewport =
-    rect.top >= 0 + scrollTop &&
-    rect.left >= 0 + scrollLeft &&
-    rect.bottom <= (window.innerHeight || document.documentElement.clientHeight) + scrollTop &&
-    rect.right <= (window.innerWidth || document.documentElement.clientWidth) + scrollLeft;
-
-  if (!isInViewport) {
-    logger('Element is not in the viewport');
-    logger(rect, window.innerHeight, window.innerWidth, scrollTop, scrollLeft);
-    return false;
-  }
-
-  let parent: HTMLElement | null = el;
-  while (parent && parent !== document.body) {
-    const parentStyle = window.getComputedStyle(parent);
-    if (parentStyle.overflow === 'hidden') {
-      const parentRect = parent.getBoundingClientRect();
-      const tolerance = 10;
-      if (
-        rect.top < parentRect.top - tolerance ||
-        rect.left < parentRect.left - tolerance ||
-        rect.bottom > parentRect.bottom + tolerance ||
-        rect.right > parentRect.right + tolerance
-      ) {
-        logger('Element is clipped by an ancestor', parent, rect, parentRect);
-        return false;
-      }
-    }
-    parent = parent.parentElement;
-  }
-
-  return {
-    left: Math.round(rect.left - scrollLeft),
-    top: Math.round(rect.top - scrollTop),
-    width: Math.round(rect.width),
-    height: Math.round(rect.height),
-  };
-}
-
-export function validTextNodeContent(node: Node): string | false {
-  if (!node) {
-    return false;
-  }
-  console.log('node', node);
-  if (node.nodeType === Node.COMMENT_NODE) {
-    return false;
-  }
-
-  const everyChildNodeIsText = Array.from(node.childNodes).findIndex(
-    (child) => child.nodeType === Node.TEXT_NODE,
-  );
-
-  if (everyChildNodeIsText === -1) {
-    return false;
-  }
-
-  const content = node.textContent || (node as HTMLElement).innerText;
-  if (content && !/^\s*$/.test(content)) {
-    return content.trim();
-  }
-
-  return false;
-}
-
-export function getNodeAttributes(node: HTMLElement | Node): Record<string, string> {
-  if (!node || !(node instanceof HTMLElement) || !node.attributes) {
-    return {};
-  }
-
-  const attributesList = Array.from(node.attributes).map((attr) => {
-    if (attr.name === 'class') {
-      return [attr.name, `.${attr.value.split(' ').join('.')}`];
-    }
-    if (!attr.value) {
-      return [];
-    }
-    return [attr.name, attr.value];
-  });
-
-  return Object.fromEntries(attributesList);
-}

package/src/img/img.ts

@@ -1,132 +0,0 @@
-import assert from 'assert';
-import { Buffer } from 'node:buffer';
-import sharp from 'sharp';
-import { NodeType } from '@/html-element/constants';
-
-// Define picture path
-type ElementType = {
-  x: number;
-  y: number;
-  width: number;
-  height: number;
-  label: string;
-  attributes: {
-    [key: string]: string;
-    nodeType: NodeType;
-  };
-};
-
-const createSvgOverlay = (elements: Array<ElementType>, imageWidth: number, imageHeight: number) => {
-  let svgContent = `<svg width="${imageWidth}" height="${imageHeight}" xmlns="http://www.w3.org/2000/svg">`;
-
-  // Define color array
-  const colors = [
-    { rect: 'blue', text: 'white' },
-    { rect: 'green', text: 'white' },
-  ];
-
-  // Define clipping path
-  svgContent += `<defs>`;
-  elements.forEach((element, index) => {
-    svgContent += `
-      <clipPath id="clip${index}">
-        <rect x="${element.x}" y="${element.y}" width="${element.width}" height="${element.height}" />
-      </clipPath>
-    `;
-  });
-  svgContent += `</defs>`;
-
-  elements.forEach((element, index) => {
-    // Calculate the width and height of the text
-    const textWidth = element.label.length * 8; // Assume that each character is 8px wide
-    const textHeight = 12; // Assume that the text height is 20px
-
-    // Calculates the position of the initial color block so that it wraps and centers the text
-    const rectWidth = textWidth + 5;
-    const rectHeight = textHeight + 4;
-    let rectX = element.x - rectWidth;
-    let rectY = element.y + element.height / 2 - textHeight / 2 - 2;
-
-    // Initial text position
-    let textX = rectX + rectWidth / 2;
-    let textY = rectY + rectHeight / 2 + 6;
-
-    // Check to see if it's obscured by the left
-    if (rectX < 0) {
-      rectX = element.x;
-      rectY = element.y - rectHeight;
-      textX = rectX + rectWidth / 2;
-      textY = rectY + rectHeight / 2 + 6;
-    }
-
-    // Choose color
-    const color = colors[index % colors.length];
-
-    // Draw boxes and text
-    svgContent += `
-      <rect x="${element.x}" y="${element.y}" width="${element.width}" height="${element.height}" 
-            style="fill:none;stroke:${color.rect};stroke-width:4" clip-path="url(#clip${index})" />
-      <rect x="${rectX}" y="${rectY}" width="${rectWidth}" height="${rectHeight}" style="fill:${color.rect};" />
-      <text x="${textX}" y="${textY}" 
-            text-anchor="middle" dominant-baseline="middle" style="fill:${color.text};font-size:12px;font-weight:bold;">
-        ${element.label}
-      </text>
-    `;
-  });
-
-  svgContent += `</svg>`;
-  return Buffer.from(svgContent);
-};
-
-export const processImageElementInfo = async (options: {
-  inputImgBase64: string;
-  elementsPostionInfo: Array<ElementType>;
-  elementsPostionInfoWithoutText: Array<ElementType>;
-}) => {
-  // Get the size of the original image
-  const base64Image = options.inputImgBase64.split(';base64,').pop();
-  assert(base64Image, 'base64Image is undefined');
-
-  const imageBuffer = Buffer.from(base64Image, 'base64');
-  const metadata = await sharp(imageBuffer).metadata();
-  const { width, height } = metadata;
-
-  if (width && height) {
-    // Create svg overlay
-    const svgOverlay = createSvgOverlay(options.elementsPostionInfo, width, height);
-    const svgOverlayWithoutText = createSvgOverlay(options.elementsPostionInfoWithoutText, width, height);
-
-    // Composite picture
-    const compositeElementInfoImgBase64 = await sharp(imageBuffer)
-      // .resize(newDimensions.width, newDimensions.height)
-      .composite([{ input: svgOverlay, blend: 'over' }])
-      .toBuffer()
-      .then((data) => {
-        // Convert image data to base64 encoding
-        return data.toString('base64');
-      })
-      .catch((err) => {
-        throw err;
-      });
-
-    // Composite picture withtoutText
-    const compositeElementInfoImgWithoutTextBase64 = await sharp(imageBuffer)
-      // .resize(newDimensions.width, newDimensions.height)
-      .composite([{ input: svgOverlayWithoutText, blend: 'over' }])
-      .toBuffer()
-      .then((data) => {
-        // Convert image data to base64 encoding
-        return data.toString('base64');
-      })
-      .catch((err) => {
-        throw err;
-      });
-
-    return {
-      compositeElementInfoImgBase64,
-      compositeElementInfoImgWithoutTextBase64,
-    };
-  } else {
-    throw Error('Image processing failed because width or height is undefined');
-  }
-};

package/src/img/util.ts

@@ -1,28 +0,0 @@
-import { getElementInfosFromPage } from '../playwright/utils';
-import { NodeType } from '@/html-element/constants';
-import { ElementInfo } from '@/html-element/extractInfo';
-
-export async function getElementInfos(page: any) {
-  const captureElementSnapshot: Array<ElementInfo> = await getElementInfosFromPage(page);
-  const elementsPostionInfo = captureElementSnapshot.map((elementInfo) => {
-    return {
-      label: elementInfo.id.toString(),
-      x: elementInfo.rect.left,
-      y: elementInfo.rect.top,
-      width: elementInfo.rect.width,
-      height: elementInfo.rect.height,
-      attributes: elementInfo.attributes,
-    };
-  });
-  const elementsPostionInfoWithoutText = elementsPostionInfo.filter((elementInfo) => {
-    if (elementInfo.attributes.nodeType === NodeType.TEXT) {
-      return false;
-    }
-    return true;
-  });
-  return {
-    elementsPostionInfo,
-    captureElementSnapshot,
-    elementsPostionInfoWithoutText,
-  };
-}

package/src/index.ts

@@ -1,2 +0,0 @@
-export { PlaywrightAiFixture } from './playwright';
-export type { PlayWrightAiFixtureType } from './playwright';

package/src/playwright/actions.ts

@@ -1,276 +0,0 @@
-import assert from 'assert';
-import type { Page as PlaywrightPage } from 'playwright';
-import Insight, {
-  DumpSubscriber,
-  ExecutionDump,
-  ExecutionRecorderItem,
-  ExecutionTaskActionApply,
-  ExecutionTaskApply,
-  ExecutionTaskInsightLocateApply,
-  ExecutionTaskInsightQueryApply,
-  ExecutionTaskPlanningApply,
-  Executor,
-  InsightDump,
-  InsightExtractParam,
-  PlanningAction,
-  PlanningActionParamHover,
-  PlanningActionParamInputOrKeyPress,
-  PlanningActionParamScroll,
-  PlanningActionParamTap,
-  plan,
-} from '@midscene/core';
-import { commonScreenshotParam, getTmpFile, sleep } from '@midscene/core/utils';
-import { base64Encoded } from '@midscene/core/image';
-import { parseContextFromPlaywrightPage } from './utils';
-import { WebElementInfo } from './element';
-
-export class PlayWrightActionAgent {
-  page: PlaywrightPage;
-
-  insight: Insight<WebElementInfo>;
-
-  executor: Executor;
-
-  actionDump?: ExecutionDump;
-
-  constructor(page: PlaywrightPage, opt?: { taskName?: string }) {
-    this.page = page;
-    this.insight = new Insight<WebElementInfo>(async () => {
-      return await parseContextFromPlaywrightPage(page);
-    });
-    this.executor = new Executor(opt?.taskName || 'MidScene - PlayWrightAI');
-  }
-
-  private async recordScreenshot(timing: ExecutionRecorderItem['timing']) {
-    const file = getTmpFile('jpeg');
-    await this.page.screenshot({
-      ...commonScreenshotParam,
-      path: file,
-    });
-    const item: ExecutionRecorderItem = {
-      type: 'screenshot',
-      ts: Date.now(),
-      screenshot: base64Encoded(file),
-      timing,
-    };
-    return item;
-  }
-
-  private wrapExecutorWithScreenshot(taskApply: ExecutionTaskApply): ExecutionTaskApply {
-    const taskWithScreenshot: ExecutionTaskApply = {
-      ...taskApply,
-      executor: async (param, context, ...args) => {
-        const recorder: ExecutionRecorderItem[] = [];
-        const { task } = context;
-        // set the recorder before executor in case of error
-        task.recorder = recorder;
-        const shot = await this.recordScreenshot(`before ${task.type}`);
-        recorder.push(shot);
-        const result = await taskApply.executor(param, context, ...args);
-        if (taskApply.type === 'Action') {
-          await sleep(1000);
-          const shot2 = await this.recordScreenshot('after Action');
-          recorder.push(shot2);
-        }
-        return result;
-      },
-    };
-    return taskWithScreenshot;
-  }
-
-  private async convertPlanToExecutable(plans: PlanningAction[]) {
-    const tasks: ExecutionTaskApply[] = plans
-      .map((plan) => {
-        if (plan.type === 'Locate') {
-          const taskFind: ExecutionTaskInsightLocateApply = {
-            type: 'Insight',
-            subType: 'Locate',
-            param: {
-              prompt: plan.thought,
-            },
-            executor: async (param) => {
-              let insightDump: InsightDump | undefined;
-              const dumpCollector: DumpSubscriber = (dump) => {
-                insightDump = dump;
-              };
-              this.insight.onceDumpUpdatedFn = dumpCollector;
-              const element = await this.insight.locate(param.prompt);
-              assert(element, `Element not found: ${param.prompt}`);
-              return {
-                output: {
-                  element,
-                },
-                log: {
-                  dump: insightDump,
-                },
-              };
-            },
-          };
-          return taskFind;
-        } else if (plan.type === 'Input') {
-          const taskActionInput: ExecutionTaskActionApply<PlanningActionParamInputOrKeyPress> = {
-            type: 'Action',
-            subType: 'Input',
-            param: plan.param,
-            executor: async (taskParam) => {
-              assert(taskParam.value, 'No value to input');
-              await this.page.keyboard.type(taskParam.value);
-            },
-          };
-          // TODO: return a recorder Object
-          return taskActionInput;
-        } else if (plan.type === 'KeyboardPress') {
-          const taskActionKeyboardPress: ExecutionTaskActionApply<PlanningActionParamInputOrKeyPress> = {
-            type: 'Action',
-            subType: 'KeyboardPress',
-            param: plan.param,
-            executor: async (taskParam) => {
-              assert(taskParam.value, 'No key to press');
-              await this.page.keyboard.press(taskParam.value);
-            },
-          };
-          return taskActionKeyboardPress;
-        } else if (plan.type === 'Tap') {
-          const taskActionTap: ExecutionTaskActionApply<PlanningActionParamTap> = {
-            type: 'Action',
-            subType: 'Tap',
-            executor: async (param, { element }) => {
-              assert(element, 'Element not found, cannot tap');
-              await this.page.mouse.click(element.center[0], element.center[1]);
-            },
-          };
-          return taskActionTap;
-        } else if (plan.type === 'Hover') {
-          const taskActionHover: ExecutionTaskActionApply<PlanningActionParamHover> = {
-            type: 'Action',
-            subType: 'Hover',
-            executor: async (param, { element }) => {
-              // console.log('executor args', param, element);
-              assert(element, 'Element not found, cannot hover');
-              await this.page.mouse.move(element.center[0], element.center[1]);
-            },
-          };
-          return taskActionHover;
-        } else if (plan.type === 'Scroll') {
-          const taskActionScroll: ExecutionTaskActionApply<PlanningActionParamScroll> = {
-            type: 'Action',
-            subType: 'Scroll',
-            param: plan.param,
-            executor: async (taskParam) => {
-              const scrollToEventName = taskParam.scrollType;
-              const innerHeight = await this.page.evaluate(() => window.innerHeight);
-
-              switch (scrollToEventName) {
-                case 'ScrollUntilTop':
-                  await this.page.mouse.wheel(0, -9999999);
-                  break;
-                case 'ScrollUntilBottom':
-                  await this.page.mouse.wheel(0, 9999999);
-                  break;
-                case 'ScrollUp':
-                  await this.page.mouse.wheel(0, -innerHeight);
-                  break;
-                case 'ScrollDown':
-                  await this.page.mouse.wheel(0, innerHeight);
-                  break;
-                default:
-                  console.error('Unknown scroll event type:', scrollToEventName);
-              }
-            },
-          };
-          return taskActionScroll;
-        } else if (plan.type === 'Error') {
-          throw new Error(`Got a task plan with type Error: ${plan.thought}`);
-        } else {
-          throw new Error(`Unknown or Unsupported task type: ${plan.type}`);
-        }
-      })
-      .map((task: ExecutionTaskApply) => {
-        return this.wrapExecutorWithScreenshot(task);
-      });
-
-    return tasks;
-  }
-
-  async action(userPrompt: string /* , actionInfo?: { actionType?: EventActions[number]['action'] } */) {
-    this.executor.description = userPrompt;
-    const pageContext = await this.insight.contextRetrieverFn();
-
-    let plans: PlanningAction[] = [];
-    const planningTask: ExecutionTaskPlanningApply = {
-      type: 'Planning',
-      param: {
-        userPrompt,
-      },
-      async executor(param) {
-        const planResult = await plan(pageContext, param.userPrompt);
-        assert(planResult.plans.length > 0, 'No plans found');
-        // eslint-disable-next-line prefer-destructuring
-        plans = planResult.plans;
-        return {
-          output: planResult,
-        };
-      },
-    };
-
-    try {
-      // plan
-      await this.executor.append(this.wrapExecutorWithScreenshot(planningTask));
-      await this.executor.flush();
-      this.actionDump = this.executor.dump();
-
-      // append tasks
-      const executables = await this.convertPlanToExecutable(plans);
-      await this.executor.append(executables);
-
-      // flush actions
-      await this.executor.flush();
-      this.actionDump = this.executor.dump();
-
-      assert(
-        this.executor.status !== 'error',
-        `failed to execute tasks: ${this.executor.status}, msg: ${this.executor.errorMsg || ''}`,
-      );
-    } catch (e: any) {
-      // keep the dump before throwing
-      this.actionDump = this.executor.dump();
-      const err = new Error(e.message, { cause: e });
-      throw err;
-    }
-  }
-
-  async query(demand: InsightExtractParam) {
-    this.executor.description = JSON.stringify(demand);
-    let data: any;
-    const queryTask: ExecutionTaskInsightQueryApply = {
-      type: 'Insight',
-      subType: 'Query',
-      param: {
-        dataDemand: demand,
-      },
-      executor: async (param) => {
-        let insightDump: InsightDump | undefined;
-        const dumpCollector: DumpSubscriber = (dump) => {
-          insightDump = dump;
-        };
-        this.insight.onceDumpUpdatedFn = dumpCollector;
-        data = await this.insight.extract<any>(param.dataDemand);
-        return {
-          output: data,
-          log: { dump: insightDump },
-        };
-      },
-    };
-    try {
-      await this.executor.append(this.wrapExecutorWithScreenshot(queryTask));
-      await this.executor.flush();
-      this.actionDump = this.executor.dump();
-    } catch (e: any) {
-      // keep the dump before throwing
-      this.actionDump = this.executor.dump();
-      const err = new Error(e.message, { cause: e });
-      throw err;
-    }
-    return data;
-  }
-}