@midscene/web 0.0.1 → 0.1.1

package/src/playwright/cdp.ts

@@ -1,322 +0,0 @@
-// fork from https://github.com/zerostep-ai/zerostep/blob/5ba8ca5282879f444e5dfefbcc1f03c76591469e/packages/playwright/src/cdp.ts
-import type { Page } from 'playwright';
-
-const cdpSessionByPage = new Map<Page, CDPSession>();
-
-export const WEBDRIVER_ELEMENT_KEY = 'element-6066-11e4-a52e-4f735466cecf';
-
-export type ScrollType = 'up' | 'down' | 'bottom' | 'top';
-
-/**
- * Closes the cdp session and clears the global shared reference. This
- * happens automatically when a page closes in a playwright test, so
- * should generally not be necessary.
- */
-export const detachCPDSession = async (page: Page) => {
-  if (cdpSessionByPage.has(page)) {
-    await cdpSessionByPage.get(page)!.detach();
-    cdpSessionByPage.delete(page);
-  }
-};
-
-/**
- * Returns a stable reference to a CDP session.
- */
-export const getCDPSession = async (page: Page): Promise<CDPSession> => {
-  if (!cdpSessionByPage.has(page)) {
-    try {
-      const session = await page.context().newCDPSession(page);
-      cdpSessionByPage.set(page, session);
-    } catch (e) {
-      throw Error('The ai() function can only be run against Chromium browsers.');
-    }
-  }
-
-  return cdpSessionByPage.get(page)!;
-};
-
-export const getScreenshot = async (page: Page) => {
-  const cdpSession = await getCDPSession(page);
-  const screenshot = await cdpSession.send('Page.captureScreenshot');
-  return screenshot.data; // Base64-encoded image data
-};
-
-export const scrollIntoView = async (page: Page, args: { id: string }) => {
-  const cdpSession = await getCDPSession(page);
-
-  await cdpSession.send('DOM.scrollIntoViewIfNeeded', {
-    backendNodeId: parseInt(args.id, 10),
-  });
-};
-
-export const getTitle = async (page: Page) => {
-  const cdpSession = await getCDPSession(page);
-  const returnedValue = await cdpSession.send('Runtime.evaluate', {
-    expression: 'document.title',
-    returnByValue: true,
-  });
-
-  return returnedValue.result.value;
-};
-
-export const get = async (page: Page, args: { url: string }) => {
-  const cdpSession = await getCDPSession(page);
-  await cdpSession.send('Page.navigate', {
-    url: args.url,
-  });
-};
-
-export const scrollElement = async (page: Page, args: { id: string; target: ScrollType }) => {
-  await runFunctionOn(page, {
-    functionDeclaration: `function() {
-      let element = this
-      let elementHeight = 0
-
-      switch (element.tagName) {
-        case 'BODY':
-        case 'HTML':
-          element = document.scrollingElement || document.body
-          elementHeight = window.visualViewport?.height ?? 720
-          break
-        default:
-          elementHeight = element.clientHeight ?? 720
-          break
-      }
-
-      const relativeScrollDistance = 0.75 * elementHeight
-
-      switch ("${args.target}") {
-        case 'top':
-          return element.scrollTo({ top: 0 })
-        case 'bottom':
-          return element.scrollTo({ top: element.scrollHeight })
-        case 'up':
-          return element.scrollBy({ top: -relativeScrollDistance })
-        case 'down':
-          return element.scrollBy({ top: relativeScrollDistance })
-        default:
-          throw Error('Unsupported scroll target ${args.target}')
-      }
-    }`,
-    backendNodeId: parseInt(args.id, 10),
-  });
-};
-
-export const runFunctionOn = async (
-  page: Page,
-  args: { functionDeclaration: string; backendNodeId: number },
-) => {
-  const cdpSession = await getCDPSession(page);
-  const {
-    object: { objectId },
-  } = await cdpSession.send('DOM.resolveNode', { backendNodeId: args.backendNodeId });
-  await cdpSession.send('Runtime.callFunctionOn', {
-    functionDeclaration: args.functionDeclaration,
-    objectId,
-  });
-};
-
-export const clearElement = async (page: Page, args: { id: string }) => {
-  return await runFunctionOn(page, {
-    functionDeclaration: `function() {this.value=''}`,
-    backendNodeId: parseInt(args.id, 10),
-  });
-};
-
-export const sendKeysToElement = async (page: Page, args: { id: string; value: string[] }) => {
-  const cdpSession = await getCDPSession(page);
-  const value = args.value[0];
-
-  const { nodeId } = await cdpSession.send('DOM.requestNode', { objectId: args.id });
-  await cdpSession.send('DOM.focus', { nodeId });
-
-  // eslint-disable-next-line @typescript-eslint/prefer-for-of
-  for (let i = 0; i < value.length; i++) {
-    await cdpSession.send('Input.dispatchKeyEvent', {
-      type: 'char',
-      text: value[i],
-    });
-  }
-
-  return true;
-};
-
-export const getElementAttribute = async (page: Page, args: { id: string; name: string }) => {
-  const cdpSession = await getCDPSession(page);
-
-  const { nodeId } = await cdpSession.send('DOM.requestNode', { objectId: args.id });
-  const { attributes } = await cdpSession.send('DOM.getAttributes', { nodeId });
-
-  for (let i = 0; i < attributes.length; i++) {
-    if (attributes[i] === args.name) {
-      return attributes[i + 1];
-    }
-  }
-  return attributes;
-};
-
-export const getElementTagName = async (page: Page, args: { id: string }) => {
-  const cdpSession = await getCDPSession(page);
-  const returnedValue = await cdpSession.send('Runtime.callFunctionOn', {
-    functionDeclaration: `function() {return this.tagName}`,
-    objectId: args.id,
-    returnByValue: true,
-  });
-
-  return returnedValue.result.value;
-};
-
-export const clickElement = async (page: Page, args: { id: string }) => {
-  const cdpSession = await getCDPSession(page);
-  const { centerX, centerY } = await getContentQuads(page, { backendNodeId: parseInt(args.id, 10) });
-
-  await cdpSession.send('Input.dispatchMouseEvent', {
-    type: 'mousePressed',
-    x: centerX,
-    y: centerY,
-    button: 'left',
-    clickCount: 1,
-    buttons: 1,
-  });
-
-  await cdpSession.send('Input.dispatchMouseEvent', {
-    type: 'mouseReleased',
-    x: centerX,
-    y: centerY,
-    button: 'left',
-    clickCount: 1,
-    buttons: 1,
-  });
-
-  return true;
-};
-
-export const getContentQuads = async (page: Page, args: { backendNodeId: number }) => {
-  const cdpSession = await getCDPSession(page);
-  const quadsResponse = await cdpSession.send('DOM.getContentQuads', args);
-
-  const [topLeftX, topLeftY, topRightX, topRightY, bottomRightX, bottomRightY, bottomLeftX, bottomLeftY] =
-    quadsResponse.quads[0];
-
-  const width = topRightX - topLeftX;
-  const height = bottomRightY - topRightY;
-  const centerX = topLeftX + width / 2;
-  const centerY = topRightY + height / 2;
-
-  console.log('getContentQuads', args, {
-    width,
-    height,
-    topLeftX,
-    topLeftY,
-    centerX,
-    centerY,
-  });
-  return {
-    topLeftX,
-    topLeftY,
-    topRightX,
-    topRightY,
-    bottomRightX,
-    bottomRightY,
-    bottomLeftX,
-    bottomLeftY,
-    width,
-    height,
-    centerX,
-    centerY,
-  };
-};
-
-export const focusElement = async (page: Page, args: { backendNodeId: number }) => {
-  const cdpSession = await getCDPSession(page);
-  await cdpSession.send('DOM.focus', args);
-};
-
-export const getElementRect = async (page: Page, args: { id: string }) => {
-  const cdpSession = await getCDPSession(page);
-  const returnedValue = await cdpSession.send('Runtime.callFunctionOn', {
-    functionDeclaration: `function() {return JSON.parse(JSON.stringify(this.getBoundingClientRect()))}`,
-    objectId: args.id,
-    returnByValue: true,
-  });
-
-  return returnedValue.result.value;
-};
-
-export const findElements = async (page: Page, args: { using: string; value: string }) => {
-  switch (args.using) {
-    case 'css selector':
-    case 'tag name':
-      return await querySelectorAll(page, { selector: args.value });
-    default:
-      throw Error(`Unsupported findElements strategy ${args.using}`);
-  }
-};
-
-export const querySelectorAll = async (page: Page, args: { selector: string }) => {
-  const cdpSession = await getCDPSession(page);
-  const rootDocumentNode = await cdpSession.send('DOM.getDocument', { depth: -1 });
-  const returned = await cdpSession.send('DOM.querySelectorAll', {
-    nodeId: rootDocumentNode.root.nodeId,
-    selector: args.selector,
-  });
-  const resolvedNodesPromises = returned.nodeIds.map(
-    async (nodeId) => await cdpSession.send('DOM.resolveNode', { nodeId }),
-  );
-  const resolvedNodes = await Promise.all(resolvedNodesPromises);
-  const returnValue = resolvedNodes.map((node) => ({ [WEBDRIVER_ELEMENT_KEY]: node.object.objectId }));
-  return returnValue;
-};
-
-export const getCurrentUrl = async (page: Page) => {
-  const cdpSession = await getCDPSession(page);
-  const returned = await cdpSession.send('Page.getNavigationHistory');
-  const returnValue = returned.entries[returned.currentIndex].url;
-  return returnValue;
-};
-
-export const executeScript = async (page: Page, args: { script: string; args: any[] }) => {
-  const functionDeclaration = `function() { ${args.script} }`;
-  const functionArgs = args.args.map((arg) => {
-    if (typeof arg === 'boolean' || typeof arg === 'string' || typeof arg === 'number') {
-      return { value: arg };
-    } else if (arg && typeof arg === 'object' && Reflect.has(arg, WEBDRIVER_ELEMENT_KEY)) {
-      return { objectId: arg[WEBDRIVER_ELEMENT_KEY] };
-    } else {
-      return { value: undefined };
-    }
-  });
-
-  const cdpSession = await getCDPSession(page);
-  await cdpSession.send('Runtime.enable');
-  const window = await cdpSession.send('Runtime.evaluate', { expression: 'window' });
-
-  const returnedRef = await cdpSession.send('Runtime.callFunctionOn', {
-    objectId: window.result.objectId,
-    functionDeclaration,
-    arguments: functionArgs,
-  });
-
-  if (returnedRef.result.className === 'NodeList') {
-    const nodeProperties = await cdpSession.send('Runtime.getProperties', {
-      objectId: returnedRef.result.objectId!,
-      ownProperties: true,
-    });
-    return nodeProperties.result
-      .map((e: any) => (!isNaN(parseInt(e.name, 10)) ? { [WEBDRIVER_ELEMENT_KEY]: e.value?.objectId } : null))
-      .filter((e: any) => e);
-  } else if (returnedRef.result.className === 'HTMLHtmlElement') {
-    return { [WEBDRIVER_ELEMENT_KEY]: returnedRef.result.objectId };
-  } else {
-    const returnedValue = await cdpSession.send('Runtime.callFunctionOn', {
-      objectId: window.result.objectId,
-      functionDeclaration,
-      arguments: functionArgs,
-      returnByValue: true,
-    });
-
-    return returnedValue.result.value;
-  }
-};
-
-type CDPSession = Awaited<ReturnType<ReturnType<Page['context']>['newCDPSession']>>;

package/src/playwright/element.ts

@@ -1,74 +0,0 @@
-import { Page } from 'playwright';
-import { BaseElement, Rect } from '@midscene/core';
-import { NodeType } from '../html-element/constants';
-
-export interface WebElementInfoType extends BaseElement {
-  id: string;
-  locator: string;
-  attributes: {
-    ['nodeType']: NodeType;
-    [key: string]: string;
-  };
-}
-
-export class WebElementInfo implements BaseElement {
-  content: string;
-
-  locator: string;
-
-  rect: Rect;
-
-  center: [number, number];
-
-  page: Page;
-
-  id: string;
-
-  attributes: {
-    ['nodeType']: NodeType;
-    [key: string]: string;
-  };
-
-  constructor({
-    content,
-    rect,
-    page,
-    locator,
-    id,
-    attributes,
-  }: {
-    content: string;
-    rect: Rect;
-    page: Page;
-    locator: string;
-    id: string;
-    attributes: {
-      ['nodeType']: NodeType;
-      [key: string]: string;
-    };
-  }) {
-    this.content = content;
-    this.rect = rect;
-    this.center = [Math.floor(rect.left + rect.width / 2), Math.floor(rect.top + rect.height / 2)];
-    this.page = page;
-    this.locator = locator;
-    this.id = id;
-    this.attributes = attributes;
-  }
-
-  async tap() {
-    await this.page.mouse.click(this.center[0], this.center[1]);
-  }
-
-  async hover() {
-    await this.page.mouse.move(this.center[0], this.center[1]);
-  }
-
-  async type(text: string) {
-    await this.page.keyboard.type(text);
-  }
-
-  async press(key: Parameters<typeof this.page.keyboard.press>[0]) {
-    await this.page.keyboard.press(key);
-  }
-}

package/src/playwright/index.ts

@@ -1,120 +0,0 @@
-import { TestInfo, TestType } from '@playwright/test';
-import { ExecutionDump, GroupedActionDump } from '@midscene/core';
-import { groupedActionDumpFileExt, writeDumpFile } from '@midscene/core/utils';
-import { PlayWrightActionAgent } from './actions';
-
-export { PlayWrightActionAgent } from './actions';
-
-export type APITestType = Pick<TestType<any, any>, 'step'>;
-
-export const PlaywrightAiFixture = () => {
-  const dumps: GroupedActionDump[] = [];
-
-  const appendDump = (groupName: string, execution: ExecutionDump) => {
-    let currentDump = dumps.find((dump) => dump.groupName === groupName);
-    if (!currentDump) {
-      currentDump = {
-        groupName,
-        executions: [],
-      };
-      dumps.push(currentDump);
-    }
-    currentDump.executions.push(execution);
-  };
-
-  const writeOutActionDumps = () => {
-    writeDumpFile(`playwright-${process.pid}`, groupedActionDumpFileExt, JSON.stringify(dumps));
-  };
-
-  const groupAndCaseForTest = (testInfo: TestInfo) => {
-    let groupName: string;
-    let caseName: string;
-    const titlePath = [...testInfo.titlePath];
-
-    if (titlePath.length > 1) {
-      caseName = titlePath.pop()!;
-      groupName = titlePath.join(' > ');
-    } else if (titlePath.length === 1) {
-      caseName = titlePath[0];
-      groupName = caseName;
-    } else {
-      caseName = 'unnamed';
-      groupName = 'unnamed';
-    }
-    return { groupName, caseName };
-  };
-
-  const aiAction = async (page: any, testInfo: TestInfo, taskPrompt: string) => {
-    const { groupName, caseName } = groupAndCaseForTest(testInfo);
-
-    const actionAgent = new PlayWrightActionAgent(page, { taskName: caseName });
-    let error: Error | undefined;
-    try {
-      await actionAgent.action(taskPrompt);
-    } catch (e: any) {
-      error = e;
-    }
-    if (actionAgent.actionDump) {
-      appendDump(groupName, actionAgent.actionDump);
-      writeOutActionDumps();
-    }
-    if (error) {
-      // playwright cli won't print error cause, so we print it here
-      console.error(error);
-      throw new Error(error.message, { cause: error });
-    }
-  };
-
-  const aiQuery = async (page: any, testInfo: TestInfo, demand: any) => {
-    const { groupName, caseName } = groupAndCaseForTest(testInfo);
-
-    const actionAgent = new PlayWrightActionAgent(page, { taskName: caseName });
-    let error: Error | undefined;
-    let result: any;
-    try {
-      result = await actionAgent.query(demand);
-    } catch (e: any) {
-      error = e;
-    }
-    if (actionAgent.actionDump) {
-      appendDump(groupName, actionAgent.actionDump);
-      writeOutActionDumps();
-    }
-    if (error) {
-      // playwright cli won't print error cause, so we print it here
-      console.error(error);
-      throw new Error(error.message, { cause: error });
-    }
-    return result;
-  };
-
-  return {
-    // shortcut
-    ai: async ({ page }: any, use: any, testInfo: TestInfo) => {
-      await use(async (taskPrompt: string, type = 'action') => {
-        if (type === 'action') {
-          return aiAction(page, testInfo, taskPrompt);
-        } else if (type === 'query') {
-          return aiQuery(page, testInfo, taskPrompt);
-        }
-        throw new Error(`Unknown or Unsupported task type: ${type}, only support 'action' or 'query'`);
-      });
-    },
-    aiAction: async ({ page }: any, use: any, testInfo: TestInfo) => {
-      await use(async (taskPrompt: string) => {
-        await aiAction(page, testInfo, taskPrompt);
-      });
-    },
-    aiQuery: async ({ page }: any, use: any, testInfo: TestInfo) => {
-      await use(async function (demand: any) {
-        return aiQuery(page, testInfo, demand);
-      });
-    },
-  };
-};
-
-export type PlayWrightAiFixtureType = {
-  ai: <T = any>(prompt: string, type?: 'action' | 'query') => Promise<T>;
-  aiAction: (taskPrompt: string) => ReturnType<PlayWrightActionAgent['action']>;
-  aiQuery: <T = any>(demand: any) => Promise<T>;
-};

package/src/playwright/utils.ts

@@ -1,88 +0,0 @@
-import fs, { readFileSync } from 'fs';
-import assert from 'assert';
-import { Buffer } from 'buffer';
-import path from 'path';
-import type { Page as PlaywrightPage } from 'playwright';
-import { Page } from 'puppeteer';
-import { UIContext, PlaywrightParserOpt } from '@midscene/core';
-import { alignCoordByTrim, base64Encoded, imageInfo, imageInfoOfBase64 } from '@midscene/core/image';
-import { getTmpFile } from '@midscene/core/utils';
-import { WebElementInfo, WebElementInfoType } from './element';
-
-export async function parseContextFromPlaywrightPage(
-  page: PlaywrightPage,
-  _opt?: PlaywrightParserOpt,
-): Promise<UIContext<WebElementInfo>> {
-  assert(page, 'page is required');
-  const file = '/Users/bytedance/workspace/midscene/packages/midscene/tests/fixtures/heytea.jpeg'; // getTmpFile('jpeg');
-  await page.screenshot({ path: file, type: 'jpeg', quality: 75 });
-  const screenshotBuffer = readFileSync(file);
-  const screenshotBase64 = base64Encoded(file);
-  const captureElementSnapshot = await getElementInfosFromPage(page);
-  // align element
-  const elementsInfo = await alignElements(screenshotBuffer, captureElementSnapshot, page);
-  const size = await imageInfoOfBase64(screenshotBase64);
-
-  return {
-    content: elementsInfo,
-    size,
-    screenshotBase64,
-  };
-}
-
-export async function getElementInfosFromPage(page: Page | PlaywrightPage) {
-  const pathDir = findNearestPackageJson(__dirname);
-  assert(pathDir, `can't find pathDir, with ${__dirname}`);
-  const scriptPath = path.join(pathDir, './dist/script/htmlElement.js');
-  const elementInfosScriptContent = readFileSync(scriptPath, 'utf-8');
-  const extraReturnLogic = `${elementInfosScriptContent}midscene_element_inspector.extractTextWithPositionDFS()`;
-
-  const captureElementSnapshot = await (page as any).evaluate(extraReturnLogic);
-  return captureElementSnapshot;
-}
-
-async function alignElements(
-  screenshotBuffer: Buffer,
-  elements: WebElementInfoType[],
-  page: PlaywrightPage,
-): Promise<WebElementInfo[]> {
-  const textsAligned: WebElementInfo[] = [];
-  for (const item of elements) {
-    const { rect } = item;
-    const aligned = await alignCoordByTrim(screenshotBuffer, rect);
-    item.rect = aligned;
-    item.center = [
-      Math.round(aligned.left + aligned.width / 2),
-      Math.round(aligned.top + aligned.height / 2),
-    ];
-    textsAligned.push(
-      new WebElementInfo({
-        ...item,
-        page,
-      }),
-    );
-  }
-  return textsAligned;
-}
-
-/**
- * Find the nearest package.json file recursively
- * @param {string} dir - Home directory
- * @returns {string|null} - The most recent package.json file path or null
- */
-function findNearestPackageJson(dir: string) {
-  const packageJsonPath = path.join(dir, 'package.json');
-
-  if (fs.existsSync(packageJsonPath)) {
-    return dir;
-  }
-
-  const parentDir = path.dirname(dir);
-
-  // Return null if the root directory has been reached
-  if (parentDir === dir) {
-    return null;
-  }
-
-  return findNearestPackageJson(parentDir);
-}

package/src/puppeteer/element.ts

@@ -1,49 +0,0 @@
-// import { Page } from 'puppeteer';
-// import { BaseElement, Rect } from '@/types';
-
-// export class Element implements BaseElement {
-//   id: string;
-
-//   attributes: Record<string,string>;
-
-//   nodeType: string;
-
-//   content: string;
-
-//   locator: string;
-
-//   rect: Rect;
-
-//   center: [number, number];
-
-//   page: Page;
-
-//   constructor(options: {
-//     id: string, attributes: Record<string, string>, nodeType: string, content: string, rect: Rect, page: Page, locator: string
-//   }) {
-//     this.id = options.id;
-//     this.attributes = options.attributes;
-//     this.nodeType = options.nodeType;
-//     this.content = options.content;
-//     this.rect = options.rect;
-//     this.center = [Math.floor(options.rect.left + options.rect.width / 2), Math.floor(options.rect.top + options.rect.height / 2)];
-//     this.page = options.page;
-//     this.locator = options.locator;
-//   }
-
-//   async tap() {
-//     await this.page.mouse.click(this.center[0], this.center[1]);
-//   }
-
-//   async hover() {
-//     console.log('hover');
-//   }
-
-//   async type(text: string) {
-//     await this.page.keyboard.type(text, { delay: 100 });
-//   }
-
-//   async press(key: string) {
-//     await this.page.keyboard.press(key as any, { delay: 100 });
-//   }
-// }

package/src/puppeteer/index.ts

@@ -1,6 +0,0 @@
-// export { Element } from './element';
-// export {
-//   parseContextFromPuppeteerBrowser,
-//   parseContextFromPuppeteerPage,
-//   parseContextFromPlaywrightPage,
-// } from './utils';