@midscene/web 0.0.1

package/src/playwright/actions.ts

@@ -0,0 +1,276 @@
+import assert from 'assert';
+import type { Page as PlaywrightPage } from 'playwright';
+import Insight, {
+  DumpSubscriber,
+  ExecutionDump,
+  ExecutionRecorderItem,
+  ExecutionTaskActionApply,
+  ExecutionTaskApply,
+  ExecutionTaskInsightLocateApply,
+  ExecutionTaskInsightQueryApply,
+  ExecutionTaskPlanningApply,
+  Executor,
+  InsightDump,
+  InsightExtractParam,
+  PlanningAction,
+  PlanningActionParamHover,
+  PlanningActionParamInputOrKeyPress,
+  PlanningActionParamScroll,
+  PlanningActionParamTap,
+  plan,
+} from '@midscene/core';
+import { commonScreenshotParam, getTmpFile, sleep } from '@midscene/core/utils';
+import { base64Encoded } from '@midscene/core/image';
+import { parseContextFromPlaywrightPage } from './utils';
+import { WebElementInfo } from './element';
+
+export class PlayWrightActionAgent {
+  page: PlaywrightPage;
+
+  insight: Insight<WebElementInfo>;
+
+  executor: Executor;
+
+  actionDump?: ExecutionDump;
+
+  constructor(page: PlaywrightPage, opt?: { taskName?: string }) {
+    this.page = page;
+    this.insight = new Insight<WebElementInfo>(async () => {
+      return await parseContextFromPlaywrightPage(page);
+    });
+    this.executor = new Executor(opt?.taskName || 'MidScene - PlayWrightAI');
+  }
+
+  private async recordScreenshot(timing: ExecutionRecorderItem['timing']) {
+    const file = getTmpFile('jpeg');
+    await this.page.screenshot({
+      ...commonScreenshotParam,
+      path: file,
+    });
+    const item: ExecutionRecorderItem = {
+      type: 'screenshot',
+      ts: Date.now(),
+      screenshot: base64Encoded(file),
+      timing,
+    };
+    return item;
+  }
+
+  private wrapExecutorWithScreenshot(taskApply: ExecutionTaskApply): ExecutionTaskApply {
+    const taskWithScreenshot: ExecutionTaskApply = {
+      ...taskApply,
+      executor: async (param, context, ...args) => {
+        const recorder: ExecutionRecorderItem[] = [];
+        const { task } = context;
+        // set the recorder before executor in case of error
+        task.recorder = recorder;
+        const shot = await this.recordScreenshot(`before ${task.type}`);
+        recorder.push(shot);
+        const result = await taskApply.executor(param, context, ...args);
+        if (taskApply.type === 'Action') {
+          await sleep(1000);
+          const shot2 = await this.recordScreenshot('after Action');
+          recorder.push(shot2);
+        }
+        return result;
+      },
+    };
+    return taskWithScreenshot;
+  }
+
+  private async convertPlanToExecutable(plans: PlanningAction[]) {
+    const tasks: ExecutionTaskApply[] = plans
+      .map((plan) => {
+        if (plan.type === 'Locate') {
+          const taskFind: ExecutionTaskInsightLocateApply = {
+            type: 'Insight',
+            subType: 'Locate',
+            param: {
+              prompt: plan.thought,
+            },
+            executor: async (param) => {
+              let insightDump: InsightDump | undefined;
+              const dumpCollector: DumpSubscriber = (dump) => {
+                insightDump = dump;
+              };
+              this.insight.onceDumpUpdatedFn = dumpCollector;
+              const element = await this.insight.locate(param.prompt);
+              assert(element, `Element not found: ${param.prompt}`);
+              return {
+                output: {
+                  element,
+                },
+                log: {
+                  dump: insightDump,
+                },
+              };
+            },
+          };
+          return taskFind;
+        } else if (plan.type === 'Input') {
+          const taskActionInput: ExecutionTaskActionApply<PlanningActionParamInputOrKeyPress> = {
+            type: 'Action',
+            subType: 'Input',
+            param: plan.param,
+            executor: async (taskParam) => {
+              assert(taskParam.value, 'No value to input');
+              await this.page.keyboard.type(taskParam.value);
+            },
+          };
+          // TODO: return a recorder Object
+          return taskActionInput;
+        } else if (plan.type === 'KeyboardPress') {
+          const taskActionKeyboardPress: ExecutionTaskActionApply<PlanningActionParamInputOrKeyPress> = {
+            type: 'Action',
+            subType: 'KeyboardPress',
+            param: plan.param,
+            executor: async (taskParam) => {
+              assert(taskParam.value, 'No key to press');
+              await this.page.keyboard.press(taskParam.value);
+            },
+          };
+          return taskActionKeyboardPress;
+        } else if (plan.type === 'Tap') {
+          const taskActionTap: ExecutionTaskActionApply<PlanningActionParamTap> = {
+            type: 'Action',
+            subType: 'Tap',
+            executor: async (param, { element }) => {
+              assert(element, 'Element not found, cannot tap');
+              await this.page.mouse.click(element.center[0], element.center[1]);
+            },
+          };
+          return taskActionTap;
+        } else if (plan.type === 'Hover') {
+          const taskActionHover: ExecutionTaskActionApply<PlanningActionParamHover> = {
+            type: 'Action',
+            subType: 'Hover',
+            executor: async (param, { element }) => {
+              // console.log('executor args', param, element);
+              assert(element, 'Element not found, cannot hover');
+              await this.page.mouse.move(element.center[0], element.center[1]);
+            },
+          };
+          return taskActionHover;
+        } else if (plan.type === 'Scroll') {
+          const taskActionScroll: ExecutionTaskActionApply<PlanningActionParamScroll> = {
+            type: 'Action',
+            subType: 'Scroll',
+            param: plan.param,
+            executor: async (taskParam) => {
+              const scrollToEventName = taskParam.scrollType;
+              const innerHeight = await this.page.evaluate(() => window.innerHeight);
+
+              switch (scrollToEventName) {
+                case 'ScrollUntilTop':
+                  await this.page.mouse.wheel(0, -9999999);
+                  break;
+                case 'ScrollUntilBottom':
+                  await this.page.mouse.wheel(0, 9999999);
+                  break;
+                case 'ScrollUp':
+                  await this.page.mouse.wheel(0, -innerHeight);
+                  break;
+                case 'ScrollDown':
+                  await this.page.mouse.wheel(0, innerHeight);
+                  break;
+                default:
+                  console.error('Unknown scroll event type:', scrollToEventName);
+              }
+            },
+          };
+          return taskActionScroll;
+        } else if (plan.type === 'Error') {
+          throw new Error(`Got a task plan with type Error: ${plan.thought}`);
+        } else {
+          throw new Error(`Unknown or Unsupported task type: ${plan.type}`);
+        }
+      })
+      .map((task: ExecutionTaskApply) => {
+        return this.wrapExecutorWithScreenshot(task);
+      });
+
+    return tasks;
+  }
+
+  async action(userPrompt: string /* , actionInfo?: { actionType?: EventActions[number]['action'] } */) {
+    this.executor.description = userPrompt;
+    const pageContext = await this.insight.contextRetrieverFn();
+
+    let plans: PlanningAction[] = [];
+    const planningTask: ExecutionTaskPlanningApply = {
+      type: 'Planning',
+      param: {
+        userPrompt,
+      },
+      async executor(param) {
+        const planResult = await plan(pageContext, param.userPrompt);
+        assert(planResult.plans.length > 0, 'No plans found');
+        // eslint-disable-next-line prefer-destructuring
+        plans = planResult.plans;
+        return {
+          output: planResult,
+        };
+      },
+    };
+
+    try {
+      // plan
+      await this.executor.append(this.wrapExecutorWithScreenshot(planningTask));
+      await this.executor.flush();
+      this.actionDump = this.executor.dump();
+
+      // append tasks
+      const executables = await this.convertPlanToExecutable(plans);
+      await this.executor.append(executables);
+
+      // flush actions
+      await this.executor.flush();
+      this.actionDump = this.executor.dump();
+
+      assert(
+        this.executor.status !== 'error',
+        `failed to execute tasks: ${this.executor.status}, msg: ${this.executor.errorMsg || ''}`,
+      );
+    } catch (e: any) {
+      // keep the dump before throwing
+      this.actionDump = this.executor.dump();
+      const err = new Error(e.message, { cause: e });
+      throw err;
+    }
+  }
+
+  async query(demand: InsightExtractParam) {
+    this.executor.description = JSON.stringify(demand);
+    let data: any;
+    const queryTask: ExecutionTaskInsightQueryApply = {
+      type: 'Insight',
+      subType: 'Query',
+      param: {
+        dataDemand: demand,
+      },
+      executor: async (param) => {
+        let insightDump: InsightDump | undefined;
+        const dumpCollector: DumpSubscriber = (dump) => {
+          insightDump = dump;
+        };
+        this.insight.onceDumpUpdatedFn = dumpCollector;
+        data = await this.insight.extract<any>(param.dataDemand);
+        return {
+          output: data,
+          log: { dump: insightDump },
+        };
+      },
+    };
+    try {
+      await this.executor.append(this.wrapExecutorWithScreenshot(queryTask));
+      await this.executor.flush();
+      this.actionDump = this.executor.dump();
+    } catch (e: any) {
+      // keep the dump before throwing
+      this.actionDump = this.executor.dump();
+      const err = new Error(e.message, { cause: e });
+      throw err;
+    }
+    return data;
+  }
+}

package/src/playwright/cdp.ts

@@ -0,0 +1,322 @@
+// fork from https://github.com/zerostep-ai/zerostep/blob/5ba8ca5282879f444e5dfefbcc1f03c76591469e/packages/playwright/src/cdp.ts
+import type { Page } from 'playwright';
+
+const cdpSessionByPage = new Map<Page, CDPSession>();
+
+export const WEBDRIVER_ELEMENT_KEY = 'element-6066-11e4-a52e-4f735466cecf';
+
+export type ScrollType = 'up' | 'down' | 'bottom' | 'top';
+
+/**
+ * Closes the cdp session and clears the global shared reference. This
+ * happens automatically when a page closes in a playwright test, so
+ * should generally not be necessary.
+ */
+export const detachCPDSession = async (page: Page) => {
+  if (cdpSessionByPage.has(page)) {
+    await cdpSessionByPage.get(page)!.detach();
+    cdpSessionByPage.delete(page);
+  }
+};
+
+/**
+ * Returns a stable reference to a CDP session.
+ */
+export const getCDPSession = async (page: Page): Promise<CDPSession> => {
+  if (!cdpSessionByPage.has(page)) {
+    try {
+      const session = await page.context().newCDPSession(page);
+      cdpSessionByPage.set(page, session);
+    } catch (e) {
+      throw Error('The ai() function can only be run against Chromium browsers.');
+    }
+  }
+
+  return cdpSessionByPage.get(page)!;
+};
+
+export const getScreenshot = async (page: Page) => {
+  const cdpSession = await getCDPSession(page);
+  const screenshot = await cdpSession.send('Page.captureScreenshot');
+  return screenshot.data; // Base64-encoded image data
+};
+
+export const scrollIntoView = async (page: Page, args: { id: string }) => {
+  const cdpSession = await getCDPSession(page);
+
+  await cdpSession.send('DOM.scrollIntoViewIfNeeded', {
+    backendNodeId: parseInt(args.id, 10),
+  });
+};
+
+export const getTitle = async (page: Page) => {
+  const cdpSession = await getCDPSession(page);
+  const returnedValue = await cdpSession.send('Runtime.evaluate', {
+    expression: 'document.title',
+    returnByValue: true,
+  });
+
+  return returnedValue.result.value;
+};
+
+export const get = async (page: Page, args: { url: string }) => {
+  const cdpSession = await getCDPSession(page);
+  await cdpSession.send('Page.navigate', {
+    url: args.url,
+  });
+};
+
+export const scrollElement = async (page: Page, args: { id: string; target: ScrollType }) => {
+  await runFunctionOn(page, {
+    functionDeclaration: `function() {
+      let element = this
+      let elementHeight = 0
+
+      switch (element.tagName) {
+        case 'BODY':
+        case 'HTML':
+          element = document.scrollingElement || document.body
+          elementHeight = window.visualViewport?.height ?? 720
+          break
+        default:
+          elementHeight = element.clientHeight ?? 720
+          break
+      }
+
+      const relativeScrollDistance = 0.75 * elementHeight
+
+      switch ("${args.target}") {
+        case 'top':
+          return element.scrollTo({ top: 0 })
+        case 'bottom':
+          return element.scrollTo({ top: element.scrollHeight })
+        case 'up':
+          return element.scrollBy({ top: -relativeScrollDistance })
+        case 'down':
+          return element.scrollBy({ top: relativeScrollDistance })
+        default:
+          throw Error('Unsupported scroll target ${args.target}')
+      }
+    }`,
+    backendNodeId: parseInt(args.id, 10),
+  });
+};
+
+export const runFunctionOn = async (
+  page: Page,
+  args: { functionDeclaration: string; backendNodeId: number },
+) => {
+  const cdpSession = await getCDPSession(page);
+  const {
+    object: { objectId },
+  } = await cdpSession.send('DOM.resolveNode', { backendNodeId: args.backendNodeId });
+  await cdpSession.send('Runtime.callFunctionOn', {
+    functionDeclaration: args.functionDeclaration,
+    objectId,
+  });
+};
+
+export const clearElement = async (page: Page, args: { id: string }) => {
+  return await runFunctionOn(page, {
+    functionDeclaration: `function() {this.value=''}`,
+    backendNodeId: parseInt(args.id, 10),
+  });
+};
+
+export const sendKeysToElement = async (page: Page, args: { id: string; value: string[] }) => {
+  const cdpSession = await getCDPSession(page);
+  const value = args.value[0];
+
+  const { nodeId } = await cdpSession.send('DOM.requestNode', { objectId: args.id });
+  await cdpSession.send('DOM.focus', { nodeId });
+
+  // eslint-disable-next-line @typescript-eslint/prefer-for-of
+  for (let i = 0; i < value.length; i++) {
+    await cdpSession.send('Input.dispatchKeyEvent', {
+      type: 'char',
+      text: value[i],
+    });
+  }
+
+  return true;
+};
+
+export const getElementAttribute = async (page: Page, args: { id: string; name: string }) => {
+  const cdpSession = await getCDPSession(page);
+
+  const { nodeId } = await cdpSession.send('DOM.requestNode', { objectId: args.id });
+  const { attributes } = await cdpSession.send('DOM.getAttributes', { nodeId });
+
+  for (let i = 0; i < attributes.length; i++) {
+    if (attributes[i] === args.name) {
+      return attributes[i + 1];
+    }
+  }
+  return attributes;
+};
+
+export const getElementTagName = async (page: Page, args: { id: string }) => {
+  const cdpSession = await getCDPSession(page);
+  const returnedValue = await cdpSession.send('Runtime.callFunctionOn', {
+    functionDeclaration: `function() {return this.tagName}`,
+    objectId: args.id,
+    returnByValue: true,
+  });
+
+  return returnedValue.result.value;
+};
+
+export const clickElement = async (page: Page, args: { id: string }) => {
+  const cdpSession = await getCDPSession(page);
+  const { centerX, centerY } = await getContentQuads(page, { backendNodeId: parseInt(args.id, 10) });
+
+  await cdpSession.send('Input.dispatchMouseEvent', {
+    type: 'mousePressed',
+    x: centerX,
+    y: centerY,
+    button: 'left',
+    clickCount: 1,
+    buttons: 1,
+  });
+
+  await cdpSession.send('Input.dispatchMouseEvent', {
+    type: 'mouseReleased',
+    x: centerX,
+    y: centerY,
+    button: 'left',
+    clickCount: 1,
+    buttons: 1,
+  });
+
+  return true;
+};
+
+export const getContentQuads = async (page: Page, args: { backendNodeId: number }) => {
+  const cdpSession = await getCDPSession(page);
+  const quadsResponse = await cdpSession.send('DOM.getContentQuads', args);
+
+  const [topLeftX, topLeftY, topRightX, topRightY, bottomRightX, bottomRightY, bottomLeftX, bottomLeftY] =
+    quadsResponse.quads[0];
+
+  const width = topRightX - topLeftX;
+  const height = bottomRightY - topRightY;
+  const centerX = topLeftX + width / 2;
+  const centerY = topRightY + height / 2;
+
+  console.log('getContentQuads', args, {
+    width,
+    height,
+    topLeftX,
+    topLeftY,
+    centerX,
+    centerY,
+  });
+  return {
+    topLeftX,
+    topLeftY,
+    topRightX,
+    topRightY,
+    bottomRightX,
+    bottomRightY,
+    bottomLeftX,
+    bottomLeftY,
+    width,
+    height,
+    centerX,
+    centerY,
+  };
+};
+
+export const focusElement = async (page: Page, args: { backendNodeId: number }) => {
+  const cdpSession = await getCDPSession(page);
+  await cdpSession.send('DOM.focus', args);
+};
+
+export const getElementRect = async (page: Page, args: { id: string }) => {
+  const cdpSession = await getCDPSession(page);
+  const returnedValue = await cdpSession.send('Runtime.callFunctionOn', {
+    functionDeclaration: `function() {return JSON.parse(JSON.stringify(this.getBoundingClientRect()))}`,
+    objectId: args.id,
+    returnByValue: true,
+  });
+
+  return returnedValue.result.value;
+};
+
+export const findElements = async (page: Page, args: { using: string; value: string }) => {
+  switch (args.using) {
+    case 'css selector':
+    case 'tag name':
+      return await querySelectorAll(page, { selector: args.value });
+    default:
+      throw Error(`Unsupported findElements strategy ${args.using}`);
+  }
+};
+
+export const querySelectorAll = async (page: Page, args: { selector: string }) => {
+  const cdpSession = await getCDPSession(page);
+  const rootDocumentNode = await cdpSession.send('DOM.getDocument', { depth: -1 });
+  const returned = await cdpSession.send('DOM.querySelectorAll', {
+    nodeId: rootDocumentNode.root.nodeId,
+    selector: args.selector,
+  });
+  const resolvedNodesPromises = returned.nodeIds.map(
+    async (nodeId) => await cdpSession.send('DOM.resolveNode', { nodeId }),
+  );
+  const resolvedNodes = await Promise.all(resolvedNodesPromises);
+  const returnValue = resolvedNodes.map((node) => ({ [WEBDRIVER_ELEMENT_KEY]: node.object.objectId }));
+  return returnValue;
+};
+
+export const getCurrentUrl = async (page: Page) => {
+  const cdpSession = await getCDPSession(page);
+  const returned = await cdpSession.send('Page.getNavigationHistory');
+  const returnValue = returned.entries[returned.currentIndex].url;
+  return returnValue;
+};
+
+export const executeScript = async (page: Page, args: { script: string; args: any[] }) => {
+  const functionDeclaration = `function() { ${args.script} }`;
+  const functionArgs = args.args.map((arg) => {
+    if (typeof arg === 'boolean' || typeof arg === 'string' || typeof arg === 'number') {
+      return { value: arg };
+    } else if (arg && typeof arg === 'object' && Reflect.has(arg, WEBDRIVER_ELEMENT_KEY)) {
+      return { objectId: arg[WEBDRIVER_ELEMENT_KEY] };
+    } else {
+      return { value: undefined };
+    }
+  });
+
+  const cdpSession = await getCDPSession(page);
+  await cdpSession.send('Runtime.enable');
+  const window = await cdpSession.send('Runtime.evaluate', { expression: 'window' });
+
+  const returnedRef = await cdpSession.send('Runtime.callFunctionOn', {
+    objectId: window.result.objectId,
+    functionDeclaration,
+    arguments: functionArgs,
+  });
+
+  if (returnedRef.result.className === 'NodeList') {
+    const nodeProperties = await cdpSession.send('Runtime.getProperties', {
+      objectId: returnedRef.result.objectId!,
+      ownProperties: true,
+    });
+    return nodeProperties.result
+      .map((e: any) => (!isNaN(parseInt(e.name, 10)) ? { [WEBDRIVER_ELEMENT_KEY]: e.value?.objectId } : null))
+      .filter((e: any) => e);
+  } else if (returnedRef.result.className === 'HTMLHtmlElement') {
+    return { [WEBDRIVER_ELEMENT_KEY]: returnedRef.result.objectId };
+  } else {
+    const returnedValue = await cdpSession.send('Runtime.callFunctionOn', {
+      objectId: window.result.objectId,
+      functionDeclaration,
+      arguments: functionArgs,
+      returnByValue: true,
+    });
+
+    return returnedValue.result.value;
+  }
+};
+
+type CDPSession = Awaited<ReturnType<ReturnType<Page['context']>['newCDPSession']>>;

package/src/playwright/element.ts

@@ -0,0 +1,74 @@
+import { Page } from 'playwright';
+import { BaseElement, Rect } from '@midscene/core';
+import { NodeType } from '../html-element/constants';
+
+export interface WebElementInfoType extends BaseElement {
+  id: string;
+  locator: string;
+  attributes: {
+    ['nodeType']: NodeType;
+    [key: string]: string;
+  };
+}
+
+export class WebElementInfo implements BaseElement {
+  content: string;
+
+  locator: string;
+
+  rect: Rect;
+
+  center: [number, number];
+
+  page: Page;
+
+  id: string;
+
+  attributes: {
+    ['nodeType']: NodeType;
+    [key: string]: string;
+  };
+
+  constructor({
+    content,
+    rect,
+    page,
+    locator,
+    id,
+    attributes,
+  }: {
+    content: string;
+    rect: Rect;
+    page: Page;
+    locator: string;
+    id: string;
+    attributes: {
+      ['nodeType']: NodeType;
+      [key: string]: string;
+    };
+  }) {
+    this.content = content;
+    this.rect = rect;
+    this.center = [Math.floor(rect.left + rect.width / 2), Math.floor(rect.top + rect.height / 2)];
+    this.page = page;
+    this.locator = locator;
+    this.id = id;
+    this.attributes = attributes;
+  }
+
+  async tap() {
+    await this.page.mouse.click(this.center[0], this.center[1]);
+  }
+
+  async hover() {
+    await this.page.mouse.move(this.center[0], this.center[1]);
+  }
+
+  async type(text: string) {
+    await this.page.keyboard.type(text);
+  }
+
+  async press(key: Parameters<typeof this.page.keyboard.press>[0]) {
+    await this.page.keyboard.press(key);
+  }
+}