@midscene/web 0.0.1 → 0.1.0

package/dist/script/types/htmlElement.d.ts

@@ -7,6 +7,8 @@ declare enum NodeType {
 
 interface ElementInfo {
     id: string;
+    indexId: string;
+    nodeHashId: string;
     locator: string | void;
     attributes: {
         nodeType: NodeType;

package/dist/types/index.d.ts

@@ -1,6 +1,7 @@
-import { TestInfo } from '@playwright/test';
 import { Page } from 'playwright';
-import Insight, { BaseElement, Rect, Executor, ExecutionDump, InsightExtractParam } from '@midscene/core';
+import { TestInfo } from '@playwright/test';
+import Insight, { BaseElement, Rect, UIContext, PlanningAction, AIElementParseResponse, ExecutionDump, InsightExtractParam, GroupedActionDump } from '@midscene/core';
+import { Page as Page$1 } from 'puppeteer';
 
 declare enum NodeType {
     'INPUT' = "INPUT Node",
@@ -9,12 +10,14 @@ declare enum NodeType {
     'TEXT' = "TEXT Node"
 }
 
+type WebPage = Page | Page$1;
+
 declare class WebElementInfo implements BaseElement {
     content: string;
     locator: string;
     rect: Rect;
     center: [number, number];
-    page: Page;
+    page: WebPage;
     id: string;
     attributes: {
         ['nodeType']: NodeType;
@@ -23,7 +26,7 @@ declare class WebElementInfo implements BaseElement {
     constructor({ content, rect, page, locator, id, attributes, }: {
         content: string;
         rect: Rect;
-        page: Page;
+        page: WebPage;
         locator: string;
         id: string;
         attributes: {
@@ -31,19 +34,85 @@ declare class WebElementInfo implements BaseElement {
             [key: string]: string;
         };
     });
-    tap(): Promise<void>;
-    hover(): Promise<void>;
-    type(text: string): Promise<void>;
-    press(key: Parameters<typeof this$1.page.keyboard.press>[0]): Promise<void>;
 }
 
-declare class PlayWrightActionAgent {
-    page: Page;
-    insight: Insight<WebElementInfo>;
-    executor: Executor;
-    actionDump?: ExecutionDump;
-    constructor(page: Page, opt?: {
-        taskName?: string;
+type WebUIContext = UIContext<WebElementInfo> & {
+    url: string;
+};
+
+type PlanTask = {
+    type: 'plan';
+    prompt: string;
+    pageContext: {
+        url: string;
+        size: {
+            width: number;
+            height: number;
+        };
+    };
+    response: {
+        plans: PlanningAction[];
+    };
+};
+type LocateTask = {
+    type: 'locate';
+    prompt: string;
+    pageContext: {
+        url: string;
+        size: {
+            width: number;
+            height: number;
+        };
+    };
+    response: AIElementParseResponse;
+};
+type AiTasks = Array<PlanTask | LocateTask>;
+type AiTaskCache = {
+    aiTasks: AiTasks;
+};
+declare class TaskCache {
+    cache: AiTaskCache | undefined;
+    newCache: AiTaskCache;
+    constructor(opts?: {
+        cache: AiTaskCache;
+    });
+    /**
+     * Read and return cached responses asynchronously based on specific criteria
+     * This function is mainly used to read cached responses from a certain storage medium.
+     * It accepts three parameters: the page context information, the task type, and the user's prompt information.
+     * In the function, it first checks whether there is cached data. If there is, it retrieves the first task response from the cache.
+     * It then checks whether the task type is 'locate' and whether the corresponding element can be found in the new context.
+     * If the element cannot be found, it returns false, indicating that the cache is invalid.
+     * If the task type is correct and the user prompt matches, it checks whether the page context is the same.
+     * If the page context is the same, it returns the cached response, indicating that the cache hit is successful.
+     * If there is no cached data or the conditions are not met, the function returns false, indicating that no cache is available or the cache is not hit.
+     *
+     * @param pageContext UIContext<WebElementInfo> type, representing the context information of the current page
+     * @param type String type, specifying the task type, can be 'plan' or 'locate'
+     * @param userPrompt String type, representing user prompt information
+     * @return Returns a Promise object that resolves to a boolean or object
+     */
+    readCache(pageContext: WebUIContext, type: 'plan', userPrompt: string): PlanTask['response'];
+    readCache(pageContext: WebUIContext, type: 'locate', userPrompt: string): LocateTask['response'];
+    saveCache(cache: PlanTask | LocateTask): void;
+    pageContextEqual(taskPageContext: LocateTask['pageContext'], pageContext: WebUIContext): boolean;
+    /**
+     * Generate task cache data.
+     * This method is mainly used to create or obtain some cached data for tasks, and it returns a new cache object.
+     * In the cache object, it may contain task-related information, states, or other necessary data.
+     * It is assumed that the `newCache` property already exists in the current class or object and is a data structure used to store task cache.
+     * @returns {Object} Returns a new cache object, which may include task cache data.
+     */
+    generateTaskCache(): AiTaskCache;
+}
+
+declare class PageTaskExecutor {
+    page: WebPage;
+    insight: Insight<WebElementInfo, WebUIContext>;
+    executionDump?: ExecutionDump;
+    taskCache: TaskCache;
+    constructor(page: WebPage, opts: {
+        cache: AiTaskCache;
     });
     private recordScreenshot;
     private wrapExecutorWithScreenshot;
@@ -53,14 +122,40 @@ declare class PlayWrightActionAgent {
 }
 
 declare const PlaywrightAiFixture: () => {
-    ai: ({ page }: any, use: any, testInfo: TestInfo) => Promise<void>;
-    aiAction: ({ page }: any, use: any, testInfo: TestInfo) => Promise<void>;
-    aiQuery: ({ page }: any, use: any, testInfo: TestInfo) => Promise<void>;
+    ai: ({ page }: {
+        page: Page;
+    }, use: any, testInfo: TestInfo) => Promise<void>;
+    aiAction: ({ page }: {
+        page: Page;
+    }, use: any, testInfo: TestInfo) => Promise<void>;
+    aiQuery: ({ page }: {
+        page: Page;
+    }, use: any, testInfo: TestInfo) => Promise<void>;
 };
 type PlayWrightAiFixtureType = {
-    ai: <T = any>(prompt: string, type?: 'action' | 'query') => Promise<T>;
-    aiAction: (taskPrompt: string) => ReturnType<PlayWrightActionAgent['action']>;
+    ai: <T = any>(prompt: string, opts?: {
+        type?: 'action' | 'query';
+    }) => Promise<T>;
+    aiAction: (taskPrompt: string) => ReturnType<PageTaskExecutor['action']>;
     aiQuery: <T = any>(demand: any) => Promise<T>;
 };
 
-export { type PlayWrightAiFixtureType, PlaywrightAiFixture };
+declare class PageAgent {
+    page: WebPage;
+    dumps: GroupedActionDump[];
+    testId: string;
+    dumpFile?: string;
+    actionAgent: PageTaskExecutor;
+    constructor(page: WebPage, opts: {
+        testId?: string;
+        taskFile?: string;
+        cache?: AiTaskCache;
+    });
+    appendDump(execution: ExecutionDump): void;
+    writeOutActionDumps(): void;
+    aiAction(taskPrompt: string): Promise<void>;
+    aiQuery(demand: any): Promise<any>;
+    ai(taskPrompt: string, type?: string): Promise<any>;
+}
+
+export { type PlayWrightAiFixtureType, PlaywrightAiFixture, PageAgent as PuppeteerAgent };

package/dist/types/playwright-report.d.ts

@@ -0,0 +1,10 @@
+import { Reporter, FullConfig, Suite, TestCase, TestResult, FullResult } from '@playwright/test/reporter';
+
+declare class MidSceneReporter implements Reporter {
+    onBegin(config: FullConfig, suite: Suite): Promise<void>;
+    onTestBegin(test: TestCase, _result: TestResult): void;
+    onTestEnd(test: TestCase, result: TestResult): void;
+    onEnd(result: FullResult): void;
+}
+
+export { MidSceneReporter as default };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/web",
   "description": "Web integration for MidScene.js",
-  "version": "0.0.1",
+  "version": "0.1.0",
   "jsnext:source": "./src/index.ts",
   "main": "./dist/lib/index.js",
   "module": "./dist/es/index.js",
@@ -12,6 +12,11 @@
       "import": "./dist/es/index.js",
       "require": "./dist/lib/index.js"
     },
+    "./playwright-report": {
+      "types": "./dist/types/playwright-report.d.ts",
+      "import": "./dist/es/playwright-report.js",
+      "require": "./dist/lib/inplaywright-report.js"
+    },
     "./constants": {
       "types": "./dist/types/constants.d.ts",
       "import": "./dist/es/constants.js",
@@ -28,6 +33,9 @@
       ".": [
         "./dist/types/index.d.ts"
       ],
+      "playwright-report": [
+        "./dist/types/playwright-report.d.ts"
+      ],
       "constants": [
         "./dist/types/constants.d.ts"
       ],
@@ -36,19 +44,28 @@
       ]
     }
   },
+  "files": [
+    "dist",
+    "README.md"
+  ],
   "dependencies": {
     "openai": "4.47.1",
     "sharp": "0.33.3",
-    "@midscene/core": "0.0.1"
+    "inquirer": "10.1.5",
+    "@midscene/core": "0.1.0",
+    "@midscene/visualizer-report": "0.1.0"
   },
   "devDependencies": {
     "@modern-js/module-tools": "^2.54.2",
+    "js-sha256": "0.11.0",
     "@types/node": "^18.0.0",
     "typescript": "~5.0.4",
     "vitest": "^1.6.0",
     "playwright": "1.44.1",
     "puppeteer": "^22.8.0",
-    "@playwright/test": "1.44.1"
+    "@playwright/test": "1.44.1",
+    "fs-extra": "11.2.0",
+    "@types/fs-extra": "11.0.4"
   },
   "peerDependencies": {
     "@playwright/test": "^1.44.1",
@@ -77,9 +94,13 @@
     "build:pkg": "modern build -c ./modern.config.ts",
     "build:script": "modern build -c ./modern.inspect.config.ts",
     "build:watch": "modern build -w -c ./modern.config.ts & modern build -w -c ./modern.inspect.config.ts",
+    "test": "vitest --run",
+    "test:all": "AITEST=true vitest --run",
     "new": "modern new",
     "upgrade": "modern upgrade",
     "e2e": "playwright test --config=playwright.config.ts",
-    "e2e:ui": "playwright test --config=playwright.config.ts --ui"
+    "e2e:cache": "MIDSCENE_CACHE=true playwright test --config=playwright.config.ts",
+    "e2e:ui": "playwright test --config=playwright.config.ts --ui",
+    "e2e:ui-cache": "MIDSCENE_CACHE=true playwright test --config=playwright.config.ts  --ui"
   }
 }

package/modern.config.ts

@@ -1,13 +0,0 @@
-import { moduleTools, defineConfig } from '@modern-js/module-tools';
-
-export default defineConfig({
-  plugins: [moduleTools()],
-  buildPreset: 'npm-library',
-  buildConfig: {
-    platform: 'node',
-    input: {
-      index: 'src/index.ts',
-    },
-    target: 'es2017',
-  },
-});

package/modern.inspect.config.ts

@@ -1,20 +0,0 @@
-import { moduleTools, defineConfig } from '@modern-js/module-tools';
-
-// It was split into two configuration files because of a bug in the build config array
-export default defineConfig({
-  plugins: [moduleTools()],
-  buildPreset: 'npm-library',
-  buildConfig: {
-    buildType: 'bundle',
-    format: 'iife',
-    input: {
-      htmlElement:'src/html-element/index.ts',
-    },
-    outDir: 'dist/script',
-    esbuildOptions: options => {
-      options.globalName = 'midscene_element_inspector'
-      return options;
-    },
-    target: 'es2017',
-  }
-});

package/playwright.config.ts

@@ -1,42 +0,0 @@
-import { defineConfig, devices } from '@playwright/test';
-
-/**
- * Read environment variables from file.
- * https://github.com/motdotla/dotenv
- */
-// require('dotenv').config();
-
-/**
- * See https://playwright.dev/docs/test-configuration.
- */
-export default defineConfig({
-  testDir: './tests/e2e',
-  timeout: 90000 * 1000,
-  /* Run tests in files in parallel */
-  fullyParallel: true,
-  /* Fail the build on CI if you accidentally left test.only in the source code. */
-  forbidOnly: !!process.env.CI,
-  /* Retry on CI only */
-  retries: process.env.CI ? 2 : 0,
-  /* Opt out of parallel tests on CI. */
-  workers: process.env.CI ? 1 : undefined,
-  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-  reporter: 'html',
-  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
-  use: {
-    /* Base URL to use in actions like `await page.goto('/')`. */
-    // baseURL: 'http://127.0.0.1:3000',
-
-    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
-    trace: 'on-first-retry',
-  },
-
-  /* Configure projects for major browsers */
-  projects: [
-    {
-      name: 'chromium',
-      use: { ...devices['Desktop Chrome'] },
-    },
-  ],
-
-});

package/src/html-element/constants.ts

@@ -1,10 +0,0 @@
-export const TEXT_SIZE_THRESHOLD = 9;
-
-export const TEXT_MAX_SIZE = 40;
-
-export enum NodeType {
-  'INPUT' = 'INPUT Node',
-  'BUTTON' = 'BUTTON Node',
-  'IMG' = 'IMG Node',
-  'TEXT' = 'TEXT Node',
-}

package/src/html-element/debug.ts

@@ -1,3 +0,0 @@
-import { extractTextWithPositionDFS } from '.';
-
-console.log(extractTextWithPositionDFS());

package/src/html-element/dom-util.ts

@@ -1,11 +0,0 @@
-export function isInputElement(node: Node): node is HTMLInputElement {
-  return node instanceof HTMLElement && node.tagName.toLowerCase() === 'input';
-}
-
-export function isButtonElement(node: Node): node is HTMLButtonElement {
-  return node instanceof HTMLElement && node.tagName.toLowerCase() === 'button';
-}
-
-export function isImgElement(node: Node): node is HTMLImageElement {
-  return node instanceof HTMLElement && node.tagName.toLowerCase() === 'img';
-}

package/src/html-element/extractInfo.ts

@@ -1,168 +0,0 @@
-import {
-  getNodeAttributes,
-  getPseudoElementContent,
-  logger,
-  setDataForNode,
-  validTextNodeContent,
-  visibleRect,
-} from './util';
-import { NodeType, TEXT_SIZE_THRESHOLD } from './constants';
-import { isButtonElement, isImgElement, isInputElement } from './dom-util';
-
-interface NodeDescriptor {
-  node: Node;
-  childrens: NodeDescriptor[];
-}
-
-export interface ElementInfo {
-  id: string;
-  locator: string | void;
-  attributes: {
-    nodeType: NodeType;
-    [key: string]: string;
-  };
-  content: string;
-  rect: { left: number; top: number; width: number; height: number };
-  center: [number, number];
-}
-
-const container: HTMLElement = document.body;
-
-function generateId(numberId: number) {
-  //   const letters = 'ABCDEFGHIJKLMNPRSTUVXYZ';
-  //   const numbers = '0123456789';
-  //   const randomLetter = letters.charAt(Math.floor(Math.random() * letters.length)).toUpperCase();
-  // const randomNumber = numbers.charAt(Math.floor(Math.random() * numbers.length));
-  // return randomLetter + numberId;
-  return `${numberId}`;
-}
-
-export function extractTextWithPositionDFS(initNode: Node = container): ElementInfo[] {
-  const elementInfoArray: ElementInfo[] = [];
-  const nodeMapTree: NodeDescriptor = { node: initNode, childrens: [] };
-  let nodeIndex = 1;
-
-  function dfs(node: Node, parentNode: NodeDescriptor | null = null): void {
-    if (!node) {
-      return;
-    }
-
-    const currentNodeDes: NodeDescriptor = { node, childrens: [] };
-    if (parentNode?.childrens) {
-      parentNode.childrens.push(currentNodeDes);
-    }
-
-    collectElementInfo(node);
-
-    // eslint-disable-next-line @typescript-eslint/prefer-for-of
-    for (let i = 0; i < node.childNodes.length; i++) {
-      logger('will dfs', node.childNodes[i]);
-      dfs(node.childNodes[i], currentNodeDes);
-    }
-  }
-
-  function collectElementInfo(node: Node) {
-    const rect = visibleRect(node);
-    if (!rect) {
-      logger('Element is not visible', node);
-      return;
-    }
-
-    if (isInputElement(node)) {
-      const attributes = getNodeAttributes(node);
-      const selector = setDataForNode(node, nodeIndex);
-      elementInfoArray.push({
-        id: generateId(nodeIndex++),
-        locator: selector,
-        attributes: {
-          ...attributes,
-          nodeType: NodeType.INPUT,
-        },
-        content: attributes.placeholder || '',
-        rect,
-        center: [Math.round(rect.left + rect.width / 2), Math.round(rect.top + rect.height / 2)],
-      });
-      return;
-    }
-
-    if (isButtonElement(node)) {
-      const attributes = getNodeAttributes(node);
-      const pseudo = getPseudoElementContent(node);
-      const selector = setDataForNode(node, nodeIndex);
-      elementInfoArray.push({
-        id: generateId(nodeIndex++),
-        locator: selector,
-        attributes: {
-          ...attributes,
-          nodeType: NodeType.BUTTON,
-        },
-        content: node.innerText || pseudo.before || pseudo.after || '',
-        rect,
-        center: [Math.round(rect.left + rect.width / 2), Math.round(rect.top + rect.height / 2)],
-      });
-      return;
-    }
-
-    if (isImgElement(node)) {
-      const attributes = getNodeAttributes(node);
-      const selector = setDataForNode(node, nodeIndex);
-      elementInfoArray.push({
-        id: generateId(nodeIndex++),
-        locator: selector,
-        attributes: {
-          ...attributes,
-          nodeType: NodeType.IMG,
-        },
-        content: '',
-        rect,
-        center: [Math.round(rect.left + rect.width / 2), Math.round(rect.top + rect.height / 2)],
-      });
-      return;
-    }
-
-    //   if (node instanceof HTMLElement && hasOverflowY(node)) {
-    //       const rect = visibleRect(node);
-    //       if (!rect || rect.height < 100) {
-    //           logger('Element is not visible', node);
-    //       } else {
-    //           const attributes = getNodeAttributes(node);
-    //           const selector = setDataForNode(node, nodeIndex);
-    //           elementInfoArray.push({
-    //               id: nodeIndex++,
-    //               nodeType: 'ScrollContainer Node',
-    //               locator: selector!,
-    //               parentIndex,
-    //               attributes,
-    //               content: "",
-    //               rect,
-    //               center: [Math.round(rect.left + rect.width / 2), Math.round(rect.top + rect.height / 2)],
-    //           });
-    //       }
-    //   }
-
-    const text = validTextNodeContent(node);
-    if (text) {
-      if (rect.width < TEXT_SIZE_THRESHOLD || rect.height < TEXT_SIZE_THRESHOLD) {
-        logger('Element is too small', text);
-        return;
-      }
-      const attributes = getNodeAttributes(node);
-      const selector = setDataForNode(node, nodeIndex);
-      elementInfoArray.push({
-        id: generateId(nodeIndex++),
-        attributes: {
-          ...attributes,
-          nodeType: NodeType.TEXT,
-        },
-        locator: selector,
-        center: [Math.round(rect.left + rect.width / 2), Math.round(rect.top + rect.height / 2)],
-        // attributes,
-        content: text,
-        rect,
-      });
-    }
-  }
-
-  dfs(initNode, nodeMapTree);
-  return elementInfoArray;
-}

package/src/html-element/index.ts

@@ -1 +0,0 @@
-export { extractTextWithPositionDFS } from './extractInfo';