@midscene/web 0.0.1 → 0.1.1

package/src/puppeteer/utils.ts

@@ -1,116 +0,0 @@
-// import { readFileSync } from 'fs';
-// import { Buffer } from 'buffer';
-// import assert from 'assert';
-// import type { Browser, Page } from 'puppeteer';
-// import type { Page as PlaywrightPage } from 'playwright';
-// import { Element } from './index';
-// import { alignCoordByTrim, base64Encoded, imageInfoOfBase64 } from '@/image';
-// import { UIContext, PuppeteerParserOpt, PlaywrightParserOpt, Rect, BaseElement } from '@/types';
-// import { getTmpFile } from '@/utils';
-// import { pageScriptToGetTexts } from '@/query';
-// import { describeUserPage } from '@/insight/prompt';
-
-// export interface TextElement {
-//   content: string;
-//   rect: Rect;
-//   center: [number, number]; // center coordinates as [rect.left + rect.width/2, rect.top + rect.height/2], use this for better control of page
-//   locator: string;
-// }
-
-// export async function alignTextElements(
-//   screenshotBuffer: Buffer,
-//   elements: TextElement[],
-// ): Promise<TextElement[]> {
-//   const textsAligned: TextElement[] = [];
-//   for (const item of elements) {
-//     const { rect } = item;
-//     const aligned = await alignCoordByTrim(screenshotBuffer, rect);
-//     item.rect = aligned;
-//     item.center = [
-//       Math.round(aligned.left + aligned.width / 2),
-//       Math.round(aligned.top + aligned.height / 2),
-//     ];
-//     textsAligned.push(item);
-//   }
-//   return textsAligned;
-// }
-
-// async function extractDataFromPage(page: Page, opt?: PuppeteerParserOpt): Promise<UIContext<Element>> {
-//   assert(page, 'page is required');
-//   const file = getTmpFile('jpeg');
-//   await page.screenshot({ path: file, type: 'jpeg', quality: 75 });
-//   const screenshotBuffer = readFileSync(file);
-//   const screenshotBase64 = base64Encoded(file);
-//   const size = await imageInfoOfBase64(screenshotBase64);
-
-//   const scripts = pageScriptToGetTexts(opt?.selector);
-//   const texts = (await page.evaluate(scripts)) as BaseElement[];
-
-//   // align texts
-//   const textsAligned = await alignTextElements(screenshotBuffer, texts);
-
-//   const baseElements = textsAligned.map((item) => {
-//     const { center, ...res } = item;
-//     return new Element(res);
-//   });
-
-//   const basicContext = {
-//     screenshotBase64,
-//     size,
-//     content: baseElements,
-//   };
-
-//   return {
-//     ...basicContext,
-//     describer: async () => {
-//       return describeUserPage(basicContext);
-//     },
-//   };
-// }
-
-// export async function parseContextFromPuppeteerPage(
-//   page: Page,
-//   opt?: PuppeteerParserOpt,
-// ): Promise<UIContext<Element>> {
-//   return extractDataFromPage(page, opt);
-// }
-
-// export async function parseContextFromPuppeteerBrowser(browser: Browser): Promise<UIContext<Element>> {
-//   const pages = await browser.pages();
-//   let visiblePage: Page;
-//   if (!pages.length) {
-//     throw new Error('No page found in the puppeteer browser');
-//   } else if (pages.length === 1) {
-//     visiblePage = pages[0];
-
-//     // filter a visible page, otherwise use the last one
-//   } else {
-//     const candidates = [];
-//     for (const page of pages) {
-//       // eslint-disable-next-line @typescript-eslint/no-loop-func
-//       const isVisible = await page.evaluate(() => document.visibilityState === 'visible');
-//       if (isVisible) {
-//         candidates.push(page);
-//       }
-//     }
-//     if (candidates.length === 0) {
-//       const lastUrl = pages[pages.length - 1].url();
-//       console.warn(`There are no visible pages, use the last one (${lastUrl})`);
-//       visiblePage = candidates[candidates.length - 1];
-//     } else if (candidates.length === 1) {
-//       visiblePage = candidates[0];
-//     } else {
-//       const lastUrl = pages[pages.length - 1].url();
-//       console.warn(`Multiple visible pages found, use the last one (${lastUrl})`);
-//       visiblePage = candidates[candidates.length - 1];
-//     }
-//   }
-//   return parseContextFromPuppeteerPage(visiblePage);
-// }
-
-// export async function parseContextFromPlaywrightPage(
-//   page: PlaywrightPage,
-//   opt?: PlaywrightParserOpt,
-// ): Promise<UIContext<Element>> {
-//   return extractDataFromPage(page as any as Page, opt); // seems key APIs are the same ?
-// }

package/tests/e2e/ai-auto-todo.spec.ts

@@ -1,24 +0,0 @@
-import { expect } from 'playwright/test';
-import { test } from './fixture';
-
-test.beforeEach(async ({ page }) => {
-  await page.goto('https://todomvc.com/examples/react/dist/');
-});
-
-test('ai todo', async ({ ai, aiQuery }) => {
-  await ai('Enter "Learn JS today" in the task box, then press Enter to create');
-  await ai('Enter "Learn Rust tomorrow" in the task box, then press Enter to create');
-  await ai('Enter "Learning AI the day after tomorrow" in the task box, then press Enter to create');
-  await ai(
-    'Move your mouse over the second item in the task list and click the Delete button to the right of the second task',
-  );
-  await ai('Click the check button to the left of the second task');
-  await ai('Click the completed Status button below the task list');
-
-  const taskList = await aiQuery<string[]>('string[], tasks in the list');
-  expect(taskList.length).toBe(1);
-  expect(taskList[0]).toBe('Learning AI the day after tomorrow');
-
-  const placeholder = await ai('string, return the placeholder text in the input box', 'query');
-  expect(placeholder).toBe('What needs to be done?');
-});

package/tests/e2e/ai-xicha.spec.ts

@@ -1,34 +0,0 @@
-import { test } from './fixture';
-
-test.beforeEach(async ({ page }) => {
-  page.setViewportSize({ width: 400, height: 905 });
-  await page.goto('https://heyteavivocity.meuu.online/home');
-  await page.waitForLoadState('networkidle');
-});
-
-test('ai order', async ({ ai }) => {
-  await ai('点击左上角语言切换按钮(英文、中文)，在弹出的下拉列表中点击中文');
-  await ai('在向下滚动一屏');
-  await ai('直接点击多肉葡萄的规格按钮');
-  await ai('点击不使用吸管、点击冰沙推荐、点击正常冰推荐');
-  await ai('在向下滚动一屏');
-  await ai('点击标准甜、点击绿妍（推荐）、点击标准口味');
-  await ai('滚动到最下面');
-  await ai('点击选好了按钮');
-  await ai('点击右上角商品图标按钮');
-
-  // 随便滚动一下
-  await ai('滚动到最下面');
-
-  // const content = await aiQuery(query('购物车商品详情', {
-  //   productName: "商品名称，在价格上面",
-  //   productPrice: "商品价格",
-  //   productDescription: "商品描述（饮品的各种参数，吸管、冰沙等），在价格下面",
-  // }));
-
-  // console.log('商品订单详情：', {
-  //   productName: content.productName,
-  //   productPrice: content.productPrice,
-  //   productDescription: content.productDescription,
-  // });
-});

package/tests/e2e/fixture.ts

@@ -1,6 +0,0 @@
-// my-test.ts
-import { test as base } from '@playwright/test';
-import type { PlayWrightAiFixtureType } from '../../src';
-import { PlaywrightAiFixture } from '../../src';
-
-export const test = base.extend<PlayWrightAiFixtureType>(PlaywrightAiFixture());

package/tests/e2e/generate-test-data.spec.ts

@@ -1,60 +0,0 @@
-import { test } from '@playwright/test';
-import { generateTestData, generateTestDataPath } from './tool';
-
-test('generate todo test data', async ({ page }) => {
-  await page.goto('https://todomvc.com/examples/react/dist/');
-  // Add data
-  await page.getByTestId('text-input').click();
-  await page.keyboard.type('Learn Python');
-  await page.keyboard.press('Enter');
-  await page.getByTestId('text-input').click();
-  await page.keyboard.type('Learn Rust');
-  await page.keyboard.press('Enter');
-  await page.getByTestId('text-input').click();
-  await page.keyboard.type('Learn AI');
-  await page.keyboard.press('Enter');
-  await page.getByText('Learn Rust').hover();
-
-  const midsceneTestDataPath = generateTestDataPath('todo');
-  const buffer = await page.screenshot();
-
-  const base64String = buffer.toString('base64');
-  await generateTestData(page, midsceneTestDataPath, base64String);
-});
-
-test('generate visualstudio test data', async ({ page }) => {
-  await page.goto('https://code.visualstudio.com/');
-  await page.waitForLoadState('networkidle');
-
-  const midsceneTestDataPath = generateTestDataPath('visualstudio');
-  const buffer = await page.screenshot();
-
-  const base64String = buffer.toString('base64');
-  await generateTestData(page, midsceneTestDataPath, base64String);
-});
-
-test('generate githubstatus test data', async ({ page }) => {
-  await page.setViewportSize({ width: 1920, height: 1080 });
-  await page.goto('https://www.githubstatus.com/');
-  await page.waitForLoadState('networkidle');
-
-  const midsceneTestDataPath = generateTestDataPath('githubstatus');
-  const buffer = await page.screenshot();
-
-  const base64String = buffer.toString('base64');
-  await generateTestData(page, midsceneTestDataPath, base64String);
-});
-
-test('generate xicha test data', async ({ page }) => {
-  page.setViewportSize({ width: 400, height: 905 });
-  await page.goto('https://heyteavivocity.meuu.online/home');
-  await page.evaluate('window.localStorage.setItem("LOCALE", "zh-CN")');
-  await page.goto('https://heyteavivocity.meuu.online/home');
-  await page.waitForLoadState('networkidle');
-
-  const midsceneTestDataPath = generateTestDataPath('xicha');
-  const buffer = await page.screenshot();
-
-  const base64String = buffer.toString('base64');
-  await generateTestData(page, midsceneTestDataPath, base64String);
-});

package/tests/e2e/todo-app-midscene.spec.ts

@@ -1,98 +0,0 @@
-// import { test, expect, type Page } from '@playwright/test';
-// import Insight, { TextElement, query } from 'midscene';
-// import { retrieveElements, retrieveOneElement } from 'midscene/query';
-
-// test.beforeEach(async ({ page }) => {
-//   await page.goto('https://todomvc.com/examples/react/dist/');
-// });
-
-// const TODO_ITEMS = ['buy some cheese', 'feed the cat', 'book a doctors appointment'];
-
-// interface InputBoxSection {
-//   element: TextElement;
-//   toggleAllBtn: TextElement;
-//   placeholder: string;
-//   inputValue: string;
-// }
-
-// interface TodoItem {
-//   name: string;
-//   finished: boolean;
-// }
-
-// interface ControlLayerSection {
-//   numbersLeft: number;
-//   tipElement: TextElement;
-//   controlElements: TextElement[];
-// }
-
-// // A comprehensive parser for page content
-// const parsePage = async (page: Page) => {
-//   const insight = await Insight.fromPlaywrightPage(page);
-//   const todoListPage = await insight.segment({
-//     'input-box': query<InputBoxSection>('an input box to type item and a "toggle-all" button', {
-//       element: retrieveOneElement('input box'),
-//       toggleAllBtn: retrieveOneElement('toggle all button, if exists'),
-//       placeholder: 'placeholder string in the input box, string, if exists',
-//       inputValue: 'the value in the input box, string, if exists',
-//     }),
-//     'todo-list': query<{ todoItems: TodoItem[] }>('a list with todo-data (if exists)', {
-//       todoItems: '{name: string, finished: boolean}[]',
-//     }),
-//     'control-layer': query<ControlLayerSection>('status and control layer of todo (if exists)', {
-//       numbersLeft: 'number',
-//       tipElement: retrieveOneElement(
-//         'the element indicates the number of remaining items, like `xx items left`',
-//       ),
-//       controlElements: retrieveElements('control elements, used to filter items'),
-//     }),
-//   });
-
-//   return todoListPage;
-// };
-
-// test.describe('New Todo', () => {
-//   test('should allow me to add todo items', async ({ page }) => {
-//     // add a todo item
-//     const todoPage = await parsePage(page);
-//     const inputBox = todoPage['input-box'];
-//     expect(inputBox).toBeTruthy();
-
-//     await page.mouse.click(...inputBox!.element.center);
-//     await page.keyboard.type(TODO_ITEMS[0], { delay: 100 });
-//     await page.keyboard.press('Enter');
-
-//     // update page parsing result, and check the interface
-//     const todoPage2 = await parsePage(page);
-//     expect(todoPage2['input-box'].inputValue).toBeFalsy();
-//     expect(todoPage2['input-box'].placeholder).toBeTruthy();
-//     expect(todoPage2['todo-list'].todoItems.length).toBe(1);
-//     expect(todoPage2['todo-list'].todoItems[0].name).toBe(TODO_ITEMS[0]);
-
-//     // add another item
-//     await page.mouse.click(...todoPage2['input-box'].element.center);
-//     await page.keyboard.type(TODO_ITEMS[1], { delay: 100 });
-//     await page.keyboard.press('Enter');
-
-//     // update page parsing result
-//     const todoPage3 = await parsePage(page);
-//     const items = todoPage3['todo-list'].todoItems;
-//     expect(items.length).toBe(2);
-//     expect(items[1].name).toEqual(TODO_ITEMS[1]);
-//     expect(items.some((item) => item.finished)).toBeFalsy();
-//     expect(todoPage3['control-layer'].numbersLeft).toBe(2);
-
-//     // will mark all as completed
-//     const toggleBtn = todoPage3['input-box'].toggleAllBtn;
-//     expect(toggleBtn).toBeTruthy();
-//     expect(todoPage3['todo-list'].todoItems.filter((item) => item.finished).length).toBe(0);
-
-//     await page.mouse.click(...toggleBtn!.center, { delay: 500 });
-//     await page.waitForTimeout(3000);
-
-//     const todoPage4 = await parsePage(page);
-//     const allItems = todoPage4['todo-list'].todoItems;
-//     expect(allItems.length).toBe(2);
-//     expect(allItems.filter((item) => item.finished).length).toBe(allItems.length);
-//   });
-// });

package/tests/e2e/tool.ts

@@ -1,63 +0,0 @@
-import { existsSync, mkdirSync, writeFileSync } from 'fs';
-import path from 'path';
-import { Page as PlaywrightPage } from '@playwright/test';
-import { resizeImg, saveBase64Image } from '@midscene/core/image';
-import { getElementInfos } from '@/img/util';
-import { processImageElementInfo } from '@/img/img';
-
-export async function generateTestData(page: PlaywrightPage, targetDir: string, inputImgBase64: string) {
-  const { elementsPostionInfo, captureElementSnapshot, elementsPostionInfoWithoutText } =
-    await getElementInfos(page);
-
-  const inputImagePath = path.join(targetDir, 'input.png');
-  const outputImagePath = path.join(targetDir, 'output.png');
-  const outputWithoutTextImgPath = path.join(targetDir, 'output_without_text.png');
-  const resizeOutputImgPath = path.join(targetDir, 'resize-output.png');
-  const snapshotJsonPath = path.join(targetDir, 'element-snapshot.json');
-
-  const { compositeElementInfoImgBase64, compositeElementInfoImgWithoutTextBase64 } =
-    await processImageElementInfo({
-      elementsPostionInfo,
-      elementsPostionInfoWithoutText,
-      inputImgBase64,
-    });
-
-  const resizeImgBase64 = await resizeImg(inputImgBase64);
-
-  writeFileSyncWithDir(snapshotJsonPath, JSON.stringify(captureElementSnapshot, null, 2));
-  await saveBase64Image({ base64Data: inputImgBase64, outputPath: inputImagePath });
-  await saveBase64Image({ base64Data: compositeElementInfoImgBase64, outputPath: outputImagePath });
-  await saveBase64Image({
-    base64Data: compositeElementInfoImgWithoutTextBase64,
-    outputPath: outputWithoutTextImgPath,
-  });
-  await saveBase64Image({ base64Data: resizeImgBase64, outputPath: resizeOutputImgPath });
-}
-
-export function generateTestDataPath(testDataName: string) {
-  // `dist/lib/index.js` Is the default export path
-  const modulePath = require.resolve('@midscene/core').replace('dist/lib/index.js', '');
-  const midsceneTestDataPath = path.join(modulePath, `tests/ai-model/inspector/test-data/${testDataName}`);
-
-  return midsceneTestDataPath;
-}
-
-function ensureDirectoryExistence(filePath: string) {
-  const dirname = path.dirname(filePath);
-  if (existsSync(dirname)) {
-    return;
-  }
-  ensureDirectoryExistence(dirname);
-  mkdirSync(dirname);
-}
-
-type WriteFileSyncParams = Parameters<typeof writeFileSync>;
-
-export function writeFileSyncWithDir(
-  filePath: string,
-  content: WriteFileSyncParams[1],
-  options: WriteFileSyncParams[2] = {},
-) {
-  ensureDirectoryExistence(filePath);
-  writeFileSync(filePath, content, options);
-}

package/tsconfig.json

@@ -1,23 +0,0 @@
-{
-  "compilerOptions": {
-    "baseUrl": ".",
-    "declaration": true,
-    "emitDeclarationOnly": true,
-    "esModuleInterop": true,
-    "forceConsistentCasingInFileNames": true,
-    "isolatedModules": true,
-    "jsx": "preserve",
-    "lib": ["DOM", "ESNext"],
-    "moduleResolution": "node",
-    "paths": {
-      "@/*": ["./src/*"]
-    },
-    "target": "ES2017",
-    "resolveJsonModule": true,
-    "rootDir": "./",
-    "skipLibCheck": true,
-    "strict": true
-  },
-  "exclude": [ "node_modules"],
-  "include": ["src", "tests", "./playwright.config.ts"]
-}

package/vitest.config.ts

@@ -1,14 +0,0 @@
-import { defineConfig } from 'vitest/config';
-import path from 'path';
-
-export default defineConfig({
-  resolve: {
-    alias: {
-      '@': path.resolve(__dirname, 'src'),
-    },
-  },
-  test: {
-    // 你的其他配置...
-    include: [],
-  },
-});