@midscene/web 0.0.1

package/dist/script/htmlElement.js

@@ -0,0 +1,272 @@
+"use strict";
+var midscene_element_inspector = (() => {
+  var __defProp = Object.defineProperty;
+  var __defProps = Object.defineProperties;
+  var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+  var __getOwnPropDescs = Object.getOwnPropertyDescriptors;
+  var __getOwnPropNames = Object.getOwnPropertyNames;
+  var __getOwnPropSymbols = Object.getOwnPropertySymbols;
+  var __hasOwnProp = Object.prototype.hasOwnProperty;
+  var __propIsEnum = Object.prototype.propertyIsEnumerable;
+  var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
+  var __spreadValues = (a, b) => {
+    for (var prop in b || (b = {}))
+      if (__hasOwnProp.call(b, prop))
+        __defNormalProp(a, prop, b[prop]);
+    if (__getOwnPropSymbols)
+      for (var prop of __getOwnPropSymbols(b)) {
+        if (__propIsEnum.call(b, prop))
+          __defNormalProp(a, prop, b[prop]);
+      }
+    return a;
+  };
+  var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
+  var __export = (target, all) => {
+    for (var name in all)
+      __defProp(target, name, { get: all[name], enumerable: true });
+  };
+  var __copyProps = (to, from, except, desc) => {
+    if (from && typeof from === "object" || typeof from === "function") {
+      for (let key of __getOwnPropNames(from))
+        if (!__hasOwnProp.call(to, key) && key !== except)
+          __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+    }
+    return to;
+  };
+  var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+  // src/html-element/index.ts
+  var html_element_exports = {};
+  __export(html_element_exports, {
+    extractTextWithPositionDFS: () => extractTextWithPositionDFS
+  });
+
+  // src/html-element/util.ts
+  function logger(...msg) {
+  }
+  var taskIdKey = "_midscene_retrieve_task_id";
+  function selectorForValue(val) {
+    return `[${taskIdKey}='${val}']`;
+  }
+  function setDataForNode(node, nodeIndex) {
+    const taskId = taskIdKey;
+    if (!(node instanceof HTMLElement)) {
+      return "";
+    }
+    if (!taskId) {
+      console.error("No task id found");
+      return "";
+    }
+    const selector = selectorForValue(nodeIndex);
+    node.setAttribute(taskIdKey, nodeIndex.toString());
+    return selector;
+  }
+  function getPseudoElementContent(element) {
+    if (!(element instanceof HTMLElement)) {
+      return { before: "", after: "" };
+    }
+    const beforeContent = window.getComputedStyle(element, "::before").getPropertyValue("content");
+    const afterContent = window.getComputedStyle(element, "::after").getPropertyValue("content");
+    return {
+      before: beforeContent === "none" ? "" : beforeContent.replace(/"/g, ""),
+      after: afterContent === "none" ? "" : afterContent.replace(/"/g, "")
+    };
+  }
+  function visibleRect(el) {
+    if (!el) {
+      logger("Element is not in the DOM hierarchy");
+      return false;
+    }
+    if (!(el instanceof HTMLElement)) {
+      logger("Element is not in the DOM hierarchy");
+      return false;
+    }
+    const style = window.getComputedStyle(el);
+    if (style.display === "none" || style.visibility === "hidden" || style.opacity === "0" && el.tagName !== "INPUT") {
+      logger("Element is hidden");
+      return false;
+    }
+    const rect = el.getBoundingClientRect();
+    if (rect.width === 0 && rect.height === 0) {
+      logger("Element has no size");
+      return false;
+    }
+    const scrollLeft = window.pageXOffset || document.documentElement.scrollLeft;
+    const scrollTop = window.pageYOffset || document.documentElement.scrollTop;
+    const isInViewport = rect.top >= 0 + scrollTop && rect.left >= 0 + scrollLeft && rect.bottom <= (window.innerHeight || document.documentElement.clientHeight) + scrollTop && rect.right <= (window.innerWidth || document.documentElement.clientWidth) + scrollLeft;
+    if (!isInViewport) {
+      logger("Element is not in the viewport");
+      logger(rect, window.innerHeight, window.innerWidth, scrollTop, scrollLeft);
+      return false;
+    }
+    let parent = el;
+    while (parent && parent !== document.body) {
+      const parentStyle = window.getComputedStyle(parent);
+      if (parentStyle.overflow === "hidden") {
+        const parentRect = parent.getBoundingClientRect();
+        const tolerance = 10;
+        if (rect.top < parentRect.top - tolerance || rect.left < parentRect.left - tolerance || rect.bottom > parentRect.bottom + tolerance || rect.right > parentRect.right + tolerance) {
+          logger("Element is clipped by an ancestor", parent, rect, parentRect);
+          return false;
+        }
+      }
+      parent = parent.parentElement;
+    }
+    return {
+      left: Math.round(rect.left - scrollLeft),
+      top: Math.round(rect.top - scrollTop),
+      width: Math.round(rect.width),
+      height: Math.round(rect.height)
+    };
+  }
+  function validTextNodeContent(node) {
+    if (!node) {
+      return false;
+    }
+    console.log("node", node);
+    if (node.nodeType === Node.COMMENT_NODE) {
+      return false;
+    }
+    const everyChildNodeIsText = Array.from(node.childNodes).findIndex(
+      (child) => child.nodeType === Node.TEXT_NODE
+    );
+    if (everyChildNodeIsText === -1) {
+      return false;
+    }
+    const content = node.textContent || node.innerText;
+    if (content && !/^\s*$/.test(content)) {
+      return content.trim();
+    }
+    return false;
+  }
+  function getNodeAttributes(node) {
+    if (!node || !(node instanceof HTMLElement) || !node.attributes) {
+      return {};
+    }
+    const attributesList = Array.from(node.attributes).map((attr) => {
+      if (attr.name === "class") {
+        return [attr.name, `.${attr.value.split(" ").join(".")}`];
+      }
+      if (!attr.value) {
+        return [];
+      }
+      return [attr.name, attr.value];
+    });
+    return Object.fromEntries(attributesList);
+  }
+
+  // src/html-element/constants.ts
+  var TEXT_SIZE_THRESHOLD = 9;
+
+  // src/html-element/dom-util.ts
+  function isInputElement(node) {
+    return node instanceof HTMLElement && node.tagName.toLowerCase() === "input";
+  }
+  function isButtonElement(node) {
+    return node instanceof HTMLElement && node.tagName.toLowerCase() === "button";
+  }
+  function isImgElement(node) {
+    return node instanceof HTMLElement && node.tagName.toLowerCase() === "img";
+  }
+
+  // src/html-element/extractInfo.ts
+  var container = document.body;
+  function generateId(numberId) {
+    return `${numberId}`;
+  }
+  function extractTextWithPositionDFS(initNode = container) {
+    const elementInfoArray = [];
+    const nodeMapTree = { node: initNode, childrens: [] };
+    let nodeIndex = 1;
+    function dfs(node, parentNode = null) {
+      if (!node) {
+        return;
+      }
+      const currentNodeDes = { node, childrens: [] };
+      if (parentNode == null ? void 0 : parentNode.childrens) {
+        parentNode.childrens.push(currentNodeDes);
+      }
+      collectElementInfo(node);
+      for (let i = 0; i < node.childNodes.length; i++) {
+        logger("will dfs", node.childNodes[i]);
+        dfs(node.childNodes[i], currentNodeDes);
+      }
+    }
+    function collectElementInfo(node) {
+      const rect = visibleRect(node);
+      if (!rect) {
+        logger("Element is not visible", node);
+        return;
+      }
+      if (isInputElement(node)) {
+        const attributes = getNodeAttributes(node);
+        const selector = setDataForNode(node, nodeIndex);
+        elementInfoArray.push({
+          id: generateId(nodeIndex++),
+          locator: selector,
+          attributes: __spreadProps(__spreadValues({}, attributes), {
+            nodeType: "INPUT Node" /* INPUT */
+          }),
+          content: attributes.placeholder || "",
+          rect,
+          center: [Math.round(rect.left + rect.width / 2), Math.round(rect.top + rect.height / 2)]
+        });
+        return;
+      }
+      if (isButtonElement(node)) {
+        const attributes = getNodeAttributes(node);
+        const pseudo = getPseudoElementContent(node);
+        const selector = setDataForNode(node, nodeIndex);
+        elementInfoArray.push({
+          id: generateId(nodeIndex++),
+          locator: selector,
+          attributes: __spreadProps(__spreadValues({}, attributes), {
+            nodeType: "BUTTON Node" /* BUTTON */
+          }),
+          content: node.innerText || pseudo.before || pseudo.after || "",
+          rect,
+          center: [Math.round(rect.left + rect.width / 2), Math.round(rect.top + rect.height / 2)]
+        });
+        return;
+      }
+      if (isImgElement(node)) {
+        const attributes = getNodeAttributes(node);
+        const selector = setDataForNode(node, nodeIndex);
+        elementInfoArray.push({
+          id: generateId(nodeIndex++),
+          locator: selector,
+          attributes: __spreadProps(__spreadValues({}, attributes), {
+            nodeType: "IMG Node" /* IMG */
+          }),
+          content: "",
+          rect,
+          center: [Math.round(rect.left + rect.width / 2), Math.round(rect.top + rect.height / 2)]
+        });
+        return;
+      }
+      const text = validTextNodeContent(node);
+      if (text) {
+        if (rect.width < TEXT_SIZE_THRESHOLD || rect.height < TEXT_SIZE_THRESHOLD) {
+          logger("Element is too small", text);
+          return;
+        }
+        const attributes = getNodeAttributes(node);
+        const selector = setDataForNode(node, nodeIndex);
+        elementInfoArray.push({
+          id: generateId(nodeIndex++),
+          attributes: __spreadProps(__spreadValues({}, attributes), {
+            nodeType: "TEXT Node" /* TEXT */
+          }),
+          locator: selector,
+          center: [Math.round(rect.left + rect.width / 2), Math.round(rect.top + rect.height / 2)],
+          // attributes,
+          content: text,
+          rect
+        });
+      }
+    }
+    dfs(initNode, nodeMapTree);
+    return elementInfoArray;
+  }
+  return __toCommonJS(html_element_exports);
+})();

package/dist/script/types/htmlElement.d.ts

@@ -0,0 +1,26 @@
+declare enum NodeType {
+    'INPUT' = "INPUT Node",
+    'BUTTON' = "BUTTON Node",
+    'IMG' = "IMG Node",
+    'TEXT' = "TEXT Node"
+}
+
+interface ElementInfo {
+    id: string;
+    locator: string | void;
+    attributes: {
+        nodeType: NodeType;
+        [key: string]: string;
+    };
+    content: string;
+    rect: {
+        left: number;
+        top: number;
+        width: number;
+        height: number;
+    };
+    center: [number, number];
+}
+declare function extractTextWithPositionDFS(initNode?: Node): ElementInfo[];
+
+export { extractTextWithPositionDFS };

package/dist/types/index.d.ts

@@ -0,0 +1,66 @@
+import { TestInfo } from '@playwright/test';
+import { Page } from 'playwright';
+import Insight, { BaseElement, Rect, Executor, ExecutionDump, InsightExtractParam } from '@midscene/core';
+
+declare enum NodeType {
+    'INPUT' = "INPUT Node",
+    'BUTTON' = "BUTTON Node",
+    'IMG' = "IMG Node",
+    'TEXT' = "TEXT Node"
+}
+
+declare class WebElementInfo implements BaseElement {
+    content: string;
+    locator: string;
+    rect: Rect;
+    center: [number, number];
+    page: Page;
+    id: string;
+    attributes: {
+        ['nodeType']: NodeType;
+        [key: string]: string;
+    };
+    constructor({ content, rect, page, locator, id, attributes, }: {
+        content: string;
+        rect: Rect;
+        page: Page;
+        locator: string;
+        id: string;
+        attributes: {
+            ['nodeType']: NodeType;
+            [key: string]: string;
+        };
+    });
+    tap(): Promise<void>;
+    hover(): Promise<void>;
+    type(text: string): Promise<void>;
+    press(key: Parameters<typeof this$1.page.keyboard.press>[0]): Promise<void>;
+}
+
+declare class PlayWrightActionAgent {
+    page: Page;
+    insight: Insight<WebElementInfo>;
+    executor: Executor;
+    actionDump?: ExecutionDump;
+    constructor(page: Page, opt?: {
+        taskName?: string;
+    });
+    private recordScreenshot;
+    private wrapExecutorWithScreenshot;
+    private convertPlanToExecutable;
+    action(userPrompt: string): Promise<void>;
+    query(demand: InsightExtractParam): Promise<any>;
+}
+
+declare const PlaywrightAiFixture: () => {
+    ai: ({ page }: any, use: any, testInfo: TestInfo) => Promise<void>;
+    aiAction: ({ page }: any, use: any, testInfo: TestInfo) => Promise<void>;
+    aiQuery: ({ page }: any, use: any, testInfo: TestInfo) => Promise<void>;
+};
+type PlayWrightAiFixtureType = {
+    ai: <T = any>(prompt: string, type?: 'action' | 'query') => Promise<T>;
+    aiAction: (taskPrompt: string) => ReturnType<PlayWrightActionAgent['action']>;
+    aiQuery: <T = any>(demand: any) => Promise<T>;
+};
+
+export { type PlayWrightAiFixtureType, PlaywrightAiFixture };

package/modern.config.ts

@@ -0,0 +1,13 @@
+import { moduleTools, defineConfig } from '@modern-js/module-tools';
+
+export default defineConfig({
+  plugins: [moduleTools()],
+  buildPreset: 'npm-library',
+  buildConfig: {
+    platform: 'node',
+    input: {
+      index: 'src/index.ts',
+    },
+    target: 'es2017',
+  },
+});

package/modern.inspect.config.ts

@@ -0,0 +1,20 @@
+import { moduleTools, defineConfig } from '@modern-js/module-tools';
+
+// It was split into two configuration files because of a bug in the build config array
+export default defineConfig({
+  plugins: [moduleTools()],
+  buildPreset: 'npm-library',
+  buildConfig: {
+    buildType: 'bundle',
+    format: 'iife',
+    input: {
+      htmlElement:'src/html-element/index.ts',
+    },
+    outDir: 'dist/script',
+    esbuildOptions: options => {
+      options.globalName = 'midscene_element_inspector'
+      return options;
+    },
+    target: 'es2017',
+  }
+});

package/package.json

@@ -0,0 +1,85 @@
+{
+  "name": "@midscene/web",
+  "description": "Web integration for MidScene.js",
+  "version": "0.0.1",
+  "jsnext:source": "./src/index.ts",
+  "main": "./dist/lib/index.js",
+  "module": "./dist/es/index.js",
+  "types": "./dist/types/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/types/index.d.ts",
+      "import": "./dist/es/index.js",
+      "require": "./dist/lib/index.js"
+    },
+    "./constants": {
+      "types": "./dist/types/constants.d.ts",
+      "import": "./dist/es/constants.js",
+      "require": "./dist/lib/constants.js"
+    },
+    "./html-element": {
+      "types": "./dist/types/html-element/index.d.ts",
+      "import": "./dist/es/html-element/index.js",
+      "require": "./dist/lib/html-element/index.js"
+    }
+  },
+  "typesVersions": {
+    "*": {
+      ".": [
+        "./dist/types/index.d.ts"
+      ],
+      "constants": [
+        "./dist/types/constants.d.ts"
+      ],
+      "html-element": [
+        "./dist/types/html-element/index.d.ts"
+      ]
+    }
+  },
+  "dependencies": {
+    "openai": "4.47.1",
+    "sharp": "0.33.3",
+    "@midscene/core": "0.0.1"
+  },
+  "devDependencies": {
+    "@modern-js/module-tools": "^2.54.2",
+    "@types/node": "^18.0.0",
+    "typescript": "~5.0.4",
+    "vitest": "^1.6.0",
+    "playwright": "1.44.1",
+    "puppeteer": "^22.8.0",
+    "@playwright/test": "1.44.1"
+  },
+  "peerDependencies": {
+    "@playwright/test": "^1.44.1",
+    "playwright": "^1.44.1",
+    "puppeteer": "^22.8.0"
+  },
+  "peerDependenciesMeta": {
+    "@playwright/test": {
+      "optional": true
+    },
+    "puppeteer": {
+      "optional": true
+    }
+  },
+  "engines": {
+    "node": ">=16.0.0"
+  },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org"
+  },
+  "license": "MIT",
+  "scripts": {
+    "dev": "modern dev",
+    "build": "npm run build:pkg && npm run build:script",
+    "build:pkg": "modern build -c ./modern.config.ts",
+    "build:script": "modern build -c ./modern.inspect.config.ts",
+    "build:watch": "modern build -w -c ./modern.config.ts & modern build -w -c ./modern.inspect.config.ts",
+    "new": "modern new",
+    "upgrade": "modern upgrade",
+    "e2e": "playwright test --config=playwright.config.ts",
+    "e2e:ui": "playwright test --config=playwright.config.ts --ui"
+  }
+}

package/playwright.config.ts

@@ -0,0 +1,42 @@
+import { defineConfig, devices } from '@playwright/test';
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+// require('dotenv').config();
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: './tests/e2e',
+  timeout: 90000 * 1000,
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: 'html',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    // baseURL: 'http://127.0.0.1:3000',
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+  ],
+
+});

package/src/html-element/constants.ts

@@ -0,0 +1,10 @@
+export const TEXT_SIZE_THRESHOLD = 9;
+
+export const TEXT_MAX_SIZE = 40;
+
+export enum NodeType {
+  'INPUT' = 'INPUT Node',
+  'BUTTON' = 'BUTTON Node',
+  'IMG' = 'IMG Node',
+  'TEXT' = 'TEXT Node',
+}

package/src/html-element/debug.ts

@@ -0,0 +1,3 @@
+import { extractTextWithPositionDFS } from '.';
+
+console.log(extractTextWithPositionDFS());

package/src/html-element/dom-util.ts

@@ -0,0 +1,11 @@
+export function isInputElement(node: Node): node is HTMLInputElement {
+  return node instanceof HTMLElement && node.tagName.toLowerCase() === 'input';
+}
+
+export function isButtonElement(node: Node): node is HTMLButtonElement {
+  return node instanceof HTMLElement && node.tagName.toLowerCase() === 'button';
+}
+
+export function isImgElement(node: Node): node is HTMLImageElement {
+  return node instanceof HTMLElement && node.tagName.toLowerCase() === 'img';
+}