@midscene/web 0.0.1

package/dist/lib/index.js

@@ -0,0 +1,473 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __defProps = Object.defineProperties;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropDescs = Object.getOwnPropertyDescriptors;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getOwnPropSymbols = Object.getOwnPropertySymbols;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __propIsEnum = Object.prototype.propertyIsEnumerable;
+var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
+var __spreadValues = (a, b) => {
+  for (var prop in b || (b = {}))
+    if (__hasOwnProp.call(b, prop))
+      __defNormalProp(a, prop, b[prop]);
+  if (__getOwnPropSymbols)
+    for (var prop of __getOwnPropSymbols(b)) {
+      if (__propIsEnum.call(b, prop))
+        __defNormalProp(a, prop, b[prop]);
+    }
+  return a;
+};
+var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var src_exports = {};
+__export(src_exports, {
+  PlaywrightAiFixture: () => PlaywrightAiFixture
+});
+module.exports = __toCommonJS(src_exports);
+
+// src/playwright/index.ts
+var import_utils3 = require("@midscene/core/utils");
+
+// src/playwright/actions.ts
+var import_assert2 = __toESM(require("assert"));
+var import_core = __toESM(require("@midscene/core"));
+var import_utils = require("@midscene/core/utils");
+var import_image2 = require("@midscene/core/image");
+
+// src/playwright/utils.ts
+var import_fs = __toESM(require("fs"));
+var import_assert = __toESM(require("assert"));
+var import_path = __toESM(require("path"));
+var import_image = require("@midscene/core/image");
+
+// src/playwright/element.ts
+var WebElementInfo = class {
+  constructor({
+    content,
+    rect,
+    page,
+    locator,
+    id,
+    attributes
+  }) {
+    this.content = content;
+    this.rect = rect;
+    this.center = [Math.floor(rect.left + rect.width / 2), Math.floor(rect.top + rect.height / 2)];
+    this.page = page;
+    this.locator = locator;
+    this.id = id;
+    this.attributes = attributes;
+  }
+  async tap() {
+    await this.page.mouse.click(this.center[0], this.center[1]);
+  }
+  async hover() {
+    await this.page.mouse.move(this.center[0], this.center[1]);
+  }
+  async type(text) {
+    await this.page.keyboard.type(text);
+  }
+  async press(key) {
+    await this.page.keyboard.press(key);
+  }
+};
+
+// src/playwright/utils.ts
+async function parseContextFromPlaywrightPage(page, _opt) {
+  (0, import_assert.default)(page, "page is required");
+  const file = "/Users/bytedance/workspace/midscene/packages/midscene/tests/fixtures/heytea.jpeg";
+  await page.screenshot({ path: file, type: "jpeg", quality: 75 });
+  const screenshotBuffer = (0, import_fs.readFileSync)(file);
+  const screenshotBase64 = (0, import_image.base64Encoded)(file);
+  const captureElementSnapshot = await getElementInfosFromPage(page);
+  const elementsInfo = await alignElements(screenshotBuffer, captureElementSnapshot, page);
+  const size = await (0, import_image.imageInfoOfBase64)(screenshotBase64);
+  return {
+    content: elementsInfo,
+    size,
+    screenshotBase64
+  };
+}
+async function getElementInfosFromPage(page) {
+  const pathDir = findNearestPackageJson(__dirname);
+  (0, import_assert.default)(pathDir, `can't find pathDir, with ${__dirname}`);
+  const scriptPath = import_path.default.join(pathDir, "./dist/script/htmlElement.js");
+  const elementInfosScriptContent = (0, import_fs.readFileSync)(scriptPath, "utf-8");
+  const extraReturnLogic = `${elementInfosScriptContent}midscene_element_inspector.extractTextWithPositionDFS()`;
+  const captureElementSnapshot = await page.evaluate(extraReturnLogic);
+  return captureElementSnapshot;
+}
+async function alignElements(screenshotBuffer, elements, page) {
+  const textsAligned = [];
+  for (const item of elements) {
+    const { rect } = item;
+    const aligned = await (0, import_image.alignCoordByTrim)(screenshotBuffer, rect);
+    item.rect = aligned;
+    item.center = [
+      Math.round(aligned.left + aligned.width / 2),
+      Math.round(aligned.top + aligned.height / 2)
+    ];
+    textsAligned.push(
+      new WebElementInfo(__spreadProps(__spreadValues({}, item), {
+        page
+      }))
+    );
+  }
+  return textsAligned;
+}
+function findNearestPackageJson(dir) {
+  const packageJsonPath = import_path.default.join(dir, "package.json");
+  if (import_fs.default.existsSync(packageJsonPath)) {
+    return dir;
+  }
+  const parentDir = import_path.default.dirname(dir);
+  if (parentDir === dir) {
+    return null;
+  }
+  return findNearestPackageJson(parentDir);
+}
+
+// src/playwright/actions.ts
+var PlayWrightActionAgent = class {
+  constructor(page, opt) {
+    this.page = page;
+    this.insight = new import_core.default(async () => {
+      return await parseContextFromPlaywrightPage(page);
+    });
+    this.executor = new import_core.Executor((opt == null ? void 0 : opt.taskName) || "MidScene - PlayWrightAI");
+  }
+  async recordScreenshot(timing) {
+    const file = (0, import_utils.getTmpFile)("jpeg");
+    await this.page.screenshot(__spreadProps(__spreadValues({}, import_utils.commonScreenshotParam), {
+      path: file
+    }));
+    const item = {
+      type: "screenshot",
+      ts: Date.now(),
+      screenshot: (0, import_image2.base64Encoded)(file),
+      timing
+    };
+    return item;
+  }
+  wrapExecutorWithScreenshot(taskApply) {
+    const taskWithScreenshot = __spreadProps(__spreadValues({}, taskApply), {
+      executor: async (param, context, ...args) => {
+        const recorder = [];
+        const { task } = context;
+        task.recorder = recorder;
+        const shot = await this.recordScreenshot(`before ${task.type}`);
+        recorder.push(shot);
+        const result = await taskApply.executor(param, context, ...args);
+        if (taskApply.type === "Action") {
+          await (0, import_utils.sleep)(1e3);
+          const shot2 = await this.recordScreenshot("after Action");
+          recorder.push(shot2);
+        }
+        return result;
+      }
+    });
+    return taskWithScreenshot;
+  }
+  async convertPlanToExecutable(plans) {
+    const tasks = plans.map((plan2) => {
+      if (plan2.type === "Locate") {
+        const taskFind = {
+          type: "Insight",
+          subType: "Locate",
+          param: {
+            prompt: plan2.thought
+          },
+          executor: async (param) => {
+            let insightDump;
+            const dumpCollector = (dump) => {
+              insightDump = dump;
+            };
+            this.insight.onceDumpUpdatedFn = dumpCollector;
+            const element = await this.insight.locate(param.prompt);
+            (0, import_assert2.default)(element, `Element not found: ${param.prompt}`);
+            return {
+              output: {
+                element
+              },
+              log: {
+                dump: insightDump
+              }
+            };
+          }
+        };
+        return taskFind;
+      } else if (plan2.type === "Input") {
+        const taskActionInput = {
+          type: "Action",
+          subType: "Input",
+          param: plan2.param,
+          executor: async (taskParam) => {
+            (0, import_assert2.default)(taskParam.value, "No value to input");
+            await this.page.keyboard.type(taskParam.value);
+          }
+        };
+        return taskActionInput;
+      } else if (plan2.type === "KeyboardPress") {
+        const taskActionKeyboardPress = {
+          type: "Action",
+          subType: "KeyboardPress",
+          param: plan2.param,
+          executor: async (taskParam) => {
+            (0, import_assert2.default)(taskParam.value, "No key to press");
+            await this.page.keyboard.press(taskParam.value);
+          }
+        };
+        return taskActionKeyboardPress;
+      } else if (plan2.type === "Tap") {
+        const taskActionTap = {
+          type: "Action",
+          subType: "Tap",
+          executor: async (param, { element }) => {
+            (0, import_assert2.default)(element, "Element not found, cannot tap");
+            await this.page.mouse.click(element.center[0], element.center[1]);
+          }
+        };
+        return taskActionTap;
+      } else if (plan2.type === "Hover") {
+        const taskActionHover = {
+          type: "Action",
+          subType: "Hover",
+          executor: async (param, { element }) => {
+            (0, import_assert2.default)(element, "Element not found, cannot hover");
+            await this.page.mouse.move(element.center[0], element.center[1]);
+          }
+        };
+        return taskActionHover;
+      } else if (plan2.type === "Scroll") {
+        const taskActionScroll = {
+          type: "Action",
+          subType: "Scroll",
+          param: plan2.param,
+          executor: async (taskParam) => {
+            const scrollToEventName = taskParam.scrollType;
+            const innerHeight = await this.page.evaluate(() => window.innerHeight);
+            switch (scrollToEventName) {
+              case "ScrollUntilTop":
+                await this.page.mouse.wheel(0, -9999999);
+                break;
+              case "ScrollUntilBottom":
+                await this.page.mouse.wheel(0, 9999999);
+                break;
+              case "ScrollUp":
+                await this.page.mouse.wheel(0, -innerHeight);
+                break;
+              case "ScrollDown":
+                await this.page.mouse.wheel(0, innerHeight);
+                break;
+              default:
+                console.error("Unknown scroll event type:", scrollToEventName);
+            }
+          }
+        };
+        return taskActionScroll;
+      } else if (plan2.type === "Error") {
+        throw new Error(`Got a task plan with type Error: ${plan2.thought}`);
+      } else {
+        throw new Error(`Unknown or Unsupported task type: ${plan2.type}`);
+      }
+    }).map((task) => {
+      return this.wrapExecutorWithScreenshot(task);
+    });
+    return tasks;
+  }
+  async action(userPrompt) {
+    this.executor.description = userPrompt;
+    const pageContext = await this.insight.contextRetrieverFn();
+    let plans = [];
+    const planningTask = {
+      type: "Planning",
+      param: {
+        userPrompt
+      },
+      async executor(param) {
+        const planResult = await (0, import_core.plan)(pageContext, param.userPrompt);
+        (0, import_assert2.default)(planResult.plans.length > 0, "No plans found");
+        plans = planResult.plans;
+        return {
+          output: planResult
+        };
+      }
+    };
+    try {
+      await this.executor.append(this.wrapExecutorWithScreenshot(planningTask));
+      await this.executor.flush();
+      this.actionDump = this.executor.dump();
+      const executables = await this.convertPlanToExecutable(plans);
+      await this.executor.append(executables);
+      await this.executor.flush();
+      this.actionDump = this.executor.dump();
+      (0, import_assert2.default)(
+        this.executor.status !== "error",
+        `failed to execute tasks: ${this.executor.status}, msg: ${this.executor.errorMsg || ""}`
+      );
+    } catch (e) {
+      this.actionDump = this.executor.dump();
+      const err = new Error(e.message, { cause: e });
+      throw err;
+    }
+  }
+  async query(demand) {
+    this.executor.description = JSON.stringify(demand);
+    let data;
+    const queryTask = {
+      type: "Insight",
+      subType: "Query",
+      param: {
+        dataDemand: demand
+      },
+      executor: async (param) => {
+        let insightDump;
+        const dumpCollector = (dump) => {
+          insightDump = dump;
+        };
+        this.insight.onceDumpUpdatedFn = dumpCollector;
+        data = await this.insight.extract(param.dataDemand);
+        return {
+          output: data,
+          log: { dump: insightDump }
+        };
+      }
+    };
+    try {
+      await this.executor.append(this.wrapExecutorWithScreenshot(queryTask));
+      await this.executor.flush();
+      this.actionDump = this.executor.dump();
+    } catch (e) {
+      this.actionDump = this.executor.dump();
+      const err = new Error(e.message, { cause: e });
+      throw err;
+    }
+    return data;
+  }
+};
+
+// src/playwright/index.ts
+var PlaywrightAiFixture = () => {
+  const dumps = [];
+  const appendDump = (groupName, execution) => {
+    let currentDump = dumps.find((dump) => dump.groupName === groupName);
+    if (!currentDump) {
+      currentDump = {
+        groupName,
+        executions: []
+      };
+      dumps.push(currentDump);
+    }
+    currentDump.executions.push(execution);
+  };
+  const writeOutActionDumps = () => {
+    (0, import_utils3.writeDumpFile)(`playwright-${process.pid}`, import_utils3.groupedActionDumpFileExt, JSON.stringify(dumps));
+  };
+  const groupAndCaseForTest = (testInfo) => {
+    let groupName;
+    let caseName;
+    const titlePath = [...testInfo.titlePath];
+    if (titlePath.length > 1) {
+      caseName = titlePath.pop();
+      groupName = titlePath.join(" > ");
+    } else if (titlePath.length === 1) {
+      caseName = titlePath[0];
+      groupName = caseName;
+    } else {
+      caseName = "unnamed";
+      groupName = "unnamed";
+    }
+    return { groupName, caseName };
+  };
+  const aiAction = async (page, testInfo, taskPrompt) => {
+    const { groupName, caseName } = groupAndCaseForTest(testInfo);
+    const actionAgent = new PlayWrightActionAgent(page, { taskName: caseName });
+    let error;
+    try {
+      await actionAgent.action(taskPrompt);
+    } catch (e) {
+      error = e;
+    }
+    if (actionAgent.actionDump) {
+      appendDump(groupName, actionAgent.actionDump);
+      writeOutActionDumps();
+    }
+    if (error) {
+      console.error(error);
+      throw new Error(error.message, { cause: error });
+    }
+  };
+  const aiQuery = async (page, testInfo, demand) => {
+    const { groupName, caseName } = groupAndCaseForTest(testInfo);
+    const actionAgent = new PlayWrightActionAgent(page, { taskName: caseName });
+    let error;
+    let result;
+    try {
+      result = await actionAgent.query(demand);
+    } catch (e) {
+      error = e;
+    }
+    if (actionAgent.actionDump) {
+      appendDump(groupName, actionAgent.actionDump);
+      writeOutActionDumps();
+    }
+    if (error) {
+      console.error(error);
+      throw new Error(error.message, { cause: error });
+    }
+    return result;
+  };
+  return {
+    // shortcut
+    ai: async ({ page }, use, testInfo) => {
+      await use(async (taskPrompt, type = "action") => {
+        if (type === "action") {
+          return aiAction(page, testInfo, taskPrompt);
+        } else if (type === "query") {
+          return aiQuery(page, testInfo, taskPrompt);
+        }
+        throw new Error(`Unknown or Unsupported task type: ${type}, only support 'action' or 'query'`);
+      });
+    },
+    aiAction: async ({ page }, use, testInfo) => {
+      await use(async (taskPrompt) => {
+        await aiAction(page, testInfo, taskPrompt);
+      });
+    },
+    aiQuery: async ({ page }, use, testInfo) => {
+      await use(async function(demand) {
+        return aiQuery(page, testInfo, demand);
+      });
+    }
+  };
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  PlaywrightAiFixture
+});