@midscene/web 0.0.1

package/dist/es/index.js

@@ -0,0 +1,442 @@
+var __defProp = Object.defineProperty;
+var __defProps = Object.defineProperties;
+var __getOwnPropDescs = Object.getOwnPropertyDescriptors;
+var __getOwnPropSymbols = Object.getOwnPropertySymbols;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __propIsEnum = Object.prototype.propertyIsEnumerable;
+var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
+var __spreadValues = (a, b) => {
+  for (var prop in b || (b = {}))
+    if (__hasOwnProp.call(b, prop))
+      __defNormalProp(a, prop, b[prop]);
+  if (__getOwnPropSymbols)
+    for (var prop of __getOwnPropSymbols(b)) {
+      if (__propIsEnum.call(b, prop))
+        __defNormalProp(a, prop, b[prop]);
+    }
+  return a;
+};
+var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
+
+// src/playwright/index.ts
+import { groupedActionDumpFileExt, writeDumpFile } from "@midscene/core/utils";
+
+// src/playwright/actions.ts
+import assert2 from "assert";
+import Insight, {
+  Executor,
+  plan
+} from "@midscene/core";
+import { commonScreenshotParam, getTmpFile, sleep } from "@midscene/core/utils";
+import { base64Encoded as base64Encoded2 } from "@midscene/core/image";
+
+// src/playwright/utils.ts
+import fs, { readFileSync } from "fs";
+import assert from "assert";
+import path from "path";
+import { alignCoordByTrim, base64Encoded, imageInfoOfBase64 } from "@midscene/core/image";
+
+// src/playwright/element.ts
+var WebElementInfo = class {
+  constructor({
+    content,
+    rect,
+    page,
+    locator,
+    id,
+    attributes
+  }) {
+    this.content = content;
+    this.rect = rect;
+    this.center = [Math.floor(rect.left + rect.width / 2), Math.floor(rect.top + rect.height / 2)];
+    this.page = page;
+    this.locator = locator;
+    this.id = id;
+    this.attributes = attributes;
+  }
+  async tap() {
+    await this.page.mouse.click(this.center[0], this.center[1]);
+  }
+  async hover() {
+    await this.page.mouse.move(this.center[0], this.center[1]);
+  }
+  async type(text) {
+    await this.page.keyboard.type(text);
+  }
+  async press(key) {
+    await this.page.keyboard.press(key);
+  }
+};
+
+// src/playwright/utils.ts
+async function parseContextFromPlaywrightPage(page, _opt) {
+  assert(page, "page is required");
+  const file = "/Users/bytedance/workspace/midscene/packages/midscene/tests/fixtures/heytea.jpeg";
+  await page.screenshot({ path: file, type: "jpeg", quality: 75 });
+  const screenshotBuffer = readFileSync(file);
+  const screenshotBase64 = base64Encoded(file);
+  const captureElementSnapshot = await getElementInfosFromPage(page);
+  const elementsInfo = await alignElements(screenshotBuffer, captureElementSnapshot, page);
+  const size = await imageInfoOfBase64(screenshotBase64);
+  return {
+    content: elementsInfo,
+    size,
+    screenshotBase64
+  };
+}
+async function getElementInfosFromPage(page) {
+  const pathDir = findNearestPackageJson(__dirname);
+  assert(pathDir, `can't find pathDir, with ${__dirname}`);
+  const scriptPath = path.join(pathDir, "./dist/script/htmlElement.js");
+  const elementInfosScriptContent = readFileSync(scriptPath, "utf-8");
+  const extraReturnLogic = `${elementInfosScriptContent}midscene_element_inspector.extractTextWithPositionDFS()`;
+  const captureElementSnapshot = await page.evaluate(extraReturnLogic);
+  return captureElementSnapshot;
+}
+async function alignElements(screenshotBuffer, elements, page) {
+  const textsAligned = [];
+  for (const item of elements) {
+    const { rect } = item;
+    const aligned = await alignCoordByTrim(screenshotBuffer, rect);
+    item.rect = aligned;
+    item.center = [
+      Math.round(aligned.left + aligned.width / 2),
+      Math.round(aligned.top + aligned.height / 2)
+    ];
+    textsAligned.push(
+      new WebElementInfo(__spreadProps(__spreadValues({}, item), {
+        page
+      }))
+    );
+  }
+  return textsAligned;
+}
+function findNearestPackageJson(dir) {
+  const packageJsonPath = path.join(dir, "package.json");
+  if (fs.existsSync(packageJsonPath)) {
+    return dir;
+  }
+  const parentDir = path.dirname(dir);
+  if (parentDir === dir) {
+    return null;
+  }
+  return findNearestPackageJson(parentDir);
+}
+
+// src/playwright/actions.ts
+var PlayWrightActionAgent = class {
+  constructor(page, opt) {
+    this.page = page;
+    this.insight = new Insight(async () => {
+      return await parseContextFromPlaywrightPage(page);
+    });
+    this.executor = new Executor((opt == null ? void 0 : opt.taskName) || "MidScene - PlayWrightAI");
+  }
+  async recordScreenshot(timing) {
+    const file = getTmpFile("jpeg");
+    await this.page.screenshot(__spreadProps(__spreadValues({}, commonScreenshotParam), {
+      path: file
+    }));
+    const item = {
+      type: "screenshot",
+      ts: Date.now(),
+      screenshot: base64Encoded2(file),
+      timing
+    };
+    return item;
+  }
+  wrapExecutorWithScreenshot(taskApply) {
+    const taskWithScreenshot = __spreadProps(__spreadValues({}, taskApply), {
+      executor: async (param, context, ...args) => {
+        const recorder = [];
+        const { task } = context;
+        task.recorder = recorder;
+        const shot = await this.recordScreenshot(`before ${task.type}`);
+        recorder.push(shot);
+        const result = await taskApply.executor(param, context, ...args);
+        if (taskApply.type === "Action") {
+          await sleep(1e3);
+          const shot2 = await this.recordScreenshot("after Action");
+          recorder.push(shot2);
+        }
+        return result;
+      }
+    });
+    return taskWithScreenshot;
+  }
+  async convertPlanToExecutable(plans) {
+    const tasks = plans.map((plan2) => {
+      if (plan2.type === "Locate") {
+        const taskFind = {
+          type: "Insight",
+          subType: "Locate",
+          param: {
+            prompt: plan2.thought
+          },
+          executor: async (param) => {
+            let insightDump;
+            const dumpCollector = (dump) => {
+              insightDump = dump;
+            };
+            this.insight.onceDumpUpdatedFn = dumpCollector;
+            const element = await this.insight.locate(param.prompt);
+            assert2(element, `Element not found: ${param.prompt}`);
+            return {
+              output: {
+                element
+              },
+              log: {
+                dump: insightDump
+              }
+            };
+          }
+        };
+        return taskFind;
+      } else if (plan2.type === "Input") {
+        const taskActionInput = {
+          type: "Action",
+          subType: "Input",
+          param: plan2.param,
+          executor: async (taskParam) => {
+            assert2(taskParam.value, "No value to input");
+            await this.page.keyboard.type(taskParam.value);
+          }
+        };
+        return taskActionInput;
+      } else if (plan2.type === "KeyboardPress") {
+        const taskActionKeyboardPress = {
+          type: "Action",
+          subType: "KeyboardPress",
+          param: plan2.param,
+          executor: async (taskParam) => {
+            assert2(taskParam.value, "No key to press");
+            await this.page.keyboard.press(taskParam.value);
+          }
+        };
+        return taskActionKeyboardPress;
+      } else if (plan2.type === "Tap") {
+        const taskActionTap = {
+          type: "Action",
+          subType: "Tap",
+          executor: async (param, { element }) => {
+            assert2(element, "Element not found, cannot tap");
+            await this.page.mouse.click(element.center[0], element.center[1]);
+          }
+        };
+        return taskActionTap;
+      } else if (plan2.type === "Hover") {
+        const taskActionHover = {
+          type: "Action",
+          subType: "Hover",
+          executor: async (param, { element }) => {
+            assert2(element, "Element not found, cannot hover");
+            await this.page.mouse.move(element.center[0], element.center[1]);
+          }
+        };
+        return taskActionHover;
+      } else if (plan2.type === "Scroll") {
+        const taskActionScroll = {
+          type: "Action",
+          subType: "Scroll",
+          param: plan2.param,
+          executor: async (taskParam) => {
+            const scrollToEventName = taskParam.scrollType;
+            const innerHeight = await this.page.evaluate(() => window.innerHeight);
+            switch (scrollToEventName) {
+              case "ScrollUntilTop":
+                await this.page.mouse.wheel(0, -9999999);
+                break;
+              case "ScrollUntilBottom":
+                await this.page.mouse.wheel(0, 9999999);
+                break;
+              case "ScrollUp":
+                await this.page.mouse.wheel(0, -innerHeight);
+                break;
+              case "ScrollDown":
+                await this.page.mouse.wheel(0, innerHeight);
+                break;
+              default:
+                console.error("Unknown scroll event type:", scrollToEventName);
+            }
+          }
+        };
+        return taskActionScroll;
+      } else if (plan2.type === "Error") {
+        throw new Error(`Got a task plan with type Error: ${plan2.thought}`);
+      } else {
+        throw new Error(`Unknown or Unsupported task type: ${plan2.type}`);
+      }
+    }).map((task) => {
+      return this.wrapExecutorWithScreenshot(task);
+    });
+    return tasks;
+  }
+  async action(userPrompt) {
+    this.executor.description = userPrompt;
+    const pageContext = await this.insight.contextRetrieverFn();
+    let plans = [];
+    const planningTask = {
+      type: "Planning",
+      param: {
+        userPrompt
+      },
+      async executor(param) {
+        const planResult = await plan(pageContext, param.userPrompt);
+        assert2(planResult.plans.length > 0, "No plans found");
+        plans = planResult.plans;
+        return {
+          output: planResult
+        };
+      }
+    };
+    try {
+      await this.executor.append(this.wrapExecutorWithScreenshot(planningTask));
+      await this.executor.flush();
+      this.actionDump = this.executor.dump();
+      const executables = await this.convertPlanToExecutable(plans);
+      await this.executor.append(executables);
+      await this.executor.flush();
+      this.actionDump = this.executor.dump();
+      assert2(
+        this.executor.status !== "error",
+        `failed to execute tasks: ${this.executor.status}, msg: ${this.executor.errorMsg || ""}`
+      );
+    } catch (e) {
+      this.actionDump = this.executor.dump();
+      const err = new Error(e.message, { cause: e });
+      throw err;
+    }
+  }
+  async query(demand) {
+    this.executor.description = JSON.stringify(demand);
+    let data;
+    const queryTask = {
+      type: "Insight",
+      subType: "Query",
+      param: {
+        dataDemand: demand
+      },
+      executor: async (param) => {
+        let insightDump;
+        const dumpCollector = (dump) => {
+          insightDump = dump;
+        };
+        this.insight.onceDumpUpdatedFn = dumpCollector;
+        data = await this.insight.extract(param.dataDemand);
+        return {
+          output: data,
+          log: { dump: insightDump }
+        };
+      }
+    };
+    try {
+      await this.executor.append(this.wrapExecutorWithScreenshot(queryTask));
+      await this.executor.flush();
+      this.actionDump = this.executor.dump();
+    } catch (e) {
+      this.actionDump = this.executor.dump();
+      const err = new Error(e.message, { cause: e });
+      throw err;
+    }
+    return data;
+  }
+};
+
+// src/playwright/index.ts
+var PlaywrightAiFixture = () => {
+  const dumps = [];
+  const appendDump = (groupName, execution) => {
+    let currentDump = dumps.find((dump) => dump.groupName === groupName);
+    if (!currentDump) {
+      currentDump = {
+        groupName,
+        executions: []
+      };
+      dumps.push(currentDump);
+    }
+    currentDump.executions.push(execution);
+  };
+  const writeOutActionDumps = () => {
+    writeDumpFile(`playwright-${process.pid}`, groupedActionDumpFileExt, JSON.stringify(dumps));
+  };
+  const groupAndCaseForTest = (testInfo) => {
+    let groupName;
+    let caseName;
+    const titlePath = [...testInfo.titlePath];
+    if (titlePath.length > 1) {
+      caseName = titlePath.pop();
+      groupName = titlePath.join(" > ");
+    } else if (titlePath.length === 1) {
+      caseName = titlePath[0];
+      groupName = caseName;
+    } else {
+      caseName = "unnamed";
+      groupName = "unnamed";
+    }
+    return { groupName, caseName };
+  };
+  const aiAction = async (page, testInfo, taskPrompt) => {
+    const { groupName, caseName } = groupAndCaseForTest(testInfo);
+    const actionAgent = new PlayWrightActionAgent(page, { taskName: caseName });
+    let error;
+    try {
+      await actionAgent.action(taskPrompt);
+    } catch (e) {
+      error = e;
+    }
+    if (actionAgent.actionDump) {
+      appendDump(groupName, actionAgent.actionDump);
+      writeOutActionDumps();
+    }
+    if (error) {
+      console.error(error);
+      throw new Error(error.message, { cause: error });
+    }
+  };
+  const aiQuery = async (page, testInfo, demand) => {
+    const { groupName, caseName } = groupAndCaseForTest(testInfo);
+    const actionAgent = new PlayWrightActionAgent(page, { taskName: caseName });
+    let error;
+    let result;
+    try {
+      result = await actionAgent.query(demand);
+    } catch (e) {
+      error = e;
+    }
+    if (actionAgent.actionDump) {
+      appendDump(groupName, actionAgent.actionDump);
+      writeOutActionDumps();
+    }
+    if (error) {
+      console.error(error);
+      throw new Error(error.message, { cause: error });
+    }
+    return result;
+  };
+  return {
+    // shortcut
+    ai: async ({ page }, use, testInfo) => {
+      await use(async (taskPrompt, type = "action") => {
+        if (type === "action") {
+          return aiAction(page, testInfo, taskPrompt);
+        } else if (type === "query") {
+          return aiQuery(page, testInfo, taskPrompt);
+        }
+        throw new Error(`Unknown or Unsupported task type: ${type}, only support 'action' or 'query'`);
+      });
+    },
+    aiAction: async ({ page }, use, testInfo) => {
+      await use(async (taskPrompt) => {
+        await aiAction(page, testInfo, taskPrompt);
+      });
+    },
+    aiQuery: async ({ page }, use, testInfo) => {
+      await use(async function(demand) {
+        return aiQuery(page, testInfo, demand);
+      });
+    }
+  };
+};
+export {
+  PlaywrightAiFixture
+};