@midscene/web 0.0.1 → 0.1.0

package/dist/lib/index.js

@@ -47,26 +47,27 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/index.ts
 var src_exports = {};
 __export(src_exports, {
-  PlaywrightAiFixture: () => PlaywrightAiFixture
+  PlaywrightAiFixture: () => PlaywrightAiFixture,
+  PuppeteerAgent: () => PageAgent
 });
 module.exports = __toCommonJS(src_exports);
 
 // src/playwright/index.ts
-var import_utils3 = require("@midscene/core/utils");
+var import_crypto = require("crypto");
 
-// src/playwright/actions.ts
-var import_assert2 = __toESM(require("assert"));
-var import_core = __toESM(require("@midscene/core"));
-var import_utils = require("@midscene/core/utils");
-var import_image2 = require("@midscene/core/image");
+// src/playwright/cache.ts
+var import_path2 = __toESM(require("path"));
+var import_fs2 = __toESM(require("fs"));
+var import_utils2 = require("@midscene/core/utils");
 
-// src/playwright/utils.ts
+// src/common/utils.ts
 var import_fs = __toESM(require("fs"));
 var import_assert = __toESM(require("assert"));
 var import_path = __toESM(require("path"));
 var import_image = require("@midscene/core/image");
+var import_utils = require("@midscene/core/utils");
 
-// src/playwright/element.ts
+// src/web-element.ts
 var WebElementInfo = class {
   constructor({
     content,
@@ -84,24 +85,13 @@ var WebElementInfo = class {
     this.id = id;
     this.attributes = attributes;
   }
-  async tap() {
-    await this.page.mouse.click(this.center[0], this.center[1]);
-  }
-  async hover() {
-    await this.page.mouse.move(this.center[0], this.center[1]);
-  }
-  async type(text) {
-    await this.page.keyboard.type(text);
-  }
-  async press(key) {
-    await this.page.keyboard.press(key);
-  }
 };
 
-// src/playwright/utils.ts
-async function parseContextFromPlaywrightPage(page, _opt) {
+// src/common/utils.ts
+async function parseContextFromWebPage(page, _opt) {
   (0, import_assert.default)(page, "page is required");
-  const file = "/Users/bytedance/workspace/midscene/packages/midscene/tests/fixtures/heytea.jpeg";
+  const url = page.url();
+  const file = (0, import_utils.getTmpFile)("jpeg");
   await page.screenshot({ path: file, type: "jpeg", quality: 75 });
   const screenshotBuffer = (0, import_fs.readFileSync)(file);
   const screenshotBase64 = (0, import_image.base64Encoded)(file);
@@ -111,7 +101,8 @@ async function parseContextFromPlaywrightPage(page, _opt) {
   return {
     content: elementsInfo,
     size,
-    screenshotBase64
+    screenshotBase64,
+    url
   };
 }
 async function getElementInfosFromPage(page) {
@@ -153,18 +144,136 @@ function findNearestPackageJson(dir) {
   return findNearestPackageJson(parentDir);
 }
 
-// src/playwright/actions.ts
-var PlayWrightActionAgent = class {
-  constructor(page, opt) {
+// src/playwright/cache.ts
+function writeTestCache(taskFile, taskTitle, taskCacheJson) {
+  const packageJson = getPkgInfo();
+  (0, import_utils2.writeDumpFile)({
+    fileName: `${taskFile}(${taskTitle})`,
+    fileExt: "json",
+    fileContent: JSON.stringify(
+      __spreadValues({
+        pkgName: packageJson.name,
+        pkgVersion: packageJson.version,
+        taskFile,
+        taskTitle
+      }, taskCacheJson),
+      null,
+      2
+    ),
+    type: "cache"
+  });
+}
+function readTestCache(taskFile, taskTitle) {
+  const cacheFile = (0, import_path2.join)((0, import_utils2.getDumpDirPath)("cache"), `${taskFile}(${taskTitle}).json`);
+  const pkgInfo = getPkgInfo();
+  if (process.env.MIDSCENE_CACHE === "true" && import_fs2.default.existsSync(cacheFile)) {
+    try {
+      const data = import_fs2.default.readFileSync(cacheFile, "utf8");
+      const jsonData = JSON.parse(data);
+      if (jsonData.pkgName !== pkgInfo.name || jsonData.pkgVersion !== pkgInfo.version) {
+        return void 0;
+      }
+      return jsonData;
+    } catch (err) {
+      return void 0;
+    }
+  }
+  return void 0;
+}
+function getPkgInfo() {
+  const packageJsonDir = findNearestPackageJson(__dirname);
+  if (!packageJsonDir) {
+    console.error("Cannot find package.json directory: ", __dirname);
+    return {
+      name: "@midscene/web",
+      version: "0.0.0"
+    };
+  }
+  const packageJsonPath = import_path2.default.join(packageJsonDir, "package.json");
+  const data = import_fs2.default.readFileSync(packageJsonPath, "utf8");
+  const packageJson = JSON.parse(data);
+  return {
+    name: packageJson.name,
+    version: packageJson.version
+  };
+}
+
+// src/common/agent.ts
+var import_utils6 = require("@midscene/core/utils");
+
+// src/common/tasks.ts
+var import_assert2 = __toESM(require("assert"));
+var import_core = __toESM(require("@midscene/core"));
+var import_utils4 = require("@midscene/core/utils");
+var import_image2 = require("@midscene/core/image");
+
+// src/common/task-cache.ts
+var TaskCache = class {
+  constructor(opts) {
+    this.cache = opts == null ? void 0 : opts.cache;
+    this.newCache = {
+      aiTasks: []
+    };
+  }
+  readCache(pageContext, type, userPrompt) {
+    var _a;
+    if (this.cache) {
+      const { aiTasks } = this.cache;
+      const index = aiTasks.findIndex((item) => item.prompt === userPrompt);
+      if (index === -1) {
+        return false;
+      }
+      const taskRes = aiTasks.splice(index, 1)[0];
+      if ((taskRes == null ? void 0 : taskRes.type) === "locate" && !((_a = taskRes.response) == null ? void 0 : _a.elements.every((element) => {
+        const findIndex = pageContext.content.findIndex(
+          (contentElement) => contentElement.id === element.id
+        );
+        if (findIndex === -1) {
+          return false;
+        }
+        return true;
+      }))) {
+        return false;
+      }
+      if (taskRes && taskRes.type === type && taskRes.prompt === userPrompt && this.pageContextEqual(taskRes.pageContext, pageContext)) {
+        return taskRes.response;
+      }
+    }
+    return false;
+  }
+  saveCache(cache) {
+    var _a;
+    if (cache) {
+      (_a = this.newCache) == null ? void 0 : _a.aiTasks.push(cache);
+    }
+  }
+  pageContextEqual(taskPageContext, pageContext) {
+    return taskPageContext.size.width === pageContext.size.width && taskPageContext.size.height === pageContext.size.height;
+  }
+  /**
+   * Generate task cache data.
+   * This method is mainly used to create or obtain some cached data for tasks, and it returns a new cache object.
+   * In the cache object, it may contain task-related information, states, or other necessary data.
+   * It is assumed that the `newCache` property already exists in the current class or object and is a data structure used to store task cache.
+   * @returns {Object} Returns a new cache object, which may include task cache data.
+   */
+  generateTaskCache() {
+    return this.newCache;
+  }
+};
+
+// src/common/tasks.ts
+var PageTaskExecutor = class {
+  constructor(page, opts) {
     this.page = page;
     this.insight = new import_core.default(async () => {
-      return await parseContextFromPlaywrightPage(page);
+      return await parseContextFromWebPage(page);
     });
-    this.executor = new import_core.Executor((opt == null ? void 0 : opt.taskName) || "MidScene - PlayWrightAI");
+    this.taskCache = new TaskCache(opts);
   }
   async recordScreenshot(timing) {
-    const file = (0, import_utils.getTmpFile)("jpeg");
-    await this.page.screenshot(__spreadProps(__spreadValues({}, import_utils.commonScreenshotParam), {
+    const file = (0, import_utils4.getTmpFile)("jpeg");
+    await this.page.screenshot(__spreadProps(__spreadValues({}, import_utils4.commonScreenshotParam), {
       path: file
     }));
     const item = {
@@ -185,7 +294,7 @@ var PlayWrightActionAgent = class {
         recorder.push(shot);
         const result = await taskApply.executor(param, context, ...args);
         if (taskApply.type === "Action") {
-          await (0, import_utils.sleep)(1e3);
+          await (0, import_utils4.sleep)(1e3);
           const shot2 = await this.recordScreenshot("after Action");
           recorder.push(shot2);
         }
@@ -209,14 +318,41 @@ var PlayWrightActionAgent = class {
               insightDump = dump;
             };
             this.insight.onceDumpUpdatedFn = dumpCollector;
-            const element = await this.insight.locate(param.prompt);
+            const pageContext = await this.insight.contextRetrieverFn();
+            const locateCache = this.taskCache.readCache(pageContext, "locate", param.prompt);
+            let locateResult;
+            const callAI = this.insight.aiVendorFn;
+            const element = await this.insight.locate(param.prompt, {
+              callAI: async (message) => {
+                if (locateCache) {
+                  locateResult = locateCache;
+                  return Promise.resolve(locateCache);
+                }
+                locateResult = await callAI(message);
+                return locateResult;
+              }
+            });
             (0, import_assert2.default)(element, `Element not found: ${param.prompt}`);
+            if (locateResult) {
+              this.taskCache.saveCache({
+                type: "locate",
+                pageContext: {
+                  url: pageContext.url,
+                  size: pageContext.size
+                },
+                prompt: param.prompt,
+                response: locateResult
+              });
+            }
             return {
               output: {
                 element
               },
               log: {
                 dump: insightDump
+              },
+              cache: {
+                hit: Boolean(locateResult)
               }
             };
           }
@@ -302,43 +438,66 @@ var PlayWrightActionAgent = class {
     return tasks;
   }
   async action(userPrompt) {
-    this.executor.description = userPrompt;
-    const pageContext = await this.insight.contextRetrieverFn();
+    const taskExecutor = new import_core.Executor(userPrompt);
+    taskExecutor.description = userPrompt;
     let plans = [];
     const planningTask = {
       type: "Planning",
       param: {
         userPrompt
       },
-      async executor(param) {
-        const planResult = await (0, import_core.plan)(pageContext, param.userPrompt);
+      executor: async (param) => {
+        const pageContext = await this.insight.contextRetrieverFn();
+        let planResult;
+        const planCache = this.taskCache.readCache(pageContext, "plan", userPrompt);
+        if (planCache) {
+          planResult = planCache;
+        } else {
+          planResult = await (0, import_core.plan)(param.userPrompt, {
+            context: pageContext
+          });
+        }
         (0, import_assert2.default)(planResult.plans.length > 0, "No plans found");
         plans = planResult.plans;
+        this.taskCache.saveCache({
+          type: "plan",
+          pageContext: {
+            url: pageContext.url,
+            size: pageContext.size
+          },
+          prompt: userPrompt,
+          response: planResult
+        });
         return {
-          output: planResult
+          output: planResult,
+          cache: {
+            hint: Boolean(planCache)
+          }
         };
       }
     };
     try {
-      await this.executor.append(this.wrapExecutorWithScreenshot(planningTask));
-      await this.executor.flush();
-      this.actionDump = this.executor.dump();
+      await taskExecutor.append(this.wrapExecutorWithScreenshot(planningTask));
+      await taskExecutor.flush();
+      this.executionDump = taskExecutor.dump();
       const executables = await this.convertPlanToExecutable(plans);
-      await this.executor.append(executables);
-      await this.executor.flush();
-      this.actionDump = this.executor.dump();
+      await taskExecutor.append(executables);
+      await taskExecutor.flush();
+      this.executionDump = taskExecutor.dump();
       (0, import_assert2.default)(
-        this.executor.status !== "error",
-        `failed to execute tasks: ${this.executor.status}, msg: ${this.executor.errorMsg || ""}`
+        taskExecutor.status !== "error",
+        `failed to execute tasks: ${taskExecutor.status}, msg: ${taskExecutor.errorMsg || ""}`
       );
     } catch (e) {
-      this.actionDump = this.executor.dump();
+      this.executionDump = taskExecutor.dump();
       const err = new Error(e.message, { cause: e });
       throw err;
     }
   }
   async query(demand) {
-    this.executor.description = JSON.stringify(demand);
+    const description = JSON.stringify(demand);
+    const taskExecutor = new import_core.Executor(description);
+    taskExecutor.description = description;
     let data;
     const queryTask = {
       type: "Insight",
@@ -360,11 +519,11 @@ var PlayWrightActionAgent = class {
       }
     };
     try {
-      await this.executor.append(this.wrapExecutorWithScreenshot(queryTask));
-      await this.executor.flush();
-      this.actionDump = this.executor.dump();
+      await taskExecutor.append(this.wrapExecutorWithScreenshot(queryTask));
+      await taskExecutor.flush();
+      this.executionDump = taskExecutor.dump();
     } catch (e) {
-      this.actionDump = this.executor.dump();
+      this.executionDump = taskExecutor.dump();
       const err = new Error(e.message, { cause: e });
       throw err;
     }
@@ -372,102 +531,171 @@ var PlayWrightActionAgent = class {
   }
 };
 
-// src/playwright/index.ts
-var PlaywrightAiFixture = () => {
-  const dumps = [];
-  const appendDump = (groupName, execution) => {
-    let currentDump = dumps.find((dump) => dump.groupName === groupName);
-    if (!currentDump) {
-      currentDump = {
-        groupName,
+// src/common/agent.ts
+var PageAgent = class {
+  constructor(page, opts) {
+    this.page = page;
+    this.dumps = [
+      {
+        groupName: (opts == null ? void 0 : opts.taskFile) || "unnamed",
         executions: []
-      };
-      dumps.push(currentDump);
-    }
+      }
+    ];
+    this.testId = (opts == null ? void 0 : opts.testId) || String(process.pid);
+    this.actionAgent = new PageTaskExecutor(this.page, {
+      cache: (opts == null ? void 0 : opts.cache) || { aiTasks: [] }
+    });
+  }
+  appendDump(execution) {
+    const currentDump = this.dumps[0];
     currentDump.executions.push(execution);
-  };
-  const writeOutActionDumps = () => {
-    (0, import_utils3.writeDumpFile)(`playwright-${process.pid}`, import_utils3.groupedActionDumpFileExt, JSON.stringify(dumps));
-  };
-  const groupAndCaseForTest = (testInfo) => {
-    let groupName;
-    let caseName;
-    const titlePath = [...testInfo.titlePath];
-    if (titlePath.length > 1) {
-      caseName = titlePath.pop();
-      groupName = titlePath.join(" > ");
-    } else if (titlePath.length === 1) {
-      caseName = titlePath[0];
-      groupName = caseName;
-    } else {
-      caseName = "unnamed";
-      groupName = "unnamed";
-    }
-    return { groupName, caseName };
-  };
-  const aiAction = async (page, testInfo, taskPrompt) => {
-    const { groupName, caseName } = groupAndCaseForTest(testInfo);
-    const actionAgent = new PlayWrightActionAgent(page, { taskName: caseName });
+  }
+  writeOutActionDumps() {
+    this.dumpFile = (0, import_utils6.writeDumpFile)({
+      fileName: `playwright-${this.testId}`,
+      fileExt: import_utils6.groupedActionDumpFileExt,
+      fileContent: JSON.stringify(this.dumps)
+    });
+  }
+  async aiAction(taskPrompt) {
     let error;
     try {
-      await actionAgent.action(taskPrompt);
+      await this.actionAgent.action(taskPrompt);
     } catch (e) {
       error = e;
     }
-    if (actionAgent.actionDump) {
-      appendDump(groupName, actionAgent.actionDump);
-      writeOutActionDumps();
+    if (this.actionAgent.executionDump) {
+      this.appendDump(this.actionAgent.executionDump);
+      this.writeOutActionDumps();
     }
     if (error) {
       console.error(error);
       throw new Error(error.message, { cause: error });
     }
-  };
-  const aiQuery = async (page, testInfo, demand) => {
-    const { groupName, caseName } = groupAndCaseForTest(testInfo);
-    const actionAgent = new PlayWrightActionAgent(page, { taskName: caseName });
+  }
+  async aiQuery(demand) {
     let error;
     let result;
     try {
-      result = await actionAgent.query(demand);
+      result = await this.actionAgent.query(demand);
     } catch (e) {
       error = e;
     }
-    if (actionAgent.actionDump) {
-      appendDump(groupName, actionAgent.actionDump);
-      writeOutActionDumps();
+    if (this.actionAgent.executionDump) {
+      this.appendDump(this.actionAgent.executionDump);
+      this.writeOutActionDumps();
     }
     if (error) {
       console.error(error);
       throw new Error(error.message, { cause: error });
     }
     return result;
+  }
+  async ai(taskPrompt, type = "action") {
+    if (type === "action") {
+      return this.aiAction(taskPrompt);
+    } else if (type === "query") {
+      return this.aiQuery(taskPrompt);
+    }
+    throw new Error(`Unknown or Unsupported task type: ${type}, only support 'action' or 'query'`);
+  }
+};
+
+// src/playwright/index.ts
+var groupAndCaseForTest = (testInfo) => {
+  let taskFile;
+  let taskTitle;
+  const titlePath = [...testInfo.titlePath];
+  if (titlePath.length > 1) {
+    taskTitle = titlePath.pop();
+    taskFile = `${titlePath.join(" > ")}:${testInfo.line}`;
+  } else if (titlePath.length === 1) {
+    taskTitle = titlePath[0];
+    taskFile = `${taskTitle}:${testInfo.line}`;
+  } else {
+    taskTitle = "unnamed";
+    taskFile = "unnamed";
+  }
+  return { taskFile, taskTitle };
+};
+var midSceneAgentKeyId = "_midSceneAgentId";
+var PlaywrightAiFixture = () => {
+  const pageAgentMap = {};
+  const agentForPage = (page, opts) => {
+    let idForPage = page[midSceneAgentKeyId];
+    if (!idForPage) {
+      idForPage = (0, import_crypto.randomUUID)();
+      page[midSceneAgentKeyId] = idForPage;
+      const testCase = readTestCache(opts.taskFile, opts.taskTitle) || { aiTasks: [] };
+      pageAgentMap[idForPage] = new PageAgent(page, {
+        testId: `${opts.testId}-${idForPage}`,
+        taskFile: opts.taskFile,
+        cache: testCase
+      });
+    }
+    return pageAgentMap[idForPage];
   };
   return {
-    // shortcut
     ai: async ({ page }, use, testInfo) => {
-      await use(async (taskPrompt, type = "action") => {
-        if (type === "action") {
-          return aiAction(page, testInfo, taskPrompt);
-        } else if (type === "query") {
-          return aiQuery(page, testInfo, taskPrompt);
-        }
-        throw new Error(`Unknown or Unsupported task type: ${type}, only support 'action' or 'query'`);
+      const { taskFile, taskTitle } = groupAndCaseForTest(testInfo);
+      const agent = agentForPage(page, { testId: testInfo.testId, taskFile, taskTitle });
+      await use(async (taskPrompt, opts) => {
+        await page.waitForLoadState("networkidle");
+        const actionType = (opts == null ? void 0 : opts.type) || "action";
+        const result = await agent.ai(taskPrompt, actionType);
+        return result;
       });
+      const taskCacheJson = agent.actionAgent.taskCache.generateTaskCache();
+      writeTestCache(taskFile, taskTitle, taskCacheJson);
+      if (agent.dumpFile) {
+        testInfo.annotations.push({
+          type: "MIDSCENE_AI_ACTION",
+          description: JSON.stringify({
+            testId: testInfo.testId,
+            dumpPath: agent.dumpFile
+          })
+        });
+      }
     },
     aiAction: async ({ page }, use, testInfo) => {
+      const { taskFile, taskTitle } = groupAndCaseForTest(testInfo);
+      const agent = agentForPage(page, { testId: testInfo.testId, taskFile, taskTitle });
       await use(async (taskPrompt) => {
-        await aiAction(page, testInfo, taskPrompt);
+        await page.waitForLoadState("networkidle");
+        await agent.aiAction(taskPrompt);
       });
+      if (agent.dumpFile) {
+        testInfo.annotations.push({
+          type: "MIDSCENE_AI_ACTION",
+          description: JSON.stringify({
+            testId: testInfo.testId,
+            dumpPath: agent.dumpFile
+          })
+        });
+      }
     },
     aiQuery: async ({ page }, use, testInfo) => {
+      const { taskFile, taskTitle } = groupAndCaseForTest(testInfo);
+      const agent = agentForPage(page, { testId: testInfo.testId, taskFile, taskTitle });
       await use(async function(demand) {
-        return aiQuery(page, testInfo, demand);
+        await page.waitForLoadState("networkidle");
+        const result = await agent.aiQuery(demand);
+        return result;
       });
+      if (agent.dumpFile) {
+        testInfo.annotations.push({
+          type: "MIDSCENE_AI_ACTION",
+          description: JSON.stringify({
+            testId: testInfo.testId,
+            dumpPath: agent.dumpFile
+          })
+        });
+      }
     }
   };
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  PlaywrightAiFixture
+  PlaywrightAiFixture,
+  PuppeteerAgent
 });