@midscene/web 0.0.1 → 0.1.1

package/dist/es/index.js

@@ -19,24 +19,21 @@ var __spreadValues = (a, b) => {
 var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
 
 // src/playwright/index.ts
-import { groupedActionDumpFileExt, writeDumpFile } from "@midscene/core/utils";
+import { randomUUID } from "crypto";
 
-// src/playwright/actions.ts
-import assert2 from "assert";
-import Insight, {
-  Executor,
-  plan
-} from "@midscene/core";
-import { commonScreenshotParam, getTmpFile, sleep } from "@midscene/core/utils";
-import { base64Encoded as base64Encoded2 } from "@midscene/core/image";
+// src/playwright/cache.ts
+import path2, { join } from "path";
+import fs2 from "fs";
+import { writeDumpFile, getDumpDirPath } from "@midscene/core/utils";
 
-// src/playwright/utils.ts
+// src/common/utils.ts
 import fs, { readFileSync } from "fs";
 import assert from "assert";
 import path from "path";
 import { alignCoordByTrim, base64Encoded, imageInfoOfBase64 } from "@midscene/core/image";
+import { getTmpFile } from "@midscene/core/utils";
 
-// src/playwright/element.ts
+// src/web-element.ts
 var WebElementInfo = class {
   constructor({
     content,
@@ -54,24 +51,13 @@ var WebElementInfo = class {
     this.id = id;
     this.attributes = attributes;
   }
-  async tap() {
-    await this.page.mouse.click(this.center[0], this.center[1]);
-  }
-  async hover() {
-    await this.page.mouse.move(this.center[0], this.center[1]);
-  }
-  async type(text) {
-    await this.page.keyboard.type(text);
-  }
-  async press(key) {
-    await this.page.keyboard.press(key);
-  }
 };
 
-// src/playwright/utils.ts
-async function parseContextFromPlaywrightPage(page, _opt) {
+// src/common/utils.ts
+async function parseContextFromWebPage(page, _opt) {
   assert(page, "page is required");
-  const file = "/Users/bytedance/workspace/midscene/packages/midscene/tests/fixtures/heytea.jpeg";
+  const url = page.url();
+  const file = getTmpFile("jpeg");
   await page.screenshot({ path: file, type: "jpeg", quality: 75 });
   const screenshotBuffer = readFileSync(file);
   const screenshotBase64 = base64Encoded(file);
@@ -81,7 +67,8 @@ async function parseContextFromPlaywrightPage(page, _opt) {
   return {
     content: elementsInfo,
     size,
-    screenshotBase64
+    screenshotBase64,
+    url
   };
 }
 async function getElementInfosFromPage(page) {
@@ -123,17 +110,138 @@ function findNearestPackageJson(dir) {
   return findNearestPackageJson(parentDir);
 }
 
-// src/playwright/actions.ts
-var PlayWrightActionAgent = class {
-  constructor(page, opt) {
+// src/playwright/cache.ts
+function writeTestCache(taskFile, taskTitle, taskCacheJson) {
+  const packageJson = getPkgInfo();
+  writeDumpFile({
+    fileName: `${taskFile}(${taskTitle})`,
+    fileExt: "json",
+    fileContent: JSON.stringify(
+      __spreadValues({
+        pkgName: packageJson.name,
+        pkgVersion: packageJson.version,
+        taskFile,
+        taskTitle
+      }, taskCacheJson),
+      null,
+      2
+    ),
+    type: "cache"
+  });
+}
+function readTestCache(taskFile, taskTitle) {
+  const cacheFile = join(getDumpDirPath("cache"), `${taskFile}(${taskTitle}).json`);
+  const pkgInfo = getPkgInfo();
+  if (process.env.MIDSCENE_CACHE === "true" && fs2.existsSync(cacheFile)) {
+    try {
+      const data = fs2.readFileSync(cacheFile, "utf8");
+      const jsonData = JSON.parse(data);
+      if (jsonData.pkgName !== pkgInfo.name || jsonData.pkgVersion !== pkgInfo.version) {
+        return void 0;
+      }
+      return jsonData;
+    } catch (err) {
+      return void 0;
+    }
+  }
+  return void 0;
+}
+function getPkgInfo() {
+  const packageJsonDir = findNearestPackageJson(__dirname);
+  if (!packageJsonDir) {
+    console.error("Cannot find package.json directory: ", __dirname);
+    return {
+      name: "@midscene/web",
+      version: "0.0.0"
+    };
+  }
+  const packageJsonPath = path2.join(packageJsonDir, "package.json");
+  const data = fs2.readFileSync(packageJsonPath, "utf8");
+  const packageJson = JSON.parse(data);
+  return {
+    name: packageJson.name,
+    version: packageJson.version
+  };
+}
+
+// src/common/agent.ts
+import { groupedActionDumpFileExt, writeDumpFile as writeDumpFile2 } from "@midscene/core/utils";
+
+// src/common/tasks.ts
+import assert2 from "assert";
+import Insight, {
+  Executor,
+  plan
+} from "@midscene/core";
+import { commonScreenshotParam, getTmpFile as getTmpFile2, sleep } from "@midscene/core/utils";
+import { base64Encoded as base64Encoded2 } from "@midscene/core/image";
+
+// src/common/task-cache.ts
+var TaskCache = class {
+  constructor(opts) {
+    this.cache = opts == null ? void 0 : opts.cache;
+    this.newCache = {
+      aiTasks: []
+    };
+  }
+  readCache(pageContext, type, userPrompt) {
+    var _a;
+    if (this.cache) {
+      const { aiTasks } = this.cache;
+      const index = aiTasks.findIndex((item) => item.prompt === userPrompt);
+      if (index === -1) {
+        return false;
+      }
+      const taskRes = aiTasks.splice(index, 1)[0];
+      if ((taskRes == null ? void 0 : taskRes.type) === "locate" && !((_a = taskRes.response) == null ? void 0 : _a.elements.every((element) => {
+        const findIndex = pageContext.content.findIndex(
+          (contentElement) => contentElement.id === element.id
+        );
+        if (findIndex === -1) {
+          return false;
+        }
+        return true;
+      }))) {
+        return false;
+      }
+      if (taskRes && taskRes.type === type && taskRes.prompt === userPrompt && this.pageContextEqual(taskRes.pageContext, pageContext)) {
+        return taskRes.response;
+      }
+    }
+    return false;
+  }
+  saveCache(cache) {
+    var _a;
+    if (cache) {
+      (_a = this.newCache) == null ? void 0 : _a.aiTasks.push(cache);
+    }
+  }
+  pageContextEqual(taskPageContext, pageContext) {
+    return taskPageContext.size.width === pageContext.size.width && taskPageContext.size.height === pageContext.size.height;
+  }
+  /**
+   * Generate task cache data.
+   * This method is mainly used to create or obtain some cached data for tasks, and it returns a new cache object.
+   * In the cache object, it may contain task-related information, states, or other necessary data.
+   * It is assumed that the `newCache` property already exists in the current class or object and is a data structure used to store task cache.
+   * @returns {Object} Returns a new cache object, which may include task cache data.
+   */
+  generateTaskCache() {
+    return this.newCache;
+  }
+};
+
+// src/common/tasks.ts
+var PageTaskExecutor = class {
+  constructor(page, opts) {
     this.page = page;
     this.insight = new Insight(async () => {
-      return await parseContextFromPlaywrightPage(page);
+      return await parseContextFromWebPage(page);
     });
-    this.executor = new Executor((opt == null ? void 0 : opt.taskName) || "MidScene - PlayWrightAI");
+    this.taskCache = new TaskCache(opts);
   }
   async recordScreenshot(timing) {
-    const file = getTmpFile("jpeg");
+    const file = getTmpFile2("jpeg");
     await this.page.screenshot(__spreadProps(__spreadValues({}, commonScreenshotParam), {
       path: file
     }));
@@ -170,23 +278,48 @@ var PlayWrightActionAgent = class {
         const taskFind = {
           type: "Insight",
           subType: "Locate",
-          param: {
-            prompt: plan2.thought
-          },
+          param: plan2.param,
           executor: async (param) => {
             let insightDump;
             const dumpCollector = (dump) => {
               insightDump = dump;
             };
             this.insight.onceDumpUpdatedFn = dumpCollector;
-            const element = await this.insight.locate(param.prompt);
+            const pageContext = await this.insight.contextRetrieverFn();
+            const locateCache = this.taskCache.readCache(pageContext, "locate", param.prompt);
+            let locateResult;
+            const callAI = this.insight.aiVendorFn;
+            const element = await this.insight.locate(param.prompt, {
+              callAI: async (message) => {
+                if (locateCache) {
+                  locateResult = locateCache;
+                  return Promise.resolve(locateCache);
+                }
+                locateResult = await callAI(message);
+                return locateResult;
+              }
+            });
             assert2(element, `Element not found: ${param.prompt}`);
+            if (locateResult) {
+              this.taskCache.saveCache({
+                type: "locate",
+                pageContext: {
+                  url: pageContext.url,
+                  size: pageContext.size
+                },
+                prompt: param.prompt,
+                response: locateResult
+              });
+            }
             return {
               output: {
                 element
               },
               log: {
                 dump: insightDump
+              },
+              cache: {
+                hit: Boolean(locateResult)
               }
             };
           }
@@ -197,7 +330,10 @@ var PlayWrightActionAgent = class {
           type: "Action",
           subType: "Input",
           param: plan2.param,
-          executor: async (taskParam) => {
+          executor: async (taskParam, { element }) => {
+            if (element) {
+              await this.page.mouse.click(element.center[0], element.center[1]);
+            }
             assert2(taskParam.value, "No value to input");
             await this.page.keyboard.type(taskParam.value);
           }
@@ -272,43 +408,66 @@ var PlayWrightActionAgent = class {
     return tasks;
   }
   async action(userPrompt) {
-    this.executor.description = userPrompt;
-    const pageContext = await this.insight.contextRetrieverFn();
+    const taskExecutor = new Executor(userPrompt);
+    taskExecutor.description = userPrompt;
     let plans = [];
     const planningTask = {
       type: "Planning",
       param: {
         userPrompt
       },
-      async executor(param) {
-        const planResult = await plan(pageContext, param.userPrompt);
+      executor: async (param) => {
+        const pageContext = await this.insight.contextRetrieverFn();
+        let planResult;
+        const planCache = this.taskCache.readCache(pageContext, "plan", userPrompt);
+        if (planCache) {
+          planResult = planCache;
+        } else {
+          planResult = await plan(param.userPrompt, {
+            context: pageContext
+          });
+        }
         assert2(planResult.plans.length > 0, "No plans found");
         plans = planResult.plans;
+        this.taskCache.saveCache({
+          type: "plan",
+          pageContext: {
+            url: pageContext.url,
+            size: pageContext.size
+          },
+          prompt: userPrompt,
+          response: planResult
+        });
         return {
-          output: planResult
+          output: planResult,
+          cache: {
+            hint: Boolean(planCache)
+          }
         };
       }
     };
     try {
-      await this.executor.append(this.wrapExecutorWithScreenshot(planningTask));
-      await this.executor.flush();
-      this.actionDump = this.executor.dump();
+      await taskExecutor.append(this.wrapExecutorWithScreenshot(planningTask));
+      await taskExecutor.flush();
+      this.executionDump = taskExecutor.dump();
       const executables = await this.convertPlanToExecutable(plans);
-      await this.executor.append(executables);
-      await this.executor.flush();
-      this.actionDump = this.executor.dump();
+      await taskExecutor.append(executables);
+      await taskExecutor.flush();
+      this.executionDump = taskExecutor.dump();
       assert2(
-        this.executor.status !== "error",
-        `failed to execute tasks: ${this.executor.status}, msg: ${this.executor.errorMsg || ""}`
+        taskExecutor.status !== "error",
+        `failed to execute tasks: ${taskExecutor.status}, msg: ${taskExecutor.errorMsg || ""}`
       );
     } catch (e) {
-      this.actionDump = this.executor.dump();
+      this.executionDump = taskExecutor.dump();
       const err = new Error(e.message, { cause: e });
       throw err;
     }
   }
   async query(demand) {
-    this.executor.description = JSON.stringify(demand);
+    const description = JSON.stringify(demand);
+    const taskExecutor = new Executor(description);
+    taskExecutor.description = description;
     let data;
     const queryTask = {
       type: "Insight",
@@ -330,11 +489,11 @@ var PlayWrightActionAgent = class {
       }
     };
     try {
-      await this.executor.append(this.wrapExecutorWithScreenshot(queryTask));
-      await this.executor.flush();
-      this.actionDump = this.executor.dump();
+      await taskExecutor.append(this.wrapExecutorWithScreenshot(queryTask));
+      await taskExecutor.flush();
+      this.executionDump = taskExecutor.dump();
     } catch (e) {
-      this.actionDump = this.executor.dump();
+      this.executionDump = taskExecutor.dump();
       const err = new Error(e.message, { cause: e });
       throw err;
     }
@@ -342,101 +501,170 @@ var PlayWrightActionAgent = class {
   }
 };
 
-// src/playwright/index.ts
-var PlaywrightAiFixture = () => {
-  const dumps = [];
-  const appendDump = (groupName, execution) => {
-    let currentDump = dumps.find((dump) => dump.groupName === groupName);
-    if (!currentDump) {
-      currentDump = {
-        groupName,
+// src/common/agent.ts
+var PageAgent = class {
+  constructor(page, opts) {
+    this.page = page;
+    this.dumps = [
+      {
+        groupName: (opts == null ? void 0 : opts.taskFile) || "unnamed",
         executions: []
-      };
-      dumps.push(currentDump);
-    }
+      }
+    ];
+    this.testId = (opts == null ? void 0 : opts.testId) || String(process.pid);
+    this.actionAgent = new PageTaskExecutor(this.page, {
+      cache: (opts == null ? void 0 : opts.cache) || { aiTasks: [] }
+    });
+  }
+  appendDump(execution) {
+    const currentDump = this.dumps[0];
     currentDump.executions.push(execution);
-  };
-  const writeOutActionDumps = () => {
-    writeDumpFile(`playwright-${process.pid}`, groupedActionDumpFileExt, JSON.stringify(dumps));
-  };
-  const groupAndCaseForTest = (testInfo) => {
-    let groupName;
-    let caseName;
-    const titlePath = [...testInfo.titlePath];
-    if (titlePath.length > 1) {
-      caseName = titlePath.pop();
-      groupName = titlePath.join(" > ");
-    } else if (titlePath.length === 1) {
-      caseName = titlePath[0];
-      groupName = caseName;
-    } else {
-      caseName = "unnamed";
-      groupName = "unnamed";
-    }
-    return { groupName, caseName };
-  };
-  const aiAction = async (page, testInfo, taskPrompt) => {
-    const { groupName, caseName } = groupAndCaseForTest(testInfo);
-    const actionAgent = new PlayWrightActionAgent(page, { taskName: caseName });
+  }
+  writeOutActionDumps() {
+    this.dumpFile = writeDumpFile2({
+      fileName: `playwright-${this.testId}`,
+      fileExt: groupedActionDumpFileExt,
+      fileContent: JSON.stringify(this.dumps)
+    });
+  }
+  async aiAction(taskPrompt) {
     let error;
     try {
-      await actionAgent.action(taskPrompt);
+      await this.actionAgent.action(taskPrompt);
     } catch (e) {
       error = e;
     }
-    if (actionAgent.actionDump) {
-      appendDump(groupName, actionAgent.actionDump);
-      writeOutActionDumps();
+    if (this.actionAgent.executionDump) {
+      this.appendDump(this.actionAgent.executionDump);
+      this.writeOutActionDumps();
     }
     if (error) {
       console.error(error);
       throw new Error(error.message, { cause: error });
     }
-  };
-  const aiQuery = async (page, testInfo, demand) => {
-    const { groupName, caseName } = groupAndCaseForTest(testInfo);
-    const actionAgent = new PlayWrightActionAgent(page, { taskName: caseName });
+  }
+  async aiQuery(demand) {
     let error;
     let result;
     try {
-      result = await actionAgent.query(demand);
+      result = await this.actionAgent.query(demand);
     } catch (e) {
       error = e;
     }
-    if (actionAgent.actionDump) {
-      appendDump(groupName, actionAgent.actionDump);
-      writeOutActionDumps();
+    if (this.actionAgent.executionDump) {
+      this.appendDump(this.actionAgent.executionDump);
+      this.writeOutActionDumps();
     }
     if (error) {
       console.error(error);
       throw new Error(error.message, { cause: error });
     }
     return result;
+  }
+  async ai(taskPrompt, type = "action") {
+    if (type === "action") {
+      return this.aiAction(taskPrompt);
+    } else if (type === "query") {
+      return this.aiQuery(taskPrompt);
+    }
+    throw new Error(`Unknown or Unsupported task type: ${type}, only support 'action' or 'query'`);
+  }
+};
+
+// src/playwright/index.ts
+var groupAndCaseForTest = (testInfo) => {
+  let taskFile;
+  let taskTitle;
+  const titlePath = [...testInfo.titlePath];
+  if (titlePath.length > 1) {
+    taskTitle = titlePath.pop();
+    taskFile = `${titlePath.join(" > ")}:${testInfo.line}`;
+  } else if (titlePath.length === 1) {
+    taskTitle = titlePath[0];
+    taskFile = `${taskTitle}:${testInfo.line}`;
+  } else {
+    taskTitle = "unnamed";
+    taskFile = "unnamed";
+  }
+  return { taskFile, taskTitle };
+};
+var midSceneAgentKeyId = "_midSceneAgentId";
+var PlaywrightAiFixture = () => {
+  const pageAgentMap = {};
+  const agentForPage = (page, opts) => {
+    let idForPage = page[midSceneAgentKeyId];
+    if (!idForPage) {
+      idForPage = randomUUID();
+      page[midSceneAgentKeyId] = idForPage;
+      const testCase = readTestCache(opts.taskFile, opts.taskTitle) || { aiTasks: [] };
+      pageAgentMap[idForPage] = new PageAgent(page, {
+        testId: `${opts.testId}-${idForPage}`,
+        taskFile: opts.taskFile,
+        cache: testCase
+      });
+    }
+    return pageAgentMap[idForPage];
   };
   return {
-    // shortcut
     ai: async ({ page }, use, testInfo) => {
-      await use(async (taskPrompt, type = "action") => {
-        if (type === "action") {
-          return aiAction(page, testInfo, taskPrompt);
-        } else if (type === "query") {
-          return aiQuery(page, testInfo, taskPrompt);
-        }
-        throw new Error(`Unknown or Unsupported task type: ${type}, only support 'action' or 'query'`);
+      const { taskFile, taskTitle } = groupAndCaseForTest(testInfo);
+      const agent = agentForPage(page, { testId: testInfo.testId, taskFile, taskTitle });
+      await use(async (taskPrompt, opts) => {
+        await page.waitForLoadState("networkidle");
+        const actionType = (opts == null ? void 0 : opts.type) || "action";
+        const result = await agent.ai(taskPrompt, actionType);
+        return result;
       });
+      const taskCacheJson = agent.actionAgent.taskCache.generateTaskCache();
+      writeTestCache(taskFile, taskTitle, taskCacheJson);
+      if (agent.dumpFile) {
+        testInfo.annotations.push({
+          type: "MIDSCENE_AI_ACTION",
+          description: JSON.stringify({
+            testId: testInfo.testId,
+            dumpPath: agent.dumpFile
+          })
+        });
+      }
     },
     aiAction: async ({ page }, use, testInfo) => {
+      const { taskFile, taskTitle } = groupAndCaseForTest(testInfo);
+      const agent = agentForPage(page, { testId: testInfo.testId, taskFile, taskTitle });
       await use(async (taskPrompt) => {
-        await aiAction(page, testInfo, taskPrompt);
+        await page.waitForLoadState("networkidle");
+        await agent.aiAction(taskPrompt);
       });
+      if (agent.dumpFile) {
+        testInfo.annotations.push({
+          type: "MIDSCENE_AI_ACTION",
+          description: JSON.stringify({
+            testId: testInfo.testId,
+            dumpPath: agent.dumpFile
+          })
+        });
+      }
     },
     aiQuery: async ({ page }, use, testInfo) => {
+      const { taskFile, taskTitle } = groupAndCaseForTest(testInfo);
+      const agent = agentForPage(page, { testId: testInfo.testId, taskFile, taskTitle });
       await use(async function(demand) {
-        return aiQuery(page, testInfo, demand);
+        await page.waitForLoadState("networkidle");
+        const result = await agent.aiQuery(demand);
+        return result;
       });
+      if (agent.dumpFile) {
+        testInfo.annotations.push({
+          type: "MIDSCENE_AI_ACTION",
+          description: JSON.stringify({
+            testId: testInfo.testId,
+            dumpPath: agent.dumpFile
+          })
+        });
+      }
     }
   };
 };
 export {
-  PlaywrightAiFixture
+  PlaywrightAiFixture,
+  PageAgent as PuppeteerAgent
 };