@midscene/web 0.1.4 → 0.2.0

package/README.md

@@ -0,0 +1,7 @@
+## Documentation
+
+See https://midscenejs.com/ for details.
+
+## License
+
+Midscene is MIT licensed.

package/dist/es/index.js

@@ -21,16 +21,82 @@ var __spreadProps = (a, b) => __defProps(a, __getOwnPropDescs(b));
 // src/playwright/index.ts
 import { randomUUID } from "crypto";
 
-// src/playwright/cache.ts
-import path2, { join } from "path";
-import fs2 from "fs";
-import { writeDumpFile, getDumpDirPath } from "@midscene/core/utils";
+// src/common/agent.ts
+import { groupedActionDumpFileExt, writeDumpFile } from "@midscene/core/utils";
+
+// src/common/tasks.ts
+import assert2 from "assert";
+import Insight, {
+  Executor,
+  plan
+} from "@midscene/core";
+import { base64Encoded as base64Encoded2 } from "@midscene/core/image";
+import { commonScreenshotParam, getTmpFile as getTmpFile2, sleep } from "@midscene/core/utils";
+
+// src/common/task-cache.ts
+var TaskCache = class {
+  constructor(opts) {
+    this.cache = opts == null ? void 0 : opts.cache;
+    this.newCache = {
+      aiTasks: []
+    };
+  }
+  readCache(pageContext, type, userPrompt) {
+    var _a;
+    if (this.cache) {
+      const { aiTasks } = this.cache;
+      const index = aiTasks.findIndex((item) => item.prompt === userPrompt);
+      if (index === -1) {
+        return false;
+      }
+      const taskRes = aiTasks.splice(index, 1)[0];
+      if ((taskRes == null ? void 0 : taskRes.type) === "locate" && !((_a = taskRes.response) == null ? void 0 : _a.elements.every((element) => {
+        const findIndex = pageContext.content.findIndex(
+          (contentElement) => contentElement.id === element.id
+        );
+        if (findIndex === -1) {
+          return false;
+        }
+        return true;
+      }))) {
+        return false;
+      }
+      if (taskRes && taskRes.type === type && taskRes.prompt === userPrompt && this.pageContextEqual(taskRes.pageContext, pageContext)) {
+        return taskRes.response;
+      }
+    }
+    return false;
+  }
+  saveCache(cache) {
+    var _a;
+    if (cache) {
+      (_a = this.newCache) == null ? void 0 : _a.aiTasks.push(cache);
+    }
+  }
+  pageContextEqual(taskPageContext, pageContext) {
+    return taskPageContext.size.width === pageContext.size.width && taskPageContext.size.height === pageContext.size.height;
+  }
+  /**
+   * Generate task cache data.
+   * This method is mainly used to create or obtain some cached data for tasks, and it returns a new cache object.
+   * In the cache object, it may contain task-related information, states, or other necessary data.
+   * It is assumed that the `newCache` property already exists in the current class or object and is a data structure used to store task cache.
+   * @returns {Object} Returns a new cache object, which may include task cache data.
+   */
+  generateTaskCache() {
+    return this.newCache;
+  }
+};
 
 // src/common/utils.ts
-import fs, { readFileSync } from "fs";
 import assert from "assert";
+import fs, { readFileSync } from "fs";
 import path from "path";
-import { alignCoordByTrim, base64Encoded, imageInfoOfBase64 } from "@midscene/core/image";
+import {
+  alignCoordByTrim,
+  base64Encoded,
+  imageInfoOfBase64
+} from "@midscene/core/image";
 import { getTmpFile } from "@midscene/core/utils";
 
 // src/web-element.ts
@@ -45,7 +111,10 @@ var WebElementInfo = class {
   }) {
     this.content = content;
     this.rect = rect;
-    this.center = [Math.floor(rect.left + rect.width / 2), Math.floor(rect.top + rect.height / 2)];
+    this.center = [
+      Math.floor(rect.left + rect.width / 2),
+      Math.floor(rect.top + rect.height / 2)
+    ];
     this.page = page;
     this.locator = locator;
     this.id = id;
@@ -62,7 +131,11 @@ async function parseContextFromWebPage(page, _opt) {
   const screenshotBuffer = readFileSync(file);
   const screenshotBase64 = base64Encoded(file);
   const captureElementSnapshot = await getElementInfosFromPage(page);
-  const elementsInfo = await alignElements(screenshotBuffer, captureElementSnapshot, page);
+  const elementsInfo = await alignElements(
+    screenshotBuffer,
+    captureElementSnapshot,
+    page
+  );
   const size = await imageInfoOfBase64(screenshotBase64);
   return {
     content: elementsInfo,
@@ -110,127 +183,6 @@ function findNearestPackageJson(dir) {
   return findNearestPackageJson(parentDir);
 }
 
-// src/playwright/cache.ts
-function writeTestCache(taskFile, taskTitle, taskCacheJson) {
-  const packageJson = getPkgInfo();
-  writeDumpFile({
-    fileName: `${taskFile}(${taskTitle})`,
-    fileExt: "json",
-    fileContent: JSON.stringify(
-      __spreadValues({
-        pkgName: packageJson.name,
-        pkgVersion: packageJson.version,
-        taskFile,
-        taskTitle
-      }, taskCacheJson),
-      null,
-      2
-    ),
-    type: "cache"
-  });
-}
-function readTestCache(taskFile, taskTitle) {
-  const cacheFile = join(getDumpDirPath("cache"), `${taskFile}(${taskTitle}).json`);
-  const pkgInfo = getPkgInfo();
-  if (process.env.MIDSCENE_CACHE === "true" && fs2.existsSync(cacheFile)) {
-    try {
-      const data = fs2.readFileSync(cacheFile, "utf8");
-      const jsonData = JSON.parse(data);
-      if (jsonData.pkgName !== pkgInfo.name || jsonData.pkgVersion !== pkgInfo.version) {
-        return void 0;
-      }
-      return jsonData;
-    } catch (err) {
-      return void 0;
-    }
-  }
-  return void 0;
-}
-function getPkgInfo() {
-  const packageJsonDir = findNearestPackageJson(__dirname);
-  if (!packageJsonDir) {
-    console.error("Cannot find package.json directory: ", __dirname);
-    return {
-      name: "@midscene/web",
-      version: "0.0.0"
-    };
-  }
-  const packageJsonPath = path2.join(packageJsonDir, "package.json");
-  const data = fs2.readFileSync(packageJsonPath, "utf8");
-  const packageJson = JSON.parse(data);
-  return {
-    name: packageJson.name,
-    version: packageJson.version
-  };
-}
-
-// src/common/agent.ts
-import { groupedActionDumpFileExt, writeDumpFile as writeDumpFile2 } from "@midscene/core/utils";
-
-// src/common/tasks.ts
-import assert2 from "assert";
-import Insight, {
-  Executor,
-  plan
-} from "@midscene/core";
-import { commonScreenshotParam, getTmpFile as getTmpFile2, sleep } from "@midscene/core/utils";
-import { base64Encoded as base64Encoded2 } from "@midscene/core/image";
-
-// src/common/task-cache.ts
-var TaskCache = class {
-  constructor(opts) {
-    this.cache = opts == null ? void 0 : opts.cache;
-    this.newCache = {
-      aiTasks: []
-    };
-  }
-  readCache(pageContext, type, userPrompt) {
-    var _a;
-    if (this.cache) {
-      const { aiTasks } = this.cache;
-      const index = aiTasks.findIndex((item) => item.prompt === userPrompt);
-      if (index === -1) {
-        return false;
-      }
-      const taskRes = aiTasks.splice(index, 1)[0];
-      if ((taskRes == null ? void 0 : taskRes.type) === "locate" && !((_a = taskRes.response) == null ? void 0 : _a.elements.every((element) => {
-        const findIndex = pageContext.content.findIndex(
-          (contentElement) => contentElement.id === element.id
-        );
-        if (findIndex === -1) {
-          return false;
-        }
-        return true;
-      }))) {
-        return false;
-      }
-      if (taskRes && taskRes.type === type && taskRes.prompt === userPrompt && this.pageContextEqual(taskRes.pageContext, pageContext)) {
-        return taskRes.response;
-      }
-    }
-    return false;
-  }
-  saveCache(cache) {
-    var _a;
-    if (cache) {
-      (_a = this.newCache) == null ? void 0 : _a.aiTasks.push(cache);
-    }
-  }
-  pageContextEqual(taskPageContext, pageContext) {
-    return taskPageContext.size.width === pageContext.size.width && taskPageContext.size.height === pageContext.size.height;
-  }
-  /**
-   * Generate task cache data.
-   * This method is mainly used to create or obtain some cached data for tasks, and it returns a new cache object.
-   * In the cache object, it may contain task-related information, states, or other necessary data.
-   * It is assumed that the `newCache` property already exists in the current class or object and is a data structure used to store task cache.
-   * @returns {Object} Returns a new cache object, which may include task cache data.
-   */
-  generateTaskCache() {
-    return this.newCache;
-  }
-};
-
 // src/common/tasks.ts
 var PageTaskExecutor = class {
   constructor(page, opts) {
@@ -286,7 +238,11 @@ var PageTaskExecutor = class {
             };
             this.insight.onceDumpUpdatedFn = dumpCollector;
             const pageContext = await this.insight.contextRetrieverFn();
-            const locateCache = this.taskCache.readCache(pageContext, "locate", param.prompt);
+            const locateCache = this.taskCache.readCache(
+              pageContext,
+              "locate",
+              param.prompt
+            );
             let locateResult;
             const callAI = this.insight.aiVendorFn;
             const element = await this.insight.locate(param.prompt, {
@@ -325,21 +281,51 @@ var PageTaskExecutor = class {
           }
         };
         return taskFind;
-      } else if (plan2.type === "Input") {
+      }
+      if (plan2.type === "Assert") {
+        const assertPlan = plan2;
+        const taskAssert = {
+          type: "Insight",
+          subType: "Assert",
+          param: assertPlan.param,
+          executor: async () => {
+            let insightDump;
+            const dumpCollector = (dump) => {
+              insightDump = dump;
+            };
+            this.insight.onceDumpUpdatedFn = dumpCollector;
+            const assertion = await this.insight.assert(
+              assertPlan.param.assertion
+            );
+            return {
+              output: assertion,
+              log: {
+                dump: insightDump
+              }
+            };
+          }
+        };
+        return taskAssert;
+      }
+      if (plan2.type === "Input") {
         const taskActionInput = {
           type: "Action",
           subType: "Input",
           param: plan2.param,
           executor: async (taskParam, { element }) => {
             if (element) {
-              await this.page.mouse.click(element.center[0], element.center[1]);
+              await this.page.mouse.click(
+                element.center[0],
+                element.center[1]
+              );
             }
             assert2(taskParam.value, "No value to input");
             await this.page.keyboard.type(taskParam.value);
           }
         };
         return taskActionInput;
-      } else if (plan2.type === "KeyboardPress") {
+      }
+      if (plan2.type === "KeyboardPress") {
         const taskActionKeyboardPress = {
           type: "Action",
           subType: "KeyboardPress",
@@ -350,34 +336,45 @@ var PageTaskExecutor = class {
           }
         };
         return taskActionKeyboardPress;
-      } else if (plan2.type === "Tap") {
+      }
+      if (plan2.type === "Tap") {
         const taskActionTap = {
           type: "Action",
           subType: "Tap",
           executor: async (param, { element }) => {
             assert2(element, "Element not found, cannot tap");
-            await this.page.mouse.click(element.center[0], element.center[1]);
+            await this.page.mouse.click(
+              element.center[0],
+              element.center[1]
+            );
           }
         };
         return taskActionTap;
-      } else if (plan2.type === "Hover") {
+      }
+      if (plan2.type === "Hover") {
         const taskActionHover = {
           type: "Action",
           subType: "Hover",
           executor: async (param, { element }) => {
             assert2(element, "Element not found, cannot hover");
-            await this.page.mouse.move(element.center[0], element.center[1]);
+            await this.page.mouse.move(
+              element.center[0],
+              element.center[1]
+            );
           }
         };
         return taskActionHover;
-      } else if (plan2.type === "Scroll") {
+      }
+      if (plan2.type === "Scroll") {
         const taskActionScroll = {
           type: "Action",
           subType: "Scroll",
           param: plan2.param,
           executor: async (taskParam) => {
             const scrollToEventName = taskParam.scrollType;
-            const innerHeight = await this.page.evaluate(() => window.innerHeight);
+            const innerHeight = await this.page.evaluate(
+              () => window.innerHeight
+            );
             switch (scrollToEventName) {
               case "ScrollUntilTop":
                 await this.page.mouse.wheel(0, -9999999);
@@ -392,16 +389,19 @@ var PageTaskExecutor = class {
                 await this.page.mouse.wheel(0, innerHeight);
                 break;
               default:
-                console.error("Unknown scroll event type:", scrollToEventName);
+                console.error(
+                  "Unknown scroll event type:",
+                  scrollToEventName
+                );
             }
           }
         };
         return taskActionScroll;
-      } else if (plan2.type === "Error") {
+      }
+      if (plan2.type === "Error") {
         throw new Error(`Got a task plan with type Error: ${plan2.thought}`);
-      } else {
-        throw new Error(`Unknown or Unsupported task type: ${plan2.type}`);
       }
+      throw new Error(`Unknown or Unsupported task type: ${plan2.type}`);
     }).map((task) => {
       return this.wrapExecutorWithScreenshot(task);
     });
@@ -419,7 +419,11 @@ var PageTaskExecutor = class {
       executor: async (param) => {
         const pageContext = await this.insight.contextRetrieverFn();
         let planResult;
-        const planCache = this.taskCache.readCache(pageContext, "plan", userPrompt);
+        const planCache = this.taskCache.readCache(
+          pageContext,
+          "plan",
+          userPrompt
+        );
         if (planCache) {
           planResult = planCache;
         } else {
@@ -441,7 +445,7 @@ var PageTaskExecutor = class {
         return {
           output: planResult,
           cache: {
-            hint: Boolean(planCache)
+            hit: Boolean(planCache)
           }
         };
       }
@@ -499,6 +503,22 @@ var PageTaskExecutor = class {
     }
     return data;
   }
+  async assert(assertion) {
+    const description = assertion;
+    const taskExecutor = new Executor(description);
+    taskExecutor.description = description;
+    const assertionPlan = {
+      type: "Assert",
+      param: {
+        assertion
+      }
+    };
+    const assertTask = await this.convertPlanToExecutable([assertionPlan]);
+    await taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
+    const assertionResult = await taskExecutor.flush();
+    this.executionDump = taskExecutor.dump();
+    return assertionResult;
+  }
 };
 
 // src/common/agent.ts
@@ -512,7 +532,7 @@ var PageAgent = class {
       }
     ];
     this.testId = (opts == null ? void 0 : opts.testId) || String(process.pid);
-    this.actionAgent = new PageTaskExecutor(this.page, {
+    this.taskExecutor = new PageTaskExecutor(this.page, {
       cache: (opts == null ? void 0 : opts.cache) || { aiTasks: [] }
     });
   }
@@ -521,7 +541,7 @@ var PageAgent = class {
     currentDump.executions.push(execution);
   }
   writeOutActionDumps() {
-    this.dumpFile = writeDumpFile2({
+    this.dumpFile = writeDumpFile({
       fileName: `playwright-${this.testId}`,
       fileExt: groupedActionDumpFileExt,
       fileContent: JSON.stringify(this.dumps)
@@ -530,12 +550,12 @@ var PageAgent = class {
   async aiAction(taskPrompt) {
     let error;
     try {
-      await this.actionAgent.action(taskPrompt);
+      await this.taskExecutor.action(taskPrompt);
     } catch (e) {
       error = e;
     }
-    if (this.actionAgent.executionDump) {
-      this.appendDump(this.actionAgent.executionDump);
+    if (this.taskExecutor.executionDump) {
+      this.appendDump(this.taskExecutor.executionDump);
       this.writeOutActionDumps();
     }
     if (error) {
@@ -547,12 +567,12 @@ var PageAgent = class {
     let error;
     let result;
     try {
-      result = await this.actionAgent.query(demand);
+      result = await this.taskExecutor.query(demand);
     } catch (e) {
       error = e;
     }
-    if (this.actionAgent.executionDump) {
-      this.appendDump(this.actionAgent.executionDump);
+    if (this.taskExecutor.executionDump) {
+      this.appendDump(this.taskExecutor.executionDump);
       this.writeOutActionDumps();
     }
     if (error) {
@@ -561,23 +581,102 @@ var PageAgent = class {
     }
     return result;
   }
+  async aiAssert(assertion, msg) {
+    const assertionResult = await this.taskExecutor.assert(assertion);
+    if (this.taskExecutor.executionDump) {
+      this.appendDump(this.taskExecutor.executionDump);
+      this.writeOutActionDumps();
+    }
+    if (!assertionResult.pass) {
+      const errMsg = msg || `Assertion failed: ${assertion}`;
+      const reasonMsg = `Reason: ${assertionResult.thought}`;
+      throw new Error(`${errMsg}
+${reasonMsg}`);
+    }
+  }
   async ai(taskPrompt, type = "action") {
     if (type === "action") {
       return this.aiAction(taskPrompt);
-    } else if (type === "query") {
+    }
+    if (type === "query") {
       return this.aiQuery(taskPrompt);
     }
-    throw new Error(`Unknown or Unsupported task type: ${type}, only support 'action' or 'query'`);
+    if (type === "assert") {
+      return this.aiAssert(taskPrompt);
+    }
+    throw new Error(
+      `Unknown type: ${type}, only support 'action', 'query', 'assert'`
+    );
   }
 };
 
+// src/playwright/cache.ts
+import fs2 from "fs";
+import path2, { join } from "path";
+import { getDumpDirPath, writeDumpFile as writeDumpFile2 } from "@midscene/core/utils";
+function writeTestCache(taskFile, taskTitle, taskCacheJson) {
+  const packageJson = getPkgInfo();
+  writeDumpFile2({
+    fileName: `${taskFile}(${taskTitle})`,
+    fileExt: "json",
+    fileContent: JSON.stringify(
+      __spreadValues({
+        pkgName: packageJson.name,
+        pkgVersion: packageJson.version,
+        taskFile,
+        taskTitle
+      }, taskCacheJson),
+      null,
+      2
+    ),
+    type: "cache"
+  });
+}
+function readTestCache(taskFile, taskTitle) {
+  const cacheFile = join(
+    getDumpDirPath("cache"),
+    `${taskFile}(${taskTitle}).json`
+  );
+  const pkgInfo = getPkgInfo();
+  if (process.env.MIDSCENE_CACHE === "true" && fs2.existsSync(cacheFile)) {
+    try {
+      const data = fs2.readFileSync(cacheFile, "utf8");
+      const jsonData = JSON.parse(data);
+      if (jsonData.pkgName !== pkgInfo.name || jsonData.pkgVersion !== pkgInfo.version) {
+        return void 0;
+      }
+      return jsonData;
+    } catch (err) {
+      return void 0;
+    }
+  }
+  return void 0;
+}
+function getPkgInfo() {
+  const packageJsonDir = findNearestPackageJson(__dirname);
+  if (!packageJsonDir) {
+    console.error("Cannot find package.json directory: ", __dirname);
+    return {
+      name: "@midscene/web",
+      version: "0.0.0"
+    };
+  }
+  const packageJsonPath = path2.join(packageJsonDir, "package.json");
+  const data = fs2.readFileSync(packageJsonPath, "utf8");
+  const packageJson = JSON.parse(data);
+  return {
+    name: packageJson.name,
+    version: packageJson.version
+  };
+}
+
 // src/playwright/index.ts
 var groupAndCaseForTest = (testInfo) => {
   let taskFile;
   let taskTitle;
   const titlePath = [...testInfo.titlePath];
   if (titlePath.length > 1) {
-    taskTitle = titlePath.pop();
+    taskTitle = titlePath.pop() || "unnamed";
     taskFile = `${titlePath.join(" > ")}:${testInfo.line}`;
   } else if (titlePath.length === 1) {
     taskTitle = titlePath[0];
@@ -588,15 +687,17 @@ var groupAndCaseForTest = (testInfo) => {
   }
   return { taskFile, taskTitle };
 };
-var midSceneAgentKeyId = "_midSceneAgentId";
+var midsceneAgentKeyId = "_midsceneAgentId";
 var PlaywrightAiFixture = () => {
   const pageAgentMap = {};
   const agentForPage = (page, opts) => {
-    let idForPage = page[midSceneAgentKeyId];
+    let idForPage = page[midsceneAgentKeyId];
     if (!idForPage) {
       idForPage = randomUUID();
-      page[midSceneAgentKeyId] = idForPage;
-      const testCase = readTestCache(opts.taskFile, opts.taskTitle) || { aiTasks: [] };
+      page[midsceneAgentKeyId] = idForPage;
+      const testCase = readTestCache(opts.taskFile, opts.taskTitle) || {
+        aiTasks: []
+      };
       pageAgentMap[idForPage] = new PageAgent(page, {
         testId: `${opts.testId}-${idForPage}`,
         taskFile: opts.taskFile,
@@ -608,14 +709,20 @@ var PlaywrightAiFixture = () => {
   return {
     ai: async ({ page }, use, testInfo) => {
       const { taskFile, taskTitle } = groupAndCaseForTest(testInfo);
-      const agent = agentForPage(page, { testId: testInfo.testId, taskFile, taskTitle });
-      await use(async (taskPrompt, opts) => {
-        await page.waitForLoadState("networkidle");
-        const actionType = (opts == null ? void 0 : opts.type) || "action";
-        const result = await agent.ai(taskPrompt, actionType);
-        return result;
+      const agent = agentForPage(page, {
+        testId: testInfo.testId,
+        taskFile,
+        taskTitle
       });
-      const taskCacheJson = agent.actionAgent.taskCache.generateTaskCache();
+      await use(
+        async (taskPrompt, opts) => {
+          await page.waitForLoadState("networkidle");
+          const actionType = (opts == null ? void 0 : opts.type) || "action";
+          const result = await agent.ai(taskPrompt, actionType);
+          return result;
+        }
+      );
+      const taskCacheJson = agent.taskExecutor.taskCache.generateTaskCache();
       writeTestCache(taskFile, taskTitle, taskCacheJson);
       if (agent.dumpFile) {
         testInfo.annotations.push({
@@ -629,7 +736,11 @@ var PlaywrightAiFixture = () => {
     },
     aiAction: async ({ page }, use, testInfo) => {
       const { taskFile, taskTitle } = groupAndCaseForTest(testInfo);
-      const agent = agentForPage(page, { testId: testInfo.testId, taskFile, taskTitle });
+      const agent = agentForPage(page, {
+        testId: testInfo.testId,
+        taskFile,
+        taskTitle
+      });
       await use(async (taskPrompt) => {
         await page.waitForLoadState("networkidle");
         await agent.aiAction(taskPrompt);
@@ -646,8 +757,12 @@ var PlaywrightAiFixture = () => {
     },
     aiQuery: async ({ page }, use, testInfo) => {
       const { taskFile, taskTitle } = groupAndCaseForTest(testInfo);
-      const agent = agentForPage(page, { testId: testInfo.testId, taskFile, taskTitle });
-      await use(async function(demand) {
+      const agent = agentForPage(page, {
+        testId: testInfo.testId,
+        taskFile,
+        taskTitle
+      });
+      await use(async (demand) => {
         await page.waitForLoadState("networkidle");
         const result = await agent.aiQuery(demand);
         return result;
@@ -661,6 +776,27 @@ var PlaywrightAiFixture = () => {
           })
         });
       }
+    },
+    aiAssert: async ({ page }, use, testInfo) => {
+      const { taskFile, taskTitle } = groupAndCaseForTest(testInfo);
+      const agent = agentForPage(page, {
+        testId: testInfo.testId,
+        taskFile,
+        taskTitle
+      });
+      await use(async (assertion, errorMsg) => {
+        await page.waitForLoadState("networkidle");
+        await agent.aiAssert(assertion, errorMsg);
+      });
+      if (agent.dumpFile) {
+        testInfo.annotations.push({
+          type: "MIDSCENE_AI_ACTION",
+          description: JSON.stringify({
+            testId: testInfo.testId,
+            dumpPath: agent.dumpFile
+          })
+        });
+      }
     }
   };
 };