@midscene/web 0.11.1 → 0.11.3

package/dist/es/index.js

@@ -330,7 +330,7 @@ var src_exports = {};
 __export(src_exports, {
   AppiumAgent: () => PageAgent,
   AppiumPage: () => Page2,
-  PlaywrightAgent: () => PageAgent,
+  PlaywrightAgent: () => PlaywrightAgent,
   PlaywrightAiFixture: () => PlaywrightAiFixture,
   PuppeteerAgent: () => PuppeteerAgent,
   ScriptPlayer: () => ScriptPlayer,
@@ -340,9 +340,6 @@ __export(src_exports, {
 });
 module.exports = __toCommonJS(src_exports);
 
-// src/playwright/ai-fixture.ts
-var import_node_crypto = require("crypto");
-
 // src/common/agent.ts
 var import_core2 = require("@midscene/core");
 var import_constants2 = require("@midscene/shared/constants");
@@ -3407,7 +3404,9 @@ var TaskCache = class {
         if (!this.midscenePkgInfo) {
           return void 0;
         }
-        if (jsonData.pkgName !== this.midscenePkgInfo.name || jsonData.pkgVersion !== this.midscenePkgInfo.version) {
+        const jsonDataPkgVersion = jsonData.pkgVersion.split(".");
+        const midscenePkgInfoPkgVersion = this.midscenePkgInfo.version.split(".");
+        if (jsonDataPkgVersion[0] !== midscenePkgInfoPkgVersion[0] || jsonDataPkgVersion[1] !== midscenePkgInfoPkgVersion[1]) {
           return void 0;
         }
         return jsonData;
@@ -3441,6 +3440,56 @@ var TaskCache = class {
   }
 };
 
+// src/common/ui-utils.ts
+function typeStr(task) {
+  return task.subType ? `${task.type} / ${task.subType || ""}` : task.type;
+}
+function getKeyCommands(value) {
+  const keys = Array.isArray(value) ? value : [value];
+  return keys.reduce((acc, k) => {
+    const includeMeta = keys.includes("Meta") || keys.includes("Control");
+    if (includeMeta && (k === "a" || k === "A")) {
+      return acc.concat([{ key: k, command: "SelectAll" }]);
+    }
+    if (includeMeta && (k === "c" || k === "C")) {
+      return acc.concat([{ key: k, command: "Copy" }]);
+    }
+    if (includeMeta && (k === "v" || k === "V")) {
+      return acc.concat([{ key: k, command: "Paste" }]);
+    }
+    return acc.concat([{ key: k }]);
+  }, []);
+}
+function paramStr(task) {
+  var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k;
+  let value;
+  if (task.type === "Planning") {
+    value = (_a = task == null ? void 0 : task.param) == null ? void 0 : _a.userPrompt;
+  }
+  if (task.type === "Insight") {
+    value = ((_b = task == null ? void 0 : task.param) == null ? void 0 : _b.prompt) || ((_c = task == null ? void 0 : task.param) == null ? void 0 : _c.id) || ((_d = task == null ? void 0 : task.param) == null ? void 0 : _d.dataDemand) || ((_e = task == null ? void 0 : task.param) == null ? void 0 : _e.assertion);
+  }
+  if (task.type === "Action") {
+    const sleepMs = (_f = task == null ? void 0 : task.param) == null ? void 0 : _f.timeMs;
+    const scrollType = (_g = task == null ? void 0 : task.param) == null ? void 0 : _g.scrollType;
+    if (sleepMs) {
+      value = `${sleepMs}ms`;
+    } else if (scrollType) {
+      const scrollDirection = (_h = task == null ? void 0 : task.param) == null ? void 0 : _h.direction;
+      const scrollDistance = (_i = task == null ? void 0 : task.param) == null ? void 0 : _i.distance;
+      value = `${scrollDirection || "down"}, ${scrollType || "once"}, ${scrollDistance || "distance-not-set"}`;
+    } else {
+      value = ((_j = task == null ? void 0 : task.param) == null ? void 0 : _j.value) || ((_k = task == null ? void 0 : task.param) == null ? void 0 : _k.scrollType);
+    }
+    if (!value) {
+      value = task.thought;
+    }
+  }
+  if (typeof value === "undefined")
+    return "";
+  return typeof value === "string" ? value : JSON.stringify(value, void 0, 2);
+}
+
 // src/common/tasks.ts
 var PageTaskExecutor = class {
   constructor(page, insight, opts) {
@@ -3575,6 +3624,7 @@ var PageTaskExecutor = class {
               output: {
                 element
               },
+              pageContext,
               log: {
                 dump: insightDump
               },
@@ -3654,8 +3704,8 @@ var PageTaskExecutor = class {
           thought: plan2.thought,
           locate: plan2.locate,
           executor: async (taskParam) => {
-            (0, import_node_assert3.default)(taskParam == null ? void 0 : taskParam.value, "No key to press");
-            await this.page.keyboard.press(taskParam.value);
+            const keys = getKeyCommands(taskParam.value);
+            await this.page.keyboard.press(keys);
           }
         };
         tasks.push(taskActionKeyboardPress);
@@ -4063,7 +4113,7 @@ var PageTaskExecutor = class {
     const cacheGroup = this.taskCache.getCacheGroupByPrompt(userPrompt);
     const isCompleted = false;
     let currentActionNumber = 0;
-    const maxActionNumber = 20;
+    const maxActionNumber = 40;
     while (!isCompleted && currentActionNumber < maxActionNumber) {
       currentActionNumber++;
       const planningTask = this.planningTaskToGoal(
@@ -4283,40 +4333,6 @@ var WebElementInfo = class {
   }
 };
 
-// src/common/ui-utils.ts
-function typeStr(task) {
-  return task.subType ? `${task.type} / ${task.subType || ""}` : task.type;
-}
-function paramStr(task) {
-  var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k;
-  let value;
-  if (task.type === "Planning") {
-    value = (_a = task == null ? void 0 : task.param) == null ? void 0 : _a.userPrompt;
-  }
-  if (task.type === "Insight") {
-    value = ((_b = task == null ? void 0 : task.param) == null ? void 0 : _b.prompt) || ((_c = task == null ? void 0 : task.param) == null ? void 0 : _c.id) || ((_d = task == null ? void 0 : task.param) == null ? void 0 : _d.dataDemand) || ((_e = task == null ? void 0 : task.param) == null ? void 0 : _e.assertion);
-  }
-  if (task.type === "Action") {
-    const sleepMs = (_f = task == null ? void 0 : task.param) == null ? void 0 : _f.timeMs;
-    const scrollType = (_g = task == null ? void 0 : task.param) == null ? void 0 : _g.scrollType;
-    if (sleepMs) {
-      value = `${sleepMs}ms`;
-    } else if (scrollType) {
-      const scrollDirection = (_h = task == null ? void 0 : task.param) == null ? void 0 : _h.direction;
-      const scrollDistance = (_i = task == null ? void 0 : task.param) == null ? void 0 : _i.distance;
-      value = `${scrollDirection || "down"}, ${scrollType || "once"}, ${scrollDistance || "distance-not-set"}`;
-    } else {
-      value = ((_j = task == null ? void 0 : task.param) == null ? void 0 : _j.value) || ((_k = task == null ? void 0 : task.param) == null ? void 0 : _k.scrollType);
-    }
-    if (!value) {
-      value = task.thought;
-    }
-  }
-  if (typeof value === "undefined")
-    return "";
-  return typeof value === "string" ? value : JSON.stringify(value, void 0, 2);
-}
-
 // src/common/utils.ts
 var import_node_assert4 = __toESM(require("assert"));
 var import_node_fs3 = require("fs");
@@ -4609,160 +4625,6 @@ ${errors}`);
   }
 };
 
-// src/playwright/ai-fixture.ts
-var import_test = require("@playwright/test");
-var groupAndCaseForTest = (testInfo) => {
-  let taskFile;
-  let taskTitle;
-  const titlePath = [...testInfo.titlePath];
-  if (titlePath.length > 1) {
-    taskTitle = titlePath.pop() || "unnamed";
-    taskFile = `${titlePath.join(" > ")}`;
-  } else if (titlePath.length === 1) {
-    taskTitle = titlePath[0];
-    taskFile = `${taskTitle}`;
-  } else {
-    taskTitle = "unnamed";
-    taskFile = "unnamed";
-  }
-  return { taskFile, taskTitle };
-};
-var midsceneAgentKeyId = "_midsceneAgentId";
-var midsceneDumpAnnotationId = "MIDSCENE_DUMP_ANNOTATION";
-var PlaywrightAiFixture = () => {
-  const pageAgentMap = {};
-  const agentForPage = (page, testInfo) => {
-    let idForPage = page[midsceneAgentKeyId];
-    if (!idForPage) {
-      idForPage = (0, import_node_crypto.randomUUID)();
-      page[midsceneAgentKeyId] = idForPage;
-      const { testId } = testInfo;
-      const { taskFile, taskTitle } = groupAndCaseForTest(testInfo);
-      pageAgentMap[idForPage] = new PageAgent(new WebPage(page), {
-        testId: `playwright-${testId}-${idForPage}`,
-        cacheId: `${taskFile}(${taskTitle})`,
-        groupName: taskTitle,
-        groupDescription: taskFile,
-        generateReport: false
-        // we will generate it in the reporter
-      });
-    }
-    return pageAgentMap[idForPage];
-  };
-  const updateDumpAnnotation = (test2, dump2) => {
-    const currentAnnotation = test2.annotations.find((item) => {
-      return item.type === midsceneDumpAnnotationId;
-    });
-    if (currentAnnotation) {
-      currentAnnotation.description = dump2;
-    } else {
-      test2.annotations.push({
-        type: midsceneDumpAnnotationId,
-        description: dump2
-      });
-    }
-  };
-  return {
-    ai: async ({ page }, use, testInfo) => {
-      const agent = agentForPage(page, testInfo);
-      await use(
-        async (taskPrompt, opts) => {
-          return new Promise((resolve, reject) => {
-            import_test.test.step(`ai - ${taskPrompt}`, async () => {
-              await waitForNetworkIdle(page);
-              const actionType = (opts == null ? void 0 : opts.type) || "action";
-              try {
-                const result = await agent.ai(taskPrompt, actionType);
-                resolve(result);
-              } catch (error) {
-                reject(error);
-              }
-            });
-          });
-        }
-      );
-      updateDumpAnnotation(testInfo, agent.dumpDataString());
-    },
-    aiAction: async ({ page }, use, testInfo) => {
-      const agent = agentForPage(page, testInfo);
-      await use(async (taskPrompt) => {
-        return new Promise((resolve, reject) => {
-          import_test.test.step(`aiAction - ${taskPrompt}`, async () => {
-            await waitForNetworkIdle(page);
-            try {
-              const result = await agent.aiAction(taskPrompt);
-              resolve(result);
-            } catch (error) {
-              reject(error);
-            }
-          });
-        });
-      });
-      updateDumpAnnotation(testInfo, agent.dumpDataString());
-    },
-    aiQuery: async ({ page }, use, testInfo) => {
-      const agent = agentForPage(page, testInfo);
-      await use(async (demand) => {
-        return new Promise((resolve, reject) => {
-          import_test.test.step(`aiQuery - ${JSON.stringify(demand)}`, async () => {
-            await waitForNetworkIdle(page);
-            try {
-              const result = await agent.aiQuery(demand);
-              resolve(result);
-            } catch (error) {
-              reject(error);
-            }
-          });
-        });
-      });
-      updateDumpAnnotation(testInfo, agent.dumpDataString());
-    },
-    aiAssert: async ({ page }, use, testInfo) => {
-      const agent = agentForPage(page, testInfo);
-      await use(async (assertion, errorMsg) => {
-        return new Promise((resolve, reject) => {
-          import_test.test.step(`aiAssert - ${assertion}`, async () => {
-            await waitForNetworkIdle(page);
-            try {
-              await agent.aiAssert(assertion, errorMsg);
-              resolve(null);
-            } catch (error) {
-              reject(error);
-            }
-          });
-        });
-      });
-      updateDumpAnnotation(testInfo, agent.dumpDataString());
-    },
-    aiWaitFor: async ({ page }, use, testInfo) => {
-      const agent = agentForPage(page, testInfo);
-      await use(async (assertion, opt) => {
-        return new Promise((resolve, reject) => {
-          import_test.test.step(`aiWaitFor - ${assertion}`, async () => {
-            await waitForNetworkIdle(page);
-            try {
-              await agent.aiWaitFor(assertion, opt);
-              resolve(null);
-            } catch (error) {
-              reject(error);
-            }
-          });
-        });
-      });
-      updateDumpAnnotation(testInfo, agent.dumpDataString());
-    }
-  };
-};
-async function waitForNetworkIdle(page, timeout = 2e4) {
-  try {
-    await page.waitForLoadState("networkidle", { timeout });
-  } catch (error) {
-    console.warn(
-      `Network idle timeout exceeded: ${error instanceof Error ? error.message : String(error)}`
-    );
-  }
-}
-
 // src/puppeteer/base-page.ts
 var import_utils9 = require("@midscene/core/utils");
 var import_extractor2 = require("@midscene/shared/extractor");
@@ -4778,12 +4640,19 @@ var Page = class {
     this.underlyingPage = underlyingPage;
     this.pageType = pageType;
   }
+  async waitForNavigation() {
+    if (this.pageType === "puppeteer" || this.pageType === "playwright") {
+      await this.underlyingPage.waitForSelector("html");
+    }
+  }
   // @deprecated
   async getElementsInfo() {
+    await this.waitForNavigation();
     const tree = await this.getElementsNodeTree();
     return (0, import_extractor2.treeToList)(tree);
   }
   async getElementsNodeTree() {
+    await this.waitForNavigation();
     const scripts = await getExtraReturnLogic(true);
     const captureElementSnapshot = await this.evaluate(scripts);
     return captureElementSnapshot;
@@ -4804,6 +4673,7 @@ var Page = class {
   async screenshotBase64() {
     const imgType = "jpeg";
     const path2 = (0, import_utils9.getTmpFile)(imgType);
+    await this.waitForNavigation();
     await this.underlyingPage.screenshot({
       path: path2,
       type: imgType,
@@ -4861,13 +4731,14 @@ var Page = class {
   get keyboard() {
     return {
       type: async (text) => this.underlyingPage.keyboard.type(text, { delay: 80 }),
-      press: async (key) => {
-        const keys = Array.isArray(key) ? key : [key];
-        for (const key2 of keys) {
-          await this.underlyingPage.keyboard.down(key2);
+      press: async (action) => {
+        const keys = Array.isArray(action) ? action : [action];
+        for (const k of keys) {
+          const commands = k.command ? [k.command] : [];
+          await this.underlyingPage.keyboard.down(k.key, { commands });
         }
-        for (const key2 of [...keys].reverse()) {
-          await this.underlyingPage.keyboard.up(key2);
+        for (const k of [...keys].reverse()) {
+          await this.underlyingPage.keyboard.up(k.key);
         }
       },
       down: async (key) => {
@@ -4902,7 +4773,7 @@ var Page = class {
       await this.underlyingPage.keyboard.up("Control");
     }
     await (0, import_utils9.sleep)(100);
-    await this.keyboard.press("Backspace");
+    await this.keyboard.press([{ key: "Backspace" }]);
   }
   async moveToPoint(point) {
     if (point) {
@@ -4963,8 +4834,186 @@ var WebPage = class extends Page {
   }
 };
 
+// src/playwright/ai-fixture.ts
+var import_node_crypto = require("crypto");
+var import_test = require("@playwright/test");
+var groupAndCaseForTest = (testInfo) => {
+  let taskFile;
+  let taskTitle;
+  const titlePath = [...testInfo.titlePath];
+  if (titlePath.length > 1) {
+    taskTitle = titlePath.pop() || "unnamed";
+    taskFile = `${titlePath.join(" > ")}`;
+  } else if (titlePath.length === 1) {
+    taskTitle = titlePath[0];
+    taskFile = `${taskTitle}`;
+  } else {
+    taskTitle = "unnamed";
+    taskFile = "unnamed";
+  }
+  return { taskFile, taskTitle };
+};
+var midsceneAgentKeyId = "_midsceneAgentId";
+var midsceneDumpAnnotationId = "MIDSCENE_DUMP_ANNOTATION";
+var PlaywrightAiFixture = (options) => {
+  const { forceSameTabNavigation = true } = options != null ? options : {};
+  const pageAgentMap = {};
+  const agentForPage = (page, testInfo) => {
+    let idForPage = page[midsceneAgentKeyId];
+    if (!idForPage) {
+      idForPage = (0, import_node_crypto.randomUUID)();
+      page[midsceneAgentKeyId] = idForPage;
+      const { testId } = testInfo;
+      const { taskFile, taskTitle } = groupAndCaseForTest(testInfo);
+      pageAgentMap[idForPage] = new PlaywrightAgent(page, {
+        testId: `playwright-${testId}-${idForPage}`,
+        forceSameTabNavigation,
+        cacheId: `${taskFile}(${taskTitle})`,
+        groupName: taskTitle,
+        groupDescription: taskFile,
+        generateReport: false
+        // we will generate it in the reporter
+      });
+    }
+    return pageAgentMap[idForPage];
+  };
+  const updateDumpAnnotation = (test2, dump2) => {
+    const currentAnnotation = test2.annotations.find((item) => {
+      return item.type === midsceneDumpAnnotationId;
+    });
+    if (currentAnnotation) {
+      currentAnnotation.description = dump2;
+    } else {
+      test2.annotations.push({
+        type: midsceneDumpAnnotationId,
+        description: dump2
+      });
+    }
+  };
+  return {
+    ai: async ({ page }, use, testInfo) => {
+      const agent = agentForPage(page, testInfo);
+      await use(
+        async (taskPrompt, opts) => {
+          return new Promise((resolve, reject) => {
+            const { type: type2 = "action" } = opts || {};
+            import_test.test.step(`ai - ${taskPrompt}`, async () => {
+              await waitForNetworkIdle(page);
+              try {
+                const result = await agent.ai(taskPrompt, type2);
+                resolve(result);
+              } catch (error) {
+                reject(error);
+              }
+            });
+          });
+        }
+      );
+      updateDumpAnnotation(testInfo, agent.dumpDataString());
+    },
+    aiAction: async ({ page }, use, testInfo) => {
+      const agent = agentForPage(page, testInfo);
+      await use(async (taskPrompt) => {
+        return new Promise((resolve, reject) => {
+          import_test.test.step(`aiAction - ${taskPrompt}`, async () => {
+            await waitForNetworkIdle(page);
+            try {
+              const result = await agent.aiAction(taskPrompt);
+              resolve(result);
+            } catch (error) {
+              reject(error);
+            }
+          });
+        });
+      });
+      updateDumpAnnotation(testInfo, agent.dumpDataString());
+    },
+    aiQuery: async ({ page }, use, testInfo) => {
+      const agent = agentForPage(page, testInfo);
+      await use(async (demand) => {
+        return new Promise((resolve, reject) => {
+          import_test.test.step(`aiQuery - ${JSON.stringify(demand)}`, async () => {
+            await waitForNetworkIdle(page);
+            try {
+              const result = await agent.aiQuery(demand);
+              resolve(result);
+            } catch (error) {
+              reject(error);
+            }
+          });
+        });
+      });
+      updateDumpAnnotation(testInfo, agent.dumpDataString());
+    },
+    aiAssert: async ({ page }, use, testInfo) => {
+      const agent = agentForPage(page, testInfo);
+      await use(async (assertion, errorMsg) => {
+        return new Promise((resolve, reject) => {
+          import_test.test.step(`aiAssert - ${assertion}`, async () => {
+            await waitForNetworkIdle(page);
+            try {
+              await agent.aiAssert(assertion, errorMsg);
+              resolve(null);
+            } catch (error) {
+              reject(error);
+            }
+          });
+        });
+      });
+      updateDumpAnnotation(testInfo, agent.dumpDataString());
+    },
+    aiWaitFor: async ({ page }, use, testInfo) => {
+      const agent = agentForPage(page, testInfo);
+      await use(async (assertion, opt) => {
+        return new Promise((resolve, reject) => {
+          import_test.test.step(`aiWaitFor - ${assertion}`, async () => {
+            await waitForNetworkIdle(page);
+            try {
+              await agent.aiWaitFor(assertion, opt);
+              resolve(null);
+            } catch (error) {
+              reject(error);
+            }
+          });
+        });
+      });
+      updateDumpAnnotation(testInfo, agent.dumpDataString());
+    }
+  };
+};
+async function waitForNetworkIdle(page, timeout = 2e4) {
+  try {
+    await page.waitForLoadState("networkidle", { timeout });
+  } catch (error) {
+    console.warn(
+      `Network idle timeout exceeded: ${error instanceof Error ? error.message : String(error)}`
+    );
+  }
+}
+
 // src/playwright/index.ts
 var import_env4 = require("@midscene/core/env");
+var PlaywrightAgent = class extends PageAgent {
+  constructor(page, opts) {
+    const webPage = new WebPage(page);
+    super(webPage, opts);
+    const { forceSameTabNavigation = true } = opts != null ? opts : {};
+    if (forceSameTabNavigation) {
+      page.on("popup", async (popup) => {
+        if (!popup) {
+          console.warn(
+            "got a popup event, but the popup is not ready yet, skip"
+          );
+          return;
+        }
+        const url = await popup.url();
+        console.log(`Popup opened: ${url}`);
+        await popup.close();
+        await page.goto(url);
+      });
+    }
+  }
+};
 
 // src/puppeteer/page.ts
 var WebPage2 = class extends Page {
@@ -4994,21 +5043,19 @@ var PuppeteerAgent = class extends PageAgent {
   constructor(page, opts) {
     const webPage = new WebPage2(page);
     super(webPage, opts);
-    if (opts == null ? void 0 : opts.trackingActiveTab) {
-      const browser = this.page.underlyingPage.browser();
-      browser.on("targetcreated", async (target) => {
-        if (target.type() === "page") {
-          const targetPage = await target.page();
-          if (!targetPage) {
-            console.warn(
-              "got a targetPage event, but the page is not ready yet, skip"
-            );
-            return;
-          }
-          const midscenePage = new WebPage2(targetPage);
-          this.page = midscenePage;
-          this.taskExecutor.page = midscenePage;
+    const { forceSameTabNavigation = true } = opts != null ? opts : {};
+    if (forceSameTabNavigation) {
+      page.on("popup", async (popup) => {
+        if (!popup) {
+          console.warn(
+            "got a popup event, but the popup is not ready yet, skip"
+          );
+          return;
         }
+        const url = await popup.url();
+        console.log(`Popup opened: ${url}`);
+        await popup.close();
+        await page.goto(url);
       });
     }
   }
@@ -5067,7 +5114,7 @@ var Page2 = class {
   get keyboard() {
     return {
       type: (text) => this.keyboardType(text),
-      press: (key) => this.keyboardPress(key)
+      press: (action) => this.keyboardPressAction(action)
     };
   }
   async clearInput(element) {
@@ -5181,6 +5228,33 @@ var Page2 = class {
       }
     ]);
   }
+  async keyboardPressAction(action) {
+    if (Array.isArray(action)) {
+      for (const act of action) {
+        await this.browser.performActions([
+          {
+            type: "key",
+            id: "keyboard",
+            actions: [
+              { type: "keyDown", value: act.key },
+              { type: "keyUp", value: act.key }
+            ]
+          }
+        ]);
+      }
+    } else {
+      await this.browser.performActions([
+        {
+          type: "key",
+          id: "keyboard",
+          actions: [
+            { type: "keyDown", value: action.key },
+            { type: "keyUp", value: action.key }
+          ]
+        }
+      ]);
+    }
+  }
   async mouseClick(x, y, button = "left") {
     await this.mouseMove(x, y);
     await this.browser.performActions([