@midscene/web 0.11.1 → 0.11.3

package/dist/es/appium.js

@@ -3362,7 +3362,9 @@ var TaskCache = class {
         if (!this.midscenePkgInfo) {
           return void 0;
         }
-        if (jsonData.pkgName !== this.midscenePkgInfo.name || jsonData.pkgVersion !== this.midscenePkgInfo.version) {
+        const jsonDataPkgVersion = jsonData.pkgVersion.split(".");
+        const midscenePkgInfoPkgVersion = this.midscenePkgInfo.version.split(".");
+        if (jsonDataPkgVersion[0] !== midscenePkgInfoPkgVersion[0] || jsonDataPkgVersion[1] !== midscenePkgInfoPkgVersion[1]) {
           return void 0;
         }
         return jsonData;
@@ -3396,6 +3398,56 @@ var TaskCache = class {
   }
 };
 
+// src/common/ui-utils.ts
+function typeStr(task) {
+  return task.subType ? `${task.type} / ${task.subType || ""}` : task.type;
+}
+function getKeyCommands(value) {
+  const keys = Array.isArray(value) ? value : [value];
+  return keys.reduce((acc, k) => {
+    const includeMeta = keys.includes("Meta") || keys.includes("Control");
+    if (includeMeta && (k === "a" || k === "A")) {
+      return acc.concat([{ key: k, command: "SelectAll" }]);
+    }
+    if (includeMeta && (k === "c" || k === "C")) {
+      return acc.concat([{ key: k, command: "Copy" }]);
+    }
+    if (includeMeta && (k === "v" || k === "V")) {
+      return acc.concat([{ key: k, command: "Paste" }]);
+    }
+    return acc.concat([{ key: k }]);
+  }, []);
+}
+function paramStr(task) {
+  var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k;
+  let value;
+  if (task.type === "Planning") {
+    value = (_a = task == null ? void 0 : task.param) == null ? void 0 : _a.userPrompt;
+  }
+  if (task.type === "Insight") {
+    value = ((_b = task == null ? void 0 : task.param) == null ? void 0 : _b.prompt) || ((_c = task == null ? void 0 : task.param) == null ? void 0 : _c.id) || ((_d = task == null ? void 0 : task.param) == null ? void 0 : _d.dataDemand) || ((_e = task == null ? void 0 : task.param) == null ? void 0 : _e.assertion);
+  }
+  if (task.type === "Action") {
+    const sleepMs = (_f = task == null ? void 0 : task.param) == null ? void 0 : _f.timeMs;
+    const scrollType = (_g = task == null ? void 0 : task.param) == null ? void 0 : _g.scrollType;
+    if (sleepMs) {
+      value = `${sleepMs}ms`;
+    } else if (scrollType) {
+      const scrollDirection = (_h = task == null ? void 0 : task.param) == null ? void 0 : _h.direction;
+      const scrollDistance = (_i = task == null ? void 0 : task.param) == null ? void 0 : _i.distance;
+      value = `${scrollDirection || "down"}, ${scrollType || "once"}, ${scrollDistance || "distance-not-set"}`;
+    } else {
+      value = ((_j = task == null ? void 0 : task.param) == null ? void 0 : _j.value) || ((_k = task == null ? void 0 : task.param) == null ? void 0 : _k.scrollType);
+    }
+    if (!value) {
+      value = task.thought;
+    }
+  }
+  if (typeof value === "undefined")
+    return "";
+  return typeof value === "string" ? value : JSON.stringify(value, void 0, 2);
+}
+
 // src/common/tasks.ts
 var PageTaskExecutor = class {
   constructor(page, insight, opts) {
@@ -3530,6 +3582,7 @@ var PageTaskExecutor = class {
               output: {
                 element
               },
+              pageContext,
               log: {
                 dump: insightDump
               },
@@ -3609,8 +3662,8 @@ var PageTaskExecutor = class {
           thought: plan2.thought,
           locate: plan2.locate,
           executor: async (taskParam) => {
-            (0, import_node_assert3.default)(taskParam == null ? void 0 : taskParam.value, "No key to press");
-            await this.page.keyboard.press(taskParam.value);
+            const keys = getKeyCommands(taskParam.value);
+            await this.page.keyboard.press(keys);
           }
         };
         tasks.push(taskActionKeyboardPress);
@@ -4018,7 +4071,7 @@ var PageTaskExecutor = class {
     const cacheGroup = this.taskCache.getCacheGroupByPrompt(userPrompt);
     const isCompleted = false;
     let currentActionNumber = 0;
-    const maxActionNumber = 20;
+    const maxActionNumber = 40;
     while (!isCompleted && currentActionNumber < maxActionNumber) {
       currentActionNumber++;
       const planningTask = this.planningTaskToGoal(
@@ -4238,40 +4291,6 @@ var WebElementInfo = class {
   }
 };
 
-// src/common/ui-utils.ts
-function typeStr(task) {
-  return task.subType ? `${task.type} / ${task.subType || ""}` : task.type;
-}
-function paramStr(task) {
-  var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k;
-  let value;
-  if (task.type === "Planning") {
-    value = (_a = task == null ? void 0 : task.param) == null ? void 0 : _a.userPrompt;
-  }
-  if (task.type === "Insight") {
-    value = ((_b = task == null ? void 0 : task.param) == null ? void 0 : _b.prompt) || ((_c = task == null ? void 0 : task.param) == null ? void 0 : _c.id) || ((_d = task == null ? void 0 : task.param) == null ? void 0 : _d.dataDemand) || ((_e = task == null ? void 0 : task.param) == null ? void 0 : _e.assertion);
-  }
-  if (task.type === "Action") {
-    const sleepMs = (_f = task == null ? void 0 : task.param) == null ? void 0 : _f.timeMs;
-    const scrollType = (_g = task == null ? void 0 : task.param) == null ? void 0 : _g.scrollType;
-    if (sleepMs) {
-      value = `${sleepMs}ms`;
-    } else if (scrollType) {
-      const scrollDirection = (_h = task == null ? void 0 : task.param) == null ? void 0 : _h.direction;
-      const scrollDistance = (_i = task == null ? void 0 : task.param) == null ? void 0 : _i.distance;
-      value = `${scrollDirection || "down"}, ${scrollType || "once"}, ${scrollDistance || "distance-not-set"}`;
-    } else {
-      value = ((_j = task == null ? void 0 : task.param) == null ? void 0 : _j.value) || ((_k = task == null ? void 0 : task.param) == null ? void 0 : _k.scrollType);
-    }
-    if (!value) {
-      value = task.thought;
-    }
-  }
-  if (typeof value === "undefined")
-    return "";
-  return typeof value === "string" ? value : JSON.stringify(value, void 0, 2);
-}
-
 // src/common/utils.ts
 var import_node_assert4 = __toESM(require("assert"));
 var import_node_fs3 = require("fs");
@@ -4607,7 +4626,7 @@ var Page = class {
   get keyboard() {
     return {
       type: (text) => this.keyboardType(text),
-      press: (key) => this.keyboardPress(key)
+      press: (action) => this.keyboardPressAction(action)
     };
   }
   async clearInput(element) {
@@ -4721,6 +4740,33 @@ var Page = class {
       }
     ]);
   }
+  async keyboardPressAction(action) {
+    if (Array.isArray(action)) {
+      for (const act of action) {
+        await this.browser.performActions([
+          {
+            type: "key",
+            id: "keyboard",
+            actions: [
+              { type: "keyDown", value: act.key },
+              { type: "keyUp", value: act.key }
+            ]
+          }
+        ]);
+      }
+    } else {
+      await this.browser.performActions([
+        {
+          type: "key",
+          id: "keyboard",
+          actions: [
+            { type: "keyDown", value: action.key },
+            { type: "keyUp", value: action.key }
+          ]
+        }
+      ]);
+    }
+  }
   async mouseClick(x, y, button = "left") {
     await this.mouseMove(x, y);
     await this.browser.performActions([

package/dist/es/bridge-mode-browser.js

@@ -61,6 +61,33 @@ var import_node_assert4 = __toESM(require("assert"));
 
 // src/chrome-extension/page.ts
 var import_node_assert2 = __toESM(require("assert"));
+
+// src/common/ui-utils.ts
+var limitOpenNewTabScript = `
+if (!window.__MIDSCENE_NEW_TAB_INTERCEPTOR_INITIALIZED__) {
+  window.__MIDSCENE_NEW_TAB_INTERCEPTOR_INITIALIZED__ = true;
+
+  // Intercept the window.open method (only once)
+  window.open = function(url) {
+    console.log('Blocked window.open:', url);
+    window.location.href = url;
+    return null;
+  };
+
+  // Block all a tag clicks with target="_blank" (only once)
+  document.addEventListener('click', function(e) {
+    const target = e.target.closest('a');
+    if (target && target.target === '_blank') {
+      e.preventDefault();
+      console.log('Blocked new tab:', target.href);
+      window.location.href = target.href;
+      target.removeAttribute('target');
+    }
+  }, true);
+}
+`;
+
+// src/chrome-extension/page.ts
 var import_extractor = require("@midscene/shared/extractor");
 
 // src/chrome-extension/cdpInput.ts
@@ -262,9 +289,9 @@ function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 var ChromeExtensionProxyPage = class {
-  constructor(trackingActiveTab) {
+  constructor(forceSameTabNavigation) {
     this.pageType = "chrome-extension-proxy";
-    this.version = "0.11.1";
+    this.version = "0.11.3";
     this.activeTabId = null;
     this.tabIdOfDebuggerAttached = null;
     this.attachingDebugger = null;
@@ -333,23 +360,24 @@ var ChromeExtensionProxyPage = class {
         });
         await cdpKeyboard.type(text, { delay: 0 });
       },
-      press: async (key) => {
+      press: async (action) => {
         const cdpKeyboard = new CdpKeyboard({
           send: this.sendCommandToDebugger.bind(this)
         });
-        const keys = Array.isArray(key) ? key : [key];
+        const keys = Array.isArray(action) ? action : [action];
         for (const k of keys) {
-          await cdpKeyboard.down(k);
+          const commands = k.command ? [k.command] : [];
+          await cdpKeyboard.down(k.key, { commands });
         }
         for (const k of [...keys].reverse()) {
-          await cdpKeyboard.up(k);
+          await cdpKeyboard.up(k.key);
         }
       }
     };
-    this.trackingActiveTab = trackingActiveTab;
+    this.forceSameTabNavigation = forceSameTabNavigation;
   }
   async getTabId() {
-    if (this.activeTabId && !this.trackingActiveTab) {
+    if (this.activeTabId && !this.forceSameTabNavigation) {
       return this.activeTabId;
     }
     const tabId = await chrome.tabs.query({ active: true, currentWindow: true }).then((tabs) => {
@@ -436,6 +464,15 @@ var ChromeExtensionProxyPage = class {
     this.tabIdOfDebuggerAttached = null;
   }
   async enableWaterFlowAnimation() {
+    if (this.forceSameTabNavigation) {
+      await chrome.debugger.sendCommand(
+        { tabId: this.tabIdOfDebuggerAttached },
+        "Runtime.evaluate",
+        {
+          expression: limitOpenNewTabScript
+        }
+      );
+    }
     const script = await injectWaterFlowAnimation();
     await chrome.debugger.sendCommand(
       { tabId: this.tabIdOfDebuggerAttached },
@@ -622,7 +659,9 @@ var ChromeExtensionProxyPage = class {
       commands: ["selectAll"]
     });
     await sleep(100);
-    await this.keyboard.press("Backspace");
+    await this.keyboard.press({
+      key: "Backspace"
+    });
   }
   async destroy() {
     this.activeTabId = null;
@@ -651,7 +690,7 @@ var BridgeClient = class {
       this.socket = (0, import_socket.io)(this.endpoint, {
         reconnection: false,
         query: {
-          version: "0.11.1"
+          version: "0.11.3"
         }
       });
       const timeout = setTimeout(() => {
@@ -710,8 +749,8 @@ ${(e == null ? void 0 : e.stack) || ""}`;
 var ChromeExtensionPageBrowserSide = class extends ChromeExtensionProxyPage {
   constructor(onDisconnect = () => {
   }, onLogMessage = () => {
-  }, trackingActiveTab = false) {
-    super(trackingActiveTab);
+  }, forceSameTabNavigation = true) {
+    super(forceSameTabNavigation);
     this.onDisconnect = onDisconnect;
     this.onLogMessage = onLogMessage;
     this.bridgeClient = null;
@@ -773,7 +812,7 @@ var ChromeExtensionPageBrowserSide = class extends ChromeExtensionProxyPage {
     );
     await this.bridgeClient.connect();
     this.onLogMessage(
-      `Bridge connected, cli-side version v${this.bridgeClient.serverVersion}, browser-side version v${"0.11.1"}`,
+      `Bridge connected, cli-side version v${this.bridgeClient.serverVersion}, browser-side version v${"0.11.3"}`,
       "log"
     );
   }
@@ -781,18 +820,18 @@ var ChromeExtensionPageBrowserSide = class extends ChromeExtensionProxyPage {
     return await this.setupBridgeClient();
   }
   async connectNewTabWithUrl(url, options = {
-    trackingActiveTab: true
+    forceSameTabNavigation: true
   }) {
     const tab = await chrome.tabs.create({ url });
     const tabId = tab.id;
     (0, import_node_assert4.default)(tabId, "failed to get tabId after creating a new tab");
     this.onLogMessage(`Creating new tab: ${url}`, "log");
-    if (options == null ? void 0 : options.trackingActiveTab) {
-      this.trackingActiveTab = true;
+    if (options == null ? void 0 : options.forceSameTabNavigation) {
+      this.forceSameTabNavigation = true;
     }
   }
   async connectCurrentTab(options = {
-    trackingActiveTab: true
+    forceSameTabNavigation: true
   }) {
     var _a, _b;
     const tabs = await chrome.tabs.query({ active: true, currentWindow: true });
@@ -800,8 +839,8 @@ var ChromeExtensionPageBrowserSide = class extends ChromeExtensionProxyPage {
     const tabId = (_a = tabs[0]) == null ? void 0 : _a.id;
     (0, import_node_assert4.default)(tabId, "failed to get tabId");
     this.onLogMessage(`Connected to current tab: ${(_b = tabs[0]) == null ? void 0 : _b.url}`, "log");
-    if (options == null ? void 0 : options.trackingActiveTab) {
-      this.trackingActiveTab = true;
+    if (options == null ? void 0 : options.forceSameTabNavigation) {
+      this.forceSameTabNavigation = true;
     }
   }
   async destroy() {

package/dist/es/bridge-mode.js

@@ -3365,7 +3365,9 @@ var TaskCache = class {
         if (!this.midscenePkgInfo) {
           return void 0;
         }
-        if (jsonData.pkgName !== this.midscenePkgInfo.name || jsonData.pkgVersion !== this.midscenePkgInfo.version) {
+        const jsonDataPkgVersion = jsonData.pkgVersion.split(".");
+        const midscenePkgInfoPkgVersion = this.midscenePkgInfo.version.split(".");
+        if (jsonDataPkgVersion[0] !== midscenePkgInfoPkgVersion[0] || jsonDataPkgVersion[1] !== midscenePkgInfoPkgVersion[1]) {
           return void 0;
         }
         return jsonData;
@@ -3399,6 +3401,56 @@ var TaskCache = class {
   }
 };
 
+// src/common/ui-utils.ts
+function typeStr(task) {
+  return task.subType ? `${task.type} / ${task.subType || ""}` : task.type;
+}
+function getKeyCommands(value) {
+  const keys = Array.isArray(value) ? value : [value];
+  return keys.reduce((acc, k) => {
+    const includeMeta = keys.includes("Meta") || keys.includes("Control");
+    if (includeMeta && (k === "a" || k === "A")) {
+      return acc.concat([{ key: k, command: "SelectAll" }]);
+    }
+    if (includeMeta && (k === "c" || k === "C")) {
+      return acc.concat([{ key: k, command: "Copy" }]);
+    }
+    if (includeMeta && (k === "v" || k === "V")) {
+      return acc.concat([{ key: k, command: "Paste" }]);
+    }
+    return acc.concat([{ key: k }]);
+  }, []);
+}
+function paramStr(task) {
+  var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k;
+  let value;
+  if (task.type === "Planning") {
+    value = (_a = task == null ? void 0 : task.param) == null ? void 0 : _a.userPrompt;
+  }
+  if (task.type === "Insight") {
+    value = ((_b = task == null ? void 0 : task.param) == null ? void 0 : _b.prompt) || ((_c = task == null ? void 0 : task.param) == null ? void 0 : _c.id) || ((_d = task == null ? void 0 : task.param) == null ? void 0 : _d.dataDemand) || ((_e = task == null ? void 0 : task.param) == null ? void 0 : _e.assertion);
+  }
+  if (task.type === "Action") {
+    const sleepMs = (_f = task == null ? void 0 : task.param) == null ? void 0 : _f.timeMs;
+    const scrollType = (_g = task == null ? void 0 : task.param) == null ? void 0 : _g.scrollType;
+    if (sleepMs) {
+      value = `${sleepMs}ms`;
+    } else if (scrollType) {
+      const scrollDirection = (_h = task == null ? void 0 : task.param) == null ? void 0 : _h.direction;
+      const scrollDistance = (_i = task == null ? void 0 : task.param) == null ? void 0 : _i.distance;
+      value = `${scrollDirection || "down"}, ${scrollType || "once"}, ${scrollDistance || "distance-not-set"}`;
+    } else {
+      value = ((_j = task == null ? void 0 : task.param) == null ? void 0 : _j.value) || ((_k = task == null ? void 0 : task.param) == null ? void 0 : _k.scrollType);
+    }
+    if (!value) {
+      value = task.thought;
+    }
+  }
+  if (typeof value === "undefined")
+    return "";
+  return typeof value === "string" ? value : JSON.stringify(value, void 0, 2);
+}
+
 // src/common/tasks.ts
 var PageTaskExecutor = class {
   constructor(page, insight, opts) {
@@ -3533,6 +3585,7 @@ var PageTaskExecutor = class {
               output: {
                 element
               },
+              pageContext,
               log: {
                 dump: insightDump
               },
@@ -3612,8 +3665,8 @@ var PageTaskExecutor = class {
           thought: plan2.thought,
           locate: plan2.locate,
           executor: async (taskParam) => {
-            (0, import_node_assert3.default)(taskParam == null ? void 0 : taskParam.value, "No key to press");
-            await this.page.keyboard.press(taskParam.value);
+            const keys = getKeyCommands(taskParam.value);
+            await this.page.keyboard.press(keys);
           }
         };
         tasks.push(taskActionKeyboardPress);
@@ -4021,7 +4074,7 @@ var PageTaskExecutor = class {
     const cacheGroup = this.taskCache.getCacheGroupByPrompt(userPrompt);
     const isCompleted = false;
     let currentActionNumber = 0;
-    const maxActionNumber = 20;
+    const maxActionNumber = 40;
     while (!isCompleted && currentActionNumber < maxActionNumber) {
       currentActionNumber++;
       const planningTask = this.planningTaskToGoal(
@@ -4241,40 +4294,6 @@ var WebElementInfo = class {
   }
 };
 
-// src/common/ui-utils.ts
-function typeStr(task) {
-  return task.subType ? `${task.type} / ${task.subType || ""}` : task.type;
-}
-function paramStr(task) {
-  var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k;
-  let value;
-  if (task.type === "Planning") {
-    value = (_a = task == null ? void 0 : task.param) == null ? void 0 : _a.userPrompt;
-  }
-  if (task.type === "Insight") {
-    value = ((_b = task == null ? void 0 : task.param) == null ? void 0 : _b.prompt) || ((_c = task == null ? void 0 : task.param) == null ? void 0 : _c.id) || ((_d = task == null ? void 0 : task.param) == null ? void 0 : _d.dataDemand) || ((_e = task == null ? void 0 : task.param) == null ? void 0 : _e.assertion);
-  }
-  if (task.type === "Action") {
-    const sleepMs = (_f = task == null ? void 0 : task.param) == null ? void 0 : _f.timeMs;
-    const scrollType = (_g = task == null ? void 0 : task.param) == null ? void 0 : _g.scrollType;
-    if (sleepMs) {
-      value = `${sleepMs}ms`;
-    } else if (scrollType) {
-      const scrollDirection = (_h = task == null ? void 0 : task.param) == null ? void 0 : _h.direction;
-      const scrollDistance = (_i = task == null ? void 0 : task.param) == null ? void 0 : _i.distance;
-      value = `${scrollDirection || "down"}, ${scrollType || "once"}, ${scrollDistance || "distance-not-set"}`;
-    } else {
-      value = ((_j = task == null ? void 0 : task.param) == null ? void 0 : _j.value) || ((_k = task == null ? void 0 : task.param) == null ? void 0 : _k.scrollType);
-    }
-    if (!value) {
-      value = task.thought;
-    }
-  }
-  if (typeof value === "undefined")
-    return "";
-  return typeof value === "string" ? value : JSON.stringify(value, void 0, 2);
-}
-
 // src/common/utils.ts
 var import_node_assert4 = __toESM(require("assert"));
 var import_node_fs3 = require("fs");
@@ -4619,7 +4638,7 @@ var BridgeServer = class {
           this.socket = socket;
           const clientVersion = socket.handshake.query.version;
           console.log(
-            `Bridge connected, cli-side version v${"0.11.1"}, browser-side version v${clientVersion}`
+            `Bridge connected, cli-side version v${"0.11.3"}, browser-side version v${clientVersion}`
           );
           socket.on("bridge-call-response" /* CallResponse */, (params) => {
             const id = params.id;
@@ -4652,7 +4671,7 @@ var BridgeServer = class {
             var _a;
             (_a = this.onConnect) == null ? void 0 : _a.call(this);
             const payload = {
-              version: "0.11.1"
+              version: "0.11.3"
             };
             socket.emit("bridge-connected" /* Connected */, payload);
             Promise.resolve().then(() => {
@@ -4741,7 +4760,7 @@ var BridgeServer = class {
 };
 
 // src/bridge-mode/agent-cli-side.ts
-var sleep2 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+var sleep3 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 var getBridgePageInCliSide = () => {
   const server = new BridgeServer(DefaultBridgeServerPort);
   server.listen();
@@ -4819,11 +4838,11 @@ var AgentOverChromeBridge = class extends PageAgent {
   }
   async connectNewTabWithUrl(url, options) {
     await this.page.connectNewTabWithUrl(url, options);
-    await sleep2(500);
+    await sleep3(500);
   }
   async connectCurrentTab(options) {
     await this.page.connectCurrentTab(options);
-    await sleep2(500);
+    await sleep3(500);
   }
   async aiAction(prompt, options) {
     if (options) {