@midscene/web 0.9.2 → 0.10.0

package/dist/es/bridge-mode-browser.js

@@ -57,7 +57,10 @@ __export(browser_exports, {
 module.exports = __toCommonJS(browser_exports);
 
 // src/bridge-mode/page-browser-side.ts
-var import_node_assert3 = __toESM(require("assert"));
+var import_node_assert4 = __toESM(require("assert"));
+
+// src/chrome-extension/page.ts
+var import_node_assert2 = __toESM(require("assert"));
 
 // src/chrome-extension/cdpInput.ts
 var import_node_assert = __toESM(require("assert"));
@@ -644,9 +647,12 @@ var ChromeExtensionProxyPage = class {
   constructor(trackingActiveTab) {
     this.pageType = "chrome-extension-proxy";
     this.activeTabId = null;
+    this.tabIdOfDebuggerAttached = null;
     this.attachingDebugger = null;
+    this.destroyed = false;
     this.mouse = {
       click: async (x, y) => {
+        await this.showMousePointer(x, y);
         await this.sendCommandToDebugger("Input.dispatchMouseEvent", {
           type: "mousePressed",
           x,
@@ -663,15 +669,19 @@ var ChromeExtensionProxyPage = class {
         });
       },
       wheel: async (deltaX, deltaY, startX, startY) => {
+        const finalX = startX || 50;
+        const finalY = startY || 50;
+        await this.showMousePointer(finalX, finalY);
         await this.sendCommandToDebugger("Input.dispatchMouseEvent", {
           type: "mouseWheel",
-          x: startX || 10,
-          y: startY || 10,
+          x: finalX,
+          y: finalY,
           deltaX,
           deltaY
         });
       },
       move: async (x, y) => {
+        await this.showMousePointer(x, y);
         await this.sendCommandToDebugger("Input.dispatchMouseEvent", {
           type: "mouseMoved",
           x,
@@ -696,8 +706,7 @@ var ChromeExtensionProxyPage = class {
     this.trackingActiveTab = trackingActiveTab;
   }
   async getTabId() {
-    const trackingActiveTab = this.trackingActiveTab();
-    if (this.activeTabId && !trackingActiveTab) {
+    if (this.activeTabId && !this.trackingActiveTab) {
       return this.activeTabId;
     }
     const tabId = await chrome.tabs.query({ active: true, currentWindow: true }).then((tabs) => {
@@ -708,6 +717,7 @@ var ChromeExtensionProxyPage = class {
     return this.activeTabId;
   }
   async attachDebugger() {
+    (0, import_node_assert2.default)(!this.destroyed, "Page is destroyed");
     if (this.attachingDebugger) {
       await this.attachingDebugger;
       return;
@@ -721,52 +731,89 @@ var ChromeExtensionProxyPage = class {
       }
       try {
         const currentTabId = await this.getTabId();
-        const targets = await chrome.debugger.getTargets();
-        const target = targets.find(
-          (target2) => target2.tabId === currentTabId && target2.attached === true
-        );
-        if (!target) {
-          await chrome.debugger.attach({ tabId: currentTabId }, "1.3");
-          await sleep(340);
+        if (this.tabIdOfDebuggerAttached === currentTabId) {
+          return;
         }
+        if (this.tabIdOfDebuggerAttached && this.tabIdOfDebuggerAttached !== currentTabId) {
+          console.log(
+            "detach the previous tab",
+            this.tabIdOfDebuggerAttached,
+            "->",
+            currentTabId
+          );
+          try {
+            await this.detachDebugger(this.tabIdOfDebuggerAttached);
+          } catch (error) {
+            console.error("Failed to detach debugger", error);
+          }
+        }
+        await chrome.debugger.attach({ tabId: currentTabId }, "1.3");
+        await sleep(500);
+        this.tabIdOfDebuggerAttached = currentTabId;
+        await this.enableWaterFlowAnimation();
+      } catch (error) {
+        console.error("Failed to attach debugger", error);
       } finally {
         this.attachingDebugger = null;
       }
     })();
     await this.attachingDebugger;
   }
-  async enableWaterFlowAnimation(tabId) {
-    const script = await injectWaterFlowAnimation();
-    await chrome.debugger.sendCommand({ tabId }, "Runtime.evaluate", {
-      expression: script
+  async showMousePointer(x, y) {
+    const pointerScript = `(() => {
+      if(typeof window.midsceneWaterFlowAnimation !== 'undefined') {
+        window.midsceneWaterFlowAnimation.enable();
+        window.midsceneWaterFlowAnimation.showMousePointer(${x}, ${y});
+      } else {
+        console.log('midsceneWaterFlowAnimation is not defined');
+      }
+    })()`;
+    await this.sendCommandToDebugger("Runtime.evaluate", {
+      expression: `${pointerScript}`
+    });
+  }
+  async hideMousePointer() {
+    await this.sendCommandToDebugger("Runtime.evaluate", {
+      expression: `(() => {
+        if(typeof window.midsceneWaterFlowAnimation !== 'undefined') {
+          window.midsceneWaterFlowAnimation.hideMousePointer();
+        }
+      })()`
     });
   }
+  async detachDebugger(tabId) {
+    const tabIdToDetach = tabId || this.tabIdOfDebuggerAttached;
+    if (!tabIdToDetach) {
+      console.warn("No tab id to detach");
+      return;
+    }
+    await this.disableWaterFlowAnimation(tabIdToDetach);
+    await sleep(200);
+    await chrome.debugger.detach({ tabId: tabIdToDetach });
+    this.tabIdOfDebuggerAttached = null;
+  }
+  async enableWaterFlowAnimation() {
+    const script = await injectWaterFlowAnimation();
+    await chrome.debugger.sendCommand(
+      { tabId: this.tabIdOfDebuggerAttached },
+      "Runtime.evaluate",
+      {
+        expression: script
+      }
+    );
+  }
   async disableWaterFlowAnimation(tabId) {
     const script = await injectStopWaterFlowAnimation();
     await chrome.debugger.sendCommand({ tabId }, "Runtime.evaluate", {
       expression: script
     });
   }
-  async detachDebugger() {
-    const targets = await chrome.debugger.getTargets();
-    const attendTabs = targets.filter(
-      (target) => target.attached === true && !target.url.startsWith("chrome-extension://")
-    );
-    if (attendTabs.length > 0) {
-      for (const tab of attendTabs) {
-        if (tab.tabId) {
-          await this.disableWaterFlowAnimation(tab.tabId);
-          chrome.debugger.detach({ tabId: tab.tabId });
-        }
-      }
-    }
-  }
   async sendCommandToDebugger(command, params) {
     await this.attachDebugger();
-    const tabId = await this.getTabId();
-    this.enableWaterFlowAnimation(tabId);
+    (0, import_node_assert2.default)(this.tabIdOfDebuggerAttached, "Debugger is not attached");
+    this.enableWaterFlowAnimation();
     return await chrome.debugger.sendCommand(
-      { tabId },
+      { tabId: this.tabIdOfDebuggerAttached },
       command,
       params
     );
@@ -823,6 +870,7 @@ var ChromeExtensionProxyPage = class {
     );
   }
   async getElementInfos() {
+    await this.hideMousePointer();
     const content = await this.getPageContentByCDP();
     if (content == null ? void 0 : content.size) {
       this.viewportSize = content.size;
@@ -836,6 +884,7 @@ var ChromeExtensionProxyPage = class {
     return content.size;
   }
   async screenshotBase64() {
+    await this.hideMousePointer();
     const base64 = await this.sendCommandToDebugger("Page.captureScreenshot", {
       format: "jpeg",
       quality: 70
@@ -935,6 +984,7 @@ var ChromeExtensionProxyPage = class {
   async destroy() {
     this.activeTabId = null;
     await this.detachDebugger();
+    this.destroyed = true;
   }
 };
 
@@ -943,7 +993,7 @@ var DefaultBridgeServerPort = 3766;
 var DefaultLocalEndpoint = `http://127.0.0.1:${DefaultBridgeServerPort}`;
 
 // src/bridge-mode/io-client.ts
-var import_node_assert2 = __toESM(require("assert"));
+var import_node_assert3 = __toESM(require("assert"));
 var import_socket = require("socket.io-client");
 var BridgeClient = class {
   constructor(endpoint, onBridgeCall, onDisconnect) {
@@ -958,7 +1008,7 @@ var BridgeClient = class {
       this.socket = (0, import_socket.io)(this.endpoint, {
         reconnection: false,
         query: {
-          version: "0.9.2"
+          version: "0.10.0"
         }
       });
       const timeout = setTimeout(() => {
@@ -983,7 +1033,7 @@ var BridgeClient = class {
       });
       this.socket.on("bridge-call" /* Call */, (call) => {
         const id = call.id;
-        (0, import_node_assert2.default)(typeof id !== "undefined", "call id is required");
+        (0, import_node_assert3.default)(typeof id !== "undefined", "call id is required");
         Promise.resolve().then(async () => {
           var _a, _b;
           let response;
@@ -1017,8 +1067,8 @@ ${(e == null ? void 0 : e.stack) || ""}`;
 var ChromeExtensionPageBrowserSide = class extends ChromeExtensionProxyPage {
   constructor(onDisconnect = () => {
   }, onLogMessage = () => {
-  }) {
-    super(() => true);
+  }, trackingActiveTab = false) {
+    super(trackingActiveTab);
     this.onDisconnect = onDisconnect;
     this.onLogMessage = onLogMessage;
     this.bridgeClient = null;
@@ -1074,26 +1124,32 @@ var ChromeExtensionPageBrowserSide = class extends ChromeExtensionProxyPage {
     );
     await this.bridgeClient.connect();
     this.onLogMessage(
-      `Bridge connected, cli-side version v${this.bridgeClient.serverVersion}, browser-side version v${"0.9.2"}`,
+      `Bridge connected, cli-side version v${this.bridgeClient.serverVersion}, browser-side version v${"0.10.0"}`,
       "log"
     );
   }
   async connect() {
     return await this.setupBridgeClient();
   }
-  async connectNewTabWithUrl(url) {
+  async connectNewTabWithUrl(url, options) {
     const tab = await chrome.tabs.create({ url });
     const tabId = tab.id;
-    (0, import_node_assert3.default)(tabId, "failed to get tabId after creating a new tab");
+    (0, import_node_assert4.default)(tabId, "failed to get tabId after creating a new tab");
     this.onLogMessage(`Creating new tab: ${url}`, "log");
+    if (options == null ? void 0 : options.trackingActiveTab) {
+      this.trackingActiveTab = true;
+    }
   }
-  async connectCurrentTab() {
+  async connectCurrentTab(options) {
     var _a, _b;
     const tabs = await chrome.tabs.query({ active: true, currentWindow: true });
     console.log("current tab", tabs);
     const tabId = (_a = tabs[0]) == null ? void 0 : _a.id;
-    (0, import_node_assert3.default)(tabId, "failed to get tabId");
+    (0, import_node_assert4.default)(tabId, "failed to get tabId");
     this.onLogMessage(`Connected to current tab: ${(_b = tabs[0]) == null ? void 0 : _b.url}`, "log");
+    if (options == null ? void 0 : options.trackingActiveTab) {
+      this.trackingActiveTab = true;
+    }
   }
   async destroy() {
     if (this.bridgeClient) {

package/dist/es/bridge-mode.js

@@ -1649,7 +1649,7 @@ var BridgeServer = class {
           this.socket = socket;
           const clientVersion = socket.handshake.query.version;
           console.log(
-            `Bridge connected, cli-side version v${"0.9.2"}, browser-side version v${clientVersion}`
+            `Bridge connected, cli-side version v${"0.10.0"}, browser-side version v${clientVersion}`
           );
           socket.on("bridge-call-response" /* CallResponse */, (params) => {
             const id = params.id;
@@ -1682,7 +1682,7 @@ var BridgeServer = class {
             var _a;
             (_a = this.onConnect) == null ? void 0 : _a.call(this);
             const payload = {
-              version: "0.9.2"
+              version: "0.10.0"
             };
             socket.emit("bridge-connected" /* Connected */, payload);
             Promise.resolve().then(() => {
@@ -1771,6 +1771,7 @@ var BridgeServer = class {
 };
 
 // src/bridge-mode/agent-cli-side.ts
+var sleep2 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 var getBridgePageInCliSide = () => {
   const server = new BridgeServer(DefaultBridgeServerPort);
   server.listen();
@@ -1842,11 +1843,13 @@ var AgentOverChromeBridge = class extends PageAgent {
       }
     });
   }
-  async connectNewTabWithUrl(url) {
-    await this.page.connectNewTabWithUrl(url);
+  async connectNewTabWithUrl(url, options) {
+    await this.page.connectNewTabWithUrl(url, options);
+    await sleep2(500);
   }
-  async connectCurrentTab() {
-    await this.page.connectCurrentTab();
+  async connectCurrentTab(options) {
+    await this.page.connectCurrentTab(options);
+    await sleep2(500);
   }
   async aiAction(prompt, options) {
     if (options) {

package/dist/es/chrome-extension.js

@@ -1618,6 +1618,9 @@ var ChromeExtensionProxyPageAgent = class extends PageAgent {
   }
 };
 
+// src/chrome-extension/page.ts
+var import_node_assert4 = __toESM(require("assert"));
+
 // src/chrome-extension/cdpInput.ts
 var import_node_assert3 = __toESM(require("assert"));
 
@@ -2203,9 +2206,12 @@ var ChromeExtensionProxyPage = class {
   constructor(trackingActiveTab) {
     this.pageType = "chrome-extension-proxy";
     this.activeTabId = null;
+    this.tabIdOfDebuggerAttached = null;
     this.attachingDebugger = null;
+    this.destroyed = false;
     this.mouse = {
       click: async (x, y) => {
+        await this.showMousePointer(x, y);
         await this.sendCommandToDebugger("Input.dispatchMouseEvent", {
           type: "mousePressed",
           x,
@@ -2222,15 +2228,19 @@ var ChromeExtensionProxyPage = class {
         });
       },
       wheel: async (deltaX, deltaY, startX, startY) => {
+        const finalX = startX || 50;
+        const finalY = startY || 50;
+        await this.showMousePointer(finalX, finalY);
         await this.sendCommandToDebugger("Input.dispatchMouseEvent", {
           type: "mouseWheel",
-          x: startX || 10,
-          y: startY || 10,
+          x: finalX,
+          y: finalY,
           deltaX,
           deltaY
         });
       },
       move: async (x, y) => {
+        await this.showMousePointer(x, y);
         await this.sendCommandToDebugger("Input.dispatchMouseEvent", {
           type: "mouseMoved",
           x,
@@ -2255,8 +2265,7 @@ var ChromeExtensionProxyPage = class {
     this.trackingActiveTab = trackingActiveTab;
   }
   async getTabId() {
-    const trackingActiveTab = this.trackingActiveTab();
-    if (this.activeTabId && !trackingActiveTab) {
+    if (this.activeTabId && !this.trackingActiveTab) {
       return this.activeTabId;
     }
     const tabId = await chrome.tabs.query({ active: true, currentWindow: true }).then((tabs) => {
@@ -2267,6 +2276,7 @@ var ChromeExtensionProxyPage = class {
     return this.activeTabId;
   }
   async attachDebugger() {
+    (0, import_node_assert4.default)(!this.destroyed, "Page is destroyed");
     if (this.attachingDebugger) {
       await this.attachingDebugger;
       return;
@@ -2280,52 +2290,89 @@ var ChromeExtensionProxyPage = class {
       }
       try {
         const currentTabId = await this.getTabId();
-        const targets = await chrome.debugger.getTargets();
-        const target = targets.find(
-          (target2) => target2.tabId === currentTabId && target2.attached === true
-        );
-        if (!target) {
-          await chrome.debugger.attach({ tabId: currentTabId }, "1.3");
-          await sleep2(340);
+        if (this.tabIdOfDebuggerAttached === currentTabId) {
+          return;
         }
+        if (this.tabIdOfDebuggerAttached && this.tabIdOfDebuggerAttached !== currentTabId) {
+          console.log(
+            "detach the previous tab",
+            this.tabIdOfDebuggerAttached,
+            "->",
+            currentTabId
+          );
+          try {
+            await this.detachDebugger(this.tabIdOfDebuggerAttached);
+          } catch (error) {
+            console.error("Failed to detach debugger", error);
+          }
+        }
+        await chrome.debugger.attach({ tabId: currentTabId }, "1.3");
+        await sleep2(500);
+        this.tabIdOfDebuggerAttached = currentTabId;
+        await this.enableWaterFlowAnimation();
+      } catch (error) {
+        console.error("Failed to attach debugger", error);
       } finally {
         this.attachingDebugger = null;
       }
     })();
     await this.attachingDebugger;
   }
-  async enableWaterFlowAnimation(tabId) {
-    const script = await injectWaterFlowAnimation();
-    await chrome.debugger.sendCommand({ tabId }, "Runtime.evaluate", {
-      expression: script
+  async showMousePointer(x, y) {
+    const pointerScript = `(() => {
+      if(typeof window.midsceneWaterFlowAnimation !== 'undefined') {
+        window.midsceneWaterFlowAnimation.enable();
+        window.midsceneWaterFlowAnimation.showMousePointer(${x}, ${y});
+      } else {
+        console.log('midsceneWaterFlowAnimation is not defined');
+      }
+    })()`;
+    await this.sendCommandToDebugger("Runtime.evaluate", {
+      expression: `${pointerScript}`
+    });
+  }
+  async hideMousePointer() {
+    await this.sendCommandToDebugger("Runtime.evaluate", {
+      expression: `(() => {
+        if(typeof window.midsceneWaterFlowAnimation !== 'undefined') {
+          window.midsceneWaterFlowAnimation.hideMousePointer();
+        }
+      })()`
     });
   }
+  async detachDebugger(tabId) {
+    const tabIdToDetach = tabId || this.tabIdOfDebuggerAttached;
+    if (!tabIdToDetach) {
+      console.warn("No tab id to detach");
+      return;
+    }
+    await this.disableWaterFlowAnimation(tabIdToDetach);
+    await sleep2(200);
+    await chrome.debugger.detach({ tabId: tabIdToDetach });
+    this.tabIdOfDebuggerAttached = null;
+  }
+  async enableWaterFlowAnimation() {
+    const script = await injectWaterFlowAnimation();
+    await chrome.debugger.sendCommand(
+      { tabId: this.tabIdOfDebuggerAttached },
+      "Runtime.evaluate",
+      {
+        expression: script
+      }
+    );
+  }
   async disableWaterFlowAnimation(tabId) {
     const script = await injectStopWaterFlowAnimation();
     await chrome.debugger.sendCommand({ tabId }, "Runtime.evaluate", {
       expression: script
     });
   }
-  async detachDebugger() {
-    const targets = await chrome.debugger.getTargets();
-    const attendTabs = targets.filter(
-      (target) => target.attached === true && !target.url.startsWith("chrome-extension://")
-    );
-    if (attendTabs.length > 0) {
-      for (const tab of attendTabs) {
-        if (tab.tabId) {
-          await this.disableWaterFlowAnimation(tab.tabId);
-          chrome.debugger.detach({ tabId: tab.tabId });
-        }
-      }
-    }
-  }
   async sendCommandToDebugger(command, params) {
     await this.attachDebugger();
-    const tabId = await this.getTabId();
-    this.enableWaterFlowAnimation(tabId);
+    (0, import_node_assert4.default)(this.tabIdOfDebuggerAttached, "Debugger is not attached");
+    this.enableWaterFlowAnimation();
     return await chrome.debugger.sendCommand(
-      { tabId },
+      { tabId: this.tabIdOfDebuggerAttached },
       command,
       params
     );
@@ -2382,6 +2429,7 @@ var ChromeExtensionProxyPage = class {
     );
   }
   async getElementInfos() {
+    await this.hideMousePointer();
     const content = await this.getPageContentByCDP();
     if (content == null ? void 0 : content.size) {
       this.viewportSize = content.size;
@@ -2395,6 +2443,7 @@ var ChromeExtensionProxyPage = class {
     return content.size;
   }
   async screenshotBase64() {
+    await this.hideMousePointer();
     const base64 = await this.sendCommandToDebugger("Page.captureScreenshot", {
       format: "jpeg",
       quality: 70
@@ -2494,6 +2543,7 @@ var ChromeExtensionProxyPage = class {
   async destroy() {
     this.activeTabId = null;
     await this.detachDebugger();
+    this.destroyed = true;
   }
 };
 // Annotate the CommonJS export names for ESM import in node:

package/dist/es/index.js

@@ -1845,61 +1845,53 @@ var Page = class {
     }
     await this.keyboard.press("Backspace");
   }
-  async scrollUntilTop(startingPoint) {
-    if (startingPoint) {
-      await this.mouse.move(startingPoint.left, startingPoint.top);
+  async moveToPoint(point) {
+    if (point) {
+      await this.mouse.move(point.left, point.top);
+    } else {
+      const size = await this.size();
+      await this.mouse.move(size.width / 2, size.height / 2);
     }
+  }
+  async scrollUntilTop(startingPoint) {
+    await this.moveToPoint(startingPoint);
     return this.mouse.wheel(0, -9999999);
   }
   async scrollUntilBottom(startingPoint) {
-    if (startingPoint) {
-      await this.mouse.move(startingPoint.left, startingPoint.top);
-    }
+    await this.moveToPoint(startingPoint);
     return this.mouse.wheel(0, 9999999);
   }
   async scrollUntilLeft(startingPoint) {
-    if (startingPoint) {
-      await this.mouse.move(startingPoint.left, startingPoint.top);
-    }
+    await this.moveToPoint(startingPoint);
     return this.mouse.wheel(-9999999, 0);
   }
   async scrollUntilRight(startingPoint) {
-    if (startingPoint) {
-      await this.mouse.move(startingPoint.left, startingPoint.top);
-    }
+    await this.moveToPoint(startingPoint);
     return this.mouse.wheel(9999999, 0);
   }
   async scrollUp(distance, startingPoint) {
     const innerHeight = await this.evaluate(() => window.innerHeight);
     const scrollDistance = distance || innerHeight * 0.7;
-    if (startingPoint) {
-      await this.mouse.move(startingPoint.left, startingPoint.top);
-    }
-    await this.mouse.wheel(0, -scrollDistance);
+    await this.moveToPoint(startingPoint);
+    return this.mouse.wheel(0, -scrollDistance);
   }
   async scrollDown(distance, startingPoint) {
     const innerHeight = await this.evaluate(() => window.innerHeight);
     const scrollDistance = distance || innerHeight * 0.7;
-    if (startingPoint) {
-      await this.mouse.move(startingPoint.left, startingPoint.top);
-    }
-    await this.mouse.wheel(0, scrollDistance);
+    await this.moveToPoint(startingPoint);
+    return this.mouse.wheel(0, scrollDistance);
   }
   async scrollLeft(distance, startingPoint) {
     const innerWidth = await this.evaluate(() => window.innerWidth);
     const scrollDistance = distance || innerWidth * 0.7;
-    if (startingPoint) {
-      await this.mouse.move(startingPoint.left, startingPoint.top);
-    }
-    await this.mouse.wheel(-scrollDistance, 0);
+    await this.moveToPoint(startingPoint);
+    return this.mouse.wheel(-scrollDistance, 0);
   }
   async scrollRight(distance, startingPoint) {
     const innerWidth = await this.evaluate(() => window.innerWidth);
     const scrollDistance = distance || innerWidth * 0.7;
-    if (startingPoint) {
-      await this.mouse.move(startingPoint.left, startingPoint.top);
-    }
-    await this.mouse.wheel(scrollDistance, 0);
+    await this.moveToPoint(startingPoint);
+    return this.mouse.wheel(scrollDistance, 0);
   }
   async destroy() {
   }