@midscene/android 0.14.2 → 0.14.3-beta-20250409031306.0

package/dist/lib/playground.js

@@ -0,0 +1,872 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+
+// src/playground/bin.ts
+var import_node_path2 = __toESM(require("path"));
+var import_midscene_server = __toESM(require("@midscene/web/midscene-server"));
+var import_open = __toESM(require("open"));
+
+// src/page/index.ts
+var import_node_assert = __toESM(require("assert"));
+var import_node_fs = __toESM(require("fs"));
+var import_node_path = __toESM(require("path"));
+var import_utils = require("@midscene/core/utils");
+var import_img = require("@midscene/shared/img");
+var import_logger = require("@midscene/shared/logger");
+var import_appium_adb = require("appium-adb");
+var androidScreenshotPath = "/data/local/tmp/midscene_screenshot.png";
+var debugPage = (0, import_logger.getDebug)("android:device");
+var AndroidDevice = class {
+  constructor(deviceId) {
+    this.screenSize = null;
+    this.yadbPushed = false;
+    this.deviceRatio = 1;
+    this.adb = null;
+    this.connectingAdb = null;
+    this.pageType = "android";
+    (0, import_node_assert.default)(deviceId, "deviceId is required for AndroidDevice");
+    this.deviceId = deviceId;
+  }
+  async connect() {
+    return this.getAdb();
+  }
+  async getAdb() {
+    if (this.adb) {
+      return this.adb;
+    }
+    if (this.connectingAdb) {
+      return this.connectingAdb;
+    }
+    this.connectingAdb = (async () => {
+      let error = null;
+      debugPage(`Initializing ADB with device ID: ${this.deviceId}`);
+      try {
+        this.adb = await import_appium_adb.ADB.createADB({
+          udid: this.deviceId,
+          adbExecTimeout: 6e4
+        });
+        debugPage("ADB initialized successfully");
+        return this.adb;
+      } catch (e) {
+        debugPage(`Failed to initialize ADB: ${e}`);
+        error = new Error(`Unable to connect to device ${this.deviceId}: ${e}`);
+      } finally {
+        this.connectingAdb = null;
+      }
+      if (error) {
+        throw error;
+      }
+      throw new Error("ADB initialization failed unexpectedly");
+    })();
+    return this.connectingAdb;
+  }
+  async launch(uri) {
+    const adb = await this.getAdb();
+    try {
+      if (uri.startsWith("http://") || uri.startsWith("https://") || uri.includes("://")) {
+        await adb.startUri(uri);
+      } else if (uri.includes("/")) {
+        const [appPackage, appActivity] = uri.split("/");
+        await adb.startApp({
+          pkg: appPackage,
+          activity: appActivity
+        });
+      } else {
+        await adb.startApp({
+          pkg: uri
+        });
+      }
+      debugPage(`Successfully launched: ${uri}`);
+    } catch (error) {
+      debugPage(`Error launching ${uri}: ${error}`);
+      throw new Error(`Failed to launch ${uri}: ${error}`, { cause: error });
+    }
+    return this;
+  }
+  async execYadb(keyboardContent) {
+    await this.ensureYadb();
+    const adb = await this.getAdb();
+    await adb.shell(
+      `app_process -Djava.class.path=/data/local/tmp/yadb /data/local/tmp com.ysbing.yadb.Main -keyboard "${keyboardContent}"`
+    );
+  }
+  async getElementsInfo() {
+    return [];
+  }
+  async getElementsNodeTree() {
+    return {
+      node: null,
+      children: []
+    };
+  }
+  async size() {
+    if (this.screenSize) {
+      return this.screenSize;
+    }
+    const adb = await this.getAdb();
+    const screenSize = await adb.getScreenSize();
+    let width;
+    let height;
+    if (typeof screenSize === "string") {
+      const match = screenSize.match(/(\d+)x(\d+)/);
+      if (!match || match.length < 3) {
+        throw new Error(`Unable to parse screen size: ${screenSize}`);
+      }
+      width = Number.parseInt(match[1], 10);
+      height = Number.parseInt(match[2], 10);
+    } else if (typeof screenSize === "object" && screenSize !== null) {
+      const sizeObj = screenSize;
+      if ("width" in sizeObj && "height" in sizeObj) {
+        width = Number(sizeObj.width);
+        height = Number(sizeObj.height);
+      } else {
+        throw new Error(
+          `Invalid screen size object: ${JSON.stringify(screenSize)}`
+        );
+      }
+    } else {
+      throw new Error(`Invalid screen size format: ${screenSize}`);
+    }
+    const densityNum = await adb.getScreenDensity();
+    this.deviceRatio = Number(densityNum) / 160;
+    const { x: logicalWidth, y: logicalHeight } = this.reverseAdjustCoordinates(
+      width,
+      height
+    );
+    this.screenSize = {
+      width: logicalWidth,
+      height: logicalHeight
+    };
+    return this.screenSize;
+  }
+  /**
+   * Convert logical coordinates to physical coordinates, handling device ratio
+   * @param x Logical X coordinate
+   * @param y Logical Y coordinate
+   * @returns Physical coordinate point
+   */
+  adjustCoordinates(x, y) {
+    const ratio = this.deviceRatio;
+    return {
+      x: Math.round(x * ratio),
+      y: Math.round(y * ratio)
+    };
+  }
+  /**
+   * Convert physical coordinates to logical coordinates, handling device ratio
+   * @param x Physical X coordinate
+   * @param y Physical Y coordinate
+   * @returns Logical coordinate point
+   */
+  reverseAdjustCoordinates(x, y) {
+    const ratio = this.deviceRatio;
+    return {
+      x: Math.round(x / ratio),
+      y: Math.round(y / ratio)
+    };
+  }
+  async screenshotBase64() {
+    debugPage("screenshotBase64 begin");
+    const { width, height } = await this.size();
+    const adb = await this.getAdb();
+    let screenshotBuffer;
+    try {
+      screenshotBuffer = await adb.takeScreenshot(null);
+    } catch (error) {
+      const screenshotPath = (0, import_utils.getTmpFile)("png");
+      try {
+        await adb.shell(`screencap -p ${androidScreenshotPath}`);
+      } catch (error2) {
+        await this.forceScreenshot(androidScreenshotPath);
+      }
+      await adb.pull(androidScreenshotPath, screenshotPath);
+      screenshotBuffer = await import_node_fs.default.promises.readFile(screenshotPath);
+    }
+    const resizedScreenshotBuffer = await (0, import_img.resizeImg)(screenshotBuffer, {
+      width,
+      height
+    });
+    const result = `data:image/jpeg;base64,${resizedScreenshotBuffer.toString("base64")}`;
+    debugPage("screenshotBase64 end");
+    return result;
+  }
+  get mouse() {
+    return {
+      click: (x, y) => this.mouseClick(x, y),
+      wheel: (deltaX, deltaY) => this.mouseWheel(deltaX, deltaY),
+      move: (x, y) => this.mouseMove(x, y),
+      drag: (from, to) => this.mouseDrag(from, to)
+    };
+  }
+  get keyboard() {
+    return {
+      type: (text) => this.keyboardType(text),
+      press: (action) => this.keyboardPressAction(action)
+    };
+  }
+  async clearInput(element) {
+    if (!element) {
+      return;
+    }
+    await this.ensureYadb();
+    const adb = await this.getAdb();
+    await this.mouse.click(element.center[0], element.center[1]);
+    await adb.shell(
+      'app_process -Djava.class.path=/data/local/tmp/yadb /data/local/tmp com.ysbing.yadb.Main -keyboard "~CLEAR~"'
+    );
+    await this.mouse.click(element.center[0], element.center[1]);
+  }
+  async forceScreenshot(path3) {
+    await this.ensureYadb();
+    const adb = await this.getAdb();
+    await adb.shell(
+      `app_process -Djava.class.path=/data/local/tmp/yadb /data/local/tmp com.ysbing.yadb.Main -screenshot ${path3}`
+    );
+  }
+  async url() {
+    const adb = await this.getAdb();
+    const { appPackage, appActivity } = await adb.getFocusedPackageAndActivity();
+    return `${appPackage}/${appActivity}`;
+  }
+  async scrollUntilTop(startPoint) {
+    if (startPoint) {
+      const start = { x: startPoint.left, y: startPoint.top };
+      const end = { x: start.x, y: 0 };
+      await this.mouseDrag(start, end);
+      return;
+    }
+    await this.mouseWheel(0, 9999999, 100);
+  }
+  async scrollUntilBottom(startPoint) {
+    if (startPoint) {
+      const { height } = await this.size();
+      const start = { x: startPoint.left, y: startPoint.top };
+      const end = { x: start.x, y: height };
+      await this.mouseDrag(start, end);
+      return;
+    }
+    await this.mouseWheel(0, -9999999, 100);
+  }
+  async scrollUntilLeft(startPoint) {
+    if (startPoint) {
+      const start = { x: startPoint.left, y: startPoint.top };
+      const end = { x: 0, y: start.y };
+      await this.mouseDrag(start, end);
+      return;
+    }
+    await this.mouseWheel(9999999, 0, 100);
+  }
+  async scrollUntilRight(startPoint) {
+    if (startPoint) {
+      const { width } = await this.size();
+      const start = { x: startPoint.left, y: startPoint.top };
+      const end = { x: width, y: start.y };
+      await this.mouseDrag(start, end);
+      return;
+    }
+    await this.mouseWheel(-9999999, 0, 100);
+  }
+  async scrollUp(distance, startPoint) {
+    const { height } = await this.size();
+    const scrollDistance = distance || height;
+    if (startPoint) {
+      const start = { x: startPoint.left, y: startPoint.top };
+      const endY = Math.max(0, start.y - scrollDistance);
+      const end = { x: start.x, y: endY };
+      await this.mouseDrag(start, end);
+      return;
+    }
+    await this.mouseWheel(0, scrollDistance, 1e3);
+  }
+  async scrollDown(distance, startPoint) {
+    const { height } = await this.size();
+    const scrollDistance = distance || height;
+    if (startPoint) {
+      const start = { x: startPoint.left, y: startPoint.top };
+      const endY = Math.min(height, start.y + scrollDistance);
+      const end = { x: start.x, y: endY };
+      await this.mouseDrag(start, end);
+      return;
+    }
+    await this.mouseWheel(0, -scrollDistance, 1e3);
+  }
+  async scrollLeft(distance, startPoint) {
+    const { width } = await this.size();
+    const scrollDistance = distance || width;
+    if (startPoint) {
+      const start = { x: startPoint.left, y: startPoint.top };
+      const endX = Math.max(0, start.x - scrollDistance);
+      const end = { x: endX, y: start.y };
+      await this.mouseDrag(start, end);
+      return;
+    }
+    await this.mouseWheel(scrollDistance, 0, 1e3);
+  }
+  async scrollRight(distance, startPoint) {
+    const { width } = await this.size();
+    const scrollDistance = distance || width;
+    if (startPoint) {
+      const start = { x: startPoint.left, y: startPoint.top };
+      const endX = Math.min(width, start.x + scrollDistance);
+      const end = { x: endX, y: start.y };
+      await this.mouseDrag(start, end);
+      return;
+    }
+    await this.mouseWheel(-scrollDistance, 0, 1e3);
+  }
+  async ensureYadb() {
+    if (!this.yadbPushed) {
+      const adb = await this.getAdb();
+      const yadbBin = import_node_path.default.join(__dirname, "../../bin/yadb");
+      await adb.push(yadbBin, "/data/local/tmp");
+      this.yadbPushed = true;
+    }
+  }
+  async keyboardType(text) {
+    if (!text)
+      return;
+    const adb = await this.getAdb();
+    const isChinese = /[\p{Script=Han}\p{sc=Hani}]/u.test(text);
+    if (!isChinese) {
+      await adb.inputText(text);
+      return;
+    }
+    await this.execYadb(text);
+  }
+  async keyboardPress(key) {
+    const keyCodeMap = {
+      Enter: 66,
+      Backspace: 67,
+      Tab: 61,
+      ArrowUp: 19,
+      ArrowDown: 20,
+      ArrowLeft: 21,
+      ArrowRight: 22,
+      Escape: 111,
+      Home: 3,
+      End: 123
+    };
+    const adb = await this.getAdb();
+    const keyCode = keyCodeMap[key];
+    if (keyCode !== void 0) {
+      await adb.keyevent(keyCode);
+    } else {
+      if (key.length === 1) {
+        const asciiCode = key.toUpperCase().charCodeAt(0);
+        if (asciiCode >= 65 && asciiCode <= 90) {
+          await adb.keyevent(asciiCode - 36);
+        }
+      }
+    }
+  }
+  async keyboardPressAction(action) {
+    if (Array.isArray(action)) {
+      for (const act of action) {
+        await this.keyboardPress(act.key);
+      }
+    } else {
+      await this.keyboardPress(action.key);
+    }
+  }
+  async mouseClick(x, y) {
+    const adb = await this.getAdb();
+    const { x: adjustedX, y: adjustedY } = this.adjustCoordinates(x, y);
+    await adb.shell(`input tap ${adjustedX} ${adjustedY}`);
+  }
+  async mouseMove(x, y) {
+    return Promise.resolve();
+  }
+  async mouseDrag(from, to) {
+    const adb = await this.getAdb();
+    const { x: fromX, y: fromY } = this.adjustCoordinates(from.x, from.y);
+    const { x: toX, y: toY } = this.adjustCoordinates(to.x, to.y);
+    await adb.shell(`input swipe ${fromX} ${fromY} ${toX} ${toY} 300`);
+  }
+  async mouseWheel(deltaX, deltaY, duration = 1e3) {
+    const { width, height } = await this.size();
+    const n = 4;
+    const startX = deltaX < 0 ? (n - 1) * (width / n) : width / n;
+    const startY = deltaY < 0 ? (n - 1) * (height / n) : height / n;
+    const maxNegativeDeltaX = startX;
+    const maxPositiveDeltaX = (n - 1) * (width / n);
+    const maxNegativeDeltaY = startY;
+    const maxPositiveDeltaY = (n - 1) * (height / n);
+    deltaX = Math.max(-maxNegativeDeltaX, Math.min(deltaX, maxPositiveDeltaX));
+    deltaY = Math.max(-maxNegativeDeltaY, Math.min(deltaY, maxPositiveDeltaY));
+    const endX = startX + deltaX;
+    const endY = startY + deltaY;
+    const { x: adjustedStartX, y: adjustedStartY } = this.adjustCoordinates(
+      startX,
+      startY
+    );
+    const { x: adjustedEndX, y: adjustedEndY } = this.adjustCoordinates(
+      endX,
+      endY
+    );
+    const adb = await this.getAdb();
+    await adb.shell(
+      `input swipe ${adjustedStartX} ${adjustedStartY} ${adjustedEndX} ${adjustedEndY} ${duration}`
+    );
+  }
+  async destroy() {
+    try {
+      const adb = await this.getAdb();
+      await adb.shell(`rm -f ${androidScreenshotPath}`);
+    } catch (error) {
+      console.error("Error during cleanup:", error);
+    }
+  }
+};
+
+// src/agent/index.ts
+var import_agent = require("@midscene/web/agent");
+
+// src/utils/index.ts
+var import_appium_adb2 = require("appium-adb");
+
+// src/agent/index.ts
+var AndroidAgent = class extends import_agent.PageAgent {
+  async launch(uri) {
+    const device = this.page;
+    await device.launch(uri);
+  }
+};
+
+// src/playground/scrcpy-server.ts
+var import_node_child_process = require("child_process");
+var import_node_fs2 = require("fs");
+var import_node_http = require("http");
+var import_node_util = require("util");
+var import_adb = require("@yume-chan/adb");
+var import_adb_scrcpy = require("@yume-chan/adb-scrcpy");
+var import_adb_server_node_tcp = require("@yume-chan/adb-server-node-tcp");
+var import_fetch_scrcpy_server = require("@yume-chan/fetch-scrcpy-server");
+var import_scrcpy = require("@yume-chan/scrcpy");
+var import_stream_extra = require("@yume-chan/stream-extra");
+var import_cors = __toESM(require("cors"));
+var import_express = __toESM(require("express"));
+var import_socket = require("socket.io");
+var promiseExec = (0, import_node_util.promisify)(import_node_child_process.exec);
+var ScrcpyServer = class {
+  // 用于保存上次设备列表的JSON字符串，用于比较变化
+  constructor() {
+    this.defaultPort = 5700;
+    this.adbClient = null;
+    this.currentDeviceId = null;
+    this.devicePollInterval = null;
+    this.lastDeviceList = "";
+    this.app = (0, import_express.default)();
+    this.httpServer = (0, import_node_http.createServer)(this.app);
+    this.io = new import_socket.Server(this.httpServer, {
+      cors: {
+        origin: [
+          /^http:\/\/localhost(:\d+)?$/,
+          /^http:\/\/127\.0\.0\.1(:\d+)?$/
+        ],
+        methods: ["GET", "POST"],
+        credentials: true
+      }
+    });
+    this.app.use(
+      (0, import_cors.default)({
+        origin: "*",
+        credentials: true
+      })
+    );
+    this.setupSocketHandlers();
+    this.setupApiRoutes();
+  }
+  // setup API routes
+  setupApiRoutes() {
+    this.app.get("/api/devices", async (req, res) => {
+      try {
+        const devices = await this.getDevicesList();
+        res.json({ devices, currentDeviceId: this.currentDeviceId });
+      } catch (error) {
+        res.status(500).json({ error: error.message || "Failed to get devices list" });
+      }
+    });
+  }
+  // get devices list
+  async getDevicesList() {
+    try {
+      debugPage("start to get devices list");
+      const client = await this.getAdbClient();
+      if (!client) {
+        console.warn("failed to get adb client");
+        return [];
+      }
+      debugPage("success to get adb client, start to request devices list");
+      let devices;
+      try {
+        devices = await client.getDevices();
+        debugPage("original devices list:", devices);
+      } catch (error) {
+        console.error("failed to get devices list:", error);
+        return [];
+      }
+      if (!devices || devices.length === 0) {
+        return [];
+      }
+      const formattedDevices = devices.map((device) => {
+        const result = {
+          id: device.serial,
+          name: device.product || device.model || device.serial,
+          status: device.state || "device"
+        };
+        return result;
+      });
+      return formattedDevices;
+    } catch (error) {
+      console.error("failed to get devices list:", error);
+      return [];
+    }
+  }
+  // get adb client
+  async getAdbClient() {
+    try {
+      if (!this.adbClient) {
+        await promiseExec("adb start-server");
+        debugPage("adb server started");
+        debugPage("initialize adb client");
+        this.adbClient = new import_adb.AdbServerClient(
+          new import_adb_server_node_tcp.AdbServerNodeTcpConnector({
+            host: "localhost",
+            port: 5037
+          })
+        );
+        await debugPage("success to initialize adb client");
+      } else {
+        debugPage("use existing adb client");
+      }
+      return this.adbClient;
+    } catch (error) {
+      console.error("failed to get adb client:", error);
+      return null;
+    }
+  }
+  // get adb object
+  async getAdb(deviceId) {
+    try {
+      const client = await this.getAdbClient();
+      if (!client) {
+        return null;
+      }
+      if (deviceId) {
+        this.currentDeviceId = deviceId;
+        return new import_adb.Adb(await client.createTransport({ serial: deviceId }));
+      }
+      const devices = await client.getDevices();
+      if (devices.length === 0) {
+        return null;
+      }
+      this.currentDeviceId = devices[0].serial;
+      return new import_adb.Adb(await client.createTransport(devices[0]));
+    } catch (error) {
+      console.error("failed to get adb client:", error);
+      return null;
+    }
+  }
+  // start scrcpy
+  async startScrcpy(adb, options = {}) {
+    try {
+      await import_adb_scrcpy.AdbScrcpyClient.pushServer(
+        adb,
+        import_stream_extra.ReadableStream.from((0, import_node_fs2.createReadStream)(import_fetch_scrcpy_server.BIN))
+      );
+      const scrcpyOptions = new import_scrcpy.ScrcpyOptions3_1({
+        // default options
+        audio: false,
+        control: true,
+        maxSize: 1024,
+        // use videoBitRate as property name
+        videoBitRate: 2e6,
+        // override default values with user provided options
+        ...options
+      });
+      return await import_adb_scrcpy.AdbScrcpyClient.start(
+        adb,
+        import_scrcpy.DefaultServerPath,
+        new import_adb_scrcpy.AdbScrcpyOptions2_1(scrcpyOptions)
+      );
+    } catch (error) {
+      console.error("failed to start scrcpy:", error);
+      throw error;
+    }
+  }
+  // setup Socket.IO connection handlers
+  setupSocketHandlers() {
+    this.io.on("connection", async (socket) => {
+      debugPage(
+        "client connected, id: %s, client address: %s",
+        socket.id,
+        socket.handshake.address
+      );
+      let scrcpyClient = null;
+      let adb = null;
+      const sendDevicesList = async () => {
+        try {
+          debugPage("Socket request to get devices list");
+          const devices = await this.getDevicesList();
+          debugPage("send devices list to client:", devices);
+          socket.emit("devices-list", {
+            devices,
+            currentDeviceId: this.currentDeviceId
+          });
+        } catch (error) {
+          console.error("failed to send devices list:", error);
+          socket.emit("error", { message: "failed to get devices list" });
+        }
+      };
+      await sendDevicesList();
+      socket.on("get-devices", async () => {
+        debugPage("received client request to get devices list");
+        await sendDevicesList();
+      });
+      socket.on("switch-device", async (deviceId) => {
+        debugPage("received client request to switch device:", deviceId);
+        try {
+          if (scrcpyClient) {
+            await scrcpyClient.close();
+            scrcpyClient = null;
+          }
+          this.currentDeviceId = deviceId;
+          debugPage("device switched to:", deviceId);
+          socket.emit("device-switched", { deviceId });
+          this.io.emit("global-device-switched", {
+            deviceId,
+            timestamp: Date.now()
+          });
+        } catch (error) {
+          console.error("failed to switch device:", error);
+          socket.emit("error", {
+            message: `Failed to switch device: ${error?.message || "Unknown error"}`
+          });
+        }
+      });
+      socket.on("connect-device", async (options) => {
+        try {
+          debugPage(
+            "received device connection request, options: %s, client id: %s",
+            options,
+            socket.id
+          );
+          adb = await this.getAdb(this.currentDeviceId || void 0);
+          if (!adb) {
+            console.error("no available device found");
+            socket.emit("error", { message: "No device found" });
+            return;
+          }
+          debugPage(
+            "starting scrcpy service, device id: %s",
+            this.currentDeviceId
+          );
+          scrcpyClient = await this.startScrcpy(adb, options);
+          debugPage("scrcpy service started successfully");
+          debugPage(
+            "check scrcpyClient object structure: %s",
+            Object.getOwnPropertyNames(scrcpyClient).map((name) => {
+              const type = typeof scrcpyClient[name];
+              const isPromise = type === "object" && scrcpyClient[name] && typeof scrcpyClient[name].then === "function";
+              return `${name}: ${type}${isPromise ? " (Promise)" : ""}`;
+            })
+          );
+          try {
+            if (scrcpyClient.videoStream) {
+              debugPage(
+                "videoStream exists, type: %s",
+                typeof scrcpyClient.videoStream
+              );
+              let videoStream;
+              if (typeof scrcpyClient.videoStream === "object" && typeof scrcpyClient.videoStream.then === "function") {
+                debugPage(
+                  "videoStream is a Promise, waiting for resolution..."
+                );
+                videoStream = await scrcpyClient.videoStream;
+              } else {
+                debugPage("videoStream is not a Promise, directly use");
+                videoStream = scrcpyClient.videoStream;
+              }
+              debugPage(
+                "video stream fetched successfully, metadata: %s",
+                videoStream.metadata
+              );
+              const metadata = videoStream.metadata || {};
+              debugPage("original metadata: %s", metadata);
+              if (!metadata.codec) {
+                debugPage(
+                  "metadata does not have codec field, use H264 by default"
+                );
+                metadata.codec = import_scrcpy.ScrcpyVideoCodecId.H264;
+              }
+              if (!metadata.width || !metadata.height) {
+                debugPage(
+                  "metadata does not have width or height field, use default values"
+                );
+                metadata.width = metadata.width || 1080;
+                metadata.height = metadata.height || 1920;
+              }
+              debugPage(
+                "prepare to send video-metadata event to client, data: %s",
+                JSON.stringify(metadata)
+              );
+              socket.emit("video-metadata", metadata);
+              debugPage(
+                "video-metadata event sent to client, id: %s",
+                socket.id
+              );
+              const { stream } = videoStream;
+              const reader = stream.getReader();
+              const processStream = async () => {
+                try {
+                  while (true) {
+                    const { done, value } = await reader.read();
+                    if (done)
+                      break;
+                    const frameType = value.type || "data";
+                    socket.emit("video-data", {
+                      data: Array.from(value.data),
+                      type: frameType,
+                      timestamp: Date.now(),
+                      // fix keyframe access
+                      keyFrame: value.keyFrame
+                    });
+                  }
+                } catch (error) {
+                  console.error("error processing video stream:", error);
+                  socket.emit("error", {
+                    message: "video stream processing error"
+                  });
+                }
+              };
+              processStream();
+            } else {
+              console.error(
+                "scrcpyClient object does not have videoStream property"
+              );
+              socket.emit("error", {
+                message: "Video stream not available in scrcpy client"
+              });
+            }
+          } catch (error) {
+            console.error("error processing video stream:", error);
+            socket.emit("error", {
+              message: `Video stream processing error: ${error.message}`
+            });
+          }
+          if (scrcpyClient?.controller) {
+            socket.emit("control-ready");
+          }
+        } catch (error) {
+          console.error("failed to connect device:", error);
+          socket.emit("error", {
+            message: `Failed to connect device: ${error?.message || "Unknown error"}`
+          });
+        }
+      });
+      socket.on("disconnect", async (reason) => {
+        debugPage("client disconnected, id: %s, reason: %s", socket.id, reason);
+        if (scrcpyClient) {
+          try {
+            debugPage("closing scrcpy client");
+            await scrcpyClient.close();
+          } catch (error) {
+            console.error("failed to close scrcpy client:", error);
+          }
+          scrcpyClient = null;
+        }
+      });
+    });
+  }
+  // launch server
+  async launch(port) {
+    this.port = port || this.defaultPort;
+    return new Promise((resolve) => {
+      this.httpServer.listen(this.port, () => {
+        console.log(`Scrcpy server running at: http://localhost:${this.port}`);
+        this.startDeviceMonitoring();
+        resolve(this);
+      });
+    });
+  }
+  // start device monitoring
+  startDeviceMonitoring() {
+    this.devicePollInterval = setInterval(async () => {
+      try {
+        const devices = await this.getDevicesList();
+        const currentDevicesJson = JSON.stringify(devices);
+        if (this.lastDeviceList !== currentDevicesJson) {
+          debugPage("devices list changed, push to all connected clients");
+          this.lastDeviceList = currentDevicesJson;
+          if (!this.currentDeviceId && devices.length > 0) {
+            const onlineDevices = devices.filter(
+              (device) => device.status.toLowerCase() === "device"
+            );
+            if (onlineDevices.length > 0) {
+              this.currentDeviceId = onlineDevices[0].id;
+              debugPage(
+                "auto select the first online device:",
+                this.currentDeviceId
+              );
+            }
+          }
+          this.io.emit("devices-list", {
+            devices,
+            currentDeviceId: this.currentDeviceId
+          });
+        }
+      } catch (error) {
+        console.error("device monitoring error:", error);
+      }
+    }, 3e3);
+  }
+  // close server
+  close() {
+    if (this.devicePollInterval) {
+      clearInterval(this.devicePollInterval);
+      this.devicePollInterval = null;
+    }
+    if (this.httpServer) {
+      return this.httpServer.close();
+    }
+  }
+};
+
+// src/playground/bin.ts
+var staticDir = import_node_path2.default.join(__dirname, "../../static");
+var playgroundServer = new import_midscene_server.default(
+  AndroidDevice,
+  AndroidAgent,
+  staticDir
+);
+var scrcpyServer = new ScrcpyServer();
+Promise.all([playgroundServer.launch(5800), scrcpyServer.launch(5700)]).then(() => {
+  console.log(
+    `Midscene playground server is running on http://localhost:${playgroundServer.port}`
+  );
+  (0, import_open.default)(`http://localhost:${playgroundServer.port}`);
+}).catch((error) => {
+  console.error("Failed to start servers:", error);
+  process.exit(1);
+});