@aiscene/aiserver 2.0.2 → 2.0.4

package/dist/scrcpy/server.js

@@ -48,7 +48,7 @@
  *   （视频流走二进制帧，不走 JSON）
  */
 import { createReadStream, existsSync } from 'node:fs';
-import { createServer } from 'node:http';
+import { createServer, get as httpGet, } from 'node:http';
 import path from 'node:path';
 import { createRequire } from 'node:module';
 import { randomUUID } from 'node:crypto';
@@ -61,6 +61,8 @@ import { withTimeout } from './timeout.js';
 import { androidRecordingManager } from '../recorder/android-recording-manager.js';
 import { replayAndroidLoginScript } from '../recorder/android-login-replay.js';
 import { captureAndroidElementState, captureAndroidScreenshotState, getAndroidScreenSize, runAdb, } from '../recorder/android-ui-hierarchy.js';
+import { deviceDetector } from '../device/detector.js';
+import { ensureIOSWda } from '../executor/ios-wda-manager.js';
 const logger = createLogger('ScrcpyServer');
 const requireCjs = createRequire(import.meta.url);
 function resolveRequestedDeviceId(options, currentDeviceId) {
@@ -225,6 +227,7 @@ export class ScrcpyServer {
                 id: d.serial,
                 name: d.product || d.model || d.serial,
                 status: d.state || 'device',
+                platform: 'android',
             }));
         }
         catch (error) {
@@ -232,6 +235,22 @@ export class ScrcpyServer {
             return [];
         }
     }
+    async loadDevicesList() {
+        const [androidDevices, iosDevices] = await Promise.all([
+            this.loadDevicesListFromAdb(),
+            deviceDetector.detectIOSDevices().catch((error) => {
+                logger.debug(`Failed to detect iOS devices for projection: ${error.message}`);
+                return [];
+            }),
+        ]);
+        const listedIOSDevices = iosDevices.map((device) => ({
+            id: device.udid,
+            name: device.name || device.model || device.udid,
+            status: device.status || 'device',
+            platform: 'ios',
+        }));
+        return [...androidDevices, ...listedIOSDevices];
+    }
     async getDevicesList(forceRefresh = false) {
         const now = Date.now();
         const cacheTtlMs = 2000;
@@ -243,7 +262,7 @@ export class ScrcpyServer {
         if (this.deviceListInFlight) {
             return this.deviceListInFlight;
         }
-        this.deviceListInFlight = this.loadDevicesListFromAdb()
+        this.deviceListInFlight = this.loadDevicesList()
             .then((devices) => {
             this.lastDeviceList = devices;
             this.lastDeviceListAt = Date.now();
@@ -443,6 +462,8 @@ export class ScrcpyServer {
                 videoWidth: 0,
                 videoHeight: 0,
                 streamReader: null,
+                platform: null,
+                iosProjection: null,
             };
             this.sessions.set(ws, state);
             this.attachSocketHandlers(state);
@@ -490,8 +511,324 @@ export class ScrcpyServer {
             }
             state.scrcpyClient = null;
         }
+        if (state.iosProjection) {
+            const projection = state.iosProjection;
+            projection.stopped = true;
+            try {
+                projection.cleanup?.();
+            }
+            catch {
+                // ignore
+            }
+            await this.deleteIOSWdaSession(projection);
+            state.iosProjection = null;
+        }
         state.videoWidth = 0;
         state.videoHeight = 0;
+        state.platform = null;
+    }
+    async connectIOSDeviceForSession(state, deviceId, emitStatus) {
+        const { ws } = state;
+        emitStatus('starting-service');
+        const wda = await ensureIOSWda({
+            udid: deviceId,
+            useXcodebuild: false,
+            log: (message) => logger.info(`[iOSProjection] ${message}`),
+        });
+        if (!this.isSessionOpen(state))
+            return;
+        const wdaSession = await this.createIOSWdaSession(wda.host, wda.port);
+        const projection = {
+            stopped: false,
+            wdaHost: wda.host,
+            wdaPort: wda.port,
+            sessionId: wdaSession.sessionId,
+            controlWidth: wdaSession.width,
+            controlHeight: wdaSession.height,
+        };
+        state.iosProjection = projection;
+        state.platform = 'ios';
+        state.deviceId = deviceId;
+        this.currentDeviceId = deviceId;
+        emitStatus('waiting-for-video');
+        logger.info(`iOS projection started for device=${deviceId},wda=http://${wda.host}:${wda.port},mjpeg=http://${wda.host}:${wda.mjpegPort},client=${state.id}`);
+        const initialFrame = await this.captureIOSWdaScreenshot(wda.host, wda.port);
+        if (!this.isSessionOpen(state) || state.iosProjection !== projection || projection.stopped)
+            return;
+        state.videoWidth = initialFrame.width;
+        state.videoHeight = initialFrame.height;
+        wsSend(ws, 'video-metadata', {
+            codec: 'mjpeg',
+            width: initialFrame.width,
+            height: initialFrame.height,
+            platform: 'ios',
+        });
+        this.sendIOSImageFrame(ws, {
+            ...initialFrame,
+            mime: 'image/png',
+            timestamp: Date.now(),
+            source: 'screenshot',
+        });
+        const startScreenshotFallback = () => {
+            void this.startIOSWdaScreenshotLoop(state, projection, wda.host, wda.port);
+        };
+        const nativeStarted = await this.startIOSNativeMjpegStream(state, projection, wda.host, wda.mjpegPort, startScreenshotFallback);
+        if (!nativeStarted) {
+            logger.info(`iOS native MJPEG unavailable, falling back to WDA screenshots: device=${deviceId}`);
+            startScreenshotFallback();
+        }
+    }
+    sendIOSImageFrame(ws, frame) {
+        wsSend(ws, 'ios-screenshot-frame', frame);
+    }
+    async createIOSWdaSession(host, port) {
+        let sessionId = null;
+        try {
+            const response = await this.iosWdaRequest(host, port, 'POST', '/session', {
+                capabilities: {
+                    alwaysMatch: {
+                        platformName: 'iOS',
+                        automationName: 'XCUITest',
+                        shouldUseSingletonTestManager: false,
+                        shouldUseTestManagerForVisibilityDetection: false,
+                    },
+                },
+            });
+            sessionId = this.extractIOSWdaSessionId(response);
+            if (sessionId) {
+                logger.info(`iOS WDA session created for projection: ${sessionId}`);
+                await this.configureIOSWdaSession(host, port, sessionId);
+            }
+            else {
+                logger.warn('iOS WDA session response did not include sessionId; touch control will be disabled');
+            }
+        }
+        catch (error) {
+            logger.warn(`iOS WDA session creation failed, projection will be view-only: ${error.message}`);
+        }
+        const size = sessionId
+            ? await this.getIOSWdaWindowSize(host, port, sessionId).catch((error) => {
+                logger.warn(`iOS WDA window size failed, using screenshot size for touch mapping: ${error.message}`);
+                return { width: 0, height: 0 };
+            })
+            : { width: 0, height: 0 };
+        return { sessionId, width: size.width, height: size.height };
+    }
+    async configureIOSWdaSession(host, port, sessionId) {
+        try {
+            await this.iosWdaRequest(host, port, 'POST', `/session/${sessionId}/appium/settings`, {
+                snapshotMaxDepth: 50,
+                elementResponseAttributes: 'type,label,name,value,rect,enabled,visible',
+                mjpegServerScreenshotQuality: 50,
+                mjpegServerFramerate: 30,
+                mjpegScalingFactor: 50,
+            });
+        }
+        catch (error) {
+            logger.debug(`iOS WDA settings update skipped: ${error.message}`);
+        }
+    }
+    async getIOSWdaWindowSize(host, port, sessionId) {
+        const response = await this.iosWdaRequest(host, port, 'GET', `/session/${sessionId}/window/size`);
+        const value = response?.value || response;
+        const width = Number(value?.width);
+        const height = Number(value?.height);
+        if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) {
+            throw new Error(`invalid WDA window size: ${JSON.stringify(response)}`);
+        }
+        return { width, height };
+    }
+    extractIOSWdaSessionId(response) {
+        const candidates = [
+            response?.sessionId,
+            response?.value?.sessionId,
+            response?.value?.session_id,
+        ];
+        const matched = candidates.find((item) => typeof item === 'string' && item.trim());
+        return matched ? matched.trim() : null;
+    }
+    async iosWdaRequest(host, port, method, endpoint, body) {
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), 8000);
+        try {
+            const response = await fetch(`http://${host}:${port}${endpoint}`, {
+                method,
+                signal: controller.signal,
+                headers: body === undefined ? undefined : { 'Content-Type': 'application/json' },
+                body: body === undefined ? undefined : JSON.stringify(body),
+            });
+            const text = await response.text();
+            const data = text ? JSON.parse(text) : {};
+            if (!response.ok) {
+                throw new Error(`WDA ${method} ${endpoint} HTTP ${response.status}: ${text}`);
+            }
+            return data;
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    }
+    async deleteIOSWdaSession(projection) {
+        if (!projection.sessionId)
+            return;
+        try {
+            await this.iosWdaRequest(projection.wdaHost, projection.wdaPort, 'DELETE', `/session/${projection.sessionId}`);
+        }
+        catch (error) {
+            logger.debug(`delete iOS WDA session skipped: ${error.message}`);
+        }
+    }
+    async startIOSWdaScreenshotLoop(state, projection, host, port) {
+        const { ws } = state;
+        try {
+            while (this.isSessionOpen(state) && state.iosProjection === projection && !projection.stopped) {
+                try {
+                    const frame = await this.captureIOSWdaScreenshot(host, port);
+                    this.sendIOSImageFrame(ws, {
+                        ...frame,
+                        mime: 'image/png',
+                        timestamp: Date.now(),
+                        source: 'screenshot',
+                    });
+                }
+                catch (error) {
+                    logger.warn(`iOS screenshot frame failed: ${error.message}`);
+                    wsSend(ws, 'error', { message: `iOS投屏截图失败：${error.message}` });
+                    break;
+                }
+                await new Promise((resolve) => setTimeout(resolve, 500));
+            }
+        }
+        finally {
+            if (state.iosProjection === projection) {
+                state.iosProjection = null;
+            }
+        }
+    }
+    startIOSNativeMjpegStream(state, projection, host, port, onFallback) {
+        const url = `http://${host}:${port}`;
+        const { ws } = state;
+        return new Promise((resolve) => {
+            let settled = false;
+            let sentFrames = 0;
+            let buffer = Buffer.alloc(0);
+            let startupTimer = null;
+            const settle = (started) => {
+                if (settled)
+                    return;
+                settled = true;
+                if (startupTimer)
+                    clearTimeout(startupTimer);
+                resolve(started);
+            };
+            const req = httpGet(url, (res) => {
+                const statusCode = res.statusCode || 0;
+                if (statusCode >= 400) {
+                    res.resume();
+                    settle(false);
+                    return;
+                }
+                res.on('data', (chunk) => {
+                    if (!this.isSessionOpen(state) || state.iosProjection !== projection || projection.stopped) {
+                        req.destroy();
+                        return;
+                    }
+                    buffer = Buffer.concat([buffer, chunk]);
+                    if (buffer.length > 8 * 1024 * 1024) {
+                        const lastStart = buffer.lastIndexOf(Buffer.from([0xff, 0xd8]));
+                        buffer = lastStart >= 0 ? buffer.subarray(lastStart) : Buffer.alloc(0);
+                    }
+                    while (true) {
+                        const start = buffer.indexOf(Buffer.from([0xff, 0xd8]));
+                        if (start < 0) {
+                            if (buffer.length > 1024 * 1024)
+                                buffer = Buffer.alloc(0);
+                            break;
+                        }
+                        const end = buffer.indexOf(Buffer.from([0xff, 0xd9]), start + 2);
+                        if (end < 0) {
+                            if (start > 0)
+                                buffer = buffer.subarray(start);
+                            break;
+                        }
+                        const jpeg = buffer.subarray(start, end + 2);
+                        buffer = buffer.subarray(end + 2);
+                        sentFrames += 1;
+                        if (sentFrames === 1) {
+                            logger.info(`iOS projection streaming via native WDA MJPEG: ${url}`);
+                            settle(true);
+                        }
+                        this.sendIOSImageFrame(ws, {
+                            mime: 'image/jpeg',
+                            base64: jpeg.toString('base64'),
+                            width: state.videoWidth,
+                            height: state.videoHeight,
+                            timestamp: Date.now(),
+                            source: 'mjpeg',
+                        });
+                    }
+                });
+                res.on('end', () => {
+                    if (sentFrames === 0) {
+                        settle(false);
+                        return;
+                    }
+                    if (this.isSessionOpen(state) && state.iosProjection === projection && !projection.stopped) {
+                        logger.warn(`iOS native MJPEG ended, falling back to WDA screenshots: ${url}`);
+                        onFallback();
+                    }
+                });
+            });
+            startupTimer = setTimeout(() => {
+                req.destroy();
+                settle(false);
+            }, 1500);
+            projection.cleanup = () => req.destroy();
+            req.on('error', (error) => {
+                if (sentFrames === 0) {
+                    logger.debug(`iOS native MJPEG unavailable: ${error.message}`);
+                    settle(false);
+                    return;
+                }
+                if (this.isSessionOpen(state) && state.iosProjection === projection && !projection.stopped) {
+                    logger.warn(`iOS native MJPEG failed, falling back to WDA screenshots: ${error.message}`);
+                    onFallback();
+                }
+            });
+        });
+    }
+    async captureIOSWdaScreenshot(host, port) {
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), 5000);
+        try {
+            const response = await fetch(`http://${host}:${port}/screenshot`, { signal: controller.signal });
+            if (!response.ok) {
+                throw new Error(`WDA screenshot HTTP ${response.status}`);
+            }
+            const body = await response.json();
+            const raw = String(body.value || '').replace(/^data:image\/png;base64,/, '');
+            if (!raw)
+                throw new Error('empty WDA screenshot');
+            const buffer = Buffer.from(raw, 'base64');
+            const size = this.readPngSize(buffer);
+            return { base64: raw, ...size };
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    }
+    readPngSize(buffer) {
+        const isPng = buffer.length >= 24 &&
+            buffer[0] === 0x89 &&
+            buffer[1] === 0x50 &&
+            buffer[2] === 0x4e &&
+            buffer[3] === 0x47;
+        if (!isPng)
+            return { width: 0, height: 0 };
+        return {
+            width: buffer.readUInt32BE(16),
+            height: buffer.readUInt32BE(20),
+        };
     }
     async connectDeviceForSession(state, options = {}, emitStatus) {
         const { ws } = state;
@@ -500,16 +837,16 @@ export class ScrcpyServer {
         try {
             emitStatus('connecting-device');
             // 重复 connect-device 时，先释放旧资源避免 scrcpy 进程残留。
-            if (state.scrcpyClient) {
+            if (state.scrcpyClient || state.iosProjection) {
                 await this.releaseSession(state);
             }
+            if (!this.isSessionOpen(state))
+                return;
+            let devices = await this.getDevicesList();
             if (!this.isSessionOpen(state))
                 return;
             let targetDeviceId = resolveRequestedDeviceId(safeOptions, this.currentDeviceId);
             if (!targetDeviceId) {
-                const devices = await this.getDevicesList();
-                if (!this.isSessionOpen(state))
-                    return;
                 targetDeviceId = this.selectAvailableDeviceId(devices) || undefined;
             }
             if (!targetDeviceId) {
@@ -533,6 +870,13 @@ export class ScrcpyServer {
             this.deviceConnectLocks.set(connectLockKey, state.id);
             this.currentDeviceId = targetDeviceId;
             state.deviceId = targetDeviceId;
+            const targetDevice = devices.find((device) => device.id === targetDeviceId);
+            const targetPlatform = targetDevice?.platform === 'ios' ? 'ios' : 'android';
+            state.platform = targetPlatform;
+            if (targetPlatform === 'ios') {
+                await this.connectIOSDeviceForSession(state, targetDeviceId, emitStatus);
+                return;
+            }
             state.adb = await this.getAdb(targetDeviceId);
             if (!this.isSessionOpen(state)) {
                 await this.releaseSession(state);
@@ -740,6 +1084,89 @@ export class ScrcpyServer {
                 metaState: 0,
             });
         };
+        const normalizeControlPoint = (raw, width, height) => {
+            let x = Number(raw.x);
+            let y = Number(raw.y);
+            if (!Number.isFinite(x) || !Number.isFinite(y))
+                return null;
+            if (raw.normalized === true) {
+                x = Math.max(0, Math.min(1, x)) * width;
+                y = Math.max(0, Math.min(1, y)) * height;
+            }
+            return {
+                x: Math.round(Math.max(0, Math.min(width - 1, x))),
+                y: Math.round(Math.max(0, Math.min(height - 1, y))),
+            };
+        };
+        const ensureIOSProjection = () => {
+            const projection = state.iosProjection;
+            if (!projection || !projection.sessionId) {
+                logger.debug('iOS WDA session not ready, ignoring control event');
+                return null;
+            }
+            return projection;
+        };
+        const handleIOSTouch = async (raw = {}) => {
+            const projection = ensureIOSProjection();
+            if (!projection)
+                return;
+            const width = projection.controlWidth || state.videoWidth || 390;
+            const height = projection.controlHeight || state.videoHeight || 844;
+            const point = normalizeControlPoint(raw, width, height);
+            if (!point)
+                return;
+            const actionStr = String(raw.action || 'down').toLowerCase();
+            if (actionStr === 'down') {
+                projection.touchStart = { ...point, at: Date.now() };
+                return;
+            }
+            if (actionStr !== 'up' && actionStr !== 'cancel') {
+                return;
+            }
+            const start = projection.touchStart || { ...point, at: Date.now() };
+            projection.touchStart = undefined;
+            const distance = Math.hypot(point.x - start.x, point.y - start.y);
+            if (actionStr === 'cancel')
+                return;
+            if (distance < 8) {
+                try {
+                    await this.iosWdaRequest(projection.wdaHost, projection.wdaPort, 'POST', `/session/${projection.sessionId}/wda/tap`, { x: point.x, y: point.y });
+                }
+                catch {
+                    await this.iosWdaRequest(projection.wdaHost, projection.wdaPort, 'POST', `/session/${projection.sessionId}/wda/tap/0`, { x: point.x, y: point.y });
+                }
+                return;
+            }
+            const duration = Math.max(100, Math.min(1500, Date.now() - start.at));
+            await this.iosWdaRequest(projection.wdaHost, projection.wdaPort, 'POST', `/session/${projection.sessionId}/actions`, {
+                actions: [
+                    {
+                        type: 'pointer',
+                        id: 'finger1',
+                        parameters: { pointerType: 'touch' },
+                        actions: [
+                            { type: 'pointerMove', duration: 0, x: start.x, y: start.y },
+                            { type: 'pointerDown', button: 0 },
+                            { type: 'pause', duration: 100 },
+                            { type: 'pointerMove', duration, x: point.x, y: point.y },
+                            { type: 'pointerUp', button: 0 },
+                        ],
+                    },
+                ],
+            });
+        };
+        const pressIOSButton = async (name) => {
+            const projection = ensureIOSProjection();
+            if (!projection)
+                return;
+            await this.iosWdaRequest(projection.wdaHost, projection.wdaPort, 'POST', `/session/${projection.sessionId}/wda/pressButton`, { name });
+        };
+        const typeIOSText = async (text) => {
+            const projection = ensureIOSProjection();
+            if (!projection || !text)
+                return;
+            await this.iosWdaRequest(projection.wdaHost, projection.wdaPort, 'POST', `/session/${projection.sessionId}/wda/keys`, { value: text.split('') });
+        };
         const handlers = {
             ping: () => {
                 wsSend(ws, 'pong');
@@ -796,22 +1223,20 @@ export class ScrcpyServer {
             },
             'control-touch': async (raw = {}) => {
                 try {
+                    if (state.platform === 'ios') {
+                        await handleIOSTouch(raw);
+                        return;
+                    }
                     const controller = ensureController();
                     if (!controller)
                         return;
                     const videoWidth = state.videoWidth || 1080;
                     const videoHeight = state.videoHeight || 1920;
                     const normalized = raw.normalized === true;
-                    let x = Number(raw.x);
-                    let y = Number(raw.y);
-                    if (!Number.isFinite(x) || !Number.isFinite(y))
+                    const point = normalizeControlPoint({ ...raw, normalized }, videoWidth, videoHeight);
+                    if (!point)
                         return;
-                    if (normalized) {
-                        x = Math.max(0, Math.min(1, x)) * videoWidth;
-                        y = Math.max(0, Math.min(1, y)) * videoHeight;
-                    }
-                    x = Math.round(Math.max(0, Math.min(videoWidth - 1, x)));
-                    y = Math.round(Math.max(0, Math.min(videoHeight - 1, y)));
+                    const { x, y } = point;
                     const actionStr = String(raw.action || 'down').toLowerCase();
                     const action = ACTION_MAP[actionStr] ?? 0;
                     const isUp = actionStr === 'up' || actionStr === 'cancel';
@@ -881,6 +1306,11 @@ export class ScrcpyServer {
             },
             'control-text': async (raw = {}) => {
                 try {
+                    if (state.platform === 'ios') {
+                        const text = typeof raw === 'string' ? raw : String(raw.text ?? '');
+                        await typeIOSText(text);
+                        return;
+                    }
                     const controller = ensureController();
                     if (!controller)
                         return;
@@ -912,6 +1342,10 @@ export class ScrcpyServer {
             },
             'control-home': async () => {
                 try {
+                    if (state.platform === 'ios') {
+                        await pressIOSButton('home');
+                        return;
+                    }
                     await pressKey(3);
                 }
                 catch (e) {
@@ -946,6 +1380,29 @@ export class ScrcpyServer {
             },
             'control-scroll': async (raw = {}) => {
                 try {
+                    if (state.platform === 'ios') {
+                        const projection = ensureIOSProjection();
+                        if (!projection)
+                            return;
+                        const width = projection.controlWidth || state.videoWidth || 390;
+                        const height = projection.controlHeight || state.videoHeight || 844;
+                        const center = normalizeControlPoint({
+                            x: raw.x ?? 0.5,
+                            y: raw.y ?? 0.5,
+                            normalized: raw.normalized !== false,
+                        }, width, height);
+                        if (!center)
+                            return;
+                        const dx = Number(raw.scrollX ?? 0) * 120;
+                        const dy = Number(raw.scrollY ?? -1) * 160;
+                        await handleIOSTouch({ action: 'down', x: center.x, y: center.y });
+                        await handleIOSTouch({
+                            action: 'up',
+                            x: Math.max(0, Math.min(width - 1, center.x - dx)),
+                            y: Math.max(0, Math.min(height - 1, center.y - dy)),
+                        });
+                        return;
+                    }
                     const controller = ensureController();
                     if (!controller)
                         return;