@dyyz1993/agent-browser 0.9.2 → 0.11.0

package/dist/stream-server-standalone.js

@@ -1,10 +1,18 @@
 import * as net from 'net';
 import * as fs from 'fs';
+const LOG_FILE = '/tmp/standalone-diag.log';
+function logDiag(msg) {
+    fs.appendFileSync(LOG_FILE, new Date().toISOString().substring(11, 23) + ' ' + msg + '\n');
+}
 import * as path from 'path';
 import * as http from 'http';
 import { WebSocketServer, WebSocket } from 'ws';
+import sharp from 'sharp';
 import { getViewerHtml } from './viewer-html.js';
+import { isAllowedOrigin } from './stream-server.js';
 import { getSocketDir } from './daemon.js';
+import { openApiSpec } from './openapi.js';
+import { getSwaggerUiHtml } from './swagger-ui.js';
 const DEFAULT_STREAM_PORT = parseInt(process.env.AGENT_BROWSER_STREAM_PORT || '5005', 10);
 const STREAM_SERVER_PID_FILE = 'stream-server.pid';
 const STREAM_SERVER_IPC_FILE = 'stream-server.ipc';
@@ -19,9 +27,12 @@ class StreamServerStandalone {
     sessions = new Map();
     clients = new Map();
     daemonSockets = new Map();
+    outboundSockets = new Map();
     frameBuffers = new Map();
     instanceIdToSession = new Map();
     latestFrames = new Map();
+    clientStates = new Map();
+    browser;
     constructor(port = DEFAULT_STREAM_PORT) {
         this.port = port;
     }
@@ -86,7 +97,7 @@ class StreamServerStandalone {
                     res.end(JSON.stringify({
                         status: 'ok',
                         sessions: Array.from(this.sessions.keys()),
-                        clients: this.getTotalClientCount()
+                        clients: this.getTotalClientCount(),
                     }));
                     return;
                 }
@@ -100,34 +111,101 @@ class StreamServerStandalone {
                     res.end(JSON.stringify({ sessions: Array.from(this.sessions.keys()) }));
                     return;
                 }
+                // HTTP API: Execute command
+                if (req.url === '/api/command' && req.method === 'POST') {
+                    let body = '';
+                    req.on('data', (chunk) => (body += chunk));
+                    req.on('end', async () => {
+                        try {
+                            const response = await this.sendCommandToDaemon(body);
+                            res.writeHead(200, { 'Content-Type': 'application/json' });
+                            res.end(response);
+                        }
+                        catch (err) {
+                            const error = err instanceof Error ? err.message : String(err);
+                            res.writeHead(500, { 'Content-Type': 'application/json' });
+                            res.end(JSON.stringify({ id: 'unknown', success: false, error }));
+                        }
+                    });
+                    return;
+                }
+                // HTTP API: OpenAPI specification
+                if (req.url === '/api/openapi.json' && req.method === 'GET') {
+                    res.writeHead(200, { 'Content-Type': 'application/json' });
+                    res.end(JSON.stringify(openApiSpec));
+                    return;
+                }
+                // HTTP API: Swagger UI
+                if (req.url === '/api/docs' && req.method === 'GET') {
+                    res.setHeader('Content-Type', 'text/html; charset=utf-8');
+                    res.end(getSwaggerUiHtml());
+                    return;
+                }
+                // HTTP API: Help - list available commands
+                if (req.url === '/api/help' && req.method === 'GET') {
+                    res.writeHead(200, { 'Content-Type': 'application/json' });
+                    res.end(JSON.stringify({
+                        title: 'agent-browser HTTP API',
+                        version: '0.10.0',
+                        endpoints: {
+                            'POST /api/command': {
+                                description: 'Execute a browser command',
+                                example: { id: '1', action: 'navigate', url: 'https://example.com' },
+                            },
+                            'GET /api/help': { description: 'Show this help message' },
+                            'GET /api/openapi.json': { description: 'OpenAPI 3.0 specification' },
+                            'GET /api/docs': { description: 'Swagger UI documentation' },
+                            'GET /health': { description: 'Health check' },
+                            'GET /sessions': { description: 'List active sessions' },
+                        },
+                        availableActions: [
+                            { action: 'launch', description: 'Launch browser', required: [] },
+                            { action: 'navigate', description: 'Navigate to URL', required: ['url'] },
+                            { action: 'click', description: 'Click element', required: ['selector'] },
+                            { action: 'fill', description: 'Fill form field', required: ['selector', 'value'] },
+                            { action: 'type', description: 'Type text', required: ['selector', 'text'] },
+                            { action: 'snapshot', description: 'Get page snapshot', required: [] },
+                            { action: 'screenshot', description: 'Take screenshot', required: [] },
+                            { action: 'evaluate', description: 'Execute JavaScript', required: ['script'] },
+                            { action: 'wait', description: 'Wait for element/condition', required: [] },
+                            { action: 'scroll', description: 'Scroll page', required: [] },
+                            { action: 'hover', description: 'Hover element', required: ['selector'] },
+                            { action: 'press', description: 'Press key', required: ['key'] },
+                            {
+                                action: 'select',
+                                description: 'Select dropdown option',
+                                required: ['selector', 'values'],
+                            },
+                            { action: 'back', description: 'Go back', required: [] },
+                            { action: 'forward', description: 'Go forward', required: [] },
+                            { action: 'reload', description: 'Reload page', required: [] },
+                            { action: 'close', description: 'Close browser', required: [] },
+                            { action: 'url', description: 'Get current URL', required: [] },
+                            { action: 'title', description: 'Get page title', required: [] },
+                            { action: 'cookies_get', description: 'Get cookies', required: [] },
+                            { action: 'cookies_set', description: 'Set cookie', required: ['cookies'] },
+                            { action: 'state_save', description: 'Save browser state', required: ['path'] },
+                            { action: 'state_load', description: 'Load browser state', required: ['path'] },
+                        ],
+                        docs: 'See http://localhost:5005/api/docs for interactive documentation',
+                    }));
+                    return;
+                }
                 res.writeHead(404, { 'Content-Type': 'application/json' });
                 res.end(JSON.stringify({ error: 'Not found' }));
             });
             this.wss = new WebSocketServer({
                 server: this.httpServer,
                 verifyClient: (info) => {
-                    const origin = info.origin;
-                    if (!origin)
-                        return true;
-                    if (origin.startsWith('file://'))
-                        return true;
-                    try {
-                        const url = new URL(origin);
-                        const host = url.hostname;
-                        if (host === 'localhost' || host === '127.0.0.1' || host === '::1' || host === '[::1]') {
-                            return true;
-                        }
-                    }
-                    catch { }
-                    return false;
-                }
+                    return isAllowedOrigin(info.origin);
+                },
             });
             this.wss.on('connection', (ws, req) => {
                 this.handleWebSocketConnection(ws, req);
             });
             this.wss.on('error', reject);
-            this.httpServer.listen(this.port, '127.0.0.1', () => {
-                console.log(`[StreamServer] Server listening on port ${this.port}`);
+            this.httpServer.listen(this.port, '0.0.0.0', () => {
+                console.log(`[StreamServer] Server listening on port ${this.port} (HTTP API enabled)`);
                 resolve();
             });
             this.httpServer.on('error', reject);
@@ -137,72 +215,118 @@ class StreamServerStandalone {
         const url = new URL(req.url || '/', `http://localhost:${this.port}`);
         const sessionParam = url.searchParams.get('session') || 'default';
         const instanceIdParam = url.searchParams.get('instanceId');
-        // 优先使用 instanceId 查找 session
+        const rawSelector = url.searchParams.get('selector');
+        const clientState = {};
+        if (rawSelector) {
+            clientState.selector = decodeURIComponent(rawSelector);
+        }
         let session;
+        let connected = false;
         if (instanceIdParam) {
             const foundSession = this.instanceIdToSession.get(instanceIdParam);
             if (foundSession) {
                 session = foundSession;
+                connected = true;
             }
             else {
-                // instanceId 不存在，返回错误
-                console.log(`[StreamServer] Invalid instanceId: ${instanceIdParam}`);
-                ws.send(JSON.stringify({ type: 'status', connected: false, error: 'Invalid instanceId' }));
-                ws.close();
-                return;
+                session = sessionParam;
             }
         }
         else {
             session = sessionParam;
         }
-        console.log(`[StreamServer] WebSocket client connected for session: ${session}`);
         if (!this.clients.has(session)) {
             this.clients.set(session, new Set());
         }
         const wasEmpty = this.clients.get(session).size === 0;
         this.clients.get(session).add(ws);
-        this.sendStatus(ws, session);
-        // 如果有最新帧，立即发送给新客户端
+        this.clientStates.set(ws, clientState);
+        if (clientState.selector) {
+            this.requestElementBox(session, clientState.selector);
+            clientState.elementCheckTimer = setInterval(() => {
+                if (!clientState.selector)
+                    return;
+                this.requestElementBox(session, clientState.selector);
+            }, 2500);
+        }
+        this.sendStatus(ws, session, clientState);
         const latestFrame = this.latestFrames.get(session);
         if (latestFrame) {
-            ws.send(latestFrame.header);
-            ws.send(latestFrame.data);
+            this.sendCroppedFrame(ws, latestFrame, clientState);
         }
-        // 如果这是该 session 的第一个客户端，通知 daemon 启动 screencast
+        logDiag('[WSCONN] viewer session=' +
+            session +
+            ' instanceId=' +
+            instanceIdParam +
+            ' sessions.has=' +
+            this.sessions.has(session) +
+            ' daemonSockets.has=' +
+            this.daemonSockets.has(session) +
+            ' clients before=' +
+            wasEmpty);
         if (wasEmpty && this.daemonSockets.has(session)) {
-            this.daemonSockets.get(session)?.write(JSON.stringify({ type: 'client_connected', session }) + '\n');
+            this.daemonSockets
+                .get(session)
+                ?.write(JSON.stringify({ type: 'client_connected', session }) + '\n');
         }
-        if (this.sessions.has(session) && !this.daemonSockets.has(session)) {
+        if (this.sessions.has(session)) {
             this.connectToDaemon(session);
         }
         ws.on('message', (data) => {
             try {
                 const message = JSON.parse(data.toString());
-                this.handleClientMessage(session, message);
+                if (message.type === 'status') {
+                    this.sendStatus(ws, session, clientState);
+                    if (clientState.selector) {
+                        this.requestElementBox(session, clientState.selector);
+                    }
+                }
+                else {
+                    this.handleClientMessage(session, message);
+                }
             }
             catch (error) {
                 console.error('[StreamServer] Failed to parse client message:', error);
             }
         });
         ws.on('close', () => {
-            console.log(`[StreamServer] WebSocket client disconnected for session: ${session}`);
+            if (clientState.elementCheckTimer) {
+                clearInterval(clientState.elementCheckTimer);
+                clientState.elementCheckTimer = undefined;
+            }
             this.clients.get(session)?.delete(ws);
+            this.clientStates.delete(ws);
             if (this.clients.get(session)?.size === 0) {
                 this.clients.delete(session);
-                // 如果该 session 没有客户端了，通知 daemon 停止 screencast
                 if (this.daemonSockets.has(session)) {
-                    this.daemonSockets.get(session)?.write(JSON.stringify({ type: 'client_disconnected', session }) + '\n');
+                    this.daemonSockets
+                        .get(session)
+                        ?.write(JSON.stringify({ type: 'client_disconnected', session }) + '\n');
                 }
             }
         });
         ws.on('error', (error) => {
-            console.error(`[StreamServer] WebSocket error for session ${session}:`, error);
+            if (clientState.elementCheckTimer) {
+                clearInterval(clientState.elementCheckTimer);
+                clientState.elementCheckTimer = undefined;
+            }
             this.clients.get(session)?.delete(ws);
+            this.clientStates.delete(ws);
         });
     }
     handleClientMessage(session, message) {
+        const msgType = message.type;
+        if (msgType === 'input_fill') {
+            logDiag('[CM] input_fill SESSION=' +
+                session +
+                ' socket_exists=' +
+                !!this.daemonSockets.get(session) +
+                ' text=' +
+                (message.text || ''));
+        }
         const daemonSocket = this.daemonSockets.get(session);
         if (!daemonSocket) {
+            logDiag('[CM] NO DAEMON SOCKET for session=' + session);
             return;
         }
         const forwardableTypes = [
@@ -213,17 +337,109 @@ class StreamServerStandalone {
             'user_activity',
             'keyboard_down',
             'keyboard_up',
-            'keyboard_insert_text'
+            'keyboard_insert_text',
+            'input_focused',
+            'input_value',
+            'input_blur',
+            'input_fill',
+            'input_blur_element',
         ];
         if (forwardableTypes.includes(message.type)) {
             try {
                 daemonSocket.write(JSON.stringify(message) + '\n');
+                if (msgType === 'input_fill') {
+                    logDiag('[CM] input_fill WRITTEN TO SOCKET');
+                }
             }
             catch (error) {
                 console.error(`[StreamServer] Failed to send message to daemon for session ${session}:`, error);
             }
         }
     }
+    requestElementBox(session, selector) {
+        if (!this.daemonSockets.has(session)) {
+            console.log(`[StreamServer] requestElementBox: no daemon socket for session ${session}`);
+            return;
+        }
+        const daemonSocket = this.daemonSockets.get(session);
+        if (daemonSocket) {
+            console.log(`[StreamServer] requestElementBox: session=${session} selector=${selector}`);
+            daemonSocket.write(JSON.stringify({
+                type: 'request_element_box',
+                session,
+                selector,
+            }) + '\n');
+        }
+    }
+    async sendCroppedFrame(ws, frame, clientState) {
+        if (clientState.selector && clientState.elementBox && ws.readyState === WebSocket.OPEN) {
+            try {
+                const box = clientState.elementBox;
+                const header = JSON.parse(frame.header);
+                const meta = header.metadata;
+                let left = Math.round(box.x);
+                let top = Math.round(box.y);
+                let w = Math.round(box.width);
+                let h = Math.round(box.height);
+                if (meta?.deviceWidth && meta?.deviceHeight) {
+                    const imgInfo = await sharp(frame.data).metadata();
+                    const actualW = imgInfo.width || meta.deviceWidth;
+                    const actualH = imgInfo.height || meta.deviceHeight;
+                    const scaleX = actualW / meta.deviceWidth;
+                    const scaleY = actualH / meta.deviceHeight;
+                    if (scaleX !== 1 || scaleY !== 1) {
+                        left = Math.round(box.x * scaleX);
+                        top = Math.round(box.y * scaleY);
+                        w = Math.round(box.width * scaleX);
+                        h = Math.round(box.height * scaleY);
+                    }
+                    left = Math.max(0, Math.min(left, actualW - 1));
+                    top = Math.max(0, Math.min(top, actualH - 1));
+                    w = Math.min(w, actualW - left);
+                    h = Math.min(h, actualH - top);
+                }
+                if (w <= 0 || h <= 0) {
+                    ws.send(frame.header);
+                    ws.send(frame.data);
+                    return;
+                }
+                const buf = await sharp(frame.data)
+                    .extract({ left, top, width: w, height: h })
+                    .resize(box.width, box.height)
+                    .jpeg({ quality: 80 })
+                    .toBuffer();
+                if (ws.readyState !== WebSocket.OPEN)
+                    return;
+                const croppedHeader = {
+                    ...header,
+                    metadata: {
+                        ...header.metadata,
+                        deviceWidth: box.width,
+                        deviceHeight: box.height,
+                        element: {
+                            selector: clientState.selector,
+                            x: box.x,
+                            y: box.y,
+                            width: box.width,
+                            height: box.height,
+                        },
+                    },
+                };
+                ws.send(JSON.stringify(croppedHeader));
+                ws.send(buf);
+            }
+            catch {
+                if (ws.readyState !== WebSocket.OPEN)
+                    return;
+                ws.send(frame.header);
+                ws.send(frame.data);
+            }
+        }
+        else {
+            ws.send(frame.header);
+            ws.send(frame.data);
+        }
+    }
     async startIpcServer() {
         return new Promise((resolve, reject) => {
             const ipcPath = this.getIpcPath();
@@ -270,22 +486,21 @@ class StreamServerStandalone {
                 if (s === socket) {
                     console.log(`[StreamServer] Daemon disconnected for session: ${session}`);
                     this.daemonSockets.delete(session);
-                    this.sessions.delete(session);
                     this.broadcastStatus(session, false);
                     break;
                 }
             }
         });
     }
-    handleIpcMessage(socket, message) {
+    async handleIpcMessage(socket, message) {
         switch (message.type) {
             case 'register':
                 if (message.session && message.socketPath && message.instanceId) {
-                    console.log(`[StreamServer] Session registered: ${message.session}, instanceId: ${message.instanceId}`);
+                    logDiag('[REGISTER] session=' + message.session + ' instanceId=' + message.instanceId);
                     this.sessions.set(message.session, {
                         socketPath: message.socketPath,
                         lastSeen: Date.now(),
-                        instanceId: message.instanceId
+                        instanceId: message.instanceId,
                     });
                     this.instanceIdToSession.set(message.instanceId, message.session);
                     this.daemonSockets.set(message.session, socket);
@@ -316,78 +531,320 @@ class StreamServerStandalone {
             case 'frame':
                 if (message.session) {
                     this.sessions.get(message.session).lastSeen = Date.now();
-                    this.broadcastFrame(message);
+                    await this.broadcastFrame(message);
+                }
+                break;
+            case 'selector_element':
+                if (message.session && message.selector) {
+                    console.log(`[StreamServer] Received selector_element: session=${message.session} selector=${message.selector} box=${message.elementBox ? JSON.stringify(message.elementBox) : 'null'}`);
+                    const clients = this.clients.get(message.session);
+                    if (clients) {
+                        for (const client of clients) {
+                            const state = this.clientStates.get(client);
+                            if (state?.selector === message.selector) {
+                                if (message.elementBox) {
+                                    state.elementBox = message.elementBox;
+                                    state.degraded = false;
+                                    const latestFrameForElem = this.latestFrames.get(message.session);
+                                    if (latestFrameForElem) {
+                                        this.sendCroppedFrame(client, latestFrameForElem, state);
+                                    }
+                                }
+                                else if (!state.degraded) {
+                                    state.elementBox = undefined;
+                                    state.degraded = true;
+                                }
+                                this.sendStatus(client, message.session, state);
+                            }
+                            else {
+                                console.log(`[StreamServer] selector_element mismatch: state.selector="${state?.selector}" vs message.selector="${message.selector}"`);
+                            }
+                        }
+                    }
+                }
+                break;
+            case 'input_focused':
+            case 'input_value':
+            case 'input_blur':
+                logDiag('[IPC] ' + String(message.type) + ' clients=' + this.clients.size);
+                for (const [, clients] of this.clients) {
+                    for (const client of clients) {
+                        if (client.readyState === WebSocket.OPEN) {
+                            try {
+                                client.send(JSON.stringify(message));
+                            }
+                            catch (_) { }
+                        }
+                    }
                 }
                 break;
         }
     }
     connectToDaemon(session) {
+        if (this.outboundSockets.has(session))
+            return;
         const sessionInfo = this.sessions.get(session);
         if (!sessionInfo)
             return;
         const socketPath = sessionInfo.socketPath;
-        const socket = net.createConnection({ path: socketPath }, () => {
+        const socket = net.createConnection({ path: socketPath }, async () => {
             console.log(`[StreamServer] Connected to daemon for session: ${session}`);
-            this.daemonSockets.set(session, socket);
+            const sessionClients = this.clients.get(session);
+            if (sessionClients) {
+                for (const client of sessionClients) {
+                    const state = this.clientStates.get(client);
+                    if (state?.selector) {
+                        this.requestElementBox(session, state.selector);
+                    }
+                }
+            }
         });
         socket.on('error', (error) => {
             console.error(`[StreamServer] Failed to connect to daemon for session ${session}:`, error);
+            this.outboundSockets.delete(session);
         });
         socket.on('close', () => {
-            this.daemonSockets.delete(session);
+            logDiag('[CTD] socket close session=' + session);
+            this.outboundSockets.delete(session);
+        });
+        this.outboundSockets.set(session, socket);
+        // Send inject_focus_listener command to daemon via this outbound connection
+        logDiag('[CTD] sending inject_focus_listener to daemon for session=' + session);
+        try {
+            socket.write(JSON.stringify({ id: 'inject-fl-' + Date.now(), action: 'inject_focus_listener' }) + '\n');
+        }
+        catch (e) {
+            console.error('[StreamServer] Failed to send inject_focus_listener:', e);
+        }
+        // Data handler: receive focus events from daemon's injectFocusListener callback
+        socket.on('data', (data) => {
+            const raw = data.toString();
+            logDiag('[CTD DATA] session=' +
+                session +
+                ' rawLen=' +
+                raw.length +
+                ' firstLine=' +
+                raw.substring(0, 100).replace(/\n/g, '|'));
+            const lines = raw.split('\n');
+            for (const line of lines) {
+                if (!line.trim())
+                    continue;
+                try {
+                    const msg = JSON.parse(line);
+                    // Handle inject_focus_listener response — retry on "Browser not launched"
+                    if (msg.id && String(msg.id).startsWith('inject-fl-')) {
+                        if (msg.success === false && msg.error && msg.error.includes('Browser not launched')) {
+                            logDiag('[CTD] inject_focus_listener failed: ' + msg.error + ' — retrying in 2s');
+                            setTimeout(() => {
+                                try {
+                                    socket.write(JSON.stringify({
+                                        id: 'inject-fl-retry-' + Date.now(),
+                                        action: 'inject_focus_listener',
+                                    }) + '\n');
+                                }
+                                catch (_) { }
+                            }, 2000);
+                        }
+                        continue;
+                    }
+                    if (msg.type === 'input_focused' ||
+                        msg.type === 'input_value' ||
+                        msg.type === 'input_blur') {
+                        const clients = this.clients.get(session);
+                        logDiag('[CTD DATA] broadcasting ' +
+                            msg.type +
+                            ' to ' +
+                            (clients?.size || 0) +
+                            ' viewer clients');
+                        if (clients) {
+                            for (const client of clients) {
+                                if (client.readyState === WebSocket.OPEN) {
+                                    try {
+                                        client.send(JSON.stringify(msg));
+                                    }
+                                    catch (_) { }
+                                }
+                            }
+                        }
+                    }
+                }
+                catch (_) {
+                    // Ignore parse errors (might be partial data or non-JSON responses)
+                }
+            }
         });
     }
-    broadcastFrame(message) {
+    async broadcastFrame(message) {
         const session = message.session;
         const clients = this.clients.get(session);
         if (!clients || clients.size === 0)
             return;
-        const headerMessage = {
-            type: 'frame',
-            metadata: message.metadata,
-            format: message.format,
-            fps: message.fps,
-            state: message.state
-        };
-        // 保存最新帧
-        if (message.data) {
-            this.latestFrames.set(session, {
-                header: JSON.stringify(headerMessage),
-                data: Buffer.from(message.data, 'base64')
-            });
-        }
+        const frameData = message.data ? Buffer.from(message.data, 'base64') : null;
         for (const client of clients) {
-            if (client.readyState === WebSocket.OPEN) {
-                client.send(JSON.stringify(headerMessage));
-                if (message.data) {
-                    client.send(Buffer.from(message.data, 'base64'));
+            if (client.readyState !== WebSocket.OPEN)
+                continue;
+            const clientState = this.clientStates.get(client);
+            let metadata = message.metadata;
+            let dataToSend = frameData;
+            const hasSelector = !!clientState?.selector;
+            const hasBox = !!clientState?.elementBox;
+            const hasFrame = !!frameData;
+            if (hasSelector && hasBox && hasFrame) {
+                try {
+                    const box = clientState.elementBox;
+                    const meta = message.metadata;
+                    let left = Math.round(box.x);
+                    let top = Math.round(box.y);
+                    let w = Math.round(box.width);
+                    let h = Math.round(box.height);
+                    if (meta?.deviceWidth && meta?.deviceHeight) {
+                        const imgInfo = await sharp(frameData).metadata();
+                        const actualW = imgInfo.width || meta.deviceWidth;
+                        const actualH = imgInfo.height || meta.deviceHeight;
+                        const scaleX = actualW / meta.deviceWidth;
+                        const scaleY = actualH / meta.deviceHeight;
+                        if (scaleX !== 1 || scaleY !== 1) {
+                            left = Math.round(box.x * scaleX);
+                            top = Math.round(box.y * scaleY);
+                            w = Math.round(box.width * scaleX);
+                            h = Math.round(box.height * scaleY);
+                        }
+                        left = Math.max(0, Math.min(left, actualW - 1));
+                        top = Math.max(0, Math.min(top, actualH - 1));
+                        w = Math.min(w, actualW - left);
+                        h = Math.min(h, actualH - top);
+                    }
+                    if (w <= 0 || h <= 0) {
+                        dataToSend = frameData;
+                    }
+                    else {
+                        const cropped = await sharp(frameData)
+                            .extract({ left, top, width: w, height: h })
+                            .resize(box.width, box.height)
+                            .jpeg({ quality: 80 })
+                            .toBuffer();
+                        dataToSend = Buffer.from(cropped);
+                        if (metadata) {
+                            metadata = {
+                                ...metadata,
+                                deviceWidth: box.width,
+                                deviceHeight: box.height,
+                                element: {
+                                    selector: clientState.selector,
+                                    x: box.x,
+                                    y: box.y,
+                                    width: box.width,
+                                    height: box.height,
+                                },
+                            };
+                        }
+                    }
+                }
+                catch (error) {
+                    const errMsg = error instanceof Error ? error.message : String(error);
+                    const box = clientState.elementBox;
+                    metadata = {
+                        ...(metadata || {}),
+                        _cropError: errMsg,
+                        _cropBox: {
+                            left: Math.round(box.x),
+                            top: Math.round(box.y),
+                            width: Math.round(box.width),
+                            height: Math.round(box.height),
+                        },
+                        _selector: clientState?.selector,
+                    };
+                    dataToSend = frameData;
                 }
             }
+            else if (hasSelector) {
+                metadata = {
+                    ...(metadata || {}),
+                    _skipCrop: `hasBox=${hasBox} hasFrame=${hasFrame}`,
+                    _selector: clientState?.selector,
+                };
+            }
+            const headerMessage = {
+                type: 'frame',
+                metadata,
+                format: message.format,
+                fps: message.fps,
+                state: message.state,
+            };
+            client.send(JSON.stringify(headerMessage));
+            if (dataToSend) {
+                client.send(dataToSend);
+            }
+        }
+        // 保存最新帧（原始）
+        if (frameData) {
+            this.latestFrames.set(session, {
+                header: JSON.stringify({
+                    type: 'frame',
+                    metadata: message.metadata,
+                    format: message.format,
+                    fps: message.fps,
+                    state: message.state,
+                }),
+                data: frameData,
+            });
         }
     }
     broadcastStatus(session, connected) {
         const clients = this.clients.get(session);
         if (!clients)
             return;
-        const message = {
-            type: 'status',
-            connected,
-            screencasting: connected
-        };
         for (const client of clients) {
             if (client.readyState === WebSocket.OPEN) {
-                client.send(JSON.stringify(message));
+                const state = this.clientStates.get(client);
+                const msg = {
+                    type: 'status',
+                    connected,
+                    screencasting: connected,
+                    session,
+                    version: '0.10.0',
+                };
+                if (state?.selector && state.elementBox) {
+                    msg.element = {
+                        selector: state.selector,
+                        x: state.elementBox.x,
+                        y: state.elementBox.y,
+                        width: state.elementBox.width,
+                        height: state.elementBox.height,
+                    };
+                    msg.viewportWidth = state.elementBox.width;
+                    msg.viewportHeight = state.elementBox.height;
+                }
+                if (state?.degraded) {
+                    msg.degraded = true;
+                }
+                client.send(JSON.stringify(msg));
             }
         }
     }
-    sendStatus(ws, session) {
+    sendStatus(ws, session, clientState) {
         const connected = this.sessions.has(session);
         const message = {
             type: 'status',
             connected,
             screencasting: connected,
-            session
+            session,
+            version: '0.10.0',
         };
+        if (clientState?.selector && clientState?.elementBox) {
+            message.element = {
+                selector: clientState.selector,
+                x: clientState.elementBox.x,
+                y: clientState.elementBox.y,
+                width: clientState.elementBox.width,
+                height: clientState.elementBox.height,
+            };
+            message.viewportWidth = clientState.elementBox.width;
+            message.viewportHeight = clientState.elementBox.height;
+        }
+        if (clientState?.degraded) {
+            message.degraded = true;
+        }
         if (ws.readyState === WebSocket.OPEN) {
             ws.send(JSON.stringify(message));
         }
@@ -399,6 +856,68 @@ class StreamServerStandalone {
         }
         return total;
     }
+    /**
+     * Send a command to the daemon via Unix socket and return the response
+     */
+    async sendCommandToDaemon(commandJson) {
+        return new Promise((resolve, reject) => {
+            // Get the daemon socket path from the first available session
+            // or use the default socket path
+            let socketPath;
+            // Try to find an active session's socket path
+            for (const [session, info] of this.sessions) {
+                socketPath = info.socketPath;
+                break;
+            }
+            if (!socketPath) {
+                // Fallback to default socket path
+                socketPath = path.join(getSocketDir(), 'default.sock');
+            }
+            const socket = net.createConnection({ path: socketPath }, () => {
+                socket.write(commandJson + '\n');
+            });
+            let response = '';
+            let resolved = false;
+            const timeout = setTimeout(() => {
+                if (!resolved) {
+                    resolved = true;
+                    socket.destroy();
+                    reject(new Error('Command timeout'));
+                }
+            }, 30000); // 30 second timeout
+            socket.on('data', (data) => {
+                response += data.toString();
+                // Check if we have a complete JSON response
+                try {
+                    JSON.parse(response);
+                    // If we can parse it, we have the complete response
+                    if (!resolved) {
+                        resolved = true;
+                        clearTimeout(timeout);
+                        resolve(response);
+                        socket.end();
+                    }
+                }
+                catch {
+                    // Not complete yet, keep reading
+                }
+            });
+            socket.on('end', () => {
+                if (!resolved) {
+                    resolved = true;
+                    clearTimeout(timeout);
+                    resolve(response);
+                }
+            });
+            socket.on('error', (err) => {
+                if (!resolved) {
+                    resolved = true;
+                    clearTimeout(timeout);
+                    reject(err);
+                }
+            });
+        });
+    }
     getPidFile() {
         return path.join(getSocketDir(), STREAM_SERVER_PID_FILE);
     }
@@ -484,7 +1003,8 @@ export function getStreamServerIpcPath() {
     return path.join(getSocketDir(), STREAM_SERVER_IPC_FILE);
 }
 export { StreamServerStandalone };
-if (process.argv[1]?.endsWith('stream-server-standalone.js') || process.env.AGENT_BROWSER_STREAM_SERVER === '1') {
+if (process.argv[1]?.endsWith('stream-server-standalone.js') ||
+    process.env.AGENT_BROWSER_STREAM_SERVER === '1') {
     const server = new StreamServerStandalone();
     server.start().catch((err) => {
         console.error('[StreamServer] Failed to start:', err);