@dyyz1993/agent-browser 0.9.2 → 0.11.1

package/dist/stream-server.js

@@ -8,9 +8,9 @@ import { executeCommand } from './actions.js';
 import { errorResponse, serializeResponse } from './protocol.js';
 import { getSocketDir, getSession, getInstanceId } from './daemon.js';
 export const STATE_CONFIGS = {
-    user_interacting: { format: 'jpeg', quality: 80, maxFps: 60, scale: 0.4 },
-    screen_moving: { format: 'webp', quality: 50, maxFps: 1, scale: 0.6 },
-    static: { format: 'webp', quality: 80, maxFps: 0.5, scale: 1 },
+    user_interacting: { format: 'jpeg', quality: 80, maxFps: 60, scale: 0.6 },
+    screen_moving: { format: 'jpeg', quality: 75, maxFps: 8, scale: 0.8 },
+    static: { format: 'jpeg', quality: 80, maxFps: 2, scale: 1 },
 };
 export class StreamStateManager {
     currentState = 'static';
@@ -119,10 +119,26 @@ export class FrameRateController {
     }
 }
 export class FrameProcessor {
-    async process(data, config, viewportWidth, viewportHeight) {
+    screencastFormat = 'jpeg';
+    screencastQuality = 80;
+    async process(data, config, viewportWidth, viewportHeight, cropConfig) {
         const buffer = Buffer.from(data, 'base64');
+        const needsResize = config.scale < 1 && viewportWidth && viewportHeight;
+        const needsCrop = !!cropConfig;
+        const needsReencode = config.format !== this.screencastFormat || config.quality !== this.screencastQuality;
+        if (!needsResize && !needsCrop && !needsReencode) {
+            return buffer;
+        }
         let processed = sharp(buffer);
-        if (config.scale < 1 && viewportWidth && viewportHeight) {
+        if (cropConfig) {
+            processed = processed.extract({
+                left: Math.round(cropConfig.x),
+                top: Math.round(cropConfig.y),
+                width: Math.round(cropConfig.width),
+                height: Math.round(cropConfig.height),
+            });
+        }
+        if (needsResize) {
             const newWidth = Math.round(viewportWidth * config.scale);
             const newHeight = Math.round(viewportHeight * config.scale);
             processed = processed.resize(newWidth, newHeight);
@@ -136,6 +152,25 @@ export class FrameProcessor {
         return processed.toBuffer();
     }
 }
+function isPrivateIP(hostname) {
+    if (hostname === 'localhost' ||
+        hostname === '127.0.0.1' ||
+        hostname === '::1' ||
+        hostname === '[::1]') {
+        return true;
+    }
+    const parts = hostname.split('.').map(Number);
+    if (parts.length === 4 && parts.every((p) => !isNaN(p))) {
+        const [a, b] = parts;
+        if (a === 10)
+            return true;
+        if (a === 172 && b >= 16 && b <= 31)
+            return true;
+        if (a === 192 && b === 168)
+            return true;
+    }
+    return false;
+}
 export function isAllowedOrigin(origin) {
     if (!origin) {
         return true;
@@ -145,8 +180,7 @@ export function isAllowedOrigin(origin) {
     }
     try {
         const url = new URL(origin);
-        const host = url.hostname;
-        if (host === 'localhost' || host === '127.0.0.1' || host === '::1' || host === '[::1]') {
+        if (isPrivateIP(url.hostname)) {
             return true;
         }
     }
@@ -160,7 +194,7 @@ function isCommandMessage(msg) {
 }
 export class StreamServer {
     wss = null;
-    clients = new Set();
+    clients = new Map();
     browser;
     port;
     isScreencasting = false;
@@ -188,7 +222,7 @@ export class StreamServer {
                     fps: this.frameRateController.getCurrentFps(),
                     state: newState,
                 };
-                for (const client of this.clients) {
+                for (const [client, _state] of this.clients) {
                     if (client.readyState === WebSocket.OPEN) {
                         client.send(JSON.stringify(headerMessage));
                         client.send(processedBuffer);
@@ -213,8 +247,8 @@ export class StreamServer {
                         return false;
                     },
                 });
-                this.wss.on('connection', (ws) => {
-                    this.handleConnection(ws);
+                this.wss.on('connection', (ws, req) => {
+                    this.handleConnection(ws, req);
                 });
                 this.wss.on('error', (error) => {
                     console.error('[StreamServer] WebSocket error:', error);
@@ -255,7 +289,7 @@ export class StreamServer {
         }
         setScreencastFrameCallback(null);
         setEventCallbacks({});
-        for (const client of this.clients) {
+        for (const [client, _state] of this.clients) {
             client.close();
         }
         this.clients.clear();
@@ -268,10 +302,60 @@ export class StreamServer {
             });
         }
     }
-    handleConnection(ws) {
-        console.log('[StreamServer] Client connected');
-        this.clients.add(ws);
-        this.sendStatus(ws);
+    handleConnection(ws, req) {
+        const clientState = {};
+        try {
+            const url = new URL(req.url || '/', 'http://localhost');
+            const rawSelector = url.searchParams.get('selector');
+            if (rawSelector) {
+                clientState.selector = decodeURIComponent(rawSelector);
+            }
+        }
+        catch {
+            // Invalid URL, ignore
+        }
+        this.clients.set(ws, clientState);
+        if (clientState.selector) {
+            clientState.elementCheckTimer = setInterval(() => {
+                if (!clientState.selector)
+                    return;
+                this.getElementBox(clientState.selector)
+                    .then((newBox) => {
+                    if (!newBox) {
+                        if (clientState.elementBox || !clientState.degraded) {
+                            clientState.elementBox = undefined;
+                            clientState.degraded = true;
+                            const statusMsg = {
+                                type: 'status',
+                                connected: true,
+                                screencasting: this.isScreencasting,
+                                degraded: true,
+                            };
+                            if (ws.readyState === WebSocket.OPEN) {
+                                ws.send(JSON.stringify(statusMsg));
+                            }
+                        }
+                    }
+                    else {
+                        clientState.elementBox = newBox;
+                        clientState.degraded = false;
+                    }
+                })
+                    .catch(() => { });
+            }, 2500);
+        }
+        this.sendStatus(ws, clientState);
+        this.refreshElementBox(ws, clientState)
+            .then(async () => {
+            this.sendStatus(ws, clientState);
+            if (this.isScreencasting && this.lastFrameData && this.lastFrameMetadata) {
+                await new Promise((r) => setTimeout(r, 100));
+                this.replayLastFrame(ws, clientState);
+            }
+        })
+            .catch((err) => {
+            console.error('[StreamServer] Failed to refresh element box:', err);
+        });
         if (this.clients.size === 1 && !this.isScreencasting) {
             this.startScreencast().catch((error) => {
                 console.error('[StreamServer] Failed to start screencast:', error);
@@ -288,7 +372,10 @@ export class StreamServer {
             }
         });
         ws.on('close', () => {
-            console.log('[StreamServer] Client disconnected');
+            if (clientState.elementCheckTimer) {
+                clearInterval(clientState.elementCheckTimer);
+                clientState.elementCheckTimer = undefined;
+            }
             this.clients.delete(ws);
             if (this.clients.size === 0 && this.isScreencasting) {
                 this.stopScreencast().catch((error) => {
@@ -361,11 +448,33 @@ export class StreamServer {
                     this.stateManager.onUserInteraction();
                     await this.browser.getPage().keyboard.insertText(message.text);
                     break;
+                case 'input_focused':
+                case 'input_value':
+                case 'input_blur':
+                    for (const [ws] of this.clients.entries()) {
+                        if (ws.readyState === WebSocket.OPEN) {
+                            try {
+                                ws.send(JSON.stringify(message));
+                            }
+                            catch (_) { }
+                        }
+                    }
+                    break;
                 case 'user_activity':
                     this.stateManager.onUserInteraction();
                     break;
                 case 'status':
-                    this.sendStatus(ws);
+                    const cs = this.clients.get(ws);
+                    if (cs?.selector) {
+                        this.sendStatus(ws, cs);
+                        this.refreshElementBox(ws, cs).catch(() => { });
+                    }
+                    else if (cs) {
+                        this.sendStatus(ws, cs);
+                    }
+                    else {
+                        this.sendStatus(ws);
+                    }
                     break;
             }
         }
@@ -382,36 +491,97 @@ export class StreamServer {
         if (!this.frameRateController.shouldSendFrame(config.maxFps)) {
             return;
         }
-        let processedBuffer;
-        try {
-            processedBuffer = await this.frameProcessor.process(frame.data, config, frame.metadata.deviceWidth, frame.metadata.deviceHeight);
-        }
-        catch {
-            processedBuffer = Buffer.from(frame.data, 'base64');
-        }
-        const headerMessage = {
-            type: 'frame',
-            metadata: frame.metadata,
-            format: config.format,
-            fps: this.frameRateController.getCurrentFps(),
-            state: this.stateManager.getState(),
-        };
-        for (const client of this.clients) {
-            if (client.readyState === WebSocket.OPEN) {
+        for (const [client, clientState] of this.clients) {
+            if (client.readyState !== WebSocket.OPEN) {
+                continue;
+            }
+            try {
+                let processedBuffer;
+                let metadata = { ...frame.metadata };
+                if (clientState.selector && clientState.elementBox) {
+                    processedBuffer = await this.frameProcessor.process(frame.data, config, clientState.elementBox.width, clientState.elementBox.height, clientState.elementBox);
+                    metadata.deviceWidth = clientState.elementBox.width;
+                    metadata.deviceHeight = clientState.elementBox.height;
+                }
+                else {
+                    processedBuffer = await this.frameProcessor.process(frame.data, config, frame.metadata.deviceWidth, frame.metadata.deviceHeight);
+                }
+                const headerMessage = {
+                    type: 'frame',
+                    metadata,
+                    format: config.format,
+                    fps: this.frameRateController.getCurrentFps(),
+                    state: this.stateManager.getState(),
+                };
                 client.send(JSON.stringify(headerMessage));
                 client.send(processedBuffer);
             }
+            catch (err) {
+                console.error('[StreamServer] Failed to process frame for client:', err);
+            }
+        }
+    }
+    async replayLastFrame(ws, clientState) {
+        if (!this.lastFrameData || !this.lastFrameMetadata)
+            return;
+        if (ws.readyState !== WebSocket.OPEN)
+            return;
+        try {
+            const config = this.stateManager.getConfig();
+            let processedBuffer;
+            let metadata = { ...this.lastFrameMetadata };
+            if (clientState.selector && clientState.elementBox) {
+                processedBuffer = await this.frameProcessor.process(this.lastFrameData, config, clientState.elementBox.width, clientState.elementBox.height, clientState.elementBox);
+                metadata.deviceWidth = clientState.elementBox.width;
+                metadata.deviceHeight = clientState.elementBox.height;
+            }
+            else {
+                processedBuffer = await this.frameProcessor.process(this.lastFrameData, config, this.lastFrameMetadata.deviceWidth, this.lastFrameMetadata.deviceHeight);
+            }
+            const headerMessage = {
+                type: 'frame',
+                metadata,
+                format: config.format,
+                fps: this.frameRateController.getCurrentFps(),
+                state: this.stateManager.getState(),
+            };
+            ws.send(JSON.stringify(headerMessage));
+            ws.send(processedBuffer);
+        }
+        catch (err) {
+            console.error('[StreamServer] Failed to replay last frame:', err);
         }
     }
     broadcastEvent(message) {
         const payload = JSON.stringify(message);
-        for (const client of this.clients) {
+        for (const [client, _state] of this.clients) {
             if (client.readyState === WebSocket.OPEN) {
                 client.send(payload);
             }
         }
     }
-    sendStatus(ws) {
+    async getElementBox(selector) {
+        try {
+            const page = this.browser.getPage();
+            const box = await page.evaluate((sel) => {
+                const el = document.querySelector(sel);
+                if (!el)
+                    return null;
+                const rect = el.getBoundingClientRect();
+                return {
+                    x: rect.x,
+                    y: rect.y,
+                    width: rect.width,
+                    height: rect.height,
+                };
+            }, selector);
+            return box ?? undefined;
+        }
+        catch {
+            return undefined;
+        }
+    }
+    sendStatus(ws, clientState) {
         let viewportWidth;
         let viewportHeight;
         try {
@@ -420,9 +590,7 @@ export class StreamServer {
             viewportWidth = viewport?.width;
             viewportHeight = viewport?.height;
         }
-        catch {
-            // Browser not launched yet
-        }
+        catch { }
         const message = {
             type: 'status',
             connected: true,
@@ -431,11 +599,48 @@ export class StreamServer {
             viewportHeight,
             fps: this.frameRateController.getCurrentFps(),
             state: this.stateManager.getState(),
+            version: process.env.npm_package_version || '0.9.5',
         };
+        if (clientState?.elementBox) {
+            message.element = {
+                selector: clientState.selector,
+                x: clientState.elementBox.x,
+                y: clientState.elementBox.y,
+                width: clientState.elementBox.width,
+                height: clientState.elementBox.height,
+            };
+            message.viewportWidth = clientState.elementBox.width;
+            message.viewportHeight = clientState.elementBox.height;
+        }
+        if (clientState?.degraded) {
+            message.degraded = true;
+        }
         if (ws.readyState === WebSocket.OPEN) {
             ws.send(JSON.stringify(message));
         }
     }
+    async refreshElementBox(ws, clientState) {
+        if (!clientState.selector)
+            return;
+        const box = await this.getElementBox(clientState.selector);
+        if (box) {
+            clientState.elementBox = box;
+            clientState.degraded = false;
+        }
+        else if (!clientState.degraded) {
+            clientState.elementBox = undefined;
+            clientState.degraded = true;
+            const message = {
+                type: 'status',
+                connected: true,
+                screencasting: this.isScreencasting,
+                degraded: true,
+            };
+            if (ws.readyState === WebSocket.OPEN) {
+                ws.send(JSON.stringify(message));
+            }
+        }
+    }
     sendError(ws, errorMessage) {
         const message = {
             type: 'error',
@@ -453,16 +658,58 @@ export class StreamServer {
             if (!this.browser.isLaunched()) {
                 throw new Error('Browser not launched');
             }
-            await this.browser.startScreencast((frame) => this.broadcastFrame(frame), {
-                format: 'jpeg',
-                quality: 80,
-                maxWidth: 1280,
-                maxHeight: 720,
-                everyNthFrame: 1,
-            });
-            for (const client of this.clients) {
-                this.sendStatus(client);
+            try {
+                await this.browser.startScreencast((frame) => this.broadcastFrame(frame), {
+                    format: 'jpeg',
+                    quality: 80,
+                    maxWidth: 1280,
+                    maxHeight: 720,
+                    everyNthFrame: 1,
+                });
+            }
+            catch (startError) {
+                if (startError instanceof Error && startError.message === 'Screencast already active') {
+                    await this.browser.stopScreencast();
+                    await this.browser.startScreencast((frame) => this.broadcastFrame(frame), {
+                        format: 'jpeg',
+                        quality: 80,
+                        maxWidth: 1280,
+                        maxHeight: 720,
+                        everyNthFrame: 1,
+                    });
+                }
+                else {
+                    throw startError;
+                }
+            }
+            for (const [client, state] of this.clients) {
+                this.sendStatus(client, state);
             }
+            try {
+                await this.browser.injectFocusListener((data) => {
+                    for (const [ws] of this.clients.entries()) {
+                        if (ws.readyState === WebSocket.OPEN) {
+                            try {
+                                ws.send(JSON.stringify(data));
+                            }
+                            catch (_) { }
+                        }
+                    }
+                });
+            }
+            catch (e) {
+                console.error('[StreamServer] Failed to inject focus listener:', e);
+            }
+            try {
+                const page = this.browser.getPage();
+                await page.evaluate(() => {
+                    document.body.style.opacity = '0.999';
+                    requestAnimationFrame(() => {
+                        document.body.style.opacity = '';
+                    });
+                });
+            }
+            catch { }
         }
         catch (error) {
             this.isScreencasting = false;
@@ -472,10 +719,10 @@ export class StreamServer {
     async stopScreencast() {
         if (!this.isScreencasting)
             return;
-        await this.browser.stopScreencast();
         this.isScreencasting = false;
-        for (const client of this.clients) {
-            this.sendStatus(client);
+        await this.browser.stopScreencast();
+        for (const [client, state] of this.clients) {
+            this.sendStatus(client, state);
         }
     }
     getPort() {
@@ -507,6 +754,8 @@ export class StreamServerProxy {
     reconnectTimer = null;
     lastFrameData = null;
     lastFrameMetadata = null;
+    inputFillDebounceMap = new Map();
+    static INPUT_FILL_DEBOUNCE_MS = 60;
     constructor(browser) {
         this.browser = browser;
         this.ipcPath = getStreamServerIpcPath();
@@ -587,6 +836,8 @@ export class StreamServerProxy {
                 case 'keyboard_down':
                 case 'keyboard_up':
                 case 'keyboard_insert_text':
+                case 'input_fill':
+                case 'input_blur_element':
                 case 'user_activity':
                     this.handleInputMessage(message);
                     break;
@@ -596,6 +847,25 @@ export class StreamServerProxy {
                 case 'client_disconnected':
                     this.handleClientDisconnected(message.session);
                     break;
+                case 'request_element_box':
+                    (async () => {
+                        const box = await this.getElementBox(message.selector);
+                        const response = {
+                            type: 'selector_element',
+                            session: message.session ?? this.session,
+                            selector: message.selector,
+                        };
+                        if (box) {
+                            response.elementBox = box;
+                        }
+                        this.send(response);
+                    })();
+                    break;
+                case 'input_focused':
+                case 'input_value':
+                case 'input_blur':
+                    this.send(message);
+                    break;
             }
         }
         catch (error) {
@@ -673,6 +943,23 @@ export class StreamServerProxy {
                     this.stateManager.onUserInteraction();
                     await this.browser.getPage().keyboard.insertText(message.text);
                     break;
+                case 'input_fill': {
+                    const sel = message.selector || '';
+                    const txt = message.text || '';
+                    const key = sel || '__global__';
+                    const prev = this.inputFillDebounceMap.get(key);
+                    if (prev)
+                        clearTimeout(prev.timer);
+                    const timer = setTimeout(async () => {
+                        this.inputFillDebounceMap.delete(key);
+                        await this.browser.fillValue(sel, txt);
+                    }, StreamServerProxy.INPUT_FILL_DEBOUNCE_MS);
+                    this.inputFillDebounceMap.set(key, { timer, text: txt, selector: sel });
+                    break;
+                }
+                case 'input_blur_element':
+                    await this.browser.blurElement(message.selector || '');
+                    break;
                 case 'user_activity':
                     this.stateManager.onUserInteraction();
                     break;
@@ -733,6 +1020,39 @@ export class StreamServerProxy {
             this.ipcSocket.write(JSON.stringify(message) + '\n');
         }
     }
+    sendSelectorElement(selector, elementBox) {
+        this.send({
+            type: 'selector_element',
+            session: this.session,
+            selector,
+            elementBox,
+        });
+    }
+    async getElementBox(selector) {
+        try {
+            const page = this.browser.getPage();
+            if (!page) {
+                return undefined;
+            }
+            const box = await page.evaluate((sel) => {
+                const el = document.querySelector(sel);
+                if (!el)
+                    return null;
+                const rect = el.getBoundingClientRect();
+                return {
+                    x: rect.x,
+                    y: rect.y,
+                    width: rect.width,
+                    height: rect.height,
+                };
+            }, selector);
+            return box ?? undefined;
+        }
+        catch (err) {
+            console.error('[StreamServerProxy] getElementBox error:', err);
+            return undefined;
+        }
+    }
     getDaemonSocketPath() {
         const isWindows = process.platform === 'win32';
         if (isWindows) {