crewly 1.5.11 → 1.5.12

package/dist/backend/backend/src/services/browser/browser-proxy.service.js

@@ -0,0 +1,441 @@
+/**
+ * Browser Proxy Service
+ *
+ * Connects to the Cloud Relay as an 'agent' and provides addressed messaging
+ * to browser instances. Any Crewly agent can call browser tools through this
+ * service without needing relay pairing.
+ *
+ * Architecture:
+ *   Agent → /api/browser/* → BrowserProxyService → Cloud Relay → Extension
+ *   Extension → Cloud Relay → BrowserProxyService → HTTP response → Agent
+ *
+ * @module services/browser/browser-proxy.service
+ */
+import WebSocket from 'ws';
+import { LoggerService } from '../core/logger.service.js';
+import { BROWSER_BRIDGE_CONSTANTS } from '../../constants.js';
+/** Default relay WebSocket URL. */
+const DEFAULT_RELAY_URL = 'wss://api.crewlyai.com/relay';
+/** Heartbeat interval to keep the relay connection alive (ms). */
+const HEARTBEAT_INTERVAL_MS = 25_000;
+/** Reconnect delay after disconnection (ms). */
+const RECONNECT_DELAY_MS = 5_000;
+/**
+ * BrowserProxyService connects to the Cloud Relay and routes browser commands
+ * to specific extension instances using addressed messaging (relay_to).
+ *
+ * Singleton — one relay connection per backend instance.
+ */
+export class BrowserProxyService {
+    static instance = null;
+    logger;
+    /** WebSocket connection to cloud relay */
+    ws = null;
+    /** Current relay session ID (assigned after registration) */
+    sessionId = null;
+    /** Connection state */
+    state = 'disconnected';
+    /** Map of commandId → pending command awaiting response */
+    pendingCommands = new Map();
+    /** Known browser instances from relay browser_list + browser_event */
+    instances = new Map();
+    /** Command counter for unique IDs */
+    commandCounter = 0;
+    /** Heartbeat interval timer */
+    heartbeatTimer = null;
+    /** Reconnect timer */
+    reconnectTimer = null;
+    /** Auth token for relay registration */
+    authToken = null;
+    /** Relay WebSocket URL */
+    relayUrl = DEFAULT_RELAY_URL;
+    /** Whether auto-reconnect is enabled */
+    autoReconnect = true;
+    constructor() {
+        this.logger = LoggerService.getInstance().createComponentLogger('BrowserProxy');
+    }
+    /**
+     * Get the singleton instance.
+     *
+     * @returns BrowserProxyService instance
+     */
+    static getInstance() {
+        if (!BrowserProxyService.instance) {
+            BrowserProxyService.instance = new BrowserProxyService();
+        }
+        return BrowserProxyService.instance;
+    }
+    /**
+     * Reset singleton instance (for testing).
+     */
+    static resetInstance() {
+        BrowserProxyService.instance = null;
+    }
+    /**
+     * Connect to the cloud relay WebSocket server.
+     *
+     * Registers as role 'agent' with a pairing code matching the backend identity.
+     * After connection, requests the browser instance list and subscribes to events.
+     *
+     * @param token - JWT auth token for relay registration
+     * @param relayUrl - Optional relay URL override
+     */
+    connect(token, relayUrl) {
+        if (this.state !== 'disconnected') {
+            this.logger.warn('Already connected or connecting to relay');
+            return;
+        }
+        this.authToken = token;
+        if (relayUrl)
+            this.relayUrl = relayUrl;
+        this.state = 'connecting';
+        this.autoReconnect = true;
+        this.logger.info('Connecting to cloud relay for browser proxy', {
+            url: this.relayUrl,
+        });
+        this.ws = new WebSocket(this.relayUrl);
+        this.ws.on('open', () => {
+            this.logger.info('Relay WebSocket connected, registering as agent');
+            this.sendRaw({
+                type: 'register',
+                role: 'agent',
+                pairingCode: `backend-proxy-${Date.now()}`,
+                token: this.authToken,
+            });
+        });
+        this.ws.on('message', (data) => {
+            this.handleMessage(typeof data === 'string' ? data : data.toString('utf8'));
+        });
+        this.ws.on('close', (code, reason) => {
+            this.logger.info('Relay WebSocket closed', { code, reason: reason.toString() });
+            this.handleDisconnect();
+        });
+        this.ws.on('error', (err) => {
+            this.logger.error('Relay WebSocket error', { error: err.message });
+            this.handleDisconnect();
+        });
+    }
+    /**
+     * Disconnect from the cloud relay.
+     */
+    disconnect() {
+        this.autoReconnect = false;
+        this.cleanup();
+    }
+    /**
+     * Check if the proxy is connected to the relay and has browser instances available.
+     *
+     * @returns True if connected and at least one browser instance is known
+     */
+    isAvailable() {
+        return this.state === 'connected' && this.instances.size > 0;
+    }
+    /**
+     * Check if the proxy has a live relay connection (even if no browsers registered yet).
+     *
+     * @returns True if WebSocket to relay is connected
+     */
+    isConnected() {
+        return this.state === 'connected';
+    }
+    /**
+     * Get the current connection state.
+     *
+     * @returns Connection state string
+     */
+    getState() {
+        return this.state;
+    }
+    /**
+     * Get the list of known browser instances.
+     *
+     * @returns Array of browser instance info
+     */
+    getInstances() {
+        return Array.from(this.instances.values());
+    }
+    /**
+     * Send a browser command to a specific instance (or the only connected one).
+     *
+     * Routes via relay_to addressed messaging. The command is forwarded by the
+     * relay server to the matching browser extension, which executes the tool
+     * and returns the response via relay_response.
+     *
+     * @param tool - Tool name (e.g. 'navigate', 'screenshot', 'getTabs')
+     * @param params - Tool parameters
+     * @param instance - Target instance name or ID (optional if only 1 connected)
+     * @param timeoutMs - Command timeout in milliseconds
+     * @param agentName - Agent name for extension banner display
+     * @returns Response from the Chrome Extension
+     * @throws Error if no instances available, target not found, or timeout
+     */
+    async sendCommand(tool, params, instance, timeoutMs = BROWSER_BRIDGE_CONSTANTS.COMMAND_TIMEOUT_MS, agentName) {
+        if (this.state !== 'connected' || !this.ws) {
+            throw new Error('Browser proxy not connected to relay');
+        }
+        // Resolve target instance
+        const targetInstance = this.resolveInstance(instance);
+        if (!targetInstance) {
+            const available = this.getInstances().map((i) => i.instanceName).join(', ');
+            throw new Error(instance
+                ? `Browser instance "${instance}" not found. Available: ${available || 'none'}`
+                : `No browser instances connected`);
+        }
+        const id = `proxy-${++this.commandCounter}-${Date.now()}`;
+        const payload = JSON.stringify({
+            id,
+            tool,
+            params: params ?? {},
+            agentName,
+        });
+        return new Promise((resolve, reject) => {
+            const timer = setTimeout(() => {
+                this.pendingCommands.delete(id);
+                reject(new Error(`Browser command '${tool}' timed out after ${timeoutMs}ms`));
+            }, timeoutMs);
+            this.pendingCommands.set(id, { resolve, reject, timer });
+            this.sendRaw({
+                type: 'relay_to',
+                targetInstance: targetInstance.instanceName,
+                payload,
+            });
+            this.logger.debug('Command sent via relay_to', {
+                id,
+                tool,
+                target: targetInstance.instanceName,
+            });
+        });
+    }
+    // ---------------------------------------------------------------------------
+    // Private methods
+    // ---------------------------------------------------------------------------
+    /**
+     * Resolve which browser instance to target.
+     *
+     * @param instance - Explicit instance name/ID, or undefined for auto-select
+     * @returns Matching BrowserInstanceInfo or null
+     */
+    resolveInstance(instance) {
+        if (!instance) {
+            // Auto-select: use the only instance if there's exactly one
+            if (this.instances.size === 1) {
+                return this.instances.values().next().value ?? null;
+            }
+            return null;
+        }
+        // Search by name or ID
+        for (const info of this.instances.values()) {
+            if (info.instanceName === instance || info.instanceId === instance) {
+                return info;
+            }
+        }
+        return null;
+    }
+    /**
+     * Handle incoming WebSocket message from the relay.
+     *
+     * @param raw - Raw JSON string
+     */
+    handleMessage(raw) {
+        let msg;
+        try {
+            msg = JSON.parse(raw);
+        }
+        catch {
+            this.logger.warn('Non-JSON relay message ignored');
+            return;
+        }
+        const type = msg.type;
+        switch (type) {
+            case 'registered':
+                this.sessionId = msg.sessionId;
+                this.state = 'connected';
+                this.startHeartbeat();
+                this.logger.info('Registered with relay', { sessionId: this.sessionId });
+                // Request browser instance list
+                this.sendRaw({ type: 'list_browsers' });
+                break;
+            case 'browser_list':
+                this.handleBrowserList(msg.instances);
+                break;
+            case 'browser_event':
+                this.handleBrowserEvent(msg);
+                break;
+            case 'relay': {
+                // Response from a browser instance
+                const payload = msg.payload;
+                this.handleRelayPayload(payload);
+                break;
+            }
+            case 'heartbeat_ack':
+                // Heartbeat acknowledged
+                break;
+            case 'error':
+                this.logger.warn('Relay error', {
+                    code: msg.code,
+                    message: msg.message,
+                });
+                // Resolve any pending command that might be waiting
+                if (msg.code === 'BROWSER_NOT_FOUND' || msg.code === 'BROWSER_UNAVAILABLE') {
+                    // These errors mean the browser isn't reachable
+                    this.logger.warn('Browser not reachable via relay', { code: msg.code });
+                }
+                break;
+            default:
+                this.logger.debug('Unhandled relay message', { type });
+        }
+    }
+    /**
+     * Handle browser_list message from relay.
+     *
+     * @param instances - Array of browser instance info from relay
+     */
+    handleBrowserList(instances) {
+        this.instances.clear();
+        for (const inst of instances) {
+            this.instances.set(inst.instanceId, inst);
+        }
+        this.logger.info('Browser instances updated', {
+            count: this.instances.size,
+            names: instances.map((i) => i.instanceName),
+        });
+    }
+    /**
+     * Handle browser_event (connected/disconnected/updated) from relay.
+     *
+     * @param msg - Event message
+     */
+    handleBrowserEvent(msg) {
+        const event = msg.event;
+        const instanceId = msg.instanceId;
+        const instanceName = msg.instanceName;
+        switch (event) {
+            case 'connected':
+                this.instances.set(instanceId, {
+                    instanceId,
+                    instanceName,
+                    sessionId: '', // Will be populated by next list_browsers
+                });
+                this.logger.info('Browser instance connected', { instanceId, instanceName });
+                // Refresh full list to get sessionId
+                this.sendRaw({ type: 'list_browsers' });
+                break;
+            case 'disconnected':
+                this.instances.delete(instanceId);
+                this.logger.info('Browser instance disconnected', { instanceId, instanceName });
+                break;
+            case 'updated':
+                // Update name
+                const existing = this.instances.get(instanceId);
+                if (existing) {
+                    existing.instanceName = instanceName;
+                }
+                break;
+        }
+    }
+    /**
+     * Handle a relay payload (response from a browser instance).
+     *
+     * @param payload - JSON string containing the command response
+     */
+    handleRelayPayload(payload) {
+        let response;
+        try {
+            response = JSON.parse(payload);
+        }
+        catch {
+            this.logger.warn('Failed to parse relay payload as command response');
+            return;
+        }
+        if (!response.id)
+            return;
+        const pending = this.pendingCommands.get(response.id);
+        if (pending) {
+            clearTimeout(pending.timer);
+            this.pendingCommands.delete(response.id);
+            pending.resolve(response);
+            this.logger.debug('Command response received', {
+                id: response.id,
+                success: response.success,
+            });
+        }
+    }
+    /**
+     * Send a raw JSON message to the relay WebSocket.
+     *
+     * @param msg - Message object to serialize and send
+     */
+    sendRaw(msg) {
+        if (this.ws && this.ws.readyState === WebSocket.OPEN) {
+            this.ws.send(JSON.stringify(msg));
+        }
+    }
+    /**
+     * Start heartbeat interval to keep relay connection alive.
+     */
+    startHeartbeat() {
+        this.stopHeartbeat();
+        this.heartbeatTimer = setInterval(() => {
+            this.sendRaw({ type: 'heartbeat' });
+        }, HEARTBEAT_INTERVAL_MS);
+    }
+    /**
+     * Stop heartbeat interval.
+     */
+    stopHeartbeat() {
+        if (this.heartbeatTimer) {
+            clearInterval(this.heartbeatTimer);
+            this.heartbeatTimer = null;
+        }
+    }
+    /**
+     * Handle disconnection — clean up and optionally reconnect.
+     */
+    handleDisconnect() {
+        this.state = 'disconnected';
+        this.sessionId = null;
+        this.stopHeartbeat();
+        // Reject all pending commands
+        for (const [id, pending] of this.pendingCommands) {
+            clearTimeout(pending.timer);
+            pending.reject(new Error('Relay connection lost'));
+            this.pendingCommands.delete(id);
+        }
+        // Auto-reconnect
+        if (this.autoReconnect && this.authToken) {
+            this.reconnectTimer = setTimeout(() => {
+                this.state = 'disconnected'; // Reset for connect()
+                this.connect(this.authToken);
+            }, RECONNECT_DELAY_MS);
+        }
+    }
+    /**
+     * Full cleanup — close WebSocket, stop timers, clear state.
+     */
+    cleanup() {
+        this.stopHeartbeat();
+        if (this.reconnectTimer) {
+            clearTimeout(this.reconnectTimer);
+            this.reconnectTimer = null;
+        }
+        // Reject all pending commands
+        for (const [id, pending] of this.pendingCommands) {
+            clearTimeout(pending.timer);
+            pending.reject(new Error('Browser proxy shutting down'));
+        }
+        this.pendingCommands.clear();
+        if (this.ws) {
+            try {
+                this.ws.close(1000, 'Proxy shutdown');
+            }
+            catch {
+                // Ignore close errors
+            }
+            this.ws = null;
+        }
+        this.state = 'disconnected';
+        this.sessionId = null;
+        this.instances.clear();
+    }
+}
+//# sourceMappingURL=browser-proxy.service.js.map

package/dist/backend/backend/src/services/browser/browser-proxy.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-proxy.service.js","sourceRoot":"","sources":["../../../../../../backend/src/services/browser/browser-proxy.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,SAAS,MAAM,IAAI,CAAC;AAC3B,OAAO,EAAE,aAAa,EAAwB,MAAM,2BAA2B,CAAC;AAEhF,OAAO,EAAE,wBAAwB,EAAE,MAAM,oBAAoB,CAAC;AAyB9D,mCAAmC;AACnC,MAAM,iBAAiB,GAAG,8BAA8B,CAAC;AAEzD,kEAAkE;AAClE,MAAM,qBAAqB,GAAG,MAAM,CAAC;AAErC,gDAAgD;AAChD,MAAM,kBAAkB,GAAG,KAAK,CAAC;AAEjC;;;;;GAKG;AACH,MAAM,OAAO,mBAAmB;IACtB,MAAM,CAAC,QAAQ,GAA+B,IAAI,CAAC;IAC1C,MAAM,CAAkB;IAEzC,0CAA0C;IAClC,EAAE,GAAqB,IAAI,CAAC;IACpC,6DAA6D;IACrD,SAAS,GAAkB,IAAI,CAAC;IACxC,uBAAuB;IACf,KAAK,GAAyB,cAAc,CAAC;IACrD,2DAA2D;IACnD,eAAe,GAAqC,IAAI,GAAG,EAAE,CAAC;IACtE,sEAAsE;IAC9D,SAAS,GAAqC,IAAI,GAAG,EAAE,CAAC;IAChE,qCAAqC;IAC7B,cAAc,GAAG,CAAC,CAAC;IAC3B,+BAA+B;IACvB,cAAc,GAA0C,IAAI,CAAC;IACrE,sBAAsB;IACd,cAAc,GAAyC,IAAI,CAAC;IACpE,wCAAwC;IAChC,SAAS,GAAkB,IAAI,CAAC;IACxC,0BAA0B;IAClB,QAAQ,GAAW,iBAAiB,CAAC;IAC7C,wCAAwC;IAChC,aAAa,GAAG,IAAI,CAAC;IAE7B;QACE,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,WAAW,EAAE,CAAC,qBAAqB,CAAC,cAAc,CAAC,CAAC;IAClF,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,WAAW;QAChB,IAAI,CAAC,mBAAmB,CAAC,QAAQ,EAAE,CAAC;YAClC,mBAAmB,CAAC,QAAQ,GAAG,IAAI,mBAAmB,EAAE,CAAC;QAC3D,CAAC;QACD,OAAO,mBAAmB,CAAC,QAAQ,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,aAAa;QAClB,mBAAmB,CAAC,QAAQ,GAAG,IAAI,CAAC;IACtC,CAAC;IAED;;;;;;;;OAQG;IACH,OAAO,CAAC,KAAa,EAAE,QAAiB;QACtC,IAAI,IAAI,CAAC,KAAK,KAAK,cAAc,EAAE,CAAC;YAClC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAC;YAC7D,OAAO;QACT,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,IAAI,QAAQ;YAAE,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACvC,IAAI,CAAC,KAAK,GAAG,YAAY,CAAC;QAC1B,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAE1B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6CAA6C,EAAE;YAC9D,GAAG,EAAE,IAAI,CAAC,QAAQ;SACnB,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEvC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;YACtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;YACpE,IAAI,CAAC,OAAO,CAAC;gBACX,IAAI,EAAE,UAAU;gBAChB,IAAI,EAAE,OAAO;gBACb,WAAW,EAAE,iBAAiB,IAAI,CAAC,GAAG,EAAE,EAAE;gBAC1C,KAAK,EAAE,IAAI,CAAC,SAAS;aACtB,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAqB,EAAE,EAAE;YAC9C,IAAI,CAAC,aAAa,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;QAC9E,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAY,EAAE,MAAc,EAAE,EAAE;YACnD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAChF,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE;YACjC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,uBAAuB,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;YACnE,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1B,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,UAAU;QACR,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;QAC3B,IAAI,CAAC,OAAO,EAAE,CAAC;IACjB,CAAC;IAED;;;;OAIG;IACH,WAAW;QACT,OAAO,IAAI,CAAC,KAAK,KAAK,WAAW,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,GAAG,CAAC,CAAC;IAC/D,CAAC;IAED;;;;OAIG;IACH,WAAW;QACT,OAAO,IAAI,CAAC,KAAK,KAAK,WAAW,CAAC;IACpC,CAAC;IAED;;;;OAIG;IACH,QAAQ;QACN,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED;;;;OAIG;IACH,YAAY;QACV,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;IAC7C,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,KAAK,CAAC,WAAW,CACf,IAAY,EACZ,MAAgC,EAChC,QAAiB,EACjB,YAAoB,wBAAwB,CAAC,kBAAkB,EAC/D,SAAkB;QAElB,IAAI,IAAI,CAAC,KAAK,KAAK,WAAW,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YAC3C,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;QAED,0BAA0B;QAC1B,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;QACtD,IAAI,CAAC,cAAc,EAAE,CAAC;YACpB,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5E,MAAM,IAAI,KAAK,CACb,QAAQ;gBACN,CAAC,CAAC,qBAAqB,QAAQ,2BAA2B,SAAS,IAAI,MAAM,EAAE;gBAC/E,CAAC,CAAC,gCAAgC,CACrC,CAAC;QACJ,CAAC;QAED,MAAM,EAAE,GAAG,SAAS,EAAE,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;QAC1D,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC;YAC7B,EAAE;YACF,IAAI;YACJ,MAAM,EAAE,MAAM,IAAI,EAAE;YACpB,SAAS;SACV,CAAC,CAAC;QAEH,OAAO,IAAI,OAAO,CAAyB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC7D,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC5B,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAChC,MAAM,CAAC,IAAI,KAAK,CAAC,oBAAoB,IAAI,qBAAqB,SAAS,IAAI,CAAC,CAAC,CAAC;YAChF,CAAC,EAAE,SAAS,CAAC,CAAC;YAEd,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;YAEzD,IAAI,CAAC,OAAO,CAAC;gBACX,IAAI,EAAE,UAAU;gBAChB,cAAc,EAAE,cAAc,CAAC,YAAY;gBAC3C,OAAO;aACR,CAAC,CAAC;YAEH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE;gBAC7C,EAAE;gBACF,IAAI;gBACJ,MAAM,EAAE,cAAc,CAAC,YAAY;aACpC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,8EAA8E;IAC9E,kBAAkB;IAClB,8EAA8E;IAE9E;;;;;OAKG;IACK,eAAe,CAAC,QAAiB;QACvC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,4DAA4D;YAC5D,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;gBAC9B,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,IAAI,IAAI,CAAC;YACtD,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QAED,uBAAuB;QACvB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,IAAI,IAAI,CAAC,YAAY,KAAK,QAAQ,IAAI,IAAI,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;gBACnE,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;OAIG;IACK,aAAa,CAAC,GAAW;QAC/B,IAAI,GAA4B,CAAC;QACjC,IAAI,CAAC;YACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAA4B,CAAC;QACnD,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;YACnD,OAAO;QACT,CAAC;QAED,MAAM,IAAI,GAAG,GAAG,CAAC,IAAc,CAAC;QAEhC,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,YAAY;gBACf,IAAI,CAAC,SAAS,GAAG,GAAG,CAAC,SAAmB,CAAC;gBACzC,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC;gBACzB,IAAI,CAAC,cAAc,EAAE,CAAC;gBACtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,uBAAuB,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;gBACzE,gCAAgC;gBAChC,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;gBACxC,MAAM;YAER,KAAK,cAAc;gBACjB,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,SAAkC,CAAC,CAAC;gBAC/D,MAAM;YAER,KAAK,eAAe;gBAClB,IAAI,CAAC,kBAAkB,CAAC,GAA8B,CAAC,CAAC;gBACxD,MAAM;YAER,KAAK,OAAO,CAAC,CAAC,CAAC;gBACb,mCAAmC;gBACnC,MAAM,OAAO,GAAG,GAAG,CAAC,OAAiB,CAAC;gBACtC,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;gBACjC,MAAM;YACR,CAAC;YAED,KAAK,eAAe;gBAClB,yBAAyB;gBACzB,MAAM;YAER,KAAK,OAAO;gBACV,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,EAAE;oBAC9B,IAAI,EAAE,GAAG,CAAC,IAAI;oBACd,OAAO,EAAE,GAAG,CAAC,OAAO;iBACrB,CAAC,CAAC;gBACH,oDAAoD;gBACpD,IAAI,GAAG,CAAC,IAAI,KAAK,mBAAmB,IAAI,GAAG,CAAC,IAAI,KAAK,qBAAqB,EAAE,CAAC;oBAC3E,gDAAgD;oBAChD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iCAAiC,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC1E,CAAC;gBACD,MAAM;YAER;gBACE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,yBAAyB,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAED;;;;OAIG;IACK,iBAAiB,CAAC,SAAgC;QACxD,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QACvB,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QAC5C,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE;YAC5C,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI;YAC1B,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC;SAC5C,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACK,kBAAkB,CAAC,GAA4B;QACrD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAe,CAAC;QAClC,MAAM,UAAU,GAAG,GAAG,CAAC,UAAoB,CAAC;QAC5C,MAAM,YAAY,GAAG,GAAG,CAAC,YAAsB,CAAC;QAEhD,QAAQ,KAAK,EAAE,CAAC;YACd,KAAK,WAAW;gBACd,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,EAAE;oBAC7B,UAAU;oBACV,YAAY;oBACZ,SAAS,EAAE,EAAE,EAAE,0CAA0C;iBAC1D,CAAC,CAAC;gBACH,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4B,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,CAAC,CAAC;gBAC7E,qCAAqC;gBACrC,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC,CAAC;gBACxC,MAAM;YAER,KAAK,cAAc;gBACjB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;gBAClC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE,EAAE,UAAU,EAAE,YAAY,EAAE,CAAC,CAAC;gBAChF,MAAM;YAER,KAAK,SAAS;gBACZ,cAAc;gBACd,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBAChD,IAAI,QAAQ,EAAE,CAAC;oBACb,QAAQ,CAAC,YAAY,GAAG,YAAY,CAAC;gBACvC,CAAC;gBACD,MAAM;QACV,CAAC;IACH,CAAC;IAED;;;;OAIG;IACK,kBAAkB,CAAC,OAAe;QACxC,IAAI,QAAgC,CAAC;QACrC,IAAI,CAAC;YACH,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAA2B,CAAC;QAC3D,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mDAAmD,CAAC,CAAC;YACtE,OAAO;QACT,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,EAAE;YAAE,OAAO;QAEzB,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QACtD,IAAI,OAAO,EAAE,CAAC;YACZ,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC5B,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YACzC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE;gBAC7C,EAAE,EAAE,QAAQ,CAAC,EAAE;gBACf,OAAO,EAAE,QAAQ,CAAC,OAAO;aAC1B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;;;OAIG;IACK,OAAO,CAAC,GAA4B;QAC1C,IAAI,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC,UAAU,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;YACrD,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IAED;;OAEG;IACK,cAAc;QACpB,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,IAAI,CAAC,cAAc,GAAG,WAAW,CAAC,GAAG,EAAE;YACrC,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;QACtC,CAAC,EAAE,qBAAqB,CAAC,CAAC;IAC5B,CAAC;IAED;;OAEG;IACK,aAAa;QACnB,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,aAAa,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACnC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC7B,CAAC;IACH,CAAC;IAED;;OAEG;IACK,gBAAgB;QACtB,IAAI,CAAC,KAAK,GAAG,cAAc,CAAC;QAC5B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,aAAa,EAAE,CAAC;QAErB,8BAA8B;QAC9B,KAAK,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACjD,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC5B,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC,CAAC;YACnD,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAClC,CAAC;QAED,iBAAiB;QACjB,IAAI,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACzC,IAAI,CAAC,cAAc,GAAG,UAAU,CAAC,GAAG,EAAE;gBACpC,IAAI,CAAC,KAAK,GAAG,cAAc,CAAC,CAAC,sBAAsB;gBACnD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,SAAU,CAAC,CAAC;YAChC,CAAC,EAAE,kBAAkB,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAED;;OAEG;IACK,OAAO;QACb,IAAI,CAAC,aAAa,EAAE,CAAC;QAErB,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,YAAY,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAClC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC7B,CAAC;QAED,8BAA8B;QAC9B,KAAK,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACjD,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC5B,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC,CAAC;QAC3D,CAAC;QACD,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAE7B,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;YACZ,IAAI,CAAC;gBACH,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;YACxC,CAAC;YAAC,MAAM,CAAC;gBACP,sBAAsB;YACxB,CAAC;YACD,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;QACjB,CAAC;QAED,IAAI,CAAC,KAAK,GAAG,cAAc,CAAC;QAC5B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC"}

package/dist/backend/backend/src/services/browser/browser-relay-adapter.service.d.ts

@@ -0,0 +1,162 @@
+/**
+ * Browser Relay Adapter Service
+ *
+ * Bridges BrowserBridgeService with CloudSyncService to enable browser
+ * commands to flow through the Cloud relay queue when the Chrome Extension
+ * is not directly connected via WebSocket.
+ *
+ * Flow:
+ *   Agent → BrowserBridgeService → BrowserRelayAdapter → CloudSyncService
+ *   → Cloud relay queue → Extension polls → executes → sends response
+ *   → Cloud relay queue → Droplet polls → BrowserRelayAdapter resolves
+ *
+ * The Extension never sees the droplet IP — all traffic goes through
+ * the Cloud relay URL (crewlyai.com).
+ *
+ * @module services/browser/browser-relay-adapter.service
+ */
+import type { BrowserCommandResponse } from './browser-bridge.service.js';
+/**
+ * Adapter that routes browser commands through the Cloud relay queue
+ * when no direct WebSocket connection exists to the Chrome Extension.
+ *
+ * Usage:
+ * 1. Create adapter with target Extension device ID
+ * 2. Register with BrowserBridgeService as relay fallback
+ * 3. Handle incoming `browser_response` messages from CloudSyncService
+ *
+ * @example
+ * ```typescript
+ * const adapter = BrowserRelayAdapter.getInstance();
+ * adapter.setExtensionDeviceId('ext-device-abc');
+ *
+ * // Send command via relay
+ * const response = await adapter.sendViaRelay('navigate', { url: 'https://example.com' });
+ *
+ * // Handle incoming responses (called by CloudSyncService message handler)
+ * adapter.handleRelayResponse({ id: 'cmd-1', success: true, result: {...} });
+ * ```
+ */
+export declare class BrowserRelayAdapter {
+    private static instance;
+    private readonly logger;
+    private pendingCommands;
+    private commandCounter;
+    /** Device ID of the connected Chrome Extension (from Cloud device list) */
+    private extensionDeviceId;
+    /** Whether auto-discovery is currently active */
+    private discoveryActive;
+    /** Bound handler references for cleanup */
+    private boundDeviceOnlineHandler;
+    private boundDeviceOfflineHandler;
+    private boundDevicesUpdatedHandler;
+    private constructor();
+    /**
+     * Get the singleton instance.
+     *
+     * @returns BrowserRelayAdapter instance
+     */
+    static getInstance(): BrowserRelayAdapter;
+    /**
+     * Reset singleton instance (for testing).
+     */
+    static resetInstance(): void;
+    /**
+     * Set the target Chrome Extension device ID for relay routing.
+     * This device ID comes from CloudSyncService device discovery.
+     *
+     * @param deviceId - Cloud device ID of the Extension
+     */
+    setExtensionDeviceId(deviceId: string): void;
+    /**
+     * Get the current Extension device ID.
+     *
+     * @returns Device ID or null if not set
+     */
+    getExtensionDeviceId(): string | null;
+    /**
+     * Whether the relay adapter is ready to send commands.
+     * Requires an Extension device ID and an active CloudSyncService.
+     *
+     * @returns True if relay is available
+     */
+    isAvailable(): boolean;
+    /**
+     * Send a browser command to the Chrome Extension via the Cloud relay queue.
+     *
+     * @param tool - Tool name to execute (e.g., 'navigate', 'screenshot')
+     * @param params - Tool parameters
+     * @param timeoutMs - Command timeout in milliseconds
+     * @param agentName - Optional agent name for display in Extension
+     * @returns Response from the Chrome Extension
+     * @throws Error if relay is not available or command times out
+     */
+    sendViaRelay(tool: string, params?: Record<string, unknown>, timeoutMs?: number, agentName?: string): Promise<BrowserCommandResponse>;
+    /**
+     * Handle an incoming browser_response message from the Cloud relay queue.
+     * Called by CloudSyncService message handler when a browser_response arrives.
+     *
+     * @param response - The browser command response from the Extension
+     */
+    handleRelayResponse(response: BrowserCommandResponse): void;
+    /**
+     * Get the number of pending relay commands.
+     *
+     * @returns Number of commands awaiting response
+     */
+    getPendingCount(): number;
+    /**
+     * Start listening for CloudSync device events to auto-discover Chrome
+     * Extension devices. When a device with role 'browser' or 'chrome-extension'
+     * comes online, automatically sets it as the relay target.
+     *
+     * Also scans the current device list on startup for already-online extensions.
+     */
+    startAutoDiscovery(): void;
+    /**
+     * Stop listening for device events.
+     */
+    stopAutoDiscovery(): void;
+    /**
+     * Whether auto-discovery is active.
+     *
+     * @returns True if listening for device events
+     */
+    isDiscoveryActive(): boolean;
+    /**
+     * Check if a SyncDevice represents a Chrome Extension (browser role).
+     *
+     * @param device - Device to check
+     * @returns True if the device has a browser/chrome-extension role
+     */
+    private isBrowserDevice;
+    /**
+     * Handle a device coming online — if it's a browser extension, set it as relay target.
+     *
+     * @param device - The device that came online
+     */
+    private onDeviceOnline;
+    /**
+     * Handle a device going offline — if it's our current extension, clear the target.
+     *
+     * @param device - The device that went offline
+     */
+    private onDeviceOffline;
+    /**
+     * Handle devices_updated — scan for browser extensions in the full list.
+     *
+     * @param devices - Current device list
+     */
+    private onDevicesUpdated;
+    /**
+     * Scan a device list for browser extensions and set the first online one as relay target.
+     *
+     * @param devices - Device list to scan
+     */
+    private scanForBrowserDevice;
+    /**
+     * Clean up pending commands and timers.
+     */
+    cleanup(): void;
+}
+//# sourceMappingURL=browser-relay-adapter.service.d.ts.map

package/dist/backend/backend/src/services/browser/browser-relay-adapter.service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser-relay-adapter.service.d.ts","sourceRoot":"","sources":["../../../../../../backend/src/services/browser/browser-relay-adapter.service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAKH,OAAO,KAAK,EAAkB,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AAkB1F;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAoC;IAC3D,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkB;IACzC,OAAO,CAAC,eAAe,CAA+C;IACtE,OAAO,CAAC,cAAc,CAAK;IAC3B,2EAA2E;IAC3E,OAAO,CAAC,iBAAiB,CAAuB;IAChD,iDAAiD;IACjD,OAAO,CAAC,eAAe,CAAS;IAChC,2CAA2C;IAC3C,OAAO,CAAC,wBAAwB,CAA+C;IAC/E,OAAO,CAAC,yBAAyB,CAA+C;IAChF,OAAO,CAAC,0BAA0B,CAAkD;IAEpF,OAAO;IAIP;;;;OAIG;IACH,MAAM,CAAC,WAAW,IAAI,mBAAmB;IAOzC;;OAEG;IACH,MAAM,CAAC,aAAa,IAAI,IAAI;IAO5B;;;;;OAKG;IACH,oBAAoB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAK5C;;;;OAIG;IACH,oBAAoB,IAAI,MAAM,GAAG,IAAI;IAIrC;;;;;OAKG;IACH,WAAW,IAAI,OAAO;IAUtB;;;;;;;;;OASG;IACG,YAAY,CAChB,IAAI,EAAE,MAAM,EACZ,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAChC,SAAS,GAAE,MAAoD,EAC/D,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC,sBAAsB,CAAC;IAkClC;;;;;OAKG;IACH,mBAAmB,CAAC,QAAQ,EAAE,sBAAsB,GAAG,IAAI;IAsB3D;;;;OAIG;IACH,eAAe,IAAI,MAAM;IAQzB;;;;;;OAMG;IACH,kBAAkB,IAAI,IAAI;IAsC1B;;OAEG;IACH,iBAAiB,IAAI,IAAI;IAyBzB;;;;OAIG;IACH,iBAAiB,IAAI,OAAO;IAI5B;;;;;OAKG;IACH,OAAO,CAAC,eAAe;IAMvB;;;;OAIG;IACH,OAAO,CAAC,cAAc;IAWtB;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAUvB;;;;OAIG;IACH,OAAO,CAAC,gBAAgB;IAIxB;;;;OAIG;IACH,OAAO,CAAC,oBAAoB;IAwB5B;;OAEG;IACH,OAAO,IAAI,IAAI;CAShB"}