@myrialabs/clopen 0.0.1

package/frontend/lib/components/preview/browser/core/interactions.svelte.ts

@@ -0,0 +1,113 @@
+/**
+ * Browser Interactions
+ *
+ * Utilities for sending user interactions to browser via WebSocket.
+ * All interactions work with active tab on backend.
+ */
+
+import ws from '$frontend/lib/utils/ws';
+import { debug } from '$shared/utils/logger';
+import type { DeviceSize, Rotation } from '$frontend/lib/constants/preview';
+
+export interface InteractionAction {
+	type:
+		| 'click'
+		| 'doubleclick'
+		| 'rightclick'
+		| 'mousemove'
+		| 'scroll'
+		| 'type'
+		| 'key'
+		| 'checkselectoptions'
+		| 'scale-update'
+		| 'viewport-update';
+	x?: number;
+	y?: number;
+	deltaX?: number;
+	deltaY?: number;
+	text?: string;
+	key?: string;
+	delay?: number;
+	steps?: number;
+	scale?: number;
+	width?: number;
+	height?: number;
+	deviceSize?: string;
+	rotation?: string;
+}
+
+// Store current projectId for interactions
+let currentProjectId = '';
+
+/**
+ * Set current project ID for interactions
+ * Must be called when project changes
+ */
+export function setInteractionProjectId(projectId: string): void {
+	currentProjectId = projectId;
+	debug.log('preview', `🔧 Interaction projectId set to: ${projectId}`);
+}
+
+/**
+ * Send interaction to active tab
+ */
+export function sendInteraction(action: InteractionAction): void {
+	try {
+		// Include projectId for project isolation
+		ws.emit('preview:browser-interact', { action });
+	} catch (error) {
+		debug.error('preview', 'Error sending interaction:', error);
+	}
+}
+
+/**
+ * Send scale update to active tab
+ */
+export function sendScaleUpdate(scale: number): void {
+	try {
+		ws.emit('preview:browser-interact', {
+			action: {
+				type: 'scale-update',
+				scale
+			}
+		});
+		debug.log('preview', `📐 Sent scale update: ${scale}`);
+	} catch (error) {
+		debug.error('preview', 'Error sending scale update:', error);
+	}
+}
+
+/**
+ * Send viewport update to active tab
+ */
+export async function updateViewport(
+	deviceSize: DeviceSize,
+	rotation: Rotation,
+	scale: number
+): Promise<void> {
+	// Import here to avoid circular dependency
+	const { getViewportDimensions } = await import('$frontend/lib/constants/preview.js');
+
+	// Use getViewportDimensions for consistent viewport calculation
+	const { width, height } = getViewportDimensions(deviceSize, rotation);
+
+	try {
+		ws.emit('preview:browser-interact', {
+			action: {
+				type: 'viewport-update',
+				width,
+				height,
+				scale,
+				deviceSize,
+				rotation
+			}
+		});
+
+		debug.log(
+			'preview',
+			`📱 Sent viewport update: ${width}x${height} (${deviceSize}/${rotation}, scale: ${scale})`
+		);
+	} catch (error) {
+		debug.error('preview', 'Error sending viewport update:', error);
+	}
+}

package/frontend/lib/components/preview/browser/core/mcp-handlers.svelte.ts

@@ -0,0 +1,296 @@
+/**
+ * Browser MCP Event Handlers
+ * Handles MCP (Model Context Protocol) control events for BrowserPreview
+ */
+
+import { debug } from '$shared/utils/logger';
+import { showInfo, showWarning } from '$frontend/lib/stores/ui/notification.svelte';
+import ws from '$frontend/lib/utils/ws';
+import type { TabManager } from './tab-manager.svelte';
+
+// MCP Control State interface
+export interface McpControlState {
+	isControlled: boolean;
+	controlledTabId: string | null;
+	browserSessionId: string | null;
+	startedAt: number | null;
+}
+
+export interface McpHandlerConfig {
+	tabManager: TabManager;
+	transformBrowserToDisplayCoordinates?: (x: number, y: number) => { x: number, y: number } | null;
+	onCursorUpdate?: (x: number, y: number, clicking?: boolean) => void;
+	onCursorHide?: () => void;
+	onLaunchRequest?: (url: string, deviceSize: string, rotation: string, sessionId?: string) => void;
+}
+
+/**
+ * Create MCP event handler
+ */
+export function createMcpHandler(config: McpHandlerConfig) {
+	const { tabManager, transformBrowserToDisplayCoordinates, onCursorUpdate, onCursorHide, onLaunchRequest } = config;
+
+	// MCP control state
+	let mcpControlState = $state<McpControlState>({
+		isControlled: false,
+		controlledTabId: null,
+		browserSessionId: null,
+		startedAt: null
+	});
+
+	/**
+	 * Setup WebSocket event listeners for MCP control events
+	 */
+	function setupEventListeners() {
+		debug.log('preview', '🎧 Setting up MCP event listeners...');
+
+		// Listen for MCP control start/end events
+		ws.on('preview:browser-mcp-control-start', (data) => {
+			debug.log('preview', `📥 Received mcp-control-start:`, data);
+			handleControlStart(data);
+		});
+
+		ws.on('preview:browser-mcp-control-end', (data) => {
+			debug.log('preview', `📥 Received mcp-control-end:`, data);
+			handleControlEnd(data);
+		});
+
+		// Listen for MCP cursor events
+		ws.on('preview:browser-mcp-cursor-position', (data) => {
+			handleCursorPosition(data);
+		});
+
+		ws.on('preview:browser-mcp-cursor-click', (data) => {
+			handleCursorClick(data);
+		});
+
+		ws.on('preview:browser-mcp-test-completed', (data) => {
+			handleTestCompleted(data);
+		});
+
+		// MCP Tab Management - Request/Response handlers
+		setupTabManagementListeners();
+
+		debug.log('preview', '✅ MCP event listeners registered');
+	}
+
+	/**
+	 * Setup tab management listeners
+	 * Note: MCP tab management events have been removed in the new architecture.
+	 */
+	function setupTabManagementListeners() {
+		// MCP tab management listeners removed - now uses HTTP request-response pattern
+		// defined in backend/ws/preview/browser/mcp.ts
+	}
+
+	/**
+	 * Check if current tab is MCP controlled
+	 */
+	function isCurrentTabMcpControlled(): boolean {
+		return mcpControlState.isControlled && mcpControlState.controlledTabId === tabManager.activeTabId;
+	}
+
+	/**
+	 * Get MCP control state
+	 */
+	function getControlState(): McpControlState {
+		return mcpControlState;
+	}
+
+	// Private handlers
+
+	function handleControlStart(data: { browserSessionId: string; mcpSessionId?: string; timestamp: number }) {
+		debug.log('preview', `🎮 MCP control started for session: ${data.browserSessionId}`);
+
+		// Find which tab has this session
+		const tab = tabManager.tabs.find(t => t.sessionId === data.browserSessionId);
+
+		mcpControlState = {
+			isControlled: true,
+			controlledTabId: tab?.id || null,
+			browserSessionId: data.browserSessionId,
+			startedAt: data.timestamp
+		};
+
+		// Show toast notification
+		showWarning('MCP Control Started', 'An MCP agent is now controlling the browser. User input is blocked.', 5000);
+	}
+
+	function handleControlEnd(data: { browserSessionId: string; timestamp: number }) {
+		debug.log('preview', `🎮 MCP control ended for session: ${data.browserSessionId}`);
+
+		mcpControlState = {
+			isControlled: false,
+			controlledTabId: null,
+			browserSessionId: null,
+			startedAt: null
+		};
+
+		// Hide cursor
+		if (onCursorHide) {
+			onCursorHide();
+		}
+
+		// Show toast notification
+		showInfo('MCP Control Ended', 'MCP agent released control. You can now interact with the browser.', 4000);
+	}
+
+	function handleCursorPosition(data: { sessionId: string; x: number; y: number; timestamp: number; source: 'mcp' }) {
+		// Only update cursor if this is for the active session and MCP is controlling
+		if (mcpControlState.isControlled && mcpControlState.browserSessionId === data.sessionId && transformBrowserToDisplayCoordinates) {
+			const transformedPosition = transformBrowserToDisplayCoordinates(data.x, data.y);
+			if (transformedPosition && onCursorUpdate) {
+				onCursorUpdate(transformedPosition.x, transformedPosition.y, false);
+			}
+		}
+	}
+
+	function handleCursorClick(data: { sessionId: string; x: number; y: number; timestamp: number; source: 'mcp' }) {
+		if (mcpControlState.isControlled && mcpControlState.browserSessionId === data.sessionId && transformBrowserToDisplayCoordinates) {
+			const transformedPosition = transformBrowserToDisplayCoordinates(data.x, data.y);
+			if (transformedPosition && onCursorUpdate) {
+				onCursorUpdate(transformedPosition.x, transformedPosition.y, true);
+			}
+		}
+	}
+
+	function handleTestCompleted(data: { sessionId: string; timestamp: number; source: 'mcp' }) {
+		if (mcpControlState.browserSessionId === data.sessionId && onCursorHide) {
+			onCursorHide();
+		}
+	}
+
+	function handleTabsListRequest(data: { requestId: string }) {
+		const tabList = tabManager.tabs.map(tab => ({
+			id: tab.id,
+			url: tab.url,
+			title: tab.title,
+			sessionId: tab.sessionId,
+			isActive: tab.id === tabManager.activeTabId
+		}));
+
+		ws.http('preview:mcp-tab-list', {
+			requestId: data.requestId,
+			tabs: tabList
+		});
+	}
+
+	function handleActiveTabRequest(data: { requestId: string }) {
+		const tab = tabManager.activeTab;
+		ws.http('preview:mcp-active-tab', {
+			requestId: data.requestId,
+			tab: tab ? {
+				id: tab.id,
+				url: tab.url,
+				title: tab.title,
+				sessionId: tab.sessionId,
+				isActive: true
+			} : null
+		});
+	}
+
+	async function handleSwitchTabRequest(data: { requestId: string; tabId: string }) {
+		const tab = tabManager.getTab(data.tabId);
+		if (!tab) {
+			ws.http('preview:mcp-switch-tab', {
+				requestId: data.requestId,
+				success: false,
+				error: `Tab '${data.tabId}' not found`
+			});
+			return;
+		}
+
+		tabManager.switchTab(data.tabId);
+
+		ws.http('preview:mcp-switch-tab', {
+			requestId: data.requestId,
+			success: true,
+			tab: {
+				id: tab.id,
+				url: tab.url,
+				title: tab.title,
+				sessionId: tab.sessionId,
+				isActive: true
+			}
+		});
+	}
+
+	async function handleOpenTabRequest(data: { requestId: string; url: string }) {
+		try {
+			const tabId = tabManager.createTab(data.url);
+			const tab = tabManager.getTab(tabId);
+
+			// Wait for session to be created if URL provided
+			if (data.url && tab) {
+				// Wait up to 5 seconds for session to be ready
+				let attempts = 0;
+				while (attempts < 50 && !tab.sessionId) {
+					await new Promise(resolve => setTimeout(resolve, 100));
+					attempts++;
+				}
+			}
+
+			ws.http('preview:mcp-open-tab', {
+				requestId: data.requestId,
+				success: true,
+				tab: tab ? {
+					id: tab.id,
+					url: tab.url,
+					title: tab.title,
+					sessionId: tab.sessionId,
+					isActive: true
+				} : undefined
+			});
+		} catch (error) {
+			ws.http('preview:mcp-open-tab', {
+				requestId: data.requestId,
+				success: false,
+				error: error instanceof Error ? error.message : 'Unknown error'
+			});
+		}
+	}
+
+	function handleCloseTabRequest(data: { requestId: string; tabId: string }) {
+		const tab = tabManager.getTab(data.tabId);
+		if (!tab) {
+			ws.http('preview:mcp-close-tab', {
+				requestId: data.requestId,
+				success: false,
+				error: `Tab '${data.tabId}' not found`
+			});
+			return;
+		}
+
+		const { newActiveTab } = tabManager.closeTab(data.tabId);
+
+		ws.http('preview:mcp-close-tab', {
+			requestId: data.requestId,
+			success: true,
+			closedTabId: data.tabId,
+			newActiveTab: newActiveTab ? {
+				id: newActiveTab.id,
+				url: newActiveTab.url,
+				title: newActiveTab.title,
+				sessionId: newActiveTab.sessionId,
+				isActive: true
+			} : undefined
+		});
+	}
+
+	function handleLaunchRequest(data: { url: string; deviceSize: string; rotation: string; sessionId?: string }) {
+		debug.log('preview', `🚀 MCP launch request: ${data.url}, sessionId: ${data.sessionId || 'none'}`);
+
+		if (onLaunchRequest) {
+			onLaunchRequest(data.url, data.deviceSize, data.rotation, data.sessionId);
+		}
+	}
+
+	return {
+		setupEventListeners,
+		isCurrentTabMcpControlled,
+		getControlState,
+		get mcpControlState() { return mcpControlState; }
+	};
+}
+
+export type McpHandler = ReturnType<typeof createMcpHandler>;