@myrialabs/clopen 0.0.1

package/backend/ws/preview/browser/native-ui.ts

@@ -0,0 +1,235 @@
+/**
+ * Browser Native UI WebSocket Handler
+ * Handles native browser UI interactions: dialogs, print, select dropdown, and context menu
+ * **PROJECT ISOLATION**: Uses project-specific BrowserPreviewService instances
+ *
+ * Note: Event forwarding is now handled by BrowserPreviewServiceManager
+ * per-project service instances with proper project scoping.
+ */
+
+import { t } from 'elysia';
+import { createRouter } from '$shared/utils/ws-server';
+import { debug } from '$shared/utils/logger';
+import { browserPreviewServiceManager } from '../../../lib/preview/index';
+import { ws } from '$backend/lib/utils/ws';
+
+// Event forwarding is now handled automatically by BrowserPreviewServiceManager
+// when service instances are created, ensuring proper project isolation.
+
+export const nativeUIPreviewHandler = createRouter()
+	// Action: Client responds to a dialog (alert, confirm, prompt)
+	.on('preview:browser-dialog-input', {
+		data: t.Object({
+			dialogId: t.String({ minLength: 1 }),
+			accept: t.Boolean(),
+			promptText: t.Optional(t.String())
+		})
+	}, async ({ data, conn }) => {
+		try {
+			const { dialogId, accept, promptText } = data;
+			const projectId = ws.getProjectId(conn);
+
+			// Get project-specific preview service
+			const previewService = browserPreviewServiceManager.getService(projectId);
+
+			debug.log('preview', `📬 Dialog response received from frontend - dialogId: ${dialogId}, accept: ${accept}${promptText ? `, promptText: "${promptText}"` : ''} (project: ${projectId})`);
+
+			// Get active tab
+			const tab = previewService.getActiveTab();
+			if (!tab) {
+				debug.error('preview', `❌ No active tab for dialog input (project: ${projectId})`);
+				return;
+			}
+
+			debug.log('preview', `✅ Active tab found: ${tab.id}`);
+
+			// Send response to dialog handler
+			const result = await previewService.respondToDialog({
+				tabId: tab.id,
+				dialogId,
+				accept,
+				promptText
+			});
+
+			if (result) {
+				debug.log('preview', `✅ Dialog response processed successfully - dialogId: ${dialogId}`);
+			} else {
+				debug.warn('preview', `⚠️ Dialog response failed - dialogId: ${dialogId} (dialog may not be found)`);
+			}
+		} catch (error) {
+			debug.error('preview', '💥 Error handling dialog response:', error);
+		}
+	})
+
+	// Action: Client triggers print (in response to print event or manually)
+	.on('preview:browser-print-input', {
+		data: t.Object({})
+	}, async ({ conn }) => {
+		try {
+			const projectId = ws.getProjectId(conn);
+
+			// Get project-specific preview service
+			const previewService = browserPreviewServiceManager.getService(projectId);
+
+			const tab = previewService.getActiveTab();
+			if (!tab) {
+				debug.error('preview', `No active tab for print input (project: ${projectId})`);
+				return;
+			}
+
+			debug.log('preview', `🖨️ Print trigger received for tab: ${tab.id} (project: ${projectId})`);
+
+			// For native print, we just acknowledge the trigger
+			// Frontend will handle window.print() directly
+		} catch (error) {
+			debug.error('preview', 'Error handling print trigger:', error);
+		}
+	})
+
+	// Action: Client responds to a select dropdown
+	.on('preview:browser-select-input', {
+		data: t.Object({
+			sessionId: t.String({ minLength: 1 }),
+			selectId: t.String({ minLength: 1 }),
+			selectedIndex: t.Number()
+		})
+	}, async ({ data, conn }) => {
+		try {
+			const { sessionId, selectId, selectedIndex } = data;
+			const projectId = ws.getProjectId(conn);
+
+			// Get project-specific preview service
+			const previewService = browserPreviewServiceManager.getService(projectId);
+
+			debug.log('preview', `📋 Select response received - selectId: ${selectId}, selectedIndex: ${selectedIndex} (project: ${projectId})`);
+
+			// Send response to native UI handler
+			await previewService.handleSelectResponse(sessionId, {
+				tabId: sessionId,
+				selectId,
+				selectedIndex
+			});
+		} catch (error) {
+			debug.error('preview', 'Error handling select response:', error);
+		}
+	})
+
+	// Action: Client responds to a context menu
+	.on('preview:browser-context-menu-input', {
+		data: t.Object({
+			sessionId: t.String({ minLength: 1 }),
+			menuId: t.String({ minLength: 1 }),
+			itemId: t.String({ minLength: 1 }),
+			clipboardText: t.Optional(t.String())
+		})
+	}, async ({ data, conn }) => {
+		try {
+			const { sessionId, menuId, itemId, clipboardText } = data;
+			const projectId = ws.getProjectId(conn);
+
+			// Get project-specific preview service
+			const previewService = browserPreviewServiceManager.getService(projectId);
+
+			debug.log('preview', `📜 Context menu response received - menuId: ${menuId}, itemId: ${itemId} (project: ${projectId})`);
+
+			// Send response to native UI handler
+			await previewService.handleContextMenuResponse(sessionId, {
+				tabId: sessionId,
+				menuId,
+				itemId
+			}, clipboardText);
+		} catch (error) {
+			debug.error('preview', 'Error handling context menu response:', error);
+		}
+	})
+
+	// Event declarations (Server → Client)
+	// These events are emitted by preview service when native UI interactions occur
+	.emit('preview:browser-dialog', t.Object({
+		sessionId: t.String(),
+		dialogId: t.String(),
+		type: t.Union([
+			t.Literal('alert'),
+			t.Literal('confirm'),
+			t.Literal('prompt'),
+			t.Literal('beforeunload')
+		]),
+		message: t.String(),
+		defaultValue: t.Optional(t.String()),
+		timestamp: t.Number()
+	}))
+
+	.emit('preview:browser-print', t.Object({
+		sessionId: t.String(),
+		timestamp: t.Number()
+	}))
+
+	.emit('preview:browser-select', t.Object({
+		sessionId: t.String(),
+		selectId: t.String(),
+		x: t.Number(),
+		y: t.Number(),
+		boundingBox: t.Object({
+			x: t.Number(),
+			y: t.Number(),
+			width: t.Number(),
+			height: t.Number()
+		}),
+		options: t.Array(t.Object({
+			index: t.Number(),
+			value: t.String(),
+			text: t.String(),
+			selected: t.Boolean(),
+			disabled: t.Optional(t.Boolean())
+		})),
+		selectedIndex: t.Number(),
+		timestamp: t.Number()
+	}))
+
+	.emit('preview:browser-context-menu', t.Object({
+		sessionId: t.String(),
+		menuId: t.String(),
+		x: t.Number(),
+		y: t.Number(),
+		items: t.Array(t.Object({
+			id: t.String(),
+			label: t.String(),
+			enabled: t.Boolean(),
+			type: t.Optional(t.Union([
+				t.Literal('normal'),
+				t.Literal('separator'),
+				t.Literal('submenu')
+			])),
+			icon: t.Optional(t.String())
+		})),
+		elementInfo: t.Object({
+			tagName: t.String(),
+			isLink: t.Boolean(),
+			isImage: t.Boolean(),
+			isInput: t.Boolean(),
+			isTextSelected: t.Boolean(),
+			linkUrl: t.Optional(t.String()),
+			imageUrl: t.Optional(t.String()),
+			inputType: t.Optional(t.String())
+		}),
+		timestamp: t.Number()
+	}))
+
+	.emit('preview:browser-copy-to-clipboard', t.Object({
+		text: t.String()
+	}))
+
+	.emit('preview:browser-open-url-new-tab', t.Object({
+		url: t.String()
+	}))
+
+	.emit('preview:browser-download-image', t.Object({
+		base64: t.String(),
+		type: t.String(),
+		filename: t.String()
+	}))
+
+	.emit('preview:browser-copy-image-to-clipboard', t.Object({
+		base64: t.String(),
+		type: t.String()
+	}));

package/backend/ws/preview/browser/stats.ts

@@ -0,0 +1,55 @@
+/**
+ * Browser Streaming Statistics WebSocket Handler
+ * Handles getting browser streaming statistics
+ * **PROJECT ISOLATION**: Uses project-specific BrowserPreviewService instances
+ */
+
+import { t } from 'elysia';
+import { createRouter } from '$shared/utils/ws-server';
+import { browserPreviewServiceManager } from '../../../lib/preview/index';
+import { ws } from '$backend/lib/utils/ws';
+
+export const statsPreviewHandler = createRouter()
+	.http('preview:browser-tab-stats', {
+		data: t.Object({
+			tabId: t.Optional(t.String()) // If not provided, get active tab stats
+		}),
+		response: t.Object({
+			tabId: t.String(),
+			isWebCodecsActive: t.Boolean(),
+			streamingMode: t.String(),
+			stats: t.Union([
+				t.Object({
+					videoBytesSent: t.Number(),
+					audioBytesSent: t.Number(),
+					videoFramesEncoded: t.Number(),
+					audioFramesEncoded: t.Number(),
+					connectionState: t.String()
+				}),
+				t.Null()
+			]),
+			timestamp: t.String()
+		})
+	}, async ({ data, conn }) => {
+		const { tabId } = data;
+		const projectId = ws.getProjectId(conn);
+
+		// Get project-specific preview service
+		const previewService = browserPreviewServiceManager.getService(projectId);
+
+		// Get tab (active tab if not specified)
+		const tab = tabId ? previewService.getTab(tabId) : previewService.getActiveTab();
+		if (!tab) {
+			throw new Error(tabId ? `Tab not found: ${tabId}` : 'No active tab');
+		}
+
+		const stats = await previewService.getWebCodecsStats(tab.id);
+
+		return {
+			tabId: tab.id,
+			isWebCodecsActive: previewService.isWebCodecsActive(tab.id),
+			streamingMode: 'webcodecs',
+			stats: stats ?? null,
+			timestamp: new Date().toISOString()
+		};
+	});

package/backend/ws/preview/browser/tab-info.ts

@@ -0,0 +1,126 @@
+/**
+ * Browser Tab Info WebSocket Handler
+ * Handles getting browser tab information and listing all tabs
+ * **PROJECT ISOLATION**: Uses project-specific BrowserPreviewService instances
+ */
+
+import { t } from 'elysia';
+import { createRouter } from '$shared/utils/ws-server';
+import { browserPreviewServiceManager } from '../../../lib/preview/index';
+import { ws } from '$backend/lib/utils/ws';
+import { debug } from '$shared/utils/logger';
+
+export const tabInfoPreviewHandler = createRouter()
+	// Get single tab info
+	.http('preview:browser-tab-info', {
+		data: t.Object({
+			tabId: t.Optional(t.String()) // If not provided, get active tab info
+		}),
+		response: t.Object({
+			tabId: t.String(),
+			url: t.String(),
+			title: t.String(),
+			quality: t.String(),
+			isStreaming: t.Boolean(),
+			deviceSize: t.String(),
+			rotation: t.String(),
+			isActive: t.Boolean()
+		})
+	}, async ({ data, conn }) => {
+		const { tabId } = data;
+		const projectId = ws.getProjectId(conn);
+
+		// Get project-specific preview service
+		const previewService = browserPreviewServiceManager.getService(projectId);
+
+		// Get tab (active tab if not specified)
+		const tab = tabId ? previewService.getTab(tabId) : previewService.getActiveTab();
+		if (!tab) {
+			throw new Error(tabId ? `Tab not found: ${tabId}` : 'No active tab');
+		}
+
+		const tabInfo = previewService.getTabInfo(tab.id);
+		if (!tabInfo) {
+			throw new Error('Tab info not found');
+		}
+
+		return {
+			...tabInfo,
+			tabId: tabInfo.id
+		};
+	})
+
+	// Get all active tabs (for session recovery after browser refresh)
+	.http('preview:browser-tabs-list', {
+		data: t.Object({}),
+		response: t.Object({
+			tabs: t.Array(t.Object({
+				tabId: t.String(),
+				url: t.String(),
+				title: t.String(),
+				quality: t.String(),
+				isStreaming: t.Boolean(),
+				deviceSize: t.String(),
+				rotation: t.String(),
+				isActive: t.Boolean()
+			})),
+			activeTabId: t.Union([t.String(), t.Null()]),
+			count: t.Number()
+		})
+	}, async ({ conn }) => {
+		const projectId = ws.getProjectId(conn);
+
+		// Get project-specific preview service
+		const previewService = browserPreviewServiceManager.getService(projectId);
+
+		const allTabsInfo = previewService.getAllTabsInfo();
+		const activeTab = previewService.getActiveTab();
+
+		debug.log('preview', `📋 Listing ${allTabsInfo.length} active browser tabs for session recovery (project: ${projectId})`);
+
+		return {
+			tabs: allTabsInfo.map(tab => ({
+				tabId: tab.id,
+				url: tab.url,
+				title: tab.title,
+				quality: tab.quality,
+				isStreaming: tab.isStreaming,
+				deviceSize: tab.deviceSize,
+				rotation: tab.rotation,
+				isActive: tab.isActive
+			})),
+			activeTabId: activeTab?.id || null,
+			count: allTabsInfo.length
+		};
+	})
+
+	// Switch to a specific tab (for session recovery)
+	.http('preview:browser-tab-switch', {
+		data: t.Object({
+			tabId: t.String()
+		}),
+		response: t.Object({
+			success: t.Boolean(),
+			tabId: t.String(),
+			message: t.String()
+		})
+	}, async ({ data, conn }) => {
+		const { tabId } = data;
+		const projectId = ws.getProjectId(conn);
+
+		// Get project-specific preview service
+		const previewService = browserPreviewServiceManager.getService(projectId);
+
+		const success = previewService.switchTab(tabId);
+		if (!success) {
+			throw new Error(`Failed to switch to tab: ${tabId}`);
+		}
+
+		debug.log('preview', `🔄 Switched to tab: ${tabId} (project: ${projectId})`);
+
+		return {
+			success: true,
+			tabId,
+			message: `Switched to tab ${tabId}`
+		};
+	});

package/backend/ws/preview/browser/tab.ts

@@ -0,0 +1,166 @@
+/**
+ * Browser Tab Operations WebSocket Handlers
+ * Handles tab lifecycle operations: open, navigate, close
+ * **PROJECT ISOLATION**: Each project has its own BrowserPreviewService instance
+ */
+
+import { t } from 'elysia';
+import { createRouter } from '$shared/utils/ws-server';
+import { browserPreviewServiceManager } from '../../../lib/preview/index';
+import { ws } from '$backend/lib/utils/ws';
+import { debug } from '$shared/utils/logger';
+
+// Timeout for browser tab open (60 seconds)
+const OPEN_TIMEOUT = 60000;
+
+export const tabPreviewHandler = createRouter()
+	// Open new browser tab
+	.http('preview:browser-tab-open', {
+		data: t.Object({
+			url: t.Optional(t.String()), // URL is now optional - can create blank tab
+			deviceSize: t.Optional(t.Union([
+				t.Literal('desktop'),
+				t.Literal('laptop'),
+				t.Literal('tablet'),
+				t.Literal('mobile')
+			])),
+			rotation: t.Optional(t.Union([
+				t.Literal('portrait'),
+				t.Literal('landscape')
+			])),
+		}),
+		response: t.Object({
+			tabId: t.String(),
+			quality: t.String(),
+			url: t.String(),
+			title: t.String(),
+			isActive: t.Boolean(),
+			message: t.String()
+		})
+	}, async ({ data, conn }) => {
+		const projectId = ws.getProjectId(conn);
+
+		debug.log('preview', `🔴🔴🔴 browser-tab-open - Request received for project: ${projectId} 🔴🔴🔴`);
+
+		const {
+			url,
+			deviceSize = 'laptop',
+			rotation = 'portrait'
+		} = data;
+
+		debug.log('preview', `📥 Tab open params - URL: ${url || 'about:blank'}, deviceSize: ${deviceSize}, rotation: ${rotation}`);
+
+		// Get project-specific preview service
+		const previewService = browserPreviewServiceManager.getService(projectId);
+
+		// Create browser tab
+		const tabPromise = previewService.createTab(
+			url, // Can be undefined for blank tab
+			deviceSize as 'desktop' | 'laptop' | 'tablet' | 'mobile',
+			rotation as 'portrait' | 'landscape'
+		);
+		const timeoutPromise = new Promise((_, reject) => {
+			setTimeout(() => reject(new Error('Browser tab open timeout - took longer than 60 seconds')), OPEN_TIMEOUT);
+		});
+
+		debug.log('preview', `⏳ Opening browser tab (timeout: ${OPEN_TIMEOUT}ms)...`);
+		const tab = await Promise.race([tabPromise, timeoutPromise]) as Awaited<typeof tabPromise>;
+
+		debug.log('preview', `✅ Browser tab opened successfully - tabId: ${tab.id}, URL: ${tab.url}, project: ${projectId}`);
+
+		// Tab activity is marked automatically in tab-manager
+
+		return {
+			tabId: tab.id,
+			quality: tab.quality,
+			url: tab.url,
+			title: tab.title,
+			isActive: tab.isActive,
+			message: `Browser tab opened with ${tab.quality} quality streaming`
+		};
+	})
+
+	// Navigate browser tab
+	.http('preview:browser-tab-navigate', {
+		data: t.Object({
+			url: t.String({ minLength: 1 }),
+			tabId: t.Optional(t.String()) // If not provided, navigate active tab
+		}),
+		response: t.Object({
+			tabId: t.String(),
+			finalUrl: t.String(),
+			title: t.String(),
+			message: t.String()
+		})
+	}, async ({ data, conn }) => {
+		const { url, tabId } = data;
+		const projectId = ws.getProjectId(conn);
+
+		// Get project-specific preview service
+		const previewService = browserPreviewServiceManager.getService(projectId);
+
+		// Get tab to navigate (active tab if not specified)
+		const tab = tabId ? previewService.getTab(tabId) : previewService.getActiveTab();
+		if (!tab) {
+			throw new Error(tabId ? `Tab not found: ${tabId}` : 'No active tab');
+		}
+
+		debug.log('preview', `🌐 Navigating tab ${tab.id} to: ${url} (project: ${projectId})`);
+
+		const finalUrl = await previewService.navigateTab(tab.id, url);
+
+		debug.log('preview', `✅ Navigation completed - final URL: ${finalUrl}`);
+
+		return {
+			tabId: tab.id,
+			finalUrl: finalUrl,
+			title: tab.title,
+			message: 'Navigation completed'
+		};
+	})
+
+	// Close browser tab
+	.http('preview:browser-tab-close', {
+		data: t.Object({
+			tabId: t.Optional(t.String()) // If not provided, close active tab
+		}),
+		response: t.Object({
+			success: t.Boolean(),
+			tabId: t.String(),
+			newActiveTabId: t.Union([t.String(), t.Null()]),
+			message: t.String()
+		})
+	}, async ({ data, conn }) => {
+		const { tabId } = data;
+		const projectId = ws.getProjectId(conn);
+
+		// Get project-specific preview service
+		const previewService = browserPreviewServiceManager.getService(projectId);
+
+		// Get tab to close (active tab if not specified)
+		const tab = tabId ? previewService.getTab(tabId) : previewService.getActiveTab();
+
+		if (!tab) {
+			throw new Error(tabId ? `Tab not found: ${tabId}` : 'No active tab');
+		}
+
+		const closingTabId = tab.id;
+
+		debug.log('preview', `🗑️ Closing tab: ${closingTabId} (project: ${projectId})`);
+
+		// Close tab
+		const result = await previewService.closeTab(closingTabId);
+
+		if (!result.success) {
+			throw new Error(`Failed to close tab: ${closingTabId}`);
+		}
+
+		debug.log('preview', `✅ Tab closed: ${closingTabId} (new active: ${result.newActiveTabId || 'none'})`);
+
+		return {
+			success: true,
+			tabId: closingTabId,
+			newActiveTabId: result.newActiveTabId,
+			message: `Tab ${closingTabId} closed successfully`
+		};
+	});