@myrialabs/clopen 0.0.1

package/backend/ws/preview/browser/webcodecs.ts

@@ -0,0 +1,293 @@
+/**
+ * Preview Streaming Handlers
+ *
+ * Handles video/audio streaming for ultra low-latency, low-bandwidth preview.
+ * Currently implemented using WebCodecs with DataChannel delivery.
+ * **PROJECT ISOLATION**: Uses project-specific BrowserPreviewService instances
+ */
+
+import { createRouter } from '$shared/utils/ws-server';
+import { t } from 'elysia';
+import { browserPreviewServiceManager } from '$backend/lib/preview';
+import { ws } from '$backend/lib/utils/ws';
+
+export const streamPreviewHandler = createRouter()
+	// Start streaming
+	.http(
+		'preview:browser-stream-start',
+		{
+			data: t.Object({}),
+			response: t.Object({
+				success: t.Boolean(),
+				message: t.Optional(t.String()),
+				offer: t.Optional(
+					t.Object({
+						type: t.String(),
+						sdp: t.Optional(t.String())
+					})
+				)
+			})
+		},
+		async ({ data, conn }) => {
+			const projectId = ws.getProjectId(conn);
+
+			// Get project-specific preview service
+			const previewService = browserPreviewServiceManager.getService(projectId);
+
+			const tab = previewService.getActiveTab();
+			if (!tab) {
+				throw new Error('No active tab');
+			}
+
+			const sessionId = tab.id;
+
+			// Verify session exists
+			if (!previewService.isValidTab(sessionId)) {
+				throw new Error('Preview session not found or invalid');
+			}
+
+			// Start WebCodecs streaming
+			const started = await previewService.startWebCodecsStreaming(sessionId);
+
+			if (!started) {
+				throw new Error('Failed to start WebCodecs streaming');
+			}
+
+			// Get offer from headless browser
+			const offer = await previewService.getWebCodecsOffer(sessionId);
+
+			return {
+				success: true,
+				message: 'WebCodecs streaming started',
+				offer: offer
+					? {
+							type: offer.type as string,
+							sdp: offer.sdp
+						}
+					: undefined
+			};
+		}
+	)
+
+	// Get SDP offer from headless browser
+	.http(
+		'preview:browser-stream-offer',
+		{
+			data: t.Object({}),
+			response: t.Object({
+				success: t.Boolean(),
+				offer: t.Optional(
+					t.Object({
+						type: t.String(),
+						sdp: t.Optional(t.String())
+					})
+				)
+			})
+		},
+		async ({ data, conn }) => {
+			const projectId = ws.getProjectId(conn);
+
+			// Get project-specific preview service
+			const previewService = browserPreviewServiceManager.getService(projectId);
+
+			const tab = previewService.getActiveTab();
+			if (!tab) {
+				throw new Error('No active tab');
+			}
+
+			const offer = await previewService.getWebCodecsOffer(tab.id);
+
+			return {
+				success: !!offer,
+				offer: offer
+					? {
+							type: offer.type as string,
+							sdp: offer.sdp
+						}
+					: undefined
+			};
+		}
+	)
+
+	// Handle SDP answer from client
+	.http(
+		'preview:browser-stream-answer',
+		{
+			data: t.Object({
+				answer: t.Object({
+					type: t.String(),
+					sdp: t.Optional(t.String())
+				})
+			}),
+			response: t.Object({
+				success: t.Boolean()
+			})
+		},
+		async ({ data, conn }) => {
+			const projectId = ws.getProjectId(conn);
+
+			// Get project-specific preview service
+			const previewService = browserPreviewServiceManager.getService(projectId);
+
+			const tab = previewService.getActiveTab();
+			if (!tab) {
+				throw new Error('No active tab');
+			}
+
+			const { answer } = data;
+			const success = await previewService.handleWebCodecsAnswer(tab.id, answer as RTCSessionDescriptionInit);
+
+			return { success };
+		}
+	)
+
+	// Exchange ICE candidates
+	.http(
+		'preview:browser-stream-ice',
+		{
+			data: t.Object({
+				candidate: t.Object({
+					candidate: t.Optional(t.String()),
+					sdpMid: t.Optional(t.Union([t.String(), t.Null()])),
+					sdpMLineIndex: t.Optional(t.Union([t.Number(), t.Null()]))
+				})
+			}),
+			response: t.Object({
+				success: t.Boolean()
+			})
+		},
+		async ({ data, conn }) => {
+			const projectId = ws.getProjectId(conn);
+
+			// Get project-specific preview service
+			const previewService = browserPreviewServiceManager.getService(projectId);
+
+			const tab = previewService.getActiveTab();
+			if (!tab) {
+				throw new Error('No active tab');
+			}
+
+			const { candidate } = data;
+			const success = await previewService.addWebCodecsIceCandidate(tab.id, candidate as RTCIceCandidateInit);
+
+			return { success };
+		}
+	)
+
+	// Stop streaming
+	.http(
+		'preview:browser-stream-stop',
+		{
+			data: t.Object({}),
+			response: t.Object({
+				success: t.Boolean()
+			})
+		},
+		async ({ data, conn }) => {
+			const projectId = ws.getProjectId(conn);
+
+			// Get project-specific preview service
+			const previewService = browserPreviewServiceManager.getService(projectId);
+
+			const tab = previewService.getActiveTab();
+			if (!tab) {
+				throw new Error('No active tab');
+			}
+
+			await previewService.stopWebCodecsStreaming(tab.id);
+
+			return { success: true };
+		}
+	)
+
+	// Server → Client: ICE candidate from headless browser
+	.emit(
+		'preview:browser-stream-ice',
+		t.Object({
+			sessionId: t.String(), // Internal session ID (kept for routing)
+			candidate: t.Object({
+				candidate: t.Optional(t.String()),
+				sdpMid: t.Optional(t.Union([t.String(), t.Null()])),
+				sdpMLineIndex: t.Optional(t.Union([t.Number(), t.Null()]))
+			}),
+			from: t.String() // 'headless' or 'client'
+		})
+	)
+
+	// Server → Client: Connection state update
+	.emit(
+		'preview:browser-stream-state',
+		t.Object({
+			sessionId: t.String(), // Internal session ID (kept for routing)
+			state: t.String()
+		})
+	)
+
+	// Server → Client: Cursor style update
+	.emit(
+		'preview:browser-cursor-change',
+		t.Object({
+			sessionId: t.String(), // Internal session ID (kept for routing)
+			cursor: t.String()
+		})
+	)
+
+	// Server → Client: Navigation started (loading)
+	.emit(
+		'preview:browser-navigation-loading',
+		t.Object({
+			sessionId: t.String(),
+			type: t.String(),
+			url: t.String(),
+			timestamp: t.Number()
+		})
+	)
+
+	// Server → Client: Navigation completed
+	.emit(
+		'preview:browser-navigation',
+		t.Object({
+			sessionId: t.String(),
+			type: t.String(),
+			url: t.String(),
+			timestamp: t.Number()
+		})
+	);
+
+// Setup event forwarding from preview service to WebSocket
+// This needs to be done per-project service instance
+// We'll set up a helper function that the service manager can call
+
+/**
+ * Setup event forwarding for a preview service instance
+ * Should be called when a new service is created
+ */
+export function setupEventForwarding(previewService: any, projectId: string) {
+	previewService.on('webcodecs-ice-candidate', (data: { sessionId: string; candidate: RTCIceCandidateInit; from: string }) => {
+		ws.emit.project(projectId, 'preview:browser-stream-ice', {
+			sessionId: data.sessionId,
+			candidate: data.candidate,
+			from: data.from
+		});
+	});
+
+	previewService.on('webcodecs-connection-state', (data: { sessionId: string; state: string }) => {
+		ws.emit.project(projectId, 'preview:browser-stream-state', data);
+	});
+
+	previewService.on('cursor-change', (data: { sessionId: string; cursor: string }) => {
+		ws.emit.project(projectId, 'preview:browser-cursor-change', data);
+	});
+
+	// Forward navigation events
+	previewService.on('preview:browser-navigation-loading', (data: { sessionId: string; type: string; url: string; timestamp: number }) => {
+		ws.emit.project(projectId, 'preview:browser-navigation-loading', data);
+	});
+
+	previewService.on('preview:browser-navigation', (data: { sessionId: string; type: string; url: string; timestamp: number }) => {
+		ws.emit.project(projectId, 'preview:browser-navigation', data);
+	});
+}
+
+// Note: Event forwarding is now set up per-project in BrowserPreviewService constructor
+// via the setupProjectEventForwarding method

package/backend/ws/preview/index.ts

@@ -0,0 +1,146 @@
+/**
+ * Preview Router
+ *
+ * Combines all preview WebSocket handlers into a single router.
+ *
+ * Tab-centric architecture - all operations work with browser tabs.
+ *
+ * Structure:
+ * - tab.ts: Tab lifecycle operations (open, navigate, close)
+ * - interact.ts: Mouse/keyboard interaction handlers
+ * - tab-info.ts: Tab information endpoints
+ * - stats.ts: Streaming statistics endpoints
+ * - console.ts: Console operations (get, clear, execute, toggle)
+ * - cleanup.ts: Admin cleanup endpoints (status, perform)
+ * - webcodecs.ts: WebCodecs streaming handlers
+ * - native-ui.ts: Native UI handlers (dialogs, print, select, context menu)
+ * - mcp.ts: MCP tab coordination response handlers
+ *
+ * Available endpoints:
+ * - preview:browser-tab-open - Open new browser tab (with optional URL)
+ * - preview:browser-tab-close - Close browser tab
+ * - preview:browser-tab-navigate - Navigate tab to new URL
+ * - preview:browser-interact - Execute mouse/keyboard interactions
+ * - preview:browser-tab-info - Get tab information
+ * - preview:browser-tab-stats - Get streaming statistics
+ * - preview:browser-console-get - Get console logs
+ * - preview:browser-console-clear - Clear console logs
+ * - preview:browser-console-execute - Execute console command
+ * - preview:browser-console-toggle - Toggle console logging
+ * - preview:browser-cleanup-status - Get cleanup status
+ * - preview:browser-cleanup-perform - Perform cleanup
+ * - preview:browser-stream-start - Start streaming
+ * - preview:browser-stream-offer - Get stream offer
+ * - preview:browser-stream-answer - Send stream answer
+ * - preview:browser-stream-ice - Exchange ICE candidates
+ * - preview:browser-stream-stop - Stop streaming
+ */
+
+import { t } from 'elysia';
+import { createRouter } from '$shared/utils/ws-server';
+import { tabPreviewHandler } from './browser/tab';
+import { interactPreviewHandler } from './browser/interact';
+import { tabInfoPreviewHandler } from './browser/tab-info';
+import { statsPreviewHandler } from './browser/stats';
+import { consolePreviewHandler } from './browser/console';
+import { cleanupPreviewHandler } from './browser/cleanup';
+import { streamPreviewHandler } from './browser/webcodecs';
+import { nativeUIPreviewHandler } from './browser/native-ui';
+import { mcpPreviewHandler } from './browser/mcp';
+
+export const previewRouter = createRouter()
+	.merge(tabPreviewHandler)
+	.merge(interactPreviewHandler)
+	.merge(tabInfoPreviewHandler)
+	.merge(statsPreviewHandler)
+	.merge(consolePreviewHandler)
+	.merge(cleanupPreviewHandler)
+	.merge(streamPreviewHandler)
+	.merge(nativeUIPreviewHandler)
+	.merge(mcpPreviewHandler)
+	// Server-emitted events (for type safety)
+	.emit('preview:browser-tab-opened', t.Object({
+		tabId: t.String(),
+		url: t.String(),
+		title: t.String(),
+		isActive: t.Boolean(),
+		timestamp: t.Number()
+	}))
+	.emit('preview:browser-tab-closed', t.Object({
+		tabId: t.String(),
+		newActiveTabId: t.Union([t.String(), t.Null()]),
+		timestamp: t.Number()
+	}))
+	.emit('preview:browser-tab-switched', t.Object({
+		previousTabId: t.String(),
+		newTabId: t.String(),
+		timestamp: t.Number()
+	}))
+	.emit('preview:browser-tab-navigated', t.Object({
+		tabId: t.String(),
+		url: t.String(),
+		title: t.String(),
+		timestamp: t.Number()
+	}))
+	.emit('preview:browser-console-message', t.Object({
+		sessionId: t.String(),
+		message: t.Object({
+			id: t.String(),
+			type: t.Union([
+				t.Literal('log'),
+				t.Literal('info'),
+				t.Literal('warn'),
+				t.Literal('error'),
+				t.Literal('debug'),
+				t.Literal('trace'),
+				t.Literal('clear')
+			]),
+			text: t.String(),
+			args: t.Optional(t.Array(t.Any())),
+			location: t.Optional(t.Object({
+				url: t.String(),
+				lineNumber: t.Number(),
+				columnNumber: t.Number()
+			})),
+			stackTrace: t.Optional(t.String()),
+			timestamp: t.Number()
+		})
+	}))
+	.emit('preview:browser-console-clear', t.Object({
+		sessionId: t.String(),
+		timestamp: t.Number()
+	}))
+	.emit('preview:browser-new-window', t.Object({
+		tabId: t.String(),
+		url: t.String(),
+		timestamp: t.Number()
+	}))
+	// MCP control events
+	.emit('preview:browser-mcp-control-start', t.Object({
+		browserSessionId: t.String(),
+		mcpSessionId: t.Optional(t.String()),
+		timestamp: t.Number()
+	}))
+	.emit('preview:browser-mcp-control-end', t.Object({
+		browserSessionId: t.String(),
+		timestamp: t.Number()
+	}))
+	.emit('preview:browser-mcp-cursor-position', t.Object({
+		sessionId: t.String(),
+		x: t.Number(),
+		y: t.Number(),
+		timestamp: t.Number(),
+		source: t.Literal('mcp')
+	}))
+	.emit('preview:browser-mcp-cursor-click', t.Object({
+		sessionId: t.String(),
+		x: t.Number(),
+		y: t.Number(),
+		timestamp: t.Number(),
+		source: t.Literal('mcp')
+	}))
+	.emit('preview:browser-mcp-test-completed', t.Object({
+		sessionId: t.String(),
+		timestamp: t.Number(),
+		source: t.Literal('mcp')
+	}));

package/backend/ws/projects/crud.ts

@@ -0,0 +1,113 @@
+/**
+ * Projects CRUD Operations
+ *
+ * HTTP endpoints for project management:
+ * - List all projects (per-user)
+ * - Create new project (or join existing)
+ * - Get project by ID
+ * - Update project info
+ * - Delete project (remove user association, cleanup if orphaned)
+ */
+
+import { t } from 'elysia';
+import { createRouter } from '$shared/utils/ws-server';
+import { initializeDatabase } from '../../lib/database';
+import { projectQueries } from '../../lib/database/queries';
+import { ws } from '$backend/lib/utils/ws';
+
+export const crudHandler = createRouter()
+	// List all projects for the current user
+	.http('projects:list', {
+		data: t.Object({}),
+		response: t.Array(t.Any())
+	}, async ({ conn }) => {
+		await initializeDatabase();
+		const userId = ws.getUserId(conn);
+		const projects = projectQueries.getAllForUser(userId);
+		return projects;
+	})
+
+	// Create new project (or join existing by path)
+	.http('projects:create', {
+		data: t.Object({
+			name: t.String({ minLength: 1 }),
+			path: t.String({ minLength: 1 })
+		}),
+		response: t.Any()
+	}, async ({ data, conn }) => {
+		await initializeDatabase();
+		const userId = ws.getUserId(conn);
+		const { name, path } = data;
+
+		// Check if project with this path already exists
+		const existing = projectQueries.getByPath(path);
+		if (existing) {
+			// Project exists - just add user association (join)
+			projectQueries.addUserProject(userId, existing.id);
+			return existing;
+		}
+
+		const now = new Date().toISOString();
+		const project = projectQueries.create({
+			name,
+			path,
+			created_at: now,
+			last_opened_at: now
+		});
+
+		// Associate the project with the creating user
+		projectQueries.addUserProject(userId, project.id);
+
+		return project;
+	})
+
+	// Get project by ID
+	.http('projects:get', {
+		data: t.Object({
+			id: t.String({ minLength: 1 })
+		}),
+		response: t.Any()
+	}, async ({ data }) => {
+		const project = projectQueries.getById(data.id);
+
+		if (!project) {
+			throw new Error('Project not found');
+		}
+
+		// Update last_opened_at when getting project
+		projectQueries.updateLastOpened(data.id);
+		const updatedProject = projectQueries.getById(data.id);
+
+		return updatedProject;
+	})
+
+	// Delete project (remove user association, cleanup if orphaned)
+	.http('projects:delete', {
+		data: t.Object({
+			id: t.String({ minLength: 1 })
+		}),
+		response: t.Object({
+			id: t.String(),
+			deleted: t.Boolean()
+		})
+	}, async ({ data, conn }) => {
+		const userId = ws.getUserId(conn);
+		const project = projectQueries.getById(data.id);
+		if (!project) {
+			throw new Error('Project not found');
+		}
+
+		// Remove user's association with the project
+		projectQueries.removeUserProject(userId, data.id);
+
+		// If no more users are associated, delete the project entirely
+		const remainingUsers = projectQueries.getUserCountForProject(data.id);
+		if (remainingUsers === 0) {
+			projectQueries.delete(data.id);
+		}
+
+		return {
+			id: data.id,
+			deleted: true
+		};
+	});

package/backend/ws/projects/index.ts

@@ -0,0 +1,25 @@
+/**
+ * Projects Router
+ *
+ * Combines all project WebSocket handlers into a single router.
+ *
+ * Structure:
+ * - crud.ts: HTTP endpoints for CRUD operations (list, create, get, update, delete)
+ * - status.ts: Real-time status updates and watching (get-status, watch, unwatch, events)
+ * - presence.ts: User presence management (update-presence with broadcast)
+ */
+
+import { createRouter } from '$shared/utils/ws-server';
+import { crudHandler } from './crud';
+import { statusHandler } from './status';
+import { presenceHandler } from './presence';
+
+export const projectsRouter = createRouter()
+	// CRUD Operations (HTTP)
+	.merge(crudHandler)
+
+	// Status & Watching (HTTP + Events)
+	.merge(statusHandler)
+
+	// Presence Management (HTTP + Broadcast)
+	.merge(presenceHandler);

package/backend/ws/projects/presence.ts

@@ -0,0 +1,46 @@
+/**
+ * Projects Presence Management
+ *
+ * Handles user presence updates:
+ * - projects:update-presence (HTTP) - Update user presence with broadcast
+ *
+ * Uses `projects:presence-updated` event (declared in status.ts)
+ */
+
+import { t } from 'elysia';
+import { createRouter } from '$shared/utils/ws-server';
+import { updateUserPresence, getProjectStatusData } from '../../lib/project/status-manager';
+import { streamManager } from '../../lib/chat/stream-manager';
+import { ws } from '$backend/lib/utils/ws';
+import { debug } from '$shared/utils/logger';
+
+export const presenceHandler = createRouter()
+	.http('projects:update-presence', {
+		data: t.Object({
+			userName: t.String({ minLength: 1 }),
+			action: t.Optional(t.String())
+		}),
+		response: t.Any()
+	}, async ({ data, conn }) => {
+		const projectId = ws.getProjectId(conn);
+		const userId = ws.getUserId(conn);
+		const { userName, action } = data;
+
+		updateUserPresence(projectId, userId, userName, action || 'join');
+
+		try {
+			streamManager.cleanupProjectStreams(projectId);
+		} catch (cleanupError) {
+			debug.error('project', 'Error cleaning up project streams:', cleanupError);
+		}
+
+		// Broadcast full presence to all clients
+		const allStatuses = await getProjectStatusData();
+		ws.emit.global('projects:presence-updated', {
+			type: 'presence-updated',
+			timestamp: Date.now(),
+			data: allStatuses
+		});
+
+		return await getProjectStatusData(projectId);
+	});