@myrialabs/clopen 0.2.9 → 0.2.11

package/backend/preview/browser/browser-preview-service.ts

@@ -94,6 +94,7 @@ export class BrowserPreviewService extends EventEmitter {
 		});
 
 		// Forward navigation events and handle video streaming restart
+		// Only full navigations (framenavigated) need streaming restart
 		this.navigationTracker.on('navigation', async (data) => {
 			this.emit('preview:browser-navigation', data);
 
@@ -121,6 +122,12 @@ export class BrowserPreviewService extends EventEmitter {
 			this.emit('preview:browser-navigation-loading', data);
 		});
 
+		// Forward SPA navigation events (pushState/replaceState)
+		// No streaming restart needed — page context is unchanged
+		this.navigationTracker.on('navigation-spa', (data) => {
+			this.emit('preview:browser-navigation-spa', data);
+		});
+
 		// Forward new window events
 		this.tabManager.on('new-window', (data) => {
 			this.emit('preview:browser-new-window', data);
@@ -140,6 +147,10 @@ export class BrowserPreviewService extends EventEmitter {
 			this.emit('preview:browser-tab-navigated', data);
 		});
 
+		this.tabManager.on('preview:browser-viewport-changed', (data) => {
+			this.emit('preview:browser-viewport-changed', data);
+		});
+
 		// Forward video capture events
 		this.videoCapture.on('ice-candidate', (data) => {
 			this.emit('preview:browser-webcodecs-ice-candidate', data);
@@ -249,6 +260,9 @@ export class BrowserPreviewService extends EventEmitter {
 		// Stop WebCodecs streaming first
 		await this.stopWebCodecsStreaming(tabId);
 
+		// Cleanup navigation tracker CDP session
+		await this.navigationTracker.cleanupSession(tabId);
+
 		// Clear cursor tracking for this tab
 		this.interactionHandler.clearSessionCursor(tabId);
 
@@ -650,6 +664,11 @@ class BrowserPreviewServiceManager {
 			ws.emit.project(projectId, 'preview:browser-navigation', data);
 		});
 
+		// Forward SPA navigation events (pushState/replaceState — URL-only update)
+		service.on('preview:browser-navigation-spa', (data) => {
+			ws.emit.project(projectId, 'preview:browser-navigation-spa', data);
+		});
+
 		// Forward tab events
 		service.on('preview:browser-tab-opened', (data) => {
 			debug.log('preview', `🚀 Forwarding preview:browser-tab-opened to project ${projectId}:`, data);
@@ -668,6 +687,10 @@ class BrowserPreviewServiceManager {
 			ws.emit.project(projectId, 'preview:browser-tab-navigated', data);
 		});
 
+		service.on('preview:browser-viewport-changed', (data) => {
+			ws.emit.project(projectId, 'preview:browser-viewport-changed', data);
+		});
+
 		// Forward console events
 		service.on('preview:browser-console-message', (data) => {
 			ws.emit.project(projectId, 'preview:browser-console-message', data);
@@ -846,37 +869,3 @@ class BrowserPreviewServiceManager {
 // Service manager instance (singleton)
 export const browserPreviewServiceManager = new BrowserPreviewServiceManager();
 
-// Graceful shutdown handlers
-const gracefulShutdown = async (signal: string) => {
-	try {
-		await browserPreviewServiceManager.cleanup();
-		process.exit(0);
-	} catch (error) {
-		process.exit(1);
-	}
-};
-
-// Handle various termination signals
-process.on('SIGINT', () => gracefulShutdown('SIGINT'));
-process.on('SIGTERM', () => gracefulShutdown('SIGTERM'));
-process.on('SIGHUP', () => gracefulShutdown('SIGHUP'));
-
-// Handle Windows-specific signals
-if (process.platform === 'win32') {
-	process.on('SIGBREAK', () => gracefulShutdown('SIGBREAK'));
-}
-
-// Handle uncaught exceptions and unhandled rejections
-process.on('uncaughtException', async (error) => {
-	await browserPreviewServiceManager.cleanup();
-	process.exit(1);
-});
-
-process.on('unhandledRejection', async (reason, promise) => {
-	await browserPreviewServiceManager.cleanup();
-	process.exit(1);
-});
-
-// Handle process exit
-process.on('exit', (code) => {
-});

package/backend/preview/browser/browser-tab-manager.ts

@@ -730,6 +730,21 @@ export class BrowserTabManager extends EventEmitter {
 		// await page.evaluateOnNewDocument(cursorTrackingScript);
 	}
 
+	/**
+	 * Returns true for errors where retrying is pointless because the page/session is gone.
+	 */
+	private isNonRetryableError(error: unknown): boolean {
+		if (error instanceof Error) {
+			const msg = error.message;
+			return (
+				msg.includes('Session closed') ||
+				msg.includes('detached Frame') ||
+				error.constructor.name === 'TargetCloseError'
+			);
+		}
+		return false;
+	}
+
 	/**
 	 * Navigate with retry, including Cloudflare auto-pass detection and CAPTCHA popup dismissal.
 	 */
@@ -750,7 +765,7 @@ export class BrowserTabManager extends EventEmitter {
 			} catch (error) {
 				retries--;
 				debug.warn('preview', `⚠️ Navigation failed, ${retries} retries left:`, error);
-				if (retries === 0) throw error;
+				if (retries === 0 || this.isNonRetryableError(error)) throw error;
 
 				// Wait before retry
 				await new Promise(resolve => setTimeout(resolve, 2000));

package/backend/preview/browser/browser-video-capture.ts

@@ -42,6 +42,7 @@ interface VideoStreamSession {
 	pendingCandidates: RTCIceCandidateInit[];
 	scriptInjected: boolean; // Track if persistent script was injected
 	scriptsPreInjected: boolean; // Track if scripts were pre-injected during tab creation
+	audioOnNewDocumentInjected: boolean; // Track if evaluateOnNewDocument was registered for audio
 	stats: {
 		videoBytesSent: number;
 		audioBytesSent: number;
@@ -97,6 +98,7 @@ export class BrowserVideoCapture extends EventEmitter {
 				pendingCandidates: [],
 				scriptInjected: true,
 				scriptsPreInjected: false, // Set to true only after injection completes
+				audioOnNewDocumentInjected: false,
 				stats: {
 					videoBytesSent: 0,
 					audioBytesSent: 0,
@@ -162,7 +164,16 @@ export class BrowserVideoCapture extends EventEmitter {
 			});
 		}
 
-		// Inject video encoder + audio capture scripts
+		// Register audio capture as a startup script — runs before page scripts on every new document load.
+		// Critical for SPAs that create AudioContext during initialization (before page.evaluate runs).
+		// The idempotency guard in audioCaptureScript prevents double-injection.
+		const session = this.sessions.get(sessionId);
+		if (session && !session.audioOnNewDocumentInjected) {
+			await page.evaluateOnNewDocument(audioCaptureScript, config.audio);
+			session.audioOnNewDocumentInjected = true;
+		}
+
+		// Inject video encoder + audio capture scripts into the current page context
 		await page.evaluate(videoEncoderScript, videoConfig);
 		await page.evaluate(audioCaptureScript, config.audio);
 	}
@@ -220,6 +231,7 @@ export class BrowserVideoCapture extends EventEmitter {
 					pendingCandidates: [],
 					scriptInjected: false,
 					scriptsPreInjected: false,
+					audioOnNewDocumentInjected: false,
 					stats: {
 						videoBytesSent: 0,
 						audioBytesSent: 0,
@@ -383,8 +395,8 @@ export class BrowserVideoCapture extends EventEmitter {
 			return null;
 		}
 
-		const maxRetries = 3;
-		const retryDelay = 50;
+		const maxRetries = 6;
+		const retryDelay = 150;
 
 		for (let attempt = 0; attempt < maxRetries; attempt++) {
 			try {

package/backend/preview/browser/scripts/audio-stream.ts

@@ -16,6 +16,11 @@ import type { StreamingConfig } from '../types';
  * This script intercepts AudioContext and captures all audio
  */
 export function audioCaptureScript(config: StreamingConfig['audio']) {
+	// Idempotency guard — prevent double-injection when both evaluateOnNewDocument
+	// and page.evaluate inject this script into the same page context.
+	if ((window as any).__audioCaptureInstalled) return;
+	(window as any).__audioCaptureInstalled = true;
+
 	// Check AudioEncoder support
 	if (typeof AudioEncoder === 'undefined') {
 		(window as any).__audioEncoderSupported = false;

package/backend/preview/browser/scripts/video-stream.ts

@@ -38,6 +38,23 @@ export function videoEncoderScript(config: StreamingConfig['video']) {
 	// STUN servers are unnecessary for localhost and add 100-500ms ICE gathering latency
 	const iceServers: { urls: string }[] = [];
 
+	// Create a loopback (127.0.0.1) copy of a host ICE candidate.
+	// Ensures WebRTC connects via loopback when VPN (e.g. Cloudflare WARP)
+	// interferes with host candidate connectivity between same-machine peers.
+	function createLoopbackCandidate(candidate: { candidate?: string; sdpMid?: string | null; sdpMLineIndex?: number | null }) {
+		if (!candidate.candidate) return null;
+		if (!candidate.candidate.includes('typ host')) return null;
+
+		const parts = candidate.candidate.split(' ');
+		if (parts.length < 8) return null;
+
+		const address = parts[4];
+		if (address === '127.0.0.1' || address === '::1') return null;
+
+		parts[4] = '127.0.0.1';
+		return { ...candidate, candidate: parts.join(' ') };
+	}
+
 	// Check cursor style from page
 	function checkCursor() {
 		try {
@@ -103,11 +120,18 @@ export function videoEncoderScript(config: StreamingConfig['video']) {
 		// Handle ICE candidates
 		peerConnection.onicecandidate = (event) => {
 			if (event.candidate && (window as any).__sendIceCandidate) {
-				(window as any).__sendIceCandidate({
+				const candidateInit = {
 					candidate: event.candidate.candidate,
 					sdpMid: event.candidate.sdpMid,
 					sdpMLineIndex: event.candidate.sdpMLineIndex
-				});
+				};
+				(window as any).__sendIceCandidate(candidateInit);
+
+				// Also send loopback version for VPN compatibility (same-machine peers)
+				const loopback = createLoopbackCandidate(candidateInit);
+				if (loopback) {
+					(window as any).__sendIceCandidate(loopback);
+				}
 			}
 		};
 
@@ -337,16 +361,27 @@ export function videoEncoderScript(config: StreamingConfig['video']) {
 		}
 	}
 
-	// Add ICE candidate
+	// Add ICE candidate (+ loopback variant for VPN compatibility)
 	async function addIceCandidate(candidate: RTCIceCandidateInit) {
 		if (!peerConnection) return false;
 
 		try {
 			await peerConnection.addIceCandidate(new RTCIceCandidate(candidate));
-			return true;
 		} catch (error) {
 			return false;
 		}
+
+		// Also try loopback version for VPN compatibility (same-machine peers)
+		const loopback = createLoopbackCandidate(candidate);
+		if (loopback) {
+			try {
+				await peerConnection.addIceCandidate(new RTCIceCandidate(loopback as RTCIceCandidateInit));
+			} catch {
+				// Expected to fail if loopback is not applicable
+			}
+		}
+
+		return true;
 	}
 
 	// Reconfigure video encoder with new dimensions (hot-swap)

package/backend/preview/browser/types.ts

@@ -229,10 +229,11 @@ export interface StreamingConfig {
 /**
  * Default streaming configuration
  *
- * Optimized for visual quality with reasonable bandwidth:
+ * Optimized for visual quality with reduced resource usage:
  * - Software encoding (hardwareAcceleration: 'no-preference')
- * - JPEG quality 70 preserves thin borders/text, VP8 handles bandwidth
- * - VP8 at 1.2Mbps for crisp UI/text rendering
+ * - JPEG quality 65: slightly lower than before but still preserves thin borders/text
+ * - VP8 at 1.0Mbps: ~17% reduction from 1.2Mbps, sharp edges preserved by VP8 codec
+ * - keyframeInterval 5s: less frequent large keyframes, saves bandwidth on static pages
  * - Opus for audio (efficient and widely supported)
  */
 export const DEFAULT_STREAMING_CONFIG: StreamingConfig = {
@@ -241,9 +242,9 @@ export const DEFAULT_STREAMING_CONFIG: StreamingConfig = {
 		width: 0,
 		height: 0,
 		framerate: 24,
-		bitrate: 1_200_000,
-		keyframeInterval: 3,
-		screenshotQuality: 70,
+		bitrate: 1_000_000,
+		keyframeInterval: 5,
+		screenshotQuality: 65,
 		hardwareAcceleration: 'no-preference',
 		latencyMode: 'realtime'
 	},

package/backend/ws/preview/browser/interact.ts

@@ -12,6 +12,9 @@ import type { KeyInput } from 'puppeteer';
 import { debug } from '$shared/utils/logger';
 import { sleep } from '$shared/utils/async';
 
+// Throttle cursor detection evaluate calls per session (100ms = ~10/sec is plenty)
+const lastCursorEvalTime = new Map<string, number>();
+
 // Helper function to check if error is navigation-related
 function isNavigationError(error: Error): boolean {
 	const msg = error.message.toLowerCase();
@@ -108,8 +111,9 @@ export const interactPreviewHandler = createRouter()
 				switch (action.type) {
 					case 'mousedown':
 						try {
-							// Reset mouse state first to ensure clean state
-							try { await session.page.mouse.up(); } catch { }
+							// Fire-and-forget reset — CDP processes commands in FIFO order so
+							// this completes before move/down even though we skip the await.
+							session.page.mouse.up().catch(() => {});
 							// Move to position and press button
 							await session.page.mouse.move(action.x!, action.y!, { steps: 1 });
 							await session.page.mouse.down({ button: action.button === 'right' ? 'right' : 'left' });
@@ -140,14 +144,10 @@ export const interactPreviewHandler = createRouter()
 
 					case 'click':
 						try {
-							// Reset mouse state before click to prevent "already pressed" errors
-							// This ensures a clean state for each click operation
-							try {
-								await session.page.mouse.up();
-							} catch { /* Ignore - mouse might not be pressed */ }
-
-							// IMPORTANT: Check for select element BEFORE clicking
-							// If it's a select, we'll emit event to frontend instead of clicking
+							// Check for select element BEFORE clicking.
+							// Skip the mouse.up() reset: page.mouse.click() is atomic (down+up),
+							// and Canvas.svelte always sends mouseup before sending click, so the
+							// mouse state is already clean at this point.
 							const selectInfo = await previewService.checkForSelectElement(session.id, action.x!, action.y!);
 							if (selectInfo) {
 								// Select element detected - event emitted by checkForSelectElement
@@ -243,37 +243,40 @@ export const interactPreviewHandler = createRouter()
 							await session.page.mouse.move(action.x!, action.y!, {
 								steps: action.steps || 1 // Reduced from 5 to 1 for faster response
 							});
-							// Update cursor position and detect cursor type in browser context (fire-and-forget, don't await)
-							// This replaces the disabled cursor-tracking script (blocked by CloudFlare)
-							session.page.evaluate((data) => {
-								const { x, y } = data;
-								// Detect cursor type from element under mouse
-								let cursor = 'default';
-								try {
-									const el = document.elementFromPoint(x, y);
-									if (el) {
-										cursor = window.getComputedStyle(el).cursor || 'default';
+							// Cursor detection via page.evaluate — throttled to ~10/sec per session.
+							// Running it on every mousemove queues extra CDP commands that delay clicks/keypresses.
+							const nowMs = Date.now();
+							const lastEval = lastCursorEvalTime.get(session.id) ?? 0;
+							if (nowMs - lastEval >= 100) {
+								lastCursorEvalTime.set(session.id, nowMs);
+								session.page.evaluate((data) => {
+									const { x, y } = data;
+									let cursor = 'default';
+									try {
+										const el = document.elementFromPoint(x, y);
+										if (el) {
+											cursor = window.getComputedStyle(el).cursor || 'default';
+										}
+									} catch {}
+
+									const existing = (window as any).__cursorInfo;
+									if (existing) {
+										existing.cursor = cursor;
+										existing.x = x;
+										existing.y = y;
+										existing.timestamp = Date.now();
+										existing.hasRecentInteraction = true;
+									} else {
+										(window as any).__cursorInfo = {
+											cursor,
+											x,
+											y,
+											timestamp: Date.now(),
+											hasRecentInteraction: true
+										};
 									}
-								} catch {}
-
-								// Initialize or update __cursorInfo
-								const existing = (window as any).__cursorInfo;
-								if (existing) {
-									existing.cursor = cursor;
-									existing.x = x;
-									existing.y = y;
-									existing.timestamp = Date.now();
-									existing.hasRecentInteraction = true;
-								} else {
-									(window as any).__cursorInfo = {
-										cursor,
-										x,
-										y,
-										timestamp: Date.now(),
-										hasRecentInteraction: true
-									};
-								}
-							}, { x: action.x!, y: action.y! }).catch(() => { /* Ignore evaluation errors */ });
+								}, { x: action.x!, y: action.y! }).catch(() => { /* Ignore evaluation errors */ });
+							}
 						} catch (error) {
 							if (error instanceof Error && isNavigationError(error)) {
 								ws.emit.user(userId, 'preview:browser-interacted', { action: action.type, message: 'Action deferred (navigation)', deferred: true });
@@ -297,8 +300,6 @@ export const interactPreviewHandler = createRouter()
 
 					case 'doubleclick':
 						try {
-							// Reset mouse state first
-							try { await session.page.mouse.up(); } catch { }
 							await session.page.mouse.click(action.x!, action.y!, { clickCount: 2 });
 						} catch (error) {
 							if (error instanceof Error) {
@@ -314,8 +315,8 @@ export const interactPreviewHandler = createRouter()
 
 					case 'rightclick':
 						try {
-							// Reset mouse state first
-							try { await session.page.mouse.up(); } catch { }
+							// Fire-and-forget reset (see mousedown comment for rationale)
+							session.page.mouse.up().catch(() => {});
 
 							// IMPORTANT: Check for context menu
 							// We'll emit context menu event to frontend for custom overlay
@@ -385,12 +386,7 @@ export const interactPreviewHandler = createRouter()
 								await session.page.keyboard.press(action.key as KeyInput);
 							}
 
-							if (['ArrowDown', 'ArrowUp'].includes(action.key)) {
-								try {
-									await sleep(50);
-								} catch { }
-							}
-						}
+					}
 						break;
 
 					case 'checkselectoptions':

package/backend/ws/preview/browser/webcodecs.ts

@@ -16,7 +16,9 @@ export const streamPreviewHandler = createRouter()
 	.http(
 		'preview:browser-stream-start',
 		{
-			data: t.Object({}),
+			data: t.Object({
+				tabId: t.Optional(t.String())
+			}),
 			response: t.Object({
 				success: t.Boolean(),
 				message: t.Optional(t.String()),
@@ -34,9 +36,10 @@ export const streamPreviewHandler = createRouter()
 			// Get project-specific preview service
 			const previewService = browserPreviewServiceManager.getService(projectId);
 
-			const tab = previewService.getActiveTab();
+			// Use explicit tabId if provided, otherwise fall back to active tab
+			const tab = data.tabId ? previewService.getTab(data.tabId) : previewService.getActiveTab();
 			if (!tab) {
-				throw new Error('No active tab');
+				throw new Error(data.tabId ? `Tab not found: ${data.tabId}` : 'No active tab');
 			}
 
 			const sessionId = tab.id;
@@ -73,7 +76,9 @@ export const streamPreviewHandler = createRouter()
 	.http(
 		'preview:browser-stream-offer',
 		{
-			data: t.Object({}),
+			data: t.Object({
+				tabId: t.Optional(t.String())
+			}),
 			response: t.Object({
 				success: t.Boolean(),
 				offer: t.Optional(
@@ -90,9 +95,9 @@ export const streamPreviewHandler = createRouter()
 			// Get project-specific preview service
 			const previewService = browserPreviewServiceManager.getService(projectId);
 
-			const tab = previewService.getActiveTab();
+			const tab = data.tabId ? previewService.getTab(data.tabId) : previewService.getActiveTab();
 			if (!tab) {
-				throw new Error('No active tab');
+				throw new Error(data.tabId ? `Tab not found: ${data.tabId}` : 'No active tab');
 			}
 
 			const offer = await previewService.getWebCodecsOffer(tab.id);
@@ -117,7 +122,8 @@ export const streamPreviewHandler = createRouter()
 				answer: t.Object({
 					type: t.String(),
 					sdp: t.Optional(t.String())
-				})
+				}),
+				tabId: t.Optional(t.String())
 			}),
 			response: t.Object({
 				success: t.Boolean()
@@ -129,9 +135,9 @@ export const streamPreviewHandler = createRouter()
 			// Get project-specific preview service
 			const previewService = browserPreviewServiceManager.getService(projectId);
 
-			const tab = previewService.getActiveTab();
+			const tab = data.tabId ? previewService.getTab(data.tabId) : previewService.getActiveTab();
 			if (!tab) {
-				throw new Error('No active tab');
+				throw new Error(data.tabId ? `Tab not found: ${data.tabId}` : 'No active tab');
 			}
 
 			const { answer } = data;
@@ -150,7 +156,8 @@ export const streamPreviewHandler = createRouter()
 					candidate: t.Optional(t.String()),
 					sdpMid: t.Optional(t.Union([t.String(), t.Null()])),
 					sdpMLineIndex: t.Optional(t.Union([t.Number(), t.Null()]))
-				})
+				}),
+				tabId: t.Optional(t.String())
 			}),
 			response: t.Object({
 				success: t.Boolean()
@@ -162,9 +169,9 @@ export const streamPreviewHandler = createRouter()
 			// Get project-specific preview service
 			const previewService = browserPreviewServiceManager.getService(projectId);
 
-			const tab = previewService.getActiveTab();
+			const tab = data.tabId ? previewService.getTab(data.tabId) : previewService.getActiveTab();
 			if (!tab) {
-				throw new Error('No active tab');
+				throw new Error(data.tabId ? `Tab not found: ${data.tabId}` : 'No active tab');
 			}
 
 			const { candidate } = data;
@@ -178,7 +185,9 @@ export const streamPreviewHandler = createRouter()
 	.http(
 		'preview:browser-stream-stop',
 		{
-			data: t.Object({}),
+			data: t.Object({
+				tabId: t.Optional(t.String())
+			}),
 			response: t.Object({
 				success: t.Boolean()
 			})
@@ -189,9 +198,9 @@ export const streamPreviewHandler = createRouter()
 			// Get project-specific preview service
 			const previewService = browserPreviewServiceManager.getService(projectId);
 
-			const tab = previewService.getActiveTab();
+			const tab = data.tabId ? previewService.getTab(data.tabId) : previewService.getActiveTab();
 			if (!tab) {
-				throw new Error('No active tab');
+				throw new Error(data.tabId ? `Tab not found: ${data.tabId}` : 'No active tab');
 			}
 
 			await previewService.stopWebCodecsStreaming(tab.id);
@@ -252,6 +261,17 @@ export const streamPreviewHandler = createRouter()
 			url: t.String(),
 			timestamp: t.Number()
 		})
+	)
+
+	// Server → Client: SPA navigation (pushState/replaceState — URL-only update, no page reload)
+	.emit(
+		'preview:browser-navigation-spa',
+		t.Object({
+			sessionId: t.String(),
+			type: t.String(),
+			url: t.String(),
+			timestamp: t.Number()
+		})
 	);
 
 // Setup event forwarding from preview service to WebSocket

package/backend/ws/preview/index.ts

@@ -143,4 +143,12 @@ export const previewRouter = createRouter()
 		sessionId: t.String(),
 		timestamp: t.Number(),
 		source: t.Literal('mcp')
+	}))
+	.emit('preview:browser-viewport-changed', t.Object({
+		tabId: t.String(),
+		deviceSize: t.String(),
+		rotation: t.String(),
+		width: t.Number(),
+		height: t.Number(),
+		timestamp: t.Number()
 	}));

package/frontend/components/chat/input/ChatInput.svelte

@@ -416,12 +416,12 @@
 			ondrop={fileHandling.handleDrop}
 		>
 			<div class="flex-1">
-				<!-- Engine/Model Picker -->
-				<EngineModelPicker />
-
 				<!-- Edit Mode Indicator -->
 				<EditModeIndicator onCancel={handleCancelEdit} />
 
+				<!-- Engine/Model Picker -->
+				<EngineModelPicker />
+
 				<div class="flex items-end">
 					<textarea
 						bind:this={textareaElement}