@myrialabs/clopen 0.2.6 → 0.2.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/backend/chat/stream-manager.ts +24 -13
  2. package/backend/engine/adapters/claude/stream.ts +10 -19
  3. package/backend/mcp/project-context.ts +20 -0
  4. package/backend/mcp/servers/browser-automation/actions.ts +0 -2
  5. package/backend/mcp/servers/browser-automation/browser.ts +86 -132
  6. package/backend/mcp/servers/browser-automation/inspection.ts +5 -11
  7. package/backend/preview/browser/browser-mcp-control.ts +175 -180
  8. package/backend/preview/browser/browser-pool.ts +3 -1
  9. package/backend/preview/browser/browser-preview-service.ts +3 -3
  10. package/backend/preview/browser/browser-tab-manager.ts +1 -1
  11. package/backend/preview/browser/browser-video-capture.ts +12 -14
  12. package/backend/preview/browser/scripts/audio-stream.ts +11 -0
  13. package/backend/preview/browser/scripts/video-stream.ts +14 -14
  14. package/backend/preview/browser/types.ts +7 -7
  15. package/backend/preview/index.ts +1 -1
  16. package/backend/ws/chat/stream.ts +1 -1
  17. package/backend/ws/preview/browser/tab-info.ts +5 -2
  18. package/backend/ws/preview/index.ts +3 -3
  19. package/frontend/components/chat/input/ChatInput.svelte +0 -3
  20. package/frontend/components/chat/input/composables/use-chat-actions.svelte.ts +6 -2
  21. package/frontend/components/chat/message/MessageBubble.svelte +2 -2
  22. package/frontend/components/history/HistoryModal.svelte +1 -1
  23. package/frontend/components/preview/browser/BrowserPreview.svelte +15 -1
  24. package/frontend/components/preview/browser/components/Canvas.svelte +323 -49
  25. package/frontend/components/preview/browser/components/Container.svelte +21 -0
  26. package/frontend/components/preview/browser/components/Toolbar.svelte +3 -3
  27. package/frontend/components/preview/browser/core/coordinator.svelte.ts +15 -0
  28. package/frontend/components/preview/browser/core/mcp-handlers.svelte.ts +78 -51
  29. package/frontend/components/preview/browser/core/tab-operations.svelte.ts +1 -0
  30. package/frontend/components/workspace/PanelHeader.svelte +15 -0
  31. package/frontend/components/workspace/panels/GitPanel.svelte +22 -13
  32. package/frontend/components/workspace/panels/PreviewPanel.svelte +2 -0
  33. package/frontend/services/chat/chat.service.ts +3 -7
  34. package/frontend/services/preview/browser/browser-webcodecs.service.ts +32 -135
  35. package/frontend/stores/core/app.svelte.ts +4 -3
  36. package/frontend/stores/core/presence.svelte.ts +3 -2
  37. package/frontend/stores/core/sessions.svelte.ts +2 -0
  38. package/frontend/stores/ui/notification.svelte.ts +4 -1
  39. package/package.json +1 -1
@@ -1,16 +1,17 @@
1
1
  /**
2
2
  * Browser MCP Control
3
3
  *
4
- * Manages exclusive MCP control over browser tabs.
5
- * Ensures only one MCP can control the browser at a time.
6
- * Emits events for frontend to show visual indicators.
7
- * Also handles MCP tab request/response coordination.
4
+ * Manages MCP control over browser tabs with multi-tab, session-scoped ownership.
5
+ * Each chat session can control multiple tabs simultaneously.
6
+ * A tab can only be controlled by one chat session at a time.
7
+ * All tabs are released when the chat session ends (stream complete/error/cancel).
8
8
  *
9
- * ARCHITECTURE: Event-based control management
10
- * - Control is maintained as long as the browser tab exists
11
- * - Auto-release when tab is destroyed via 'preview:browser-tab-destroyed' event
12
- * - NO timeouts or polling - 100% real-time event-driven
13
- * - Listens directly to preview service events for tab lifecycle
9
+ * ARCHITECTURE:
10
+ * - Control lifecycle follows chat sessions (no idle timeout)
11
+ * - Multiple tabs can be locked by one chat session (accumulated via switch/open)
12
+ * - Tab destroyed auto-release that single tab from its owning session
13
+ * - Stream ends releaseSession() releases all tabs owned by that session
14
+ * - Emits per-tab control-start/control-end events for frontend UI
14
15
  */
15
16
 
16
17
  import { EventEmitter } from 'events';
@@ -24,18 +25,17 @@ interface PendingTabRequest<T = any> {
24
25
  timeout: NodeJS.Timeout;
25
26
  }
26
27
 
27
- export interface McpControlState {
28
- isControlling: boolean;
29
- mcpSessionId: string | null;
30
- browserTabId: string | null;
31
- startedAt: number | null;
32
- lastActionAt: number | null;
28
+ /** Ownership info for a single tab */
29
+ interface TabOwnershipInfo {
30
+ chatSessionId: string;
31
+ projectId: string;
32
+ acquiredAt: number;
33
33
  }
34
34
 
35
35
  export interface McpControlEvent {
36
36
  type: 'mcp:control-start' | 'mcp:control-end';
37
37
  browserTabId: string;
38
- mcpSessionId?: string;
38
+ chatSessionId?: string;
39
39
  timestamp: number;
40
40
  }
41
41
 
@@ -56,17 +56,11 @@ export interface McpClickEvent {
56
56
  }
57
57
 
58
58
  export class BrowserMcpControl extends EventEmitter {
59
- private controlState: McpControlState = {
60
- isControlling: false,
61
- mcpSessionId: null,
62
- browserTabId: null,
63
- startedAt: null,
64
- lastActionAt: null
65
- };
66
-
67
- // Auto-release configuration
68
- private readonly IDLE_TIMEOUT_MS = 30000; // 30 seconds idle = auto release
69
- private idleCheckInterval: NodeJS.Timeout | null = null;
59
+ /** Tab ownership info (which chat session controls it) */
60
+ private tabOwnership = new Map<string, TabOwnershipInfo>();
61
+
62
+ /** Chat session → set of tab IDs it controls */
63
+ private sessionTabs = new Map<string, Set<string>>();
70
64
 
71
65
  // Pending tab requests (keyed by request type + timestamp)
72
66
  private pendingTabRequests = new Map<string, PendingTabRequest>();
@@ -96,13 +90,18 @@ export class BrowserMcpControl extends EventEmitter {
96
90
 
97
91
  /**
98
92
  * Handle tab destroyed event
99
- * Auto-release control if the destroyed tab was being controlled
93
+ * Auto-release control for the destroyed tab only
100
94
  */
101
95
  private handleTabDestroyed(tabId: string): void {
102
- if (this.controlState.isControlling && this.controlState.browserTabId === tabId) {
103
- debug.warn('mcp', `⚠️ Controlled tab ${tabId} was destroyed - auto-releasing control`);
104
- this.releaseControl();
105
- }
96
+ const ownership = this.tabOwnership.get(tabId);
97
+ if (!ownership) return;
98
+
99
+ // Validate project to prevent cross-project collisions
100
+ const serviceProjectId = this.previewService?.getProjectId();
101
+ if (serviceProjectId && ownership.projectId !== serviceProjectId) return;
102
+
103
+ debug.warn('mcp', `⚠️ Controlled tab ${tabId} was destroyed - auto-releasing from session ${ownership.chatSessionId}`);
104
+ this.releaseTab(tabId);
106
105
  }
107
106
 
108
107
  /**
@@ -152,118 +151,190 @@ export class BrowserMcpControl extends EventEmitter {
152
151
  return false;
153
152
  }
154
153
 
154
+ // ============================================================================
155
+ // Control State Queries
156
+ // ============================================================================
157
+
155
158
  /**
156
- * Check if MCP is currently controlling a browser session
159
+ * Check if any tab is being controlled
157
160
  */
158
161
  isControlling(): boolean {
159
- return this.controlState.isControlling;
162
+ return this.tabOwnership.size > 0;
160
163
  }
161
164
 
162
165
  /**
163
- * Get current control state
166
+ * Check if a specific tab is being controlled (by any session)
164
167
  */
165
- getControlState(): McpControlState {
166
- return { ...this.controlState };
168
+ isTabControlled(browserTabId: string, projectId?: string): boolean {
169
+ const ownership = this.tabOwnership.get(browserTabId);
170
+ if (!ownership) return false;
171
+ if (projectId && ownership.projectId !== projectId) return false;
172
+ return true;
167
173
  }
168
174
 
169
175
  /**
170
- * Check if a specific browser tab is being controlled
176
+ * Check if a tab is controlled by a specific chat session
171
177
  */
172
- isTabControlled(browserTabId: string): boolean {
173
- return this.controlState.isControlling &&
174
- this.controlState.browserTabId === browserTabId;
178
+ isTabControlledBySession(browserTabId: string, chatSessionId: string): boolean {
179
+ const ownership = this.tabOwnership.get(browserTabId);
180
+ return ownership?.chatSessionId === chatSessionId;
175
181
  }
176
182
 
177
183
  /**
178
- * Acquire control of a browser tab
179
- * Returns true if control was acquired, false if already controlled by another MCP
184
+ * Get the chat session ID that controls a specific tab
180
185
  */
181
- acquireControl(browserTabId: string, mcpSessionId?: string): boolean {
182
- // Validate tab exists before acquiring control
183
- if (this.previewService && !this.previewService.getTab(browserTabId)) {
184
- debug.warn('mcp', `❌ Cannot acquire control: tab ${browserTabId} does not exist`);
185
- return false;
186
- }
186
+ getTabOwner(browserTabId: string): string | null {
187
+ return this.tabOwnership.get(browserTabId)?.chatSessionId || null;
188
+ }
187
189
 
188
- // If already controlling the same tab, just update timestamp
189
- if (this.controlState.isControlling &&
190
- this.controlState.browserTabId === browserTabId) {
191
- this.controlState.lastActionAt = Date.now();
192
- return true;
193
- }
190
+ /**
191
+ * Get all tab IDs controlled by a specific chat session
192
+ */
193
+ getSessionTabs(chatSessionId: string): string[] {
194
+ const tabs = this.sessionTabs.get(chatSessionId);
195
+ return tabs ? Array.from(tabs) : [];
196
+ }
197
+
198
+ /**
199
+ * Get all controlled tab IDs (across all sessions)
200
+ */
201
+ getAllControlledTabs(): Map<string, TabOwnershipInfo> {
202
+ return new Map(this.tabOwnership);
203
+ }
194
204
 
195
- // If another tab is being controlled, deny
196
- if (this.controlState.isControlling) {
197
- debug.warn('mcp', `MCP control denied: another tab (${this.controlState.browserTabId}) is already being controlled`);
205
+ // ============================================================================
206
+ // Control Acquisition
207
+ // ============================================================================
208
+
209
+ /**
210
+ * Acquire control of a browser tab for a chat session.
211
+ *
212
+ * - If the tab is already owned by the same session → success (idempotent)
213
+ * - If the tab is owned by another session → denied
214
+ * - If the tab is free → acquire and add to session's controlled set
215
+ */
216
+ acquireControl(browserTabId: string, chatSessionId: string, projectId: string): boolean {
217
+ // Check existing ownership
218
+ const existingOwner = this.tabOwnership.get(browserTabId);
219
+
220
+ if (existingOwner) {
221
+ // Same session already owns it → idempotent success
222
+ if (existingOwner.chatSessionId === chatSessionId) {
223
+ return true;
224
+ }
225
+ // Different session owns it → denied
226
+ debug.warn('mcp', `❌ Tab ${browserTabId} is controlled by session ${existingOwner.chatSessionId}, denied for ${chatSessionId}`);
198
227
  return false;
199
228
  }
200
229
 
201
230
  // Acquire control
202
231
  const now = Date.now();
203
- this.controlState = {
204
- isControlling: true,
205
- mcpSessionId: mcpSessionId || null,
206
- browserTabId,
207
- startedAt: now,
208
- lastActionAt: now
209
- };
232
+ this.tabOwnership.set(browserTabId, {
233
+ chatSessionId,
234
+ projectId,
235
+ acquiredAt: now
236
+ });
210
237
 
211
- // Emit control start event to frontend
212
- this.emitControlStart(browserTabId, mcpSessionId);
238
+ // Add to session's tab set
239
+ let sessionSet = this.sessionTabs.get(chatSessionId);
240
+ if (!sessionSet) {
241
+ sessionSet = new Set();
242
+ this.sessionTabs.set(chatSessionId, sessionSet);
243
+ }
244
+ sessionSet.add(browserTabId);
213
245
 
214
- // Start idle check interval
215
- this.startIdleCheck();
246
+ // Emit control start event to frontend
247
+ this.emitControlStart(browserTabId, chatSessionId);
216
248
 
217
- debug.log('mcp', `🎮 MCP acquired control of tab: ${browserTabId} (event-based tracking)`);
249
+ debug.log('mcp', `🎮 Session ${chatSessionId.slice(0, 8)} acquired tab: ${browserTabId} (total: ${sessionSet.size} tabs)`);
218
250
  return true;
219
251
  }
220
252
 
253
+ // ============================================================================
254
+ // Control Release
255
+ // ============================================================================
256
+
221
257
  /**
222
- * Release control of a browser tab
258
+ * Release a single tab from its owning session.
259
+ * Used when a tab is closed via close_tab or destroyed.
223
260
  */
224
- releaseControl(browserTabId?: string): void {
225
- // If browserTabId provided, only release if it matches
226
- if (browserTabId && this.controlState.browserTabId !== browserTabId) {
227
- return;
261
+ releaseTab(browserTabId: string): void {
262
+ const ownership = this.tabOwnership.get(browserTabId);
263
+ if (!ownership) return;
264
+
265
+ // Remove from tab ownership
266
+ this.tabOwnership.delete(browserTabId);
267
+
268
+ // Remove from session's tab set
269
+ const sessionSet = this.sessionTabs.get(ownership.chatSessionId);
270
+ if (sessionSet) {
271
+ sessionSet.delete(browserTabId);
272
+ if (sessionSet.size === 0) {
273
+ this.sessionTabs.delete(ownership.chatSessionId);
274
+ }
228
275
  }
229
276
 
230
- if (!this.controlState.isControlling) {
277
+ // Emit control end event to frontend
278
+ this.emitControlEnd(browserTabId);
279
+
280
+ debug.log('mcp', `🎮 Released tab: ${browserTabId} (was owned by session ${ownership.chatSessionId.slice(0, 8)})`);
281
+ }
282
+
283
+ /**
284
+ * Release all tabs owned by a chat session.
285
+ * Called when chat stream ends (complete/error/cancel).
286
+ */
287
+ releaseSession(chatSessionId: string): void {
288
+ const sessionSet = this.sessionTabs.get(chatSessionId);
289
+ if (!sessionSet || sessionSet.size === 0) {
290
+ this.sessionTabs.delete(chatSessionId);
231
291
  return;
232
292
  }
233
293
 
234
- const releasedTabId = this.controlState.browserTabId;
294
+ const tabIds = Array.from(sessionSet);
295
+ debug.log('mcp', `🎮 Releasing ${tabIds.length} tabs for session ${chatSessionId.slice(0, 8)}`);
235
296
 
236
- // Reset state
237
- this.controlState = {
238
- isControlling: false,
239
- mcpSessionId: null,
240
- browserTabId: null,
241
- startedAt: null,
242
- lastActionAt: null
243
- };
297
+ for (const tabId of tabIds) {
298
+ this.tabOwnership.delete(tabId);
299
+ this.emitControlEnd(tabId);
300
+ }
244
301
 
245
- // Stop idle check interval
246
- this.stopIdleCheck();
302
+ this.sessionTabs.delete(chatSessionId);
247
303
 
248
- // Emit control end event to frontend
249
- if (releasedTabId) {
250
- this.emitControlEnd(releasedTabId);
251
- }
304
+ debug.log('mcp', `🎮 Session ${chatSessionId.slice(0, 8)} fully released`);
305
+ }
252
306
 
253
- debug.log('mcp', `🎮 MCP released control of tab: ${releasedTabId}`);
307
+ /**
308
+ * Auto-release control for a specific tab when it's closed.
309
+ * projectId is used to prevent accidental release across projects.
310
+ */
311
+ autoReleaseForTab(browserTabId: string, projectId?: string): void {
312
+ const ownership = this.tabOwnership.get(browserTabId);
313
+ if (!ownership) return;
314
+ if (projectId && ownership.projectId !== projectId) return;
315
+ debug.log('mcp', `🗑️ Auto-releasing tab: ${browserTabId} (closed)`);
316
+ this.releaseTab(browserTabId);
254
317
  }
255
318
 
256
319
  /**
257
- * Update last action timestamp (for tracking purposes only)
258
- * NOTE: This does NOT affect control lifecycle - control is maintained
259
- * as long as the session exists, regardless of action timestamps
320
+ * Force release all control (for cleanup)
260
321
  */
261
- updateLastAction(): void {
262
- if (this.controlState.isControlling) {
263
- this.controlState.lastActionAt = Date.now();
322
+ forceReleaseAll(): void {
323
+ // Emit control-end for all controlled tabs
324
+ for (const [tabId] of this.tabOwnership) {
325
+ this.emitControlEnd(tabId);
264
326
  }
327
+
328
+ this.tabOwnership.clear();
329
+ this.sessionTabs.clear();
330
+
331
+ debug.log('mcp', '🧹 Force released all MCP control');
265
332
  }
266
333
 
334
+ // ============================================================================
335
+ // Cursor Events
336
+ // ============================================================================
337
+
267
338
  /**
268
339
  * Emit cursor position event with MCP source
269
340
  */
@@ -305,14 +376,15 @@ export class BrowserMcpControl extends EventEmitter {
305
376
  });
306
377
  }
307
378
 
308
- /**
309
- * Emit control start event to frontend
310
- */
311
- private emitControlStart(browserTabId: string, mcpSessionId?: string): void {
379
+ // ============================================================================
380
+ // Private Event Emitters
381
+ // ============================================================================
382
+
383
+ private emitControlStart(browserTabId: string, chatSessionId?: string): void {
312
384
  const event: McpControlEvent = {
313
385
  type: 'mcp:control-start',
314
386
  browserTabId,
315
- mcpSessionId,
387
+ chatSessionId,
316
388
  timestamp: Date.now()
317
389
  };
318
390
 
@@ -321,9 +393,6 @@ export class BrowserMcpControl extends EventEmitter {
321
393
  debug.log('mcp', `📢 Emitted mcp:control-start for tab: ${browserTabId}`);
322
394
  }
323
395
 
324
- /**
325
- * Emit control end event to frontend
326
- */
327
396
  private emitControlEnd(browserTabId: string): void {
328
397
  const event: McpControlEvent = {
329
398
  type: 'mcp:control-end',
@@ -335,80 +404,6 @@ export class BrowserMcpControl extends EventEmitter {
335
404
 
336
405
  debug.log('mcp', `📢 Emitted mcp:control-end for tab: ${browserTabId}`);
337
406
  }
338
-
339
- /**
340
- * Start idle check interval
341
- */
342
- private startIdleCheck(): void {
343
- // Clear any existing interval
344
- this.stopIdleCheck();
345
-
346
- // Check every 10 seconds
347
- this.idleCheckInterval = setInterval(() => {
348
- this.checkAndReleaseIfIdle();
349
- }, 10000);
350
-
351
- debug.log('mcp', '⏰ Started idle check interval (30s timeout)');
352
- }
353
-
354
- /**
355
- * Stop idle check interval
356
- */
357
- private stopIdleCheck(): void {
358
- if (this.idleCheckInterval) {
359
- clearInterval(this.idleCheckInterval);
360
- this.idleCheckInterval = null;
361
- debug.log('mcp', '⏰ Stopped idle check interval');
362
- }
363
- }
364
-
365
- /**
366
- * Check if MCP control is idle and auto-release if timeout
367
- */
368
- private checkAndReleaseIfIdle(): void {
369
- if (!this.controlState.isControlling) {
370
- return;
371
- }
372
-
373
- const now = Date.now();
374
- const idleTime = now - (this.controlState.lastActionAt || this.controlState.startedAt || now);
375
-
376
- if (idleTime >= this.IDLE_TIMEOUT_MS) {
377
- debug.log('mcp', `⏰ MCP control idle for ${Math.round(idleTime / 1000)}s, auto-releasing...`);
378
- this.releaseControl();
379
- }
380
- }
381
-
382
- /**
383
- * Auto-release control for a specific browser tab (called when tab closes)
384
- */
385
- autoReleaseForTab(browserTabId: string): void {
386
- if (this.controlState.isControlling && this.controlState.browserTabId === browserTabId) {
387
- debug.log('mcp', `🗑️ Auto-releasing MCP control for closed tab: ${browserTabId}`);
388
- this.releaseControl(browserTabId);
389
- }
390
- }
391
-
392
- /**
393
- * Force release all control (for cleanup)
394
- */
395
- forceReleaseAll(): void {
396
- this.stopIdleCheck();
397
-
398
- if (this.controlState.isControlling && this.controlState.browserTabId) {
399
- this.emitControlEnd(this.controlState.browserTabId);
400
- }
401
-
402
- this.controlState = {
403
- isControlling: false,
404
- mcpSessionId: null,
405
- browserTabId: null,
406
- startedAt: null,
407
- lastActionAt: null
408
- };
409
-
410
- debug.log('mcp', '🧹 Force released all MCP control');
411
- }
412
407
  }
413
408
 
414
409
  // Singleton instance
@@ -49,7 +49,9 @@ const DEFAULT_CONFIG: PoolConfig = {
49
49
  const CHROMIUM_ARGS = [
50
50
  '--no-sandbox',
51
51
  '--disable-blink-features=AutomationControlled',
52
- '--window-size=1366,768'
52
+ '--window-size=1366,768',
53
+ '--autoplay-policy=no-user-gesture-required',
54
+ '--disable-features=AudioServiceOutOfProcess'
53
55
  ];
54
56
 
55
57
  class BrowserPool {
@@ -733,8 +733,8 @@ class BrowserPreviewServiceManager {
733
733
  browserMcpControl.on('control-start', (data) => {
734
734
  debug.log('preview', `🚀 Forwarding mcp-control-start to project ${projectId}:`, data);
735
735
  ws.emit.project(projectId, 'preview:browser-mcp-control-start', {
736
- browserSessionId: data.browserTabId,
737
- mcpSessionId: data.mcpSessionId,
736
+ browserTabId: data.browserTabId,
737
+ chatSessionId: data.chatSessionId,
738
738
  timestamp: data.timestamp
739
739
  });
740
740
  });
@@ -742,7 +742,7 @@ class BrowserPreviewServiceManager {
742
742
  browserMcpControl.on('control-end', (data) => {
743
743
  debug.log('preview', `🚀 Forwarding mcp-control-end to project ${projectId}:`, data);
744
744
  ws.emit.project(projectId, 'preview:browser-mcp-control-end', {
745
- browserSessionId: data.browserTabId,
745
+ browserTabId: data.browserTabId,
746
746
  timestamp: data.timestamp
747
747
  });
748
748
  });
@@ -273,7 +273,7 @@ export class BrowserTabManager extends EventEmitter {
273
273
  const wasActive = tab.isActive;
274
274
 
275
275
  // Auto-release MCP control if this tab is being controlled
276
- browserMcpControl.autoReleaseForTab(tabId);
276
+ browserMcpControl.autoReleaseForTab(tabId, this.projectId);
277
277
 
278
278
  // IMMEDIATELY set destroyed flag and stop streaming
279
279
  tab.isDestroyed = true;
@@ -4,9 +4,9 @@
4
4
  * Handles WebCodecs-based video streaming with WebRTC DataChannel transport.
5
5
  *
6
6
  * Video Architecture:
7
- * 1. Puppeteer CDP captures JPEG frames via Page.screencastFrame
8
- * 2. Decode JPEG to ImageBitmap in browser
9
- * 3. Encode with VideoEncoder (VP8) in browser
7
+ * 1. CDP captures JPEG frames via Page.screencastFrame
8
+ * 2. Direct CDP Runtime.evaluate sends base64 to page (bypasses Puppeteer IPC)
9
+ * 3. Page decodes JPEG via createImageBitmap, encodes with VP8 VideoEncoder
10
10
  * 4. Send encoded chunks via RTCDataChannel
11
11
  *
12
12
  * Audio Architecture:
@@ -347,17 +347,15 @@ export class BrowserVideoCapture extends EventEmitter {
347
347
  // ACK immediately
348
348
  cdp.send('Page.screencastFrameAck', { sessionId: event.sessionId }).catch(() => {});
349
349
 
350
- // Send frame to encoder
351
- page.evaluate((frameData) => {
352
- const peer = (window as any).__webCodecsPeer;
353
- if (!peer) return false;
354
- peer.encodeFrame(frameData);
355
- return true;
356
- }, event.data).catch((err) => {
357
- if (cdpFrameCount <= 5) {
358
- debug.warn('webcodecs', `Frame delivery error (frame ${cdpFrameCount}):`, err.message);
359
- }
360
- });
350
+ // Send frame to encoder via direct CDP (bypasses Puppeteer's
351
+ // ExecutionContext lookup, function serialization, Runtime.callFunctionOn
352
+ // overhead, and result deserialization). Base64 charset [A-Za-z0-9+/=]
353
+ // is safe to embed in a JS double-quoted string literal.
354
+ cdp.send('Runtime.evaluate', {
355
+ expression: `window.__webCodecsPeer?.encodeFrame("${event.data}")`,
356
+ awaitPromise: false,
357
+ returnByValue: false
358
+ }).catch(() => {});
361
359
  });
362
360
 
363
361
  // Start screencast with scaled dimensions
@@ -148,6 +148,12 @@ export function audioCaptureScript(config: StreamingConfig['audio']) {
148
148
  if (interceptedContexts.has(ctx)) return;
149
149
  interceptedContexts.add(ctx);
150
150
 
151
+ // Resume AudioContext immediately — in headless Chrome without a user gesture,
152
+ // AudioContext starts in 'suspended' state and onaudioprocess never fires.
153
+ if (ctx.state === 'suspended') {
154
+ ctx.resume().catch(() => {});
155
+ }
156
+
151
157
  // Store original destination
152
158
  const originalDestination = ctx.destination;
153
159
 
@@ -213,6 +219,11 @@ export function audioCaptureScript(config: StreamingConfig['audio']) {
213
219
  const OriginalAudioContext = (window as any).__OriginalAudioContext || window.AudioContext;
214
220
  const ctx = new OriginalAudioContext();
215
221
 
222
+ // Resume context immediately — headless Chrome requires explicit resume
223
+ if (ctx.state === 'suspended') {
224
+ ctx.resume().catch(() => {});
225
+ }
226
+
216
227
  // Create media element source
217
228
  const source = ctx.createMediaElementSource(element);
218
229
 
@@ -218,23 +218,24 @@ export function videoEncoderScript(config: StreamingConfig['video']) {
218
218
  if (!videoEncoder || !isCapturing) return;
219
219
 
220
220
  try {
221
- // Base64 to arrayBuffer (more efficient)
222
- const imageBuffer = await fetch(`data:image/jpeg;base64,${imageData}`);
223
- const arrayBuffer = await imageBuffer.arrayBuffer();
224
-
225
- // Decode JPEG to VideoFrame
226
- const decoder = new ImageDecoder({
227
- data: arrayBuffer,
228
- type: 'image/jpeg',
229
- });
221
+ // Direct base64 decode (avoids fetch() + data URL parsing overhead)
222
+ const binaryStr = atob(imageData);
223
+ const len = binaryStr.length;
224
+ const bytes = new Uint8Array(len);
225
+ for (let i = 0; i < len; i++) {
226
+ bytes[i] = binaryStr.charCodeAt(i);
227
+ }
230
228
 
231
- const { image } = await decoder.decode();
229
+ // Decode JPEG via createImageBitmap (avoids per-frame ImageDecoder
230
+ // constructor/destructor overhead)
231
+ const blob = new Blob([bytes], { type: 'image/jpeg' });
232
+ const bitmap = await createImageBitmap(blob);
232
233
 
233
234
  // Get aligned timestamp in microseconds
234
235
  const timestamp = performance.now() * 1000;
235
236
 
236
- // Create VideoFrame with aligned timestamp
237
- const frame = new VideoFrame(image, {
237
+ // Create VideoFrame from ImageBitmap
238
+ const frame = new VideoFrame(bitmap, {
238
239
  timestamp,
239
240
  alpha: 'discard'
240
241
  });
@@ -253,8 +254,7 @@ export function videoEncoderScript(config: StreamingConfig['video']) {
253
254
 
254
255
  // Close immediately to prevent memory leaks
255
256
  frame.close();
256
- image.close();
257
- decoder.close();
257
+ bitmap.close();
258
258
  } catch (error) {}
259
259
  }
260
260
 
@@ -229,10 +229,10 @@ export interface StreamingConfig {
229
229
  /**
230
230
  * Default streaming configuration
231
231
  *
232
- * Optimized for low-end servers without GPU:
233
- * - Software encoding only (hardwareAcceleration: 'no-preference')
234
- * - Lower bitrate for bandwidth efficiency
235
- * - VP8 for video (good software encoder performance)
232
+ * Optimized for visual quality with reasonable bandwidth:
233
+ * - Software encoding (hardwareAcceleration: 'no-preference')
234
+ * - JPEG quality 70 preserves thin borders/text, VP8 handles bandwidth
235
+ * - VP8 at 1.2Mbps for crisp UI/text rendering
236
236
  * - Opus for audio (efficient and widely supported)
237
237
  */
238
238
  export const DEFAULT_STREAMING_CONFIG: StreamingConfig = {
@@ -241,9 +241,9 @@ export const DEFAULT_STREAMING_CONFIG: StreamingConfig = {
241
241
  width: 0,
242
242
  height: 0,
243
243
  framerate: 24,
244
- bitrate: 600_000,
245
- keyframeInterval: 2,
246
- screenshotQuality: 35,
244
+ bitrate: 1_200_000,
245
+ keyframeInterval: 3,
246
+ screenshotQuality: 70,
247
247
  hardwareAcceleration: 'no-preference',
248
248
  latencyMode: 'realtime'
249
249
  },
@@ -2,7 +2,7 @@
2
2
  export * from './browser/types';
3
3
 
4
4
  // Export MCP control types
5
- export type { McpControlState, McpControlEvent, McpCursorEvent, McpClickEvent } from './browser/browser-mcp-control';
5
+ export type { McpControlEvent, McpCursorEvent, McpClickEvent } from './browser/browser-mcp-control';
6
6
 
7
7
  // Export the main preview service class and manager
8
8
  export {