@myrialabs/clopen 0.2.6 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/backend/chat/stream-manager.ts +24 -13
- package/backend/engine/adapters/claude/stream.ts +10 -19
- package/backend/mcp/project-context.ts +20 -0
- package/backend/mcp/servers/browser-automation/actions.ts +0 -2
- package/backend/mcp/servers/browser-automation/browser.ts +86 -132
- package/backend/mcp/servers/browser-automation/inspection.ts +5 -11
- package/backend/preview/browser/browser-mcp-control.ts +175 -180
- package/backend/preview/browser/browser-pool.ts +3 -1
- package/backend/preview/browser/browser-preview-service.ts +3 -3
- package/backend/preview/browser/browser-tab-manager.ts +1 -1
- package/backend/preview/browser/browser-video-capture.ts +12 -14
- package/backend/preview/browser/scripts/audio-stream.ts +11 -0
- package/backend/preview/browser/scripts/video-stream.ts +14 -14
- package/backend/preview/browser/types.ts +7 -7
- package/backend/preview/index.ts +1 -1
- package/backend/ws/chat/stream.ts +1 -1
- package/backend/ws/preview/browser/tab-info.ts +5 -2
- package/backend/ws/preview/index.ts +3 -3
- package/frontend/components/chat/input/ChatInput.svelte +0 -3
- package/frontend/components/chat/input/composables/use-chat-actions.svelte.ts +6 -2
- package/frontend/components/chat/message/MessageBubble.svelte +2 -2
- package/frontend/components/history/HistoryModal.svelte +1 -1
- package/frontend/components/preview/browser/BrowserPreview.svelte +15 -1
- package/frontend/components/preview/browser/components/Canvas.svelte +323 -49
- package/frontend/components/preview/browser/components/Container.svelte +21 -0
- package/frontend/components/preview/browser/components/Toolbar.svelte +3 -3
- package/frontend/components/preview/browser/core/coordinator.svelte.ts +15 -0
- package/frontend/components/preview/browser/core/mcp-handlers.svelte.ts +78 -51
- package/frontend/components/preview/browser/core/tab-operations.svelte.ts +1 -0
- package/frontend/components/workspace/PanelHeader.svelte +15 -0
- package/frontend/components/workspace/panels/GitPanel.svelte +22 -13
- package/frontend/components/workspace/panels/PreviewPanel.svelte +2 -0
- package/frontend/services/chat/chat.service.ts +3 -7
- package/frontend/services/preview/browser/browser-webcodecs.service.ts +32 -135
- package/frontend/stores/core/app.svelte.ts +4 -3
- package/frontend/stores/core/presence.svelte.ts +3 -2
- package/frontend/stores/core/sessions.svelte.ts +2 -0
- package/frontend/stores/ui/notification.svelte.ts +4 -1
- package/package.json +1 -1
|
@@ -1,16 +1,17 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Browser MCP Control
|
|
3
3
|
*
|
|
4
|
-
* Manages
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
4
|
+
* Manages MCP control over browser tabs with multi-tab, session-scoped ownership.
|
|
5
|
+
* Each chat session can control multiple tabs simultaneously.
|
|
6
|
+
* A tab can only be controlled by one chat session at a time.
|
|
7
|
+
* All tabs are released when the chat session ends (stream complete/error/cancel).
|
|
8
8
|
*
|
|
9
|
-
* ARCHITECTURE:
|
|
10
|
-
* - Control
|
|
11
|
-
* -
|
|
12
|
-
* -
|
|
13
|
-
* -
|
|
9
|
+
* ARCHITECTURE:
|
|
10
|
+
* - Control lifecycle follows chat sessions (no idle timeout)
|
|
11
|
+
* - Multiple tabs can be locked by one chat session (accumulated via switch/open)
|
|
12
|
+
* - Tab destroyed → auto-release that single tab from its owning session
|
|
13
|
+
* - Stream ends → releaseSession() releases all tabs owned by that session
|
|
14
|
+
* - Emits per-tab control-start/control-end events for frontend UI
|
|
14
15
|
*/
|
|
15
16
|
|
|
16
17
|
import { EventEmitter } from 'events';
|
|
@@ -24,18 +25,17 @@ interface PendingTabRequest<T = any> {
|
|
|
24
25
|
timeout: NodeJS.Timeout;
|
|
25
26
|
}
|
|
26
27
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
lastActionAt: number | null;
|
|
28
|
+
/** Ownership info for a single tab */
|
|
29
|
+
interface TabOwnershipInfo {
|
|
30
|
+
chatSessionId: string;
|
|
31
|
+
projectId: string;
|
|
32
|
+
acquiredAt: number;
|
|
33
33
|
}
|
|
34
34
|
|
|
35
35
|
export interface McpControlEvent {
|
|
36
36
|
type: 'mcp:control-start' | 'mcp:control-end';
|
|
37
37
|
browserTabId: string;
|
|
38
|
-
|
|
38
|
+
chatSessionId?: string;
|
|
39
39
|
timestamp: number;
|
|
40
40
|
}
|
|
41
41
|
|
|
@@ -56,17 +56,11 @@ export interface McpClickEvent {
|
|
|
56
56
|
}
|
|
57
57
|
|
|
58
58
|
export class BrowserMcpControl extends EventEmitter {
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
lastActionAt: null
|
|
65
|
-
};
|
|
66
|
-
|
|
67
|
-
// Auto-release configuration
|
|
68
|
-
private readonly IDLE_TIMEOUT_MS = 30000; // 30 seconds idle = auto release
|
|
69
|
-
private idleCheckInterval: NodeJS.Timeout | null = null;
|
|
59
|
+
/** Tab → ownership info (which chat session controls it) */
|
|
60
|
+
private tabOwnership = new Map<string, TabOwnershipInfo>();
|
|
61
|
+
|
|
62
|
+
/** Chat session → set of tab IDs it controls */
|
|
63
|
+
private sessionTabs = new Map<string, Set<string>>();
|
|
70
64
|
|
|
71
65
|
// Pending tab requests (keyed by request type + timestamp)
|
|
72
66
|
private pendingTabRequests = new Map<string, PendingTabRequest>();
|
|
@@ -96,13 +90,18 @@ export class BrowserMcpControl extends EventEmitter {
|
|
|
96
90
|
|
|
97
91
|
/**
|
|
98
92
|
* Handle tab destroyed event
|
|
99
|
-
* Auto-release control
|
|
93
|
+
* Auto-release control for the destroyed tab only
|
|
100
94
|
*/
|
|
101
95
|
private handleTabDestroyed(tabId: string): void {
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
96
|
+
const ownership = this.tabOwnership.get(tabId);
|
|
97
|
+
if (!ownership) return;
|
|
98
|
+
|
|
99
|
+
// Validate project to prevent cross-project collisions
|
|
100
|
+
const serviceProjectId = this.previewService?.getProjectId();
|
|
101
|
+
if (serviceProjectId && ownership.projectId !== serviceProjectId) return;
|
|
102
|
+
|
|
103
|
+
debug.warn('mcp', `⚠️ Controlled tab ${tabId} was destroyed - auto-releasing from session ${ownership.chatSessionId}`);
|
|
104
|
+
this.releaseTab(tabId);
|
|
106
105
|
}
|
|
107
106
|
|
|
108
107
|
/**
|
|
@@ -152,118 +151,190 @@ export class BrowserMcpControl extends EventEmitter {
|
|
|
152
151
|
return false;
|
|
153
152
|
}
|
|
154
153
|
|
|
154
|
+
// ============================================================================
|
|
155
|
+
// Control State Queries
|
|
156
|
+
// ============================================================================
|
|
157
|
+
|
|
155
158
|
/**
|
|
156
|
-
* Check if
|
|
159
|
+
* Check if any tab is being controlled
|
|
157
160
|
*/
|
|
158
161
|
isControlling(): boolean {
|
|
159
|
-
return this.
|
|
162
|
+
return this.tabOwnership.size > 0;
|
|
160
163
|
}
|
|
161
164
|
|
|
162
165
|
/**
|
|
163
|
-
*
|
|
166
|
+
* Check if a specific tab is being controlled (by any session)
|
|
164
167
|
*/
|
|
165
|
-
|
|
166
|
-
|
|
168
|
+
isTabControlled(browserTabId: string, projectId?: string): boolean {
|
|
169
|
+
const ownership = this.tabOwnership.get(browserTabId);
|
|
170
|
+
if (!ownership) return false;
|
|
171
|
+
if (projectId && ownership.projectId !== projectId) return false;
|
|
172
|
+
return true;
|
|
167
173
|
}
|
|
168
174
|
|
|
169
175
|
/**
|
|
170
|
-
* Check if a
|
|
176
|
+
* Check if a tab is controlled by a specific chat session
|
|
171
177
|
*/
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
178
|
+
isTabControlledBySession(browserTabId: string, chatSessionId: string): boolean {
|
|
179
|
+
const ownership = this.tabOwnership.get(browserTabId);
|
|
180
|
+
return ownership?.chatSessionId === chatSessionId;
|
|
175
181
|
}
|
|
176
182
|
|
|
177
183
|
/**
|
|
178
|
-
*
|
|
179
|
-
* Returns true if control was acquired, false if already controlled by another MCP
|
|
184
|
+
* Get the chat session ID that controls a specific tab
|
|
180
185
|
*/
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
debug.warn('mcp', `❌ Cannot acquire control: tab ${browserTabId} does not exist`);
|
|
185
|
-
return false;
|
|
186
|
-
}
|
|
186
|
+
getTabOwner(browserTabId: string): string | null {
|
|
187
|
+
return this.tabOwnership.get(browserTabId)?.chatSessionId || null;
|
|
188
|
+
}
|
|
187
189
|
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
190
|
+
/**
|
|
191
|
+
* Get all tab IDs controlled by a specific chat session
|
|
192
|
+
*/
|
|
193
|
+
getSessionTabs(chatSessionId: string): string[] {
|
|
194
|
+
const tabs = this.sessionTabs.get(chatSessionId);
|
|
195
|
+
return tabs ? Array.from(tabs) : [];
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/**
|
|
199
|
+
* Get all controlled tab IDs (across all sessions)
|
|
200
|
+
*/
|
|
201
|
+
getAllControlledTabs(): Map<string, TabOwnershipInfo> {
|
|
202
|
+
return new Map(this.tabOwnership);
|
|
203
|
+
}
|
|
194
204
|
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
205
|
+
// ============================================================================
|
|
206
|
+
// Control Acquisition
|
|
207
|
+
// ============================================================================
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Acquire control of a browser tab for a chat session.
|
|
211
|
+
*
|
|
212
|
+
* - If the tab is already owned by the same session → success (idempotent)
|
|
213
|
+
* - If the tab is owned by another session → denied
|
|
214
|
+
* - If the tab is free → acquire and add to session's controlled set
|
|
215
|
+
*/
|
|
216
|
+
acquireControl(browserTabId: string, chatSessionId: string, projectId: string): boolean {
|
|
217
|
+
// Check existing ownership
|
|
218
|
+
const existingOwner = this.tabOwnership.get(browserTabId);
|
|
219
|
+
|
|
220
|
+
if (existingOwner) {
|
|
221
|
+
// Same session already owns it → idempotent success
|
|
222
|
+
if (existingOwner.chatSessionId === chatSessionId) {
|
|
223
|
+
return true;
|
|
224
|
+
}
|
|
225
|
+
// Different session owns it → denied
|
|
226
|
+
debug.warn('mcp', `❌ Tab ${browserTabId} is controlled by session ${existingOwner.chatSessionId}, denied for ${chatSessionId}`);
|
|
198
227
|
return false;
|
|
199
228
|
}
|
|
200
229
|
|
|
201
230
|
// Acquire control
|
|
202
231
|
const now = Date.now();
|
|
203
|
-
this.
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
lastActionAt: now
|
|
209
|
-
};
|
|
232
|
+
this.tabOwnership.set(browserTabId, {
|
|
233
|
+
chatSessionId,
|
|
234
|
+
projectId,
|
|
235
|
+
acquiredAt: now
|
|
236
|
+
});
|
|
210
237
|
|
|
211
|
-
//
|
|
212
|
-
this.
|
|
238
|
+
// Add to session's tab set
|
|
239
|
+
let sessionSet = this.sessionTabs.get(chatSessionId);
|
|
240
|
+
if (!sessionSet) {
|
|
241
|
+
sessionSet = new Set();
|
|
242
|
+
this.sessionTabs.set(chatSessionId, sessionSet);
|
|
243
|
+
}
|
|
244
|
+
sessionSet.add(browserTabId);
|
|
213
245
|
|
|
214
|
-
//
|
|
215
|
-
this.
|
|
246
|
+
// Emit control start event to frontend
|
|
247
|
+
this.emitControlStart(browserTabId, chatSessionId);
|
|
216
248
|
|
|
217
|
-
debug.log('mcp', `🎮
|
|
249
|
+
debug.log('mcp', `🎮 Session ${chatSessionId.slice(0, 8)} acquired tab: ${browserTabId} (total: ${sessionSet.size} tabs)`);
|
|
218
250
|
return true;
|
|
219
251
|
}
|
|
220
252
|
|
|
253
|
+
// ============================================================================
|
|
254
|
+
// Control Release
|
|
255
|
+
// ============================================================================
|
|
256
|
+
|
|
221
257
|
/**
|
|
222
|
-
* Release
|
|
258
|
+
* Release a single tab from its owning session.
|
|
259
|
+
* Used when a tab is closed via close_tab or destroyed.
|
|
223
260
|
*/
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
if (
|
|
227
|
-
|
|
261
|
+
releaseTab(browserTabId: string): void {
|
|
262
|
+
const ownership = this.tabOwnership.get(browserTabId);
|
|
263
|
+
if (!ownership) return;
|
|
264
|
+
|
|
265
|
+
// Remove from tab ownership
|
|
266
|
+
this.tabOwnership.delete(browserTabId);
|
|
267
|
+
|
|
268
|
+
// Remove from session's tab set
|
|
269
|
+
const sessionSet = this.sessionTabs.get(ownership.chatSessionId);
|
|
270
|
+
if (sessionSet) {
|
|
271
|
+
sessionSet.delete(browserTabId);
|
|
272
|
+
if (sessionSet.size === 0) {
|
|
273
|
+
this.sessionTabs.delete(ownership.chatSessionId);
|
|
274
|
+
}
|
|
228
275
|
}
|
|
229
276
|
|
|
230
|
-
|
|
277
|
+
// Emit control end event to frontend
|
|
278
|
+
this.emitControlEnd(browserTabId);
|
|
279
|
+
|
|
280
|
+
debug.log('mcp', `🎮 Released tab: ${browserTabId} (was owned by session ${ownership.chatSessionId.slice(0, 8)})`);
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/**
|
|
284
|
+
* Release all tabs owned by a chat session.
|
|
285
|
+
* Called when chat stream ends (complete/error/cancel).
|
|
286
|
+
*/
|
|
287
|
+
releaseSession(chatSessionId: string): void {
|
|
288
|
+
const sessionSet = this.sessionTabs.get(chatSessionId);
|
|
289
|
+
if (!sessionSet || sessionSet.size === 0) {
|
|
290
|
+
this.sessionTabs.delete(chatSessionId);
|
|
231
291
|
return;
|
|
232
292
|
}
|
|
233
293
|
|
|
234
|
-
const
|
|
294
|
+
const tabIds = Array.from(sessionSet);
|
|
295
|
+
debug.log('mcp', `🎮 Releasing ${tabIds.length} tabs for session ${chatSessionId.slice(0, 8)}`);
|
|
235
296
|
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
browserTabId: null,
|
|
241
|
-
startedAt: null,
|
|
242
|
-
lastActionAt: null
|
|
243
|
-
};
|
|
297
|
+
for (const tabId of tabIds) {
|
|
298
|
+
this.tabOwnership.delete(tabId);
|
|
299
|
+
this.emitControlEnd(tabId);
|
|
300
|
+
}
|
|
244
301
|
|
|
245
|
-
|
|
246
|
-
this.stopIdleCheck();
|
|
302
|
+
this.sessionTabs.delete(chatSessionId);
|
|
247
303
|
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
this.emitControlEnd(releasedTabId);
|
|
251
|
-
}
|
|
304
|
+
debug.log('mcp', `🎮 Session ${chatSessionId.slice(0, 8)} fully released`);
|
|
305
|
+
}
|
|
252
306
|
|
|
253
|
-
|
|
307
|
+
/**
|
|
308
|
+
* Auto-release control for a specific tab when it's closed.
|
|
309
|
+
* projectId is used to prevent accidental release across projects.
|
|
310
|
+
*/
|
|
311
|
+
autoReleaseForTab(browserTabId: string, projectId?: string): void {
|
|
312
|
+
const ownership = this.tabOwnership.get(browserTabId);
|
|
313
|
+
if (!ownership) return;
|
|
314
|
+
if (projectId && ownership.projectId !== projectId) return;
|
|
315
|
+
debug.log('mcp', `🗑️ Auto-releasing tab: ${browserTabId} (closed)`);
|
|
316
|
+
this.releaseTab(browserTabId);
|
|
254
317
|
}
|
|
255
318
|
|
|
256
319
|
/**
|
|
257
|
-
*
|
|
258
|
-
* NOTE: This does NOT affect control lifecycle - control is maintained
|
|
259
|
-
* as long as the session exists, regardless of action timestamps
|
|
320
|
+
* Force release all control (for cleanup)
|
|
260
321
|
*/
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
322
|
+
forceReleaseAll(): void {
|
|
323
|
+
// Emit control-end for all controlled tabs
|
|
324
|
+
for (const [tabId] of this.tabOwnership) {
|
|
325
|
+
this.emitControlEnd(tabId);
|
|
264
326
|
}
|
|
327
|
+
|
|
328
|
+
this.tabOwnership.clear();
|
|
329
|
+
this.sessionTabs.clear();
|
|
330
|
+
|
|
331
|
+
debug.log('mcp', '🧹 Force released all MCP control');
|
|
265
332
|
}
|
|
266
333
|
|
|
334
|
+
// ============================================================================
|
|
335
|
+
// Cursor Events
|
|
336
|
+
// ============================================================================
|
|
337
|
+
|
|
267
338
|
/**
|
|
268
339
|
* Emit cursor position event with MCP source
|
|
269
340
|
*/
|
|
@@ -305,14 +376,15 @@ export class BrowserMcpControl extends EventEmitter {
|
|
|
305
376
|
});
|
|
306
377
|
}
|
|
307
378
|
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
379
|
+
// ============================================================================
|
|
380
|
+
// Private Event Emitters
|
|
381
|
+
// ============================================================================
|
|
382
|
+
|
|
383
|
+
private emitControlStart(browserTabId: string, chatSessionId?: string): void {
|
|
312
384
|
const event: McpControlEvent = {
|
|
313
385
|
type: 'mcp:control-start',
|
|
314
386
|
browserTabId,
|
|
315
|
-
|
|
387
|
+
chatSessionId,
|
|
316
388
|
timestamp: Date.now()
|
|
317
389
|
};
|
|
318
390
|
|
|
@@ -321,9 +393,6 @@ export class BrowserMcpControl extends EventEmitter {
|
|
|
321
393
|
debug.log('mcp', `📢 Emitted mcp:control-start for tab: ${browserTabId}`);
|
|
322
394
|
}
|
|
323
395
|
|
|
324
|
-
/**
|
|
325
|
-
* Emit control end event to frontend
|
|
326
|
-
*/
|
|
327
396
|
private emitControlEnd(browserTabId: string): void {
|
|
328
397
|
const event: McpControlEvent = {
|
|
329
398
|
type: 'mcp:control-end',
|
|
@@ -335,80 +404,6 @@ export class BrowserMcpControl extends EventEmitter {
|
|
|
335
404
|
|
|
336
405
|
debug.log('mcp', `📢 Emitted mcp:control-end for tab: ${browserTabId}`);
|
|
337
406
|
}
|
|
338
|
-
|
|
339
|
-
/**
|
|
340
|
-
* Start idle check interval
|
|
341
|
-
*/
|
|
342
|
-
private startIdleCheck(): void {
|
|
343
|
-
// Clear any existing interval
|
|
344
|
-
this.stopIdleCheck();
|
|
345
|
-
|
|
346
|
-
// Check every 10 seconds
|
|
347
|
-
this.idleCheckInterval = setInterval(() => {
|
|
348
|
-
this.checkAndReleaseIfIdle();
|
|
349
|
-
}, 10000);
|
|
350
|
-
|
|
351
|
-
debug.log('mcp', '⏰ Started idle check interval (30s timeout)');
|
|
352
|
-
}
|
|
353
|
-
|
|
354
|
-
/**
|
|
355
|
-
* Stop idle check interval
|
|
356
|
-
*/
|
|
357
|
-
private stopIdleCheck(): void {
|
|
358
|
-
if (this.idleCheckInterval) {
|
|
359
|
-
clearInterval(this.idleCheckInterval);
|
|
360
|
-
this.idleCheckInterval = null;
|
|
361
|
-
debug.log('mcp', '⏰ Stopped idle check interval');
|
|
362
|
-
}
|
|
363
|
-
}
|
|
364
|
-
|
|
365
|
-
/**
|
|
366
|
-
* Check if MCP control is idle and auto-release if timeout
|
|
367
|
-
*/
|
|
368
|
-
private checkAndReleaseIfIdle(): void {
|
|
369
|
-
if (!this.controlState.isControlling) {
|
|
370
|
-
return;
|
|
371
|
-
}
|
|
372
|
-
|
|
373
|
-
const now = Date.now();
|
|
374
|
-
const idleTime = now - (this.controlState.lastActionAt || this.controlState.startedAt || now);
|
|
375
|
-
|
|
376
|
-
if (idleTime >= this.IDLE_TIMEOUT_MS) {
|
|
377
|
-
debug.log('mcp', `⏰ MCP control idle for ${Math.round(idleTime / 1000)}s, auto-releasing...`);
|
|
378
|
-
this.releaseControl();
|
|
379
|
-
}
|
|
380
|
-
}
|
|
381
|
-
|
|
382
|
-
/**
|
|
383
|
-
* Auto-release control for a specific browser tab (called when tab closes)
|
|
384
|
-
*/
|
|
385
|
-
autoReleaseForTab(browserTabId: string): void {
|
|
386
|
-
if (this.controlState.isControlling && this.controlState.browserTabId === browserTabId) {
|
|
387
|
-
debug.log('mcp', `🗑️ Auto-releasing MCP control for closed tab: ${browserTabId}`);
|
|
388
|
-
this.releaseControl(browserTabId);
|
|
389
|
-
}
|
|
390
|
-
}
|
|
391
|
-
|
|
392
|
-
/**
|
|
393
|
-
* Force release all control (for cleanup)
|
|
394
|
-
*/
|
|
395
|
-
forceReleaseAll(): void {
|
|
396
|
-
this.stopIdleCheck();
|
|
397
|
-
|
|
398
|
-
if (this.controlState.isControlling && this.controlState.browserTabId) {
|
|
399
|
-
this.emitControlEnd(this.controlState.browserTabId);
|
|
400
|
-
}
|
|
401
|
-
|
|
402
|
-
this.controlState = {
|
|
403
|
-
isControlling: false,
|
|
404
|
-
mcpSessionId: null,
|
|
405
|
-
browserTabId: null,
|
|
406
|
-
startedAt: null,
|
|
407
|
-
lastActionAt: null
|
|
408
|
-
};
|
|
409
|
-
|
|
410
|
-
debug.log('mcp', '🧹 Force released all MCP control');
|
|
411
|
-
}
|
|
412
407
|
}
|
|
413
408
|
|
|
414
409
|
// Singleton instance
|
|
@@ -49,7 +49,9 @@ const DEFAULT_CONFIG: PoolConfig = {
|
|
|
49
49
|
const CHROMIUM_ARGS = [
|
|
50
50
|
'--no-sandbox',
|
|
51
51
|
'--disable-blink-features=AutomationControlled',
|
|
52
|
-
'--window-size=1366,768'
|
|
52
|
+
'--window-size=1366,768',
|
|
53
|
+
'--autoplay-policy=no-user-gesture-required',
|
|
54
|
+
'--disable-features=AudioServiceOutOfProcess'
|
|
53
55
|
];
|
|
54
56
|
|
|
55
57
|
class BrowserPool {
|
|
@@ -733,8 +733,8 @@ class BrowserPreviewServiceManager {
|
|
|
733
733
|
browserMcpControl.on('control-start', (data) => {
|
|
734
734
|
debug.log('preview', `🚀 Forwarding mcp-control-start to project ${projectId}:`, data);
|
|
735
735
|
ws.emit.project(projectId, 'preview:browser-mcp-control-start', {
|
|
736
|
-
|
|
737
|
-
|
|
736
|
+
browserTabId: data.browserTabId,
|
|
737
|
+
chatSessionId: data.chatSessionId,
|
|
738
738
|
timestamp: data.timestamp
|
|
739
739
|
});
|
|
740
740
|
});
|
|
@@ -742,7 +742,7 @@ class BrowserPreviewServiceManager {
|
|
|
742
742
|
browserMcpControl.on('control-end', (data) => {
|
|
743
743
|
debug.log('preview', `🚀 Forwarding mcp-control-end to project ${projectId}:`, data);
|
|
744
744
|
ws.emit.project(projectId, 'preview:browser-mcp-control-end', {
|
|
745
|
-
|
|
745
|
+
browserTabId: data.browserTabId,
|
|
746
746
|
timestamp: data.timestamp
|
|
747
747
|
});
|
|
748
748
|
});
|
|
@@ -273,7 +273,7 @@ export class BrowserTabManager extends EventEmitter {
|
|
|
273
273
|
const wasActive = tab.isActive;
|
|
274
274
|
|
|
275
275
|
// Auto-release MCP control if this tab is being controlled
|
|
276
|
-
browserMcpControl.autoReleaseForTab(tabId);
|
|
276
|
+
browserMcpControl.autoReleaseForTab(tabId, this.projectId);
|
|
277
277
|
|
|
278
278
|
// IMMEDIATELY set destroyed flag and stop streaming
|
|
279
279
|
tab.isDestroyed = true;
|
|
@@ -4,9 +4,9 @@
|
|
|
4
4
|
* Handles WebCodecs-based video streaming with WebRTC DataChannel transport.
|
|
5
5
|
*
|
|
6
6
|
* Video Architecture:
|
|
7
|
-
* 1.
|
|
8
|
-
* 2.
|
|
9
|
-
* 3.
|
|
7
|
+
* 1. CDP captures JPEG frames via Page.screencastFrame
|
|
8
|
+
* 2. Direct CDP Runtime.evaluate sends base64 to page (bypasses Puppeteer IPC)
|
|
9
|
+
* 3. Page decodes JPEG via createImageBitmap, encodes with VP8 VideoEncoder
|
|
10
10
|
* 4. Send encoded chunks via RTCDataChannel
|
|
11
11
|
*
|
|
12
12
|
* Audio Architecture:
|
|
@@ -347,17 +347,15 @@ export class BrowserVideoCapture extends EventEmitter {
|
|
|
347
347
|
// ACK immediately
|
|
348
348
|
cdp.send('Page.screencastFrameAck', { sessionId: event.sessionId }).catch(() => {});
|
|
349
349
|
|
|
350
|
-
// Send frame to encoder
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
}
|
|
360
|
-
});
|
|
350
|
+
// Send frame to encoder via direct CDP (bypasses Puppeteer's
|
|
351
|
+
// ExecutionContext lookup, function serialization, Runtime.callFunctionOn
|
|
352
|
+
// overhead, and result deserialization). Base64 charset [A-Za-z0-9+/=]
|
|
353
|
+
// is safe to embed in a JS double-quoted string literal.
|
|
354
|
+
cdp.send('Runtime.evaluate', {
|
|
355
|
+
expression: `window.__webCodecsPeer?.encodeFrame("${event.data}")`,
|
|
356
|
+
awaitPromise: false,
|
|
357
|
+
returnByValue: false
|
|
358
|
+
}).catch(() => {});
|
|
361
359
|
});
|
|
362
360
|
|
|
363
361
|
// Start screencast with scaled dimensions
|
|
@@ -148,6 +148,12 @@ export function audioCaptureScript(config: StreamingConfig['audio']) {
|
|
|
148
148
|
if (interceptedContexts.has(ctx)) return;
|
|
149
149
|
interceptedContexts.add(ctx);
|
|
150
150
|
|
|
151
|
+
// Resume AudioContext immediately — in headless Chrome without a user gesture,
|
|
152
|
+
// AudioContext starts in 'suspended' state and onaudioprocess never fires.
|
|
153
|
+
if (ctx.state === 'suspended') {
|
|
154
|
+
ctx.resume().catch(() => {});
|
|
155
|
+
}
|
|
156
|
+
|
|
151
157
|
// Store original destination
|
|
152
158
|
const originalDestination = ctx.destination;
|
|
153
159
|
|
|
@@ -213,6 +219,11 @@ export function audioCaptureScript(config: StreamingConfig['audio']) {
|
|
|
213
219
|
const OriginalAudioContext = (window as any).__OriginalAudioContext || window.AudioContext;
|
|
214
220
|
const ctx = new OriginalAudioContext();
|
|
215
221
|
|
|
222
|
+
// Resume context immediately — headless Chrome requires explicit resume
|
|
223
|
+
if (ctx.state === 'suspended') {
|
|
224
|
+
ctx.resume().catch(() => {});
|
|
225
|
+
}
|
|
226
|
+
|
|
216
227
|
// Create media element source
|
|
217
228
|
const source = ctx.createMediaElementSource(element);
|
|
218
229
|
|
|
@@ -218,23 +218,24 @@ export function videoEncoderScript(config: StreamingConfig['video']) {
|
|
|
218
218
|
if (!videoEncoder || !isCapturing) return;
|
|
219
219
|
|
|
220
220
|
try {
|
|
221
|
-
//
|
|
222
|
-
const
|
|
223
|
-
const
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
type: 'image/jpeg',
|
|
229
|
-
});
|
|
221
|
+
// Direct base64 decode (avoids fetch() + data URL parsing overhead)
|
|
222
|
+
const binaryStr = atob(imageData);
|
|
223
|
+
const len = binaryStr.length;
|
|
224
|
+
const bytes = new Uint8Array(len);
|
|
225
|
+
for (let i = 0; i < len; i++) {
|
|
226
|
+
bytes[i] = binaryStr.charCodeAt(i);
|
|
227
|
+
}
|
|
230
228
|
|
|
231
|
-
|
|
229
|
+
// Decode JPEG via createImageBitmap (avoids per-frame ImageDecoder
|
|
230
|
+
// constructor/destructor overhead)
|
|
231
|
+
const blob = new Blob([bytes], { type: 'image/jpeg' });
|
|
232
|
+
const bitmap = await createImageBitmap(blob);
|
|
232
233
|
|
|
233
234
|
// Get aligned timestamp in microseconds
|
|
234
235
|
const timestamp = performance.now() * 1000;
|
|
235
236
|
|
|
236
|
-
// Create VideoFrame
|
|
237
|
-
const frame = new VideoFrame(
|
|
237
|
+
// Create VideoFrame from ImageBitmap
|
|
238
|
+
const frame = new VideoFrame(bitmap, {
|
|
238
239
|
timestamp,
|
|
239
240
|
alpha: 'discard'
|
|
240
241
|
});
|
|
@@ -253,8 +254,7 @@ export function videoEncoderScript(config: StreamingConfig['video']) {
|
|
|
253
254
|
|
|
254
255
|
// Close immediately to prevent memory leaks
|
|
255
256
|
frame.close();
|
|
256
|
-
|
|
257
|
-
decoder.close();
|
|
257
|
+
bitmap.close();
|
|
258
258
|
} catch (error) {}
|
|
259
259
|
}
|
|
260
260
|
|
|
@@ -229,10 +229,10 @@ export interface StreamingConfig {
|
|
|
229
229
|
/**
|
|
230
230
|
* Default streaming configuration
|
|
231
231
|
*
|
|
232
|
-
* Optimized for
|
|
233
|
-
* - Software encoding
|
|
234
|
-
* -
|
|
235
|
-
* - VP8
|
|
232
|
+
* Optimized for visual quality with reasonable bandwidth:
|
|
233
|
+
* - Software encoding (hardwareAcceleration: 'no-preference')
|
|
234
|
+
* - JPEG quality 70 preserves thin borders/text, VP8 handles bandwidth
|
|
235
|
+
* - VP8 at 1.2Mbps for crisp UI/text rendering
|
|
236
236
|
* - Opus for audio (efficient and widely supported)
|
|
237
237
|
*/
|
|
238
238
|
export const DEFAULT_STREAMING_CONFIG: StreamingConfig = {
|
|
@@ -241,9 +241,9 @@ export const DEFAULT_STREAMING_CONFIG: StreamingConfig = {
|
|
|
241
241
|
width: 0,
|
|
242
242
|
height: 0,
|
|
243
243
|
framerate: 24,
|
|
244
|
-
bitrate:
|
|
245
|
-
keyframeInterval:
|
|
246
|
-
screenshotQuality:
|
|
244
|
+
bitrate: 1_200_000,
|
|
245
|
+
keyframeInterval: 3,
|
|
246
|
+
screenshotQuality: 70,
|
|
247
247
|
hardwareAcceleration: 'no-preference',
|
|
248
248
|
latencyMode: 'realtime'
|
|
249
249
|
},
|
package/backend/preview/index.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
export * from './browser/types';
|
|
3
3
|
|
|
4
4
|
// Export MCP control types
|
|
5
|
-
export type {
|
|
5
|
+
export type { McpControlEvent, McpCursorEvent, McpClickEvent } from './browser/browser-mcp-control';
|
|
6
6
|
|
|
7
7
|
// Export the main preview service class and manager
|
|
8
8
|
export {
|