@myrialabs/clopen 0.2.6 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/backend/chat/stream-manager.ts +24 -13
- package/backend/engine/adapters/claude/stream.ts +10 -19
- package/backend/mcp/project-context.ts +20 -0
- package/backend/mcp/servers/browser-automation/actions.ts +0 -2
- package/backend/mcp/servers/browser-automation/browser.ts +86 -132
- package/backend/mcp/servers/browser-automation/inspection.ts +5 -11
- package/backend/preview/browser/browser-mcp-control.ts +175 -180
- package/backend/preview/browser/browser-pool.ts +3 -1
- package/backend/preview/browser/browser-preview-service.ts +3 -3
- package/backend/preview/browser/browser-tab-manager.ts +1 -1
- package/backend/preview/browser/browser-video-capture.ts +12 -14
- package/backend/preview/browser/scripts/audio-stream.ts +11 -0
- package/backend/preview/browser/scripts/video-stream.ts +14 -14
- package/backend/preview/browser/types.ts +7 -7
- package/backend/preview/index.ts +1 -1
- package/backend/ws/chat/stream.ts +1 -1
- package/backend/ws/preview/browser/tab-info.ts +5 -2
- package/backend/ws/preview/index.ts +3 -3
- package/frontend/components/chat/input/ChatInput.svelte +0 -3
- package/frontend/components/chat/input/composables/use-chat-actions.svelte.ts +6 -2
- package/frontend/components/chat/message/MessageBubble.svelte +2 -2
- package/frontend/components/history/HistoryModal.svelte +1 -1
- package/frontend/components/preview/browser/BrowserPreview.svelte +15 -1
- package/frontend/components/preview/browser/components/Canvas.svelte +323 -49
- package/frontend/components/preview/browser/components/Container.svelte +21 -0
- package/frontend/components/preview/browser/components/Toolbar.svelte +3 -3
- package/frontend/components/preview/browser/core/coordinator.svelte.ts +15 -0
- package/frontend/components/preview/browser/core/mcp-handlers.svelte.ts +78 -51
- package/frontend/components/preview/browser/core/tab-operations.svelte.ts +1 -0
- package/frontend/components/workspace/PanelHeader.svelte +15 -0
- package/frontend/components/workspace/panels/GitPanel.svelte +22 -13
- package/frontend/components/workspace/panels/PreviewPanel.svelte +2 -0
- package/frontend/services/chat/chat.service.ts +3 -7
- package/frontend/services/preview/browser/browser-webcodecs.service.ts +32 -135
- package/frontend/stores/core/app.svelte.ts +4 -3
- package/frontend/stores/core/presence.svelte.ts +3 -2
- package/frontend/stores/core/sessions.svelte.ts +2 -0
- package/frontend/stores/ui/notification.svelte.ts +4 -1
- package/package.json +1 -1
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Browser MCP Event Handlers
|
|
3
3
|
* Handles MCP (Model Context Protocol) control events for BrowserPreview
|
|
4
|
+
*
|
|
5
|
+
* Supports multi-tab control: each chat session can control multiple tabs.
|
|
6
|
+
* Tracks controlled tabs via a Set of backend tab IDs (session IDs).
|
|
4
7
|
*/
|
|
5
8
|
|
|
6
9
|
import { debug } from '$shared/utils/logger';
|
|
@@ -8,14 +11,6 @@ import { showInfo, showWarning } from '$frontend/stores/ui/notification.svelte';
|
|
|
8
11
|
import ws from '$frontend/utils/ws';
|
|
9
12
|
import type { TabManager } from './tab-manager.svelte';
|
|
10
13
|
|
|
11
|
-
// MCP Control State interface
|
|
12
|
-
export interface McpControlState {
|
|
13
|
-
isControlled: boolean;
|
|
14
|
-
controlledTabId: string | null;
|
|
15
|
-
browserSessionId: string | null;
|
|
16
|
-
startedAt: number | null;
|
|
17
|
-
}
|
|
18
|
-
|
|
19
14
|
export interface McpHandlerConfig {
|
|
20
15
|
tabManager: TabManager;
|
|
21
16
|
transformBrowserToDisplayCoordinates?: (x: number, y: number) => { x: number, y: number } | null;
|
|
@@ -30,13 +25,8 @@ export interface McpHandlerConfig {
|
|
|
30
25
|
export function createMcpHandler(config: McpHandlerConfig) {
|
|
31
26
|
const { tabManager, transformBrowserToDisplayCoordinates, onCursorUpdate, onCursorHide, onLaunchRequest } = config;
|
|
32
27
|
|
|
33
|
-
//
|
|
34
|
-
let
|
|
35
|
-
isControlled: false,
|
|
36
|
-
controlledTabId: null,
|
|
37
|
-
browserSessionId: null,
|
|
38
|
-
startedAt: null
|
|
39
|
-
});
|
|
28
|
+
// Set of backend tab IDs (session IDs) currently controlled by MCP
|
|
29
|
+
let controlledSessionIds = $state(new Set<string>());
|
|
40
30
|
|
|
41
31
|
/**
|
|
42
32
|
* Setup WebSocket event listeners for MCP control events
|
|
@@ -44,7 +34,7 @@ export function createMcpHandler(config: McpHandlerConfig) {
|
|
|
44
34
|
function setupEventListeners() {
|
|
45
35
|
debug.log('preview', '🎧 Setting up MCP event listeners...');
|
|
46
36
|
|
|
47
|
-
// Listen for MCP control start/end events
|
|
37
|
+
// Listen for MCP control start/end events (per-tab)
|
|
48
38
|
ws.on('preview:browser-mcp-control-start', (data) => {
|
|
49
39
|
debug.log('preview', `📥 Received mcp-control-start:`, data);
|
|
50
40
|
handleControlStart(data);
|
|
@@ -93,60 +83,72 @@ export function createMcpHandler(config: McpHandlerConfig) {
|
|
|
93
83
|
}
|
|
94
84
|
|
|
95
85
|
/**
|
|
96
|
-
* Check if current tab is MCP controlled
|
|
86
|
+
* Check if current active tab is MCP controlled
|
|
97
87
|
*/
|
|
98
88
|
function isCurrentTabMcpControlled(): boolean {
|
|
99
|
-
|
|
89
|
+
const activeTab = tabManager.tabs.find(t => t.id === tabManager.activeTabId);
|
|
90
|
+
if (!activeTab?.sessionId) return false;
|
|
91
|
+
return controlledSessionIds.has(activeTab.sessionId);
|
|
100
92
|
}
|
|
101
93
|
|
|
102
94
|
/**
|
|
103
|
-
*
|
|
95
|
+
* Check if a specific frontend tab is MCP controlled (by sessionId)
|
|
104
96
|
*/
|
|
105
|
-
function
|
|
106
|
-
return
|
|
97
|
+
function isSessionControlled(sessionId: string): boolean {
|
|
98
|
+
return controlledSessionIds.has(sessionId);
|
|
107
99
|
}
|
|
108
100
|
|
|
109
|
-
|
|
101
|
+
/**
|
|
102
|
+
* Get set of frontend tab IDs that are MCP controlled
|
|
103
|
+
*/
|
|
104
|
+
function getControlledTabIds(): Set<string> {
|
|
105
|
+
const result = new Set<string>();
|
|
106
|
+
for (const tab of tabManager.tabs) {
|
|
107
|
+
if (tab.sessionId && controlledSessionIds.has(tab.sessionId)) {
|
|
108
|
+
result.add(tab.id);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return result;
|
|
112
|
+
}
|
|
110
113
|
|
|
111
|
-
|
|
112
|
-
debug.log('preview', `🎮 MCP control started for session: ${data.browserSessionId}`);
|
|
114
|
+
// Private handlers
|
|
113
115
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
+
function handleControlStart(data: { browserTabId: string; chatSessionId?: string; timestamp: number }) {
|
|
117
|
+
debug.log('preview', `🎮 MCP control started for tab: ${data.browserTabId}`);
|
|
116
118
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
controlledTabId: tab?.id || null,
|
|
120
|
-
browserSessionId: data.browserSessionId,
|
|
121
|
-
startedAt: data.timestamp
|
|
122
|
-
};
|
|
119
|
+
// Add to controlled set (reassign for Svelte reactivity)
|
|
120
|
+
controlledSessionIds = new Set([...controlledSessionIds, data.browserTabId]);
|
|
123
121
|
|
|
124
|
-
// Show toast
|
|
125
|
-
|
|
122
|
+
// Show toast only for the first controlled tab
|
|
123
|
+
if (controlledSessionIds.size === 1) {
|
|
124
|
+
showWarning('MCP Control Started', 'An MCP agent is now controlling the browser. User input is blocked.', 5000);
|
|
125
|
+
}
|
|
126
126
|
}
|
|
127
127
|
|
|
128
|
-
function handleControlEnd(data: {
|
|
129
|
-
debug.log('preview', `🎮 MCP control ended for
|
|
128
|
+
function handleControlEnd(data: { browserTabId: string; timestamp: number }) {
|
|
129
|
+
debug.log('preview', `🎮 MCP control ended for tab: ${data.browserTabId}`);
|
|
130
130
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
startedAt: null
|
|
136
|
-
};
|
|
131
|
+
// Remove from controlled set (reassign for Svelte reactivity)
|
|
132
|
+
const newSet = new Set(controlledSessionIds);
|
|
133
|
+
newSet.delete(data.browserTabId);
|
|
134
|
+
controlledSessionIds = newSet;
|
|
137
135
|
|
|
138
|
-
// Hide cursor
|
|
139
|
-
|
|
136
|
+
// Hide cursor if the released tab was the active one
|
|
137
|
+
const activeTab = tabManager.tabs.find(t => t.id === tabManager.activeTabId);
|
|
138
|
+
if (activeTab?.sessionId === data.browserTabId && onCursorHide) {
|
|
140
139
|
onCursorHide();
|
|
141
140
|
}
|
|
142
141
|
|
|
143
|
-
// Show toast
|
|
144
|
-
|
|
142
|
+
// Show toast when all tabs released
|
|
143
|
+
if (controlledSessionIds.size === 0) {
|
|
144
|
+
showInfo('MCP Control Ended', 'MCP agent released control. You can now interact with the browser.', 4000);
|
|
145
|
+
}
|
|
145
146
|
}
|
|
146
147
|
|
|
147
148
|
function handleCursorPosition(data: { sessionId: string; x: number; y: number; timestamp: number; source: 'mcp' }) {
|
|
148
|
-
// Only
|
|
149
|
-
|
|
149
|
+
// Only show cursor if this tab is controlled AND user is viewing it
|
|
150
|
+
const activeTab = tabManager.tabs.find(t => t.id === tabManager.activeTabId);
|
|
151
|
+
if (activeTab?.sessionId === data.sessionId && controlledSessionIds.has(data.sessionId) && transformBrowserToDisplayCoordinates) {
|
|
150
152
|
const transformedPosition = transformBrowserToDisplayCoordinates(data.x, data.y);
|
|
151
153
|
if (transformedPosition && onCursorUpdate) {
|
|
152
154
|
onCursorUpdate(transformedPosition.x, transformedPosition.y, false);
|
|
@@ -155,7 +157,9 @@ export function createMcpHandler(config: McpHandlerConfig) {
|
|
|
155
157
|
}
|
|
156
158
|
|
|
157
159
|
function handleCursorClick(data: { sessionId: string; x: number; y: number; timestamp: number; source: 'mcp' }) {
|
|
158
|
-
if
|
|
160
|
+
// Only show cursor click if this tab is controlled AND user is viewing it
|
|
161
|
+
const activeTab = tabManager.tabs.find(t => t.id === tabManager.activeTabId);
|
|
162
|
+
if (activeTab?.sessionId === data.sessionId && controlledSessionIds.has(data.sessionId) && transformBrowserToDisplayCoordinates) {
|
|
159
163
|
const transformedPosition = transformBrowserToDisplayCoordinates(data.x, data.y);
|
|
160
164
|
if (transformedPosition && onCursorUpdate) {
|
|
161
165
|
onCursorUpdate(transformedPosition.x, transformedPosition.y, true);
|
|
@@ -163,6 +167,26 @@ export function createMcpHandler(config: McpHandlerConfig) {
|
|
|
163
167
|
}
|
|
164
168
|
}
|
|
165
169
|
|
|
170
|
+
/**
|
|
171
|
+
* Restore MCP control state after session recovery (browser refresh or project switch)
|
|
172
|
+
* Called when recovered backend tab was previously MCP-controlled
|
|
173
|
+
*/
|
|
174
|
+
function restoreControlState(frontendTabId: string, browserSessionId: string): void {
|
|
175
|
+
debug.log('preview', `🔄 Restoring MCP control state for tab: ${frontendTabId} (session: ${browserSessionId})`);
|
|
176
|
+
controlledSessionIds = new Set([...controlledSessionIds, browserSessionId]);
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* Reset MCP control state (called before project switch recovery)
|
|
181
|
+
*/
|
|
182
|
+
function resetControlState(): void {
|
|
183
|
+
debug.log('preview', `🔄 Resetting MCP control state`);
|
|
184
|
+
controlledSessionIds = new Set();
|
|
185
|
+
if (onCursorHide) {
|
|
186
|
+
onCursorHide();
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
166
190
|
function handleTestCompleted(_data: { sessionId: string; timestamp: number; source: 'mcp' }) {
|
|
167
191
|
// Cursor is hidden via chat:complete / chat:cancelled listeners instead,
|
|
168
192
|
// because test-completed fires per-tool-call, not at end of full request.
|
|
@@ -296,8 +320,11 @@ export function createMcpHandler(config: McpHandlerConfig) {
|
|
|
296
320
|
return {
|
|
297
321
|
setupEventListeners,
|
|
298
322
|
isCurrentTabMcpControlled,
|
|
299
|
-
|
|
300
|
-
|
|
323
|
+
isSessionControlled,
|
|
324
|
+
getControlledTabIds,
|
|
325
|
+
restoreControlState,
|
|
326
|
+
resetControlState,
|
|
327
|
+
get controlledSessionIds() { return controlledSessionIds; }
|
|
301
328
|
};
|
|
302
329
|
}
|
|
303
330
|
|
|
@@ -481,6 +481,21 @@
|
|
|
481
481
|
{/if}
|
|
482
482
|
</div>
|
|
483
483
|
|
|
484
|
+
<!-- Touch mode toggle (scroll ↔ trackpad cursor) -->
|
|
485
|
+
<button
|
|
486
|
+
type="button"
|
|
487
|
+
class="flex items-center justify-center gap-1.5 {isMobile ? 'px-2 h-9' : 'px-1 h-6'} bg-transparent border-none rounded-md cursor-pointer transition-all duration-150 hover:bg-violet-500/10
|
|
488
|
+
{previewPanelRef?.panelActions?.getTouchMode() === 'cursor' ? 'text-violet-600 dark:text-violet-400' : 'text-slate-500 hover:text-slate-900 dark:hover:text-slate-100'}"
|
|
489
|
+
onclick={() => {
|
|
490
|
+
const current = previewPanelRef?.panelActions?.getTouchMode() || 'scroll';
|
|
491
|
+
previewPanelRef?.panelActions?.setTouchMode(current === 'scroll' ? 'cursor' : 'scroll');
|
|
492
|
+
}}
|
|
493
|
+
title={previewPanelRef?.panelActions?.getTouchMode() === 'cursor' ? 'Trackpad mode: 1-finger moves cursor, tap=click, 2-finger scroll/right-click' : 'Scroll mode: touch scrolls the page (tap to click)'}
|
|
494
|
+
>
|
|
495
|
+
<Icon name={previewPanelRef?.panelActions?.getTouchMode() === 'cursor' ? 'lucide:mouse-pointer-2' : 'lucide:pointer'} class={isMobile ? 'w-4.5 h-4.5' : 'w-4 h-4'} />
|
|
496
|
+
<span class="text-xs font-medium">{previewPanelRef?.panelActions?.getTouchMode() === 'cursor' ? 'Cursor' : 'Touch'}</span>
|
|
497
|
+
</button>
|
|
498
|
+
|
|
484
499
|
<!-- Rotation toggle -->
|
|
485
500
|
<button
|
|
486
501
|
type="button"
|
|
@@ -942,6 +942,16 @@
|
|
|
942
942
|
}
|
|
943
943
|
}
|
|
944
944
|
|
|
945
|
+
async function copyTagHash(hash: string, e: MouseEvent) {
|
|
946
|
+
e.stopPropagation();
|
|
947
|
+
try {
|
|
948
|
+
await navigator.clipboard.writeText(hash);
|
|
949
|
+
showInfo('Copied', `Hash ${hash.substring(0, 7)} copied to clipboard`);
|
|
950
|
+
} catch {
|
|
951
|
+
showError('Copy Failed', 'Could not copy to clipboard');
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
|
|
945
955
|
// ============================
|
|
946
956
|
// Lifecycle
|
|
947
957
|
// ============================
|
|
@@ -1323,8 +1333,8 @@
|
|
|
1323
1333
|
<div class="group flex items-center gap-2 px-2.5 py-2 rounded-md hover:bg-slate-100 dark:hover:bg-slate-800/60 transition-colors">
|
|
1324
1334
|
<Icon name="lucide:archive" class="w-4 h-4 text-slate-400 shrink-0" />
|
|
1325
1335
|
<div class="flex-1 min-w-0">
|
|
1326
|
-
<p class="text-
|
|
1327
|
-
<p class="text-
|
|
1336
|
+
<p class="text-sm font-medium text-slate-900 dark:text-slate-100 truncate">{entry.message}</p>
|
|
1337
|
+
<p class="text-xs text-slate-400 dark:text-slate-500">stash@{{entry.index}}</p>
|
|
1328
1338
|
</div>
|
|
1329
1339
|
<div class="flex items-center gap-0.5 shrink-0">
|
|
1330
1340
|
<button
|
|
@@ -1416,21 +1426,20 @@
|
|
|
1416
1426
|
<div class="space-y-1 px-1">
|
|
1417
1427
|
{#each tags as tag (tag.name)}
|
|
1418
1428
|
<div class="group flex items-center gap-2 px-2.5 py-2 rounded-md hover:bg-slate-100 dark:hover:bg-slate-800/60 transition-colors">
|
|
1419
|
-
<Icon
|
|
1420
|
-
name={tag.isAnnotated ? 'lucide:bookmark' : 'lucide:tag'}
|
|
1421
|
-
class="w-4 h-4 shrink-0 {tag.isAnnotated ? 'text-amber-500' : 'text-slate-400'}"
|
|
1422
|
-
/>
|
|
1429
|
+
<Icon name="lucide:tag" class="w-4 h-4 text-slate-400 shrink-0" />
|
|
1423
1430
|
<div class="flex-1 min-w-0">
|
|
1431
|
+
<p class="text-sm font-medium text-slate-900 dark:text-slate-100 truncate">{tag.name}</p>
|
|
1424
1432
|
<div class="flex items-center gap-1.5">
|
|
1425
|
-
<
|
|
1426
|
-
|
|
1427
|
-
|
|
1433
|
+
<button
|
|
1434
|
+
type="button"
|
|
1435
|
+
class="text-xs font-mono text-slate-400 dark:text-slate-500 hover:text-violet-600 dark:hover:text-violet-400 bg-transparent border-none cursor-pointer p-0 shrink-0 transition-colors"
|
|
1436
|
+
onclick={(e) => copyTagHash(tag.hash, e)}
|
|
1437
|
+
title="Copy tag hash"
|
|
1438
|
+
>{tag.hash.slice(0, 7)}</button>
|
|
1439
|
+
{#if tag.message}
|
|
1440
|
+
<span class="text-xs text-slate-400 dark:text-slate-500 truncate">{tag.message}</span>
|
|
1428
1441
|
{/if}
|
|
1429
1442
|
</div>
|
|
1430
|
-
{#if tag.message}
|
|
1431
|
-
<p class="text-3xs text-slate-500 dark:text-slate-400 truncate">{tag.message}</p>
|
|
1432
|
-
{/if}
|
|
1433
|
-
<p class="text-3xs text-slate-400 dark:text-slate-500 font-mono">{tag.hash}</p>
|
|
1434
1443
|
</div>
|
|
1435
1444
|
<div class="flex items-center gap-0.5 shrink-0">
|
|
1436
1445
|
<button
|
|
@@ -104,6 +104,8 @@
|
|
|
104
104
|
|
|
105
105
|
// Export actions for DesktopPanel header
|
|
106
106
|
export const panelActions = {
|
|
107
|
+
getTouchMode: () => browserPreviewRef?.browserActions?.getTouchMode() || 'scroll',
|
|
108
|
+
setTouchMode: (mode: 'scroll' | 'cursor') => { browserPreviewRef?.browserActions?.setTouchMode(mode); },
|
|
107
109
|
getDeviceSize: () => deviceSize,
|
|
108
110
|
getRotation: () => rotation,
|
|
109
111
|
getScale: () => previewDimensions?.scale || 1,
|
|
@@ -485,13 +485,9 @@ class ChatService {
|
|
|
485
485
|
// before a stale non-reasoning stream_event instead of at the end).
|
|
486
486
|
this.cleanupStreamEvents();
|
|
487
487
|
|
|
488
|
-
//
|
|
489
|
-
//
|
|
490
|
-
|
|
491
|
-
if (appState.isCancelling) {
|
|
492
|
-
appState.isCancelling = false;
|
|
493
|
-
}
|
|
494
|
-
}, 10000);
|
|
488
|
+
// No safety timeout needed — cancel completion is confirmed via WS events:
|
|
489
|
+
// chat:cancelled clears isLoading, then presence update clears isCancelling.
|
|
490
|
+
// If WS disconnects, reconnection logic re-fetches presence and clears state.
|
|
495
491
|
}
|
|
496
492
|
|
|
497
493
|
/**
|
|
@@ -164,7 +164,7 @@ export class BrowserWebCodecsService {
|
|
|
164
164
|
* Start WebCodecs streaming for a preview session
|
|
165
165
|
*/
|
|
166
166
|
async startStreaming(sessionId: string, canvas: HTMLCanvasElement): Promise<boolean> {
|
|
167
|
-
debug.log('webcodecs', `
|
|
167
|
+
debug.log('webcodecs', `[DIAG] startStreaming called: sessionId=${sessionId}, isConnected=${this.isConnected}, existingSessionId=${this.sessionId}`);
|
|
168
168
|
|
|
169
169
|
if (!BrowserWebCodecsService.isSupported()) {
|
|
170
170
|
debug.error('webcodecs', 'Not supported in this browser');
|
|
@@ -182,7 +182,11 @@ export class BrowserWebCodecsService {
|
|
|
182
182
|
this.audioContext = new AudioContext({ sampleRate: 48000 });
|
|
183
183
|
}
|
|
184
184
|
if (this.audioContext.state === 'suspended') {
|
|
185
|
-
await
|
|
185
|
+
// Fire-and-forget: don't await — after page refresh (no user gesture),
|
|
186
|
+
// resume() returns a promise that NEVER resolves until user interacts.
|
|
187
|
+
// Awaiting it would block streaming indefinitely. Audio will resume
|
|
188
|
+
// automatically on first user interaction via the safety net in playAudioFrame.
|
|
189
|
+
this.audioContext.resume().catch(() => {});
|
|
186
190
|
}
|
|
187
191
|
|
|
188
192
|
// Clean up any existing connection
|
|
@@ -204,7 +208,7 @@ export class BrowserWebCodecsService {
|
|
|
204
208
|
|
|
205
209
|
if (this.ctx) {
|
|
206
210
|
this.ctx.imageSmoothingEnabled = true;
|
|
207
|
-
this.ctx.imageSmoothingQuality = '
|
|
211
|
+
this.ctx.imageSmoothingQuality = 'medium';
|
|
208
212
|
}
|
|
209
213
|
|
|
210
214
|
this.clearCanvas();
|
|
@@ -214,7 +218,9 @@ export class BrowserWebCodecsService {
|
|
|
214
218
|
this.setupEventListeners();
|
|
215
219
|
|
|
216
220
|
// Request server to start streaming and get offer
|
|
221
|
+
debug.log('webcodecs', `[DIAG] Sending preview:browser-stream-start for session: ${sessionId}`);
|
|
217
222
|
const response = await ws.http('preview:browser-stream-start', {}, 30000);
|
|
223
|
+
debug.log('webcodecs', `[DIAG] preview:browser-stream-start response: success=${response.success}, hasOffer=${!!response.offer}, message=${response.message}`);
|
|
218
224
|
|
|
219
225
|
if (!response.success) {
|
|
220
226
|
throw new Error(response.message || 'Failed to start streaming');
|
|
@@ -225,12 +231,15 @@ export class BrowserWebCodecsService {
|
|
|
225
231
|
|
|
226
232
|
// Set remote description (offer)
|
|
227
233
|
if (response.offer) {
|
|
234
|
+
debug.log('webcodecs', `[DIAG] Using offer from stream-start response`);
|
|
228
235
|
await this.handleOffer({
|
|
229
236
|
type: response.offer.type as RTCSdpType,
|
|
230
237
|
sdp: response.offer.sdp
|
|
231
238
|
});
|
|
232
239
|
} else {
|
|
240
|
+
debug.log('webcodecs', `[DIAG] No offer in stream-start response, fetching via stream-offer`);
|
|
233
241
|
const offerResponse = await ws.http('preview:browser-stream-offer', {}, 10000);
|
|
242
|
+
debug.log('webcodecs', `[DIAG] preview:browser-stream-offer response: hasOffer=${!!offerResponse.offer}`);
|
|
234
243
|
if (offerResponse.offer) {
|
|
235
244
|
await this.handleOffer({
|
|
236
245
|
type: offerResponse.offer.type as RTCSdpType,
|
|
@@ -241,7 +250,7 @@ export class BrowserWebCodecsService {
|
|
|
241
250
|
}
|
|
242
251
|
}
|
|
243
252
|
|
|
244
|
-
debug.log('webcodecs', 'Streaming setup complete');
|
|
253
|
+
debug.log('webcodecs', '[DIAG] Streaming setup complete, waiting for ICE/DataChannel');
|
|
245
254
|
return true;
|
|
246
255
|
} catch (error) {
|
|
247
256
|
debug.error('webcodecs', 'Failed to start streaming:', error);
|
|
@@ -595,9 +604,9 @@ export class BrowserWebCodecsService {
|
|
|
595
604
|
this.audioContext = new AudioContext({ sampleRate: 48000 });
|
|
596
605
|
}
|
|
597
606
|
|
|
598
|
-
// Resume if suspended
|
|
607
|
+
// Resume if suspended — fire-and-forget, same reason as in startStreaming
|
|
599
608
|
if (this.audioContext.state === 'suspended') {
|
|
600
|
-
|
|
609
|
+
this.audioContext.resume().catch(() => {});
|
|
601
610
|
}
|
|
602
611
|
|
|
603
612
|
debug.log('webcodecs', `AudioContext initialized (state: ${this.audioContext.state})`);
|
|
@@ -748,33 +757,28 @@ export class BrowserWebCodecsService {
|
|
|
748
757
|
}
|
|
749
758
|
|
|
750
759
|
/**
|
|
751
|
-
* Play audio frame
|
|
752
|
-
*
|
|
753
|
-
* The key insight: Audio and video timestamps from the server use the same
|
|
754
|
-
* performance.now() origin. However, audio may start LATER than video if the
|
|
755
|
-
* page has no audio initially (silence is skipped).
|
|
760
|
+
* Play audio frame using simple back-to-back scheduling.
|
|
756
761
|
*
|
|
757
|
-
*
|
|
758
|
-
*
|
|
759
|
-
*
|
|
762
|
+
* Frames are scheduled immediately one after another. When a gap occurs
|
|
763
|
+
* (e.g. silence was skipped server-side and audio resumes), the schedule
|
|
764
|
+
* is reset with a 50ms lookahead so playback starts cleanly without
|
|
765
|
+
* audible pops or stutters from scheduling in the past.
|
|
760
766
|
*/
|
|
761
767
|
private playAudioFrame(audioData: AudioData): void {
|
|
762
768
|
if (!this.audioContext) return;
|
|
763
769
|
|
|
764
|
-
// Safety net: resume AudioContext if
|
|
770
|
+
// Safety net: resume AudioContext if suspended
|
|
765
771
|
if (this.audioContext.state === 'suspended') {
|
|
766
772
|
this.audioContext.resume().catch(() => {});
|
|
767
773
|
}
|
|
768
774
|
|
|
769
775
|
try {
|
|
770
|
-
// Create AudioBuffer
|
|
771
776
|
const buffer = this.audioContext.createBuffer(
|
|
772
777
|
audioData.numberOfChannels,
|
|
773
778
|
audioData.numberOfFrames,
|
|
774
779
|
audioData.sampleRate
|
|
775
780
|
);
|
|
776
781
|
|
|
777
|
-
// Copy audio data to buffer
|
|
778
782
|
for (let channel = 0; channel < audioData.numberOfChannels; channel++) {
|
|
779
783
|
const options = {
|
|
780
784
|
planeIndex: channel,
|
|
@@ -782,134 +786,27 @@ export class BrowserWebCodecsService {
|
|
|
782
786
|
frameCount: audioData.numberOfFrames,
|
|
783
787
|
format: 'f32-planar' as AudioSampleFormat
|
|
784
788
|
};
|
|
785
|
-
|
|
786
|
-
const
|
|
787
|
-
const tempBuffer = new ArrayBuffer(requiredSize);
|
|
788
|
-
const tempFloat32 = new Float32Array(tempBuffer);
|
|
789
|
+
// allocationSize() returns bytes — wrap in ArrayBuffer so Float32Array length is correct
|
|
790
|
+
const tempFloat32 = new Float32Array(new ArrayBuffer(audioData.allocationSize(options)));
|
|
789
791
|
audioData.copyTo(tempFloat32, options);
|
|
790
|
-
|
|
791
|
-
const channelData = buffer.getChannelData(channel);
|
|
792
|
-
channelData.set(tempFloat32);
|
|
792
|
+
buffer.getChannelData(channel).set(tempFloat32);
|
|
793
793
|
}
|
|
794
794
|
|
|
795
795
|
const currentTime = this.audioContext.currentTime;
|
|
796
|
-
const audioTimestamp = audioData.timestamp; // microseconds
|
|
797
|
-
const bufferDuration = buffer.duration;
|
|
798
|
-
const now = performance.now();
|
|
799
|
-
|
|
800
|
-
// Wait for video to establish sync before playing audio
|
|
801
|
-
if (!this.syncEstablished || this.lastVideoTimestamp === 0) {
|
|
802
|
-
// No video yet - skip this audio frame to prevent desync
|
|
803
|
-
return;
|
|
804
|
-
}
|
|
805
|
-
|
|
806
|
-
// Phase 1: Calibration - collect samples to determine stable offset
|
|
807
|
-
if (this.audioCalibrationSamples < this.CALIBRATION_SAMPLES) {
|
|
808
|
-
// Calculate the offset between audio timestamp and video timeline
|
|
809
|
-
// audioVideoOffset > 0 means audio is AHEAD of video in stream time
|
|
810
|
-
// audioVideoOffset < 0 means audio is BEHIND video in stream time
|
|
811
|
-
const audioVideoOffset = (audioTimestamp - this.lastVideoTimestamp) / 1000000; // seconds
|
|
812
|
-
|
|
813
|
-
// Also account for the time elapsed since video was rendered
|
|
814
|
-
const timeSinceVideoRender = (now - this.lastVideoRealTime) / 1000; // seconds
|
|
815
|
-
|
|
816
|
-
// Expected audio position relative to current video position
|
|
817
|
-
// If audio and video are in sync, audio should play at:
|
|
818
|
-
// currentTime + audioVideoOffset - timeSinceVideoRender
|
|
819
|
-
const expectedOffset = audioVideoOffset - timeSinceVideoRender;
|
|
820
|
-
|
|
821
|
-
this.audioOffsetAccumulator += expectedOffset;
|
|
822
|
-
this.audioCalibrationSamples++;
|
|
823
|
-
|
|
824
|
-
if (this.audioCalibrationSamples === this.CALIBRATION_SAMPLES) {
|
|
825
|
-
// Calibration complete - calculate average offset
|
|
826
|
-
this.calibratedAudioOffset = this.audioOffsetAccumulator / this.CALIBRATION_SAMPLES;
|
|
827
|
-
|
|
828
|
-
// Clamp the offset to reasonable bounds (-500ms to +500ms)
|
|
829
|
-
// Beyond this, something is wrong and we should just play immediately
|
|
830
|
-
if (this.calibratedAudioOffset < -0.5) {
|
|
831
|
-
debug.warn('webcodecs', `Audio calibration: offset ${(this.calibratedAudioOffset * 1000).toFixed(0)}ms too negative, clamping to -500ms`);
|
|
832
|
-
this.calibratedAudioOffset = -0.5;
|
|
833
|
-
} else if (this.calibratedAudioOffset > 0.5) {
|
|
834
|
-
debug.warn('webcodecs', `Audio calibration: offset ${(this.calibratedAudioOffset * 1000).toFixed(0)}ms too positive, clamping to +500ms`);
|
|
835
|
-
this.calibratedAudioOffset = 0.5;
|
|
836
|
-
}
|
|
837
|
-
|
|
838
|
-
// Initialize nextAudioPlayTime based on calibrated offset
|
|
839
|
-
// Add small buffer (30ms) for smooth playback
|
|
840
|
-
this.nextAudioPlayTime = currentTime + Math.max(0.03, this.calibratedAudioOffset + 0.03);
|
|
841
|
-
this.audioSyncInitialized = true;
|
|
842
|
-
|
|
843
|
-
debug.log('webcodecs', `Audio calibration complete: offset=${(this.calibratedAudioOffset * 1000).toFixed(1)}ms, startTime=${(this.nextAudioPlayTime - currentTime).toFixed(3)}s from now`);
|
|
844
|
-
} else {
|
|
845
|
-
// Still calibrating - skip this audio frame
|
|
846
|
-
return;
|
|
847
|
-
}
|
|
848
|
-
}
|
|
849
796
|
|
|
850
|
-
//
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
if (targetPlayTime < currentTime - 0.01) {
|
|
855
|
-
// We're behind by more than 10ms, need to catch up
|
|
856
|
-
targetPlayTime = currentTime + 0.02; // Small buffer to recover
|
|
857
|
-
debug.warn('webcodecs', 'Audio buffer underrun, resetting playback');
|
|
858
|
-
}
|
|
859
|
-
|
|
860
|
-
// Periodic drift check (every ~500ms worth of audio, ~25 frames)
|
|
861
|
-
if (this.stats.audioFramesDecoded % 25 === 0) {
|
|
862
|
-
// Recalculate where audio SHOULD be based on current video position
|
|
863
|
-
const audioVideoOffset = (audioTimestamp - this.lastVideoTimestamp) / 1000000;
|
|
864
|
-
const timeSinceVideoRender = (now - this.lastVideoRealTime) / 1000;
|
|
865
|
-
const expectedOffset = audioVideoOffset - timeSinceVideoRender;
|
|
866
|
-
|
|
867
|
-
// Where audio should play relative to currentTime
|
|
868
|
-
const idealTime = currentTime + expectedOffset + 0.03; // +30ms buffer
|
|
869
|
-
|
|
870
|
-
const drift = targetPlayTime - idealTime;
|
|
871
|
-
|
|
872
|
-
// Apply correction based on drift magnitude
|
|
873
|
-
if (Math.abs(drift) > 0.2) {
|
|
874
|
-
// Large drift (>200ms) - aggressive correction (80%)
|
|
875
|
-
targetPlayTime -= drift * 0.8;
|
|
876
|
-
debug.warn('webcodecs', `Large audio drift: ${(drift * 1000).toFixed(0)}ms, aggressive correction`);
|
|
877
|
-
} else if (Math.abs(drift) > 0.05) {
|
|
878
|
-
// Medium drift (50-200ms) - moderate correction (40%)
|
|
879
|
-
targetPlayTime -= drift * 0.4;
|
|
880
|
-
}
|
|
881
|
-
// Small drift (<50ms) - no correction, continuous playback handles it
|
|
882
|
-
}
|
|
883
|
-
|
|
884
|
-
// Ensure we don't schedule in the past
|
|
885
|
-
if (targetPlayTime < currentTime + 0.005) {
|
|
886
|
-
targetPlayTime = currentTime + 0.005;
|
|
797
|
+
// When the scheduler has fallen behind (gap due to silence or decode delay),
|
|
798
|
+
// reset with a 50ms lookahead so the next chunk starts cleanly.
|
|
799
|
+
if (this.nextAudioPlayTime < currentTime) {
|
|
800
|
+
this.nextAudioPlayTime = currentTime + 0.05;
|
|
887
801
|
}
|
|
888
802
|
|
|
889
|
-
// Schedule this buffer
|
|
890
803
|
const source = this.audioContext.createBufferSource();
|
|
891
804
|
source.buffer = buffer;
|
|
892
805
|
source.connect(this.audioContext.destination);
|
|
893
|
-
source.start(
|
|
806
|
+
source.start(this.nextAudioPlayTime);
|
|
894
807
|
|
|
895
|
-
//
|
|
896
|
-
this.
|
|
897
|
-
|
|
898
|
-
// Update next play time for continuous scheduling (back-to-back)
|
|
899
|
-
this.nextAudioPlayTime = targetPlayTime + bufferDuration;
|
|
900
|
-
|
|
901
|
-
// Limit queue size to prevent memory buildup
|
|
902
|
-
if (this.audioBufferQueue.length > this.maxAudioQueueSize) {
|
|
903
|
-
this.audioBufferQueue.shift();
|
|
904
|
-
}
|
|
905
|
-
|
|
906
|
-
// Cleanup old scheduled buffers
|
|
907
|
-
source.onended = () => {
|
|
908
|
-
const index = this.audioBufferQueue.findIndex(item => item.buffer === buffer);
|
|
909
|
-
if (index !== -1) {
|
|
910
|
-
this.audioBufferQueue.splice(index, 1);
|
|
911
|
-
}
|
|
912
|
-
};
|
|
808
|
+
// Back-to-back: next chunk plays immediately after this one ends
|
|
809
|
+
this.nextAudioPlayTime += buffer.duration;
|
|
913
810
|
} catch (error) {
|
|
914
811
|
debug.warn('webcodecs', 'Audio playback error:', error);
|
|
915
812
|
}
|
|
@@ -1106,7 +1003,7 @@ export class BrowserWebCodecsService {
|
|
|
1106
1003
|
|
|
1107
1004
|
if (this.ctx) {
|
|
1108
1005
|
this.ctx.imageSmoothingEnabled = true;
|
|
1109
|
-
this.ctx.imageSmoothingQuality = '
|
|
1006
|
+
this.ctx.imageSmoothingQuality = 'medium';
|
|
1110
1007
|
}
|
|
1111
1008
|
|
|
1112
1009
|
try {
|
|
@@ -214,10 +214,11 @@ export function isSessionUnread(sessionId: string): boolean {
|
|
|
214
214
|
|
|
215
215
|
/**
|
|
216
216
|
* Check if a project has any unread sessions.
|
|
217
|
+
* Optionally exclude a specific session (e.g. the currently viewed one).
|
|
217
218
|
*/
|
|
218
|
-
export function hasUnreadSessionsForProject(projectId: string): boolean {
|
|
219
|
-
for (const pId of appState.unreadSessions.
|
|
220
|
-
if (pId === projectId) return true;
|
|
219
|
+
export function hasUnreadSessionsForProject(projectId: string, excludeSessionId?: string): boolean {
|
|
220
|
+
for (const [sId, pId] of appState.unreadSessions.entries()) {
|
|
221
|
+
if (pId === projectId && sId !== excludeSessionId) return true;
|
|
221
222
|
}
|
|
222
223
|
return false;
|
|
223
224
|
}
|
|
@@ -119,8 +119,9 @@ export function getProjectStatusColor(projectId: string): string {
|
|
|
119
119
|
return 'bg-amber-500';
|
|
120
120
|
}
|
|
121
121
|
|
|
122
|
-
// Check for unread sessions in this project
|
|
123
|
-
|
|
122
|
+
// Check for unread sessions in this project, excluding the currently viewed session
|
|
123
|
+
const currentSessionId = sessionState.currentSession?.id;
|
|
124
|
+
if (hasUnreadSessionsForProject(projectId, currentSessionId)) return 'bg-blue-500';
|
|
124
125
|
|
|
125
126
|
return 'bg-slate-500/30';
|
|
126
127
|
}
|
|
@@ -295,6 +295,8 @@ export async function loadSessions() {
|
|
|
295
295
|
// wiping out any stream_event injected by catchup.
|
|
296
296
|
await loadMessagesForSession(targetSession.id);
|
|
297
297
|
sessionState.currentSession = targetSession;
|
|
298
|
+
// Clear unread status — user is actively viewing this session
|
|
299
|
+
markSessionRead(targetSession.id);
|
|
298
300
|
// Join chat session room so we receive session-scoped events
|
|
299
301
|
// (stream, input sync, edit mode, model sync).
|
|
300
302
|
// Critical after refresh — without it, connection misses all events.
|
|
@@ -6,6 +6,9 @@ export const notificationStore = $state({
|
|
|
6
6
|
maxNotifications: 5
|
|
7
7
|
});
|
|
8
8
|
|
|
9
|
+
// Monotonic counter for unique notification IDs
|
|
10
|
+
let _notificationCounter = 0;
|
|
11
|
+
|
|
9
12
|
// Derived values as functions (cannot export derived state from modules)
|
|
10
13
|
export function hasNotifications() {
|
|
11
14
|
return notificationStore.notifications.length > 0;
|
|
@@ -18,7 +21,7 @@ export function notificationCount() {
|
|
|
18
21
|
// Notification management functions
|
|
19
22
|
export function addNotification(notification: Omit<ToastNotification, 'id'>) {
|
|
20
23
|
const newNotification: ToastNotification = {
|
|
21
|
-
id: Date.now()
|
|
24
|
+
id: `${Date.now()}-${++_notificationCounter}`,
|
|
22
25
|
...notification,
|
|
23
26
|
duration: notification.duration || 5000 // Default 5 seconds
|
|
24
27
|
};
|