@myrialabs/clopen 0.2.6 → 0.2.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/backend/chat/stream-manager.ts +24 -13
  2. package/backend/engine/adapters/claude/stream.ts +10 -19
  3. package/backend/mcp/project-context.ts +20 -0
  4. package/backend/mcp/servers/browser-automation/actions.ts +0 -2
  5. package/backend/mcp/servers/browser-automation/browser.ts +86 -132
  6. package/backend/mcp/servers/browser-automation/inspection.ts +5 -11
  7. package/backend/preview/browser/browser-mcp-control.ts +175 -180
  8. package/backend/preview/browser/browser-pool.ts +3 -1
  9. package/backend/preview/browser/browser-preview-service.ts +3 -3
  10. package/backend/preview/browser/browser-tab-manager.ts +1 -1
  11. package/backend/preview/browser/browser-video-capture.ts +12 -14
  12. package/backend/preview/browser/scripts/audio-stream.ts +11 -0
  13. package/backend/preview/browser/scripts/video-stream.ts +14 -14
  14. package/backend/preview/browser/types.ts +7 -7
  15. package/backend/preview/index.ts +1 -1
  16. package/backend/ws/chat/stream.ts +1 -1
  17. package/backend/ws/preview/browser/tab-info.ts +5 -2
  18. package/backend/ws/preview/index.ts +3 -3
  19. package/frontend/components/chat/input/ChatInput.svelte +0 -3
  20. package/frontend/components/chat/input/composables/use-chat-actions.svelte.ts +6 -2
  21. package/frontend/components/chat/message/MessageBubble.svelte +2 -2
  22. package/frontend/components/history/HistoryModal.svelte +1 -1
  23. package/frontend/components/preview/browser/BrowserPreview.svelte +15 -1
  24. package/frontend/components/preview/browser/components/Canvas.svelte +323 -49
  25. package/frontend/components/preview/browser/components/Container.svelte +21 -0
  26. package/frontend/components/preview/browser/components/Toolbar.svelte +3 -3
  27. package/frontend/components/preview/browser/core/coordinator.svelte.ts +15 -0
  28. package/frontend/components/preview/browser/core/mcp-handlers.svelte.ts +78 -51
  29. package/frontend/components/preview/browser/core/tab-operations.svelte.ts +1 -0
  30. package/frontend/components/workspace/PanelHeader.svelte +15 -0
  31. package/frontend/components/workspace/panels/GitPanel.svelte +22 -13
  32. package/frontend/components/workspace/panels/PreviewPanel.svelte +2 -0
  33. package/frontend/services/chat/chat.service.ts +3 -7
  34. package/frontend/services/preview/browser/browser-webcodecs.service.ts +32 -135
  35. package/frontend/stores/core/app.svelte.ts +4 -3
  36. package/frontend/stores/core/presence.svelte.ts +3 -2
  37. package/frontend/stores/core/sessions.svelte.ts +2 -0
  38. package/frontend/stores/ui/notification.svelte.ts +4 -1
  39. package/package.json +1 -1
@@ -1,6 +1,9 @@
1
1
  /**
2
2
  * Browser MCP Event Handlers
3
3
  * Handles MCP (Model Context Protocol) control events for BrowserPreview
4
+ *
5
+ * Supports multi-tab control: each chat session can control multiple tabs.
6
+ * Tracks controlled tabs via a Set of backend tab IDs (session IDs).
4
7
  */
5
8
 
6
9
  import { debug } from '$shared/utils/logger';
@@ -8,14 +11,6 @@ import { showInfo, showWarning } from '$frontend/stores/ui/notification.svelte';
8
11
  import ws from '$frontend/utils/ws';
9
12
  import type { TabManager } from './tab-manager.svelte';
10
13
 
11
- // MCP Control State interface
12
- export interface McpControlState {
13
- isControlled: boolean;
14
- controlledTabId: string | null;
15
- browserSessionId: string | null;
16
- startedAt: number | null;
17
- }
18
-
19
14
  export interface McpHandlerConfig {
20
15
  tabManager: TabManager;
21
16
  transformBrowserToDisplayCoordinates?: (x: number, y: number) => { x: number, y: number } | null;
@@ -30,13 +25,8 @@ export interface McpHandlerConfig {
30
25
  export function createMcpHandler(config: McpHandlerConfig) {
31
26
  const { tabManager, transformBrowserToDisplayCoordinates, onCursorUpdate, onCursorHide, onLaunchRequest } = config;
32
27
 
33
- // MCP control state
34
- let mcpControlState = $state<McpControlState>({
35
- isControlled: false,
36
- controlledTabId: null,
37
- browserSessionId: null,
38
- startedAt: null
39
- });
28
+ // Set of backend tab IDs (session IDs) currently controlled by MCP
29
+ let controlledSessionIds = $state(new Set<string>());
40
30
 
41
31
  /**
42
32
  * Setup WebSocket event listeners for MCP control events
@@ -44,7 +34,7 @@ export function createMcpHandler(config: McpHandlerConfig) {
44
34
  function setupEventListeners() {
45
35
  debug.log('preview', '🎧 Setting up MCP event listeners...');
46
36
 
47
- // Listen for MCP control start/end events
37
+ // Listen for MCP control start/end events (per-tab)
48
38
  ws.on('preview:browser-mcp-control-start', (data) => {
49
39
  debug.log('preview', `📥 Received mcp-control-start:`, data);
50
40
  handleControlStart(data);
@@ -93,60 +83,72 @@ export function createMcpHandler(config: McpHandlerConfig) {
93
83
  }
94
84
 
95
85
  /**
96
- * Check if current tab is MCP controlled
86
+ * Check if current active tab is MCP controlled
97
87
  */
98
88
  function isCurrentTabMcpControlled(): boolean {
99
- return mcpControlState.isControlled && mcpControlState.controlledTabId === tabManager.activeTabId;
89
+ const activeTab = tabManager.tabs.find(t => t.id === tabManager.activeTabId);
90
+ if (!activeTab?.sessionId) return false;
91
+ return controlledSessionIds.has(activeTab.sessionId);
100
92
  }
101
93
 
102
94
  /**
103
- * Get MCP control state
95
+ * Check if a specific frontend tab is MCP controlled (by sessionId)
104
96
  */
105
- function getControlState(): McpControlState {
106
- return mcpControlState;
97
+ function isSessionControlled(sessionId: string): boolean {
98
+ return controlledSessionIds.has(sessionId);
107
99
  }
108
100
 
109
- // Private handlers
101
+ /**
102
+ * Get set of frontend tab IDs that are MCP controlled
103
+ */
104
+ function getControlledTabIds(): Set<string> {
105
+ const result = new Set<string>();
106
+ for (const tab of tabManager.tabs) {
107
+ if (tab.sessionId && controlledSessionIds.has(tab.sessionId)) {
108
+ result.add(tab.id);
109
+ }
110
+ }
111
+ return result;
112
+ }
110
113
 
111
- function handleControlStart(data: { browserSessionId: string; mcpSessionId?: string; timestamp: number }) {
112
- debug.log('preview', `🎮 MCP control started for session: ${data.browserSessionId}`);
114
+ // Private handlers
113
115
 
114
- // Find which tab has this session
115
- const tab = tabManager.tabs.find(t => t.sessionId === data.browserSessionId);
116
+ function handleControlStart(data: { browserTabId: string; chatSessionId?: string; timestamp: number }) {
117
+ debug.log('preview', `🎮 MCP control started for tab: ${data.browserTabId}`);
116
118
 
117
- mcpControlState = {
118
- isControlled: true,
119
- controlledTabId: tab?.id || null,
120
- browserSessionId: data.browserSessionId,
121
- startedAt: data.timestamp
122
- };
119
+ // Add to controlled set (reassign for Svelte reactivity)
120
+ controlledSessionIds = new Set([...controlledSessionIds, data.browserTabId]);
123
121
 
124
- // Show toast notification
125
- showWarning('MCP Control Started', 'An MCP agent is now controlling the browser. User input is blocked.', 5000);
122
+ // Show toast only for the first controlled tab
123
+ if (controlledSessionIds.size === 1) {
124
+ showWarning('MCP Control Started', 'An MCP agent is now controlling the browser. User input is blocked.', 5000);
125
+ }
126
126
  }
127
127
 
128
- function handleControlEnd(data: { browserSessionId: string; timestamp: number }) {
129
- debug.log('preview', `🎮 MCP control ended for session: ${data.browserSessionId}`);
128
+ function handleControlEnd(data: { browserTabId: string; timestamp: number }) {
129
+ debug.log('preview', `🎮 MCP control ended for tab: ${data.browserTabId}`);
130
130
 
131
- mcpControlState = {
132
- isControlled: false,
133
- controlledTabId: null,
134
- browserSessionId: null,
135
- startedAt: null
136
- };
131
+ // Remove from controlled set (reassign for Svelte reactivity)
132
+ const newSet = new Set(controlledSessionIds);
133
+ newSet.delete(data.browserTabId);
134
+ controlledSessionIds = newSet;
137
135
 
138
- // Hide cursor
139
- if (onCursorHide) {
136
+ // Hide cursor if the released tab was the active one
137
+ const activeTab = tabManager.tabs.find(t => t.id === tabManager.activeTabId);
138
+ if (activeTab?.sessionId === data.browserTabId && onCursorHide) {
140
139
  onCursorHide();
141
140
  }
142
141
 
143
- // Show toast notification
144
- showInfo('MCP Control Ended', 'MCP agent released control. You can now interact with the browser.', 4000);
142
+ // Show toast when all tabs released
143
+ if (controlledSessionIds.size === 0) {
144
+ showInfo('MCP Control Ended', 'MCP agent released control. You can now interact with the browser.', 4000);
145
+ }
145
146
  }
146
147
 
147
148
  function handleCursorPosition(data: { sessionId: string; x: number; y: number; timestamp: number; source: 'mcp' }) {
148
- // Only update cursor if this is for the active session and MCP is controlling
149
- if (mcpControlState.isControlled && mcpControlState.browserSessionId === data.sessionId && transformBrowserToDisplayCoordinates) {
149
+ // Only show cursor if this tab is controlled AND user is viewing it
150
+ const activeTab = tabManager.tabs.find(t => t.id === tabManager.activeTabId);
151
+ if (activeTab?.sessionId === data.sessionId && controlledSessionIds.has(data.sessionId) && transformBrowserToDisplayCoordinates) {
150
152
  const transformedPosition = transformBrowserToDisplayCoordinates(data.x, data.y);
151
153
  if (transformedPosition && onCursorUpdate) {
152
154
  onCursorUpdate(transformedPosition.x, transformedPosition.y, false);
@@ -155,7 +157,9 @@ export function createMcpHandler(config: McpHandlerConfig) {
155
157
  }
156
158
 
157
159
  function handleCursorClick(data: { sessionId: string; x: number; y: number; timestamp: number; source: 'mcp' }) {
158
- if (mcpControlState.isControlled && mcpControlState.browserSessionId === data.sessionId && transformBrowserToDisplayCoordinates) {
160
+ // Only show cursor click if this tab is controlled AND user is viewing it
161
+ const activeTab = tabManager.tabs.find(t => t.id === tabManager.activeTabId);
162
+ if (activeTab?.sessionId === data.sessionId && controlledSessionIds.has(data.sessionId) && transformBrowserToDisplayCoordinates) {
159
163
  const transformedPosition = transformBrowserToDisplayCoordinates(data.x, data.y);
160
164
  if (transformedPosition && onCursorUpdate) {
161
165
  onCursorUpdate(transformedPosition.x, transformedPosition.y, true);
@@ -163,6 +167,26 @@ export function createMcpHandler(config: McpHandlerConfig) {
163
167
  }
164
168
  }
165
169
 
170
+ /**
171
+ * Restore MCP control state after session recovery (browser refresh or project switch)
172
+ * Called when recovered backend tab was previously MCP-controlled
173
+ */
174
+ function restoreControlState(frontendTabId: string, browserSessionId: string): void {
175
+ debug.log('preview', `🔄 Restoring MCP control state for tab: ${frontendTabId} (session: ${browserSessionId})`);
176
+ controlledSessionIds = new Set([...controlledSessionIds, browserSessionId]);
177
+ }
178
+
179
+ /**
180
+ * Reset MCP control state (called before project switch recovery)
181
+ */
182
+ function resetControlState(): void {
183
+ debug.log('preview', `🔄 Resetting MCP control state`);
184
+ controlledSessionIds = new Set();
185
+ if (onCursorHide) {
186
+ onCursorHide();
187
+ }
188
+ }
189
+
166
190
  function handleTestCompleted(_data: { sessionId: string; timestamp: number; source: 'mcp' }) {
167
191
  // Cursor is hidden via chat:complete / chat:cancelled listeners instead,
168
192
  // because test-completed fires per-tool-call, not at end of full request.
@@ -296,8 +320,11 @@ export function createMcpHandler(config: McpHandlerConfig) {
296
320
  return {
297
321
  setupEventListeners,
298
322
  isCurrentTabMcpControlled,
299
- getControlState,
300
- get mcpControlState() { return mcpControlState; }
323
+ isSessionControlled,
324
+ getControlledTabIds,
325
+ restoreControlState,
326
+ resetControlState,
327
+ get controlledSessionIds() { return controlledSessionIds; }
301
328
  };
302
329
  }
303
330
 
@@ -44,6 +44,7 @@ export interface ExistingTabInfo {
44
44
  deviceSize: string;
45
45
  rotation: string;
46
46
  isActive: boolean;
47
+ isMcpControlled?: boolean;
47
48
  }
48
49
 
49
50
  export interface ExistingTabsResult {
@@ -481,6 +481,21 @@
481
481
  {/if}
482
482
  </div>
483
483
 
484
+ <!-- Touch mode toggle (scroll ↔ trackpad cursor) -->
485
+ <button
486
+ type="button"
487
+ class="flex items-center justify-center gap-1.5 {isMobile ? 'px-2 h-9' : 'px-1 h-6'} bg-transparent border-none rounded-md cursor-pointer transition-all duration-150 hover:bg-violet-500/10
488
+ {previewPanelRef?.panelActions?.getTouchMode() === 'cursor' ? 'text-violet-600 dark:text-violet-400' : 'text-slate-500 hover:text-slate-900 dark:hover:text-slate-100'}"
489
+ onclick={() => {
490
+ const current = previewPanelRef?.panelActions?.getTouchMode() || 'scroll';
491
+ previewPanelRef?.panelActions?.setTouchMode(current === 'scroll' ? 'cursor' : 'scroll');
492
+ }}
493
+ title={previewPanelRef?.panelActions?.getTouchMode() === 'cursor' ? 'Trackpad mode: 1-finger moves cursor, tap=click, 2-finger scroll/right-click' : 'Scroll mode: touch scrolls the page (tap to click)'}
494
+ >
495
+ <Icon name={previewPanelRef?.panelActions?.getTouchMode() === 'cursor' ? 'lucide:mouse-pointer-2' : 'lucide:pointer'} class={isMobile ? 'w-4.5 h-4.5' : 'w-4 h-4'} />
496
+ <span class="text-xs font-medium">{previewPanelRef?.panelActions?.getTouchMode() === 'cursor' ? 'Cursor' : 'Touch'}</span>
497
+ </button>
498
+
484
499
  <!-- Rotation toggle -->
485
500
  <button
486
501
  type="button"
@@ -942,6 +942,16 @@
942
942
  }
943
943
  }
944
944
 
945
+ async function copyTagHash(hash: string, e: MouseEvent) {
946
+ e.stopPropagation();
947
+ try {
948
+ await navigator.clipboard.writeText(hash);
949
+ showInfo('Copied', `Hash ${hash.substring(0, 7)} copied to clipboard`);
950
+ } catch {
951
+ showError('Copy Failed', 'Could not copy to clipboard');
952
+ }
953
+ }
954
+
945
955
  // ============================
946
956
  // Lifecycle
947
957
  // ============================
@@ -1323,8 +1333,8 @@
1323
1333
  <div class="group flex items-center gap-2 px-2.5 py-2 rounded-md hover:bg-slate-100 dark:hover:bg-slate-800/60 transition-colors">
1324
1334
  <Icon name="lucide:archive" class="w-4 h-4 text-slate-400 shrink-0" />
1325
1335
  <div class="flex-1 min-w-0">
1326
- <p class="text-xs font-medium text-slate-900 dark:text-slate-100 truncate">{entry.message}</p>
1327
- <p class="text-3xs text-slate-400 dark:text-slate-500">stash@&#123;{entry.index}&#125;</p>
1336
+ <p class="text-sm font-medium text-slate-900 dark:text-slate-100 truncate">{entry.message}</p>
1337
+ <p class="text-xs text-slate-400 dark:text-slate-500">stash@&#123;{entry.index}&#125;</p>
1328
1338
  </div>
1329
1339
  <div class="flex items-center gap-0.5 shrink-0">
1330
1340
  <button
@@ -1416,21 +1426,20 @@
1416
1426
  <div class="space-y-1 px-1">
1417
1427
  {#each tags as tag (tag.name)}
1418
1428
  <div class="group flex items-center gap-2 px-2.5 py-2 rounded-md hover:bg-slate-100 dark:hover:bg-slate-800/60 transition-colors">
1419
- <Icon
1420
- name={tag.isAnnotated ? 'lucide:bookmark' : 'lucide:tag'}
1421
- class="w-4 h-4 shrink-0 {tag.isAnnotated ? 'text-amber-500' : 'text-slate-400'}"
1422
- />
1429
+ <Icon name="lucide:tag" class="w-4 h-4 text-slate-400 shrink-0" />
1423
1430
  <div class="flex-1 min-w-0">
1431
+ <p class="text-sm font-medium text-slate-900 dark:text-slate-100 truncate">{tag.name}</p>
1424
1432
  <div class="flex items-center gap-1.5">
1425
- <p class="text-xs font-medium text-slate-900 dark:text-slate-100 truncate">{tag.name}</p>
1426
- {#if tag.isAnnotated}
1427
- <span class="text-3xs px-1 py-0.5 rounded bg-amber-500/10 text-amber-600 dark:text-amber-400 shrink-0">annotated</span>
1433
+ <button
1434
+ type="button"
1435
+ class="text-xs font-mono text-slate-400 dark:text-slate-500 hover:text-violet-600 dark:hover:text-violet-400 bg-transparent border-none cursor-pointer p-0 shrink-0 transition-colors"
1436
+ onclick={(e) => copyTagHash(tag.hash, e)}
1437
+ title="Copy tag hash"
1438
+ >{tag.hash.slice(0, 7)}</button>
1439
+ {#if tag.message}
1440
+ <span class="text-xs text-slate-400 dark:text-slate-500 truncate">{tag.message}</span>
1428
1441
  {/if}
1429
1442
  </div>
1430
- {#if tag.message}
1431
- <p class="text-3xs text-slate-500 dark:text-slate-400 truncate">{tag.message}</p>
1432
- {/if}
1433
- <p class="text-3xs text-slate-400 dark:text-slate-500 font-mono">{tag.hash}</p>
1434
1443
  </div>
1435
1444
  <div class="flex items-center gap-0.5 shrink-0">
1436
1445
  <button
@@ -104,6 +104,8 @@
104
104
 
105
105
  // Export actions for DesktopPanel header
106
106
  export const panelActions = {
107
+ getTouchMode: () => browserPreviewRef?.browserActions?.getTouchMode() || 'scroll',
108
+ setTouchMode: (mode: 'scroll' | 'cursor') => { browserPreviewRef?.browserActions?.setTouchMode(mode); },
107
109
  getDeviceSize: () => deviceSize,
108
110
  getRotation: () => rotation,
109
111
  getScale: () => previewDimensions?.scale || 1,
@@ -485,13 +485,9 @@ class ChatService {
485
485
  // before a stale non-reasoning stream_event instead of at the end).
486
486
  this.cleanupStreamEvents();
487
487
 
488
- // Safety timeout: clear isCancelling after 10s if WS confirmation never arrives
489
- // (e.g., network issues, dropped connection)
490
- setTimeout(() => {
491
- if (appState.isCancelling) {
492
- appState.isCancelling = false;
493
- }
494
- }, 10000);
488
+ // No safety timeout needed cancel completion is confirmed via WS events:
489
+ // chat:cancelled clears isLoading, then presence update clears isCancelling.
490
+ // If WS disconnects, reconnection logic re-fetches presence and clears state.
495
491
  }
496
492
 
497
493
  /**
@@ -164,7 +164,7 @@ export class BrowserWebCodecsService {
164
164
  * Start WebCodecs streaming for a preview session
165
165
  */
166
166
  async startStreaming(sessionId: string, canvas: HTMLCanvasElement): Promise<boolean> {
167
- debug.log('webcodecs', `Starting streaming for session: ${sessionId}`);
167
+ debug.log('webcodecs', `[DIAG] startStreaming called: sessionId=${sessionId}, isConnected=${this.isConnected}, existingSessionId=${this.sessionId}`);
168
168
 
169
169
  if (!BrowserWebCodecsService.isSupported()) {
170
170
  debug.error('webcodecs', 'Not supported in this browser');
@@ -182,7 +182,11 @@ export class BrowserWebCodecsService {
182
182
  this.audioContext = new AudioContext({ sampleRate: 48000 });
183
183
  }
184
184
  if (this.audioContext.state === 'suspended') {
185
- await this.audioContext.resume().catch(() => {});
185
+ // Fire-and-forget: don't await — after page refresh (no user gesture),
186
+ // resume() returns a promise that NEVER resolves until user interacts.
187
+ // Awaiting it would block streaming indefinitely. Audio will resume
188
+ // automatically on first user interaction via the safety net in playAudioFrame.
189
+ this.audioContext.resume().catch(() => {});
186
190
  }
187
191
 
188
192
  // Clean up any existing connection
@@ -204,7 +208,7 @@ export class BrowserWebCodecsService {
204
208
 
205
209
  if (this.ctx) {
206
210
  this.ctx.imageSmoothingEnabled = true;
207
- this.ctx.imageSmoothingQuality = 'low';
211
+ this.ctx.imageSmoothingQuality = 'medium';
208
212
  }
209
213
 
210
214
  this.clearCanvas();
@@ -214,7 +218,9 @@ export class BrowserWebCodecsService {
214
218
  this.setupEventListeners();
215
219
 
216
220
  // Request server to start streaming and get offer
221
+ debug.log('webcodecs', `[DIAG] Sending preview:browser-stream-start for session: ${sessionId}`);
217
222
  const response = await ws.http('preview:browser-stream-start', {}, 30000);
223
+ debug.log('webcodecs', `[DIAG] preview:browser-stream-start response: success=${response.success}, hasOffer=${!!response.offer}, message=${response.message}`);
218
224
 
219
225
  if (!response.success) {
220
226
  throw new Error(response.message || 'Failed to start streaming');
@@ -225,12 +231,15 @@ export class BrowserWebCodecsService {
225
231
 
226
232
  // Set remote description (offer)
227
233
  if (response.offer) {
234
+ debug.log('webcodecs', `[DIAG] Using offer from stream-start response`);
228
235
  await this.handleOffer({
229
236
  type: response.offer.type as RTCSdpType,
230
237
  sdp: response.offer.sdp
231
238
  });
232
239
  } else {
240
+ debug.log('webcodecs', `[DIAG] No offer in stream-start response, fetching via stream-offer`);
233
241
  const offerResponse = await ws.http('preview:browser-stream-offer', {}, 10000);
242
+ debug.log('webcodecs', `[DIAG] preview:browser-stream-offer response: hasOffer=${!!offerResponse.offer}`);
234
243
  if (offerResponse.offer) {
235
244
  await this.handleOffer({
236
245
  type: offerResponse.offer.type as RTCSdpType,
@@ -241,7 +250,7 @@ export class BrowserWebCodecsService {
241
250
  }
242
251
  }
243
252
 
244
- debug.log('webcodecs', 'Streaming setup complete');
253
+ debug.log('webcodecs', '[DIAG] Streaming setup complete, waiting for ICE/DataChannel');
245
254
  return true;
246
255
  } catch (error) {
247
256
  debug.error('webcodecs', 'Failed to start streaming:', error);
@@ -595,9 +604,9 @@ export class BrowserWebCodecsService {
595
604
  this.audioContext = new AudioContext({ sampleRate: 48000 });
596
605
  }
597
606
 
598
- // Resume if suspended (may happen without user gesture)
607
+ // Resume if suspended fire-and-forget, same reason as in startStreaming
599
608
  if (this.audioContext.state === 'suspended') {
600
- await this.audioContext.resume();
609
+ this.audioContext.resume().catch(() => {});
601
610
  }
602
611
 
603
612
  debug.log('webcodecs', `AudioContext initialized (state: ${this.audioContext.state})`);
@@ -748,33 +757,28 @@ export class BrowserWebCodecsService {
748
757
  }
749
758
 
750
759
  /**
751
- * Play audio frame with proper AV synchronization
752
- *
753
- * The key insight: Audio and video timestamps from the server use the same
754
- * performance.now() origin. However, audio may start LATER than video if the
755
- * page has no audio initially (silence is skipped).
760
+ * Play audio frame using simple back-to-back scheduling.
756
761
  *
757
- * Solution: Use lastVideoTimestamp (currently rendered video) as the reference
758
- * point, not the first video frame timestamp. This ensures we synchronize
759
- * audio to the CURRENT video position, not the initial position.
762
+ * Frames are scheduled immediately one after another. When a gap occurs
763
+ * (e.g. silence was skipped server-side and audio resumes), the schedule
764
+ * is reset with a 50ms lookahead so playback starts cleanly without
765
+ * audible pops or stutters from scheduling in the past.
760
766
  */
761
767
  private playAudioFrame(audioData: AudioData): void {
762
768
  if (!this.audioContext) return;
763
769
 
764
- // Safety net: resume AudioContext if it somehow got suspended
770
+ // Safety net: resume AudioContext if suspended
765
771
  if (this.audioContext.state === 'suspended') {
766
772
  this.audioContext.resume().catch(() => {});
767
773
  }
768
774
 
769
775
  try {
770
- // Create AudioBuffer
771
776
  const buffer = this.audioContext.createBuffer(
772
777
  audioData.numberOfChannels,
773
778
  audioData.numberOfFrames,
774
779
  audioData.sampleRate
775
780
  );
776
781
 
777
- // Copy audio data to buffer
778
782
  for (let channel = 0; channel < audioData.numberOfChannels; channel++) {
779
783
  const options = {
780
784
  planeIndex: channel,
@@ -782,134 +786,27 @@ export class BrowserWebCodecsService {
782
786
  frameCount: audioData.numberOfFrames,
783
787
  format: 'f32-planar' as AudioSampleFormat
784
788
  };
785
-
786
- const requiredSize = audioData.allocationSize(options);
787
- const tempBuffer = new ArrayBuffer(requiredSize);
788
- const tempFloat32 = new Float32Array(tempBuffer);
789
+ // allocationSize() returns bytes — wrap in ArrayBuffer so Float32Array length is correct
790
+ const tempFloat32 = new Float32Array(new ArrayBuffer(audioData.allocationSize(options)));
789
791
  audioData.copyTo(tempFloat32, options);
790
-
791
- const channelData = buffer.getChannelData(channel);
792
- channelData.set(tempFloat32);
792
+ buffer.getChannelData(channel).set(tempFloat32);
793
793
  }
794
794
 
795
795
  const currentTime = this.audioContext.currentTime;
796
- const audioTimestamp = audioData.timestamp; // microseconds
797
- const bufferDuration = buffer.duration;
798
- const now = performance.now();
799
-
800
- // Wait for video to establish sync before playing audio
801
- if (!this.syncEstablished || this.lastVideoTimestamp === 0) {
802
- // No video yet - skip this audio frame to prevent desync
803
- return;
804
- }
805
-
806
- // Phase 1: Calibration - collect samples to determine stable offset
807
- if (this.audioCalibrationSamples < this.CALIBRATION_SAMPLES) {
808
- // Calculate the offset between audio timestamp and video timeline
809
- // audioVideoOffset > 0 means audio is AHEAD of video in stream time
810
- // audioVideoOffset < 0 means audio is BEHIND video in stream time
811
- const audioVideoOffset = (audioTimestamp - this.lastVideoTimestamp) / 1000000; // seconds
812
-
813
- // Also account for the time elapsed since video was rendered
814
- const timeSinceVideoRender = (now - this.lastVideoRealTime) / 1000; // seconds
815
-
816
- // Expected audio position relative to current video position
817
- // If audio and video are in sync, audio should play at:
818
- // currentTime + audioVideoOffset - timeSinceVideoRender
819
- const expectedOffset = audioVideoOffset - timeSinceVideoRender;
820
-
821
- this.audioOffsetAccumulator += expectedOffset;
822
- this.audioCalibrationSamples++;
823
-
824
- if (this.audioCalibrationSamples === this.CALIBRATION_SAMPLES) {
825
- // Calibration complete - calculate average offset
826
- this.calibratedAudioOffset = this.audioOffsetAccumulator / this.CALIBRATION_SAMPLES;
827
-
828
- // Clamp the offset to reasonable bounds (-500ms to +500ms)
829
- // Beyond this, something is wrong and we should just play immediately
830
- if (this.calibratedAudioOffset < -0.5) {
831
- debug.warn('webcodecs', `Audio calibration: offset ${(this.calibratedAudioOffset * 1000).toFixed(0)}ms too negative, clamping to -500ms`);
832
- this.calibratedAudioOffset = -0.5;
833
- } else if (this.calibratedAudioOffset > 0.5) {
834
- debug.warn('webcodecs', `Audio calibration: offset ${(this.calibratedAudioOffset * 1000).toFixed(0)}ms too positive, clamping to +500ms`);
835
- this.calibratedAudioOffset = 0.5;
836
- }
837
-
838
- // Initialize nextAudioPlayTime based on calibrated offset
839
- // Add small buffer (30ms) for smooth playback
840
- this.nextAudioPlayTime = currentTime + Math.max(0.03, this.calibratedAudioOffset + 0.03);
841
- this.audioSyncInitialized = true;
842
-
843
- debug.log('webcodecs', `Audio calibration complete: offset=${(this.calibratedAudioOffset * 1000).toFixed(1)}ms, startTime=${(this.nextAudioPlayTime - currentTime).toFixed(3)}s from now`);
844
- } else {
845
- // Still calibrating - skip this audio frame
846
- return;
847
- }
848
- }
849
796
 
850
- // Phase 2: Synchronized playback with drift correction
851
- let targetPlayTime = this.nextAudioPlayTime;
852
-
853
- // If we've fallen too far behind (buffer underrun), reset
854
- if (targetPlayTime < currentTime - 0.01) {
855
- // We're behind by more than 10ms, need to catch up
856
- targetPlayTime = currentTime + 0.02; // Small buffer to recover
857
- debug.warn('webcodecs', 'Audio buffer underrun, resetting playback');
858
- }
859
-
860
- // Periodic drift check (every ~500ms worth of audio, ~25 frames)
861
- if (this.stats.audioFramesDecoded % 25 === 0) {
862
- // Recalculate where audio SHOULD be based on current video position
863
- const audioVideoOffset = (audioTimestamp - this.lastVideoTimestamp) / 1000000;
864
- const timeSinceVideoRender = (now - this.lastVideoRealTime) / 1000;
865
- const expectedOffset = audioVideoOffset - timeSinceVideoRender;
866
-
867
- // Where audio should play relative to currentTime
868
- const idealTime = currentTime + expectedOffset + 0.03; // +30ms buffer
869
-
870
- const drift = targetPlayTime - idealTime;
871
-
872
- // Apply correction based on drift magnitude
873
- if (Math.abs(drift) > 0.2) {
874
- // Large drift (>200ms) - aggressive correction (80%)
875
- targetPlayTime -= drift * 0.8;
876
- debug.warn('webcodecs', `Large audio drift: ${(drift * 1000).toFixed(0)}ms, aggressive correction`);
877
- } else if (Math.abs(drift) > 0.05) {
878
- // Medium drift (50-200ms) - moderate correction (40%)
879
- targetPlayTime -= drift * 0.4;
880
- }
881
- // Small drift (<50ms) - no correction, continuous playback handles it
882
- }
883
-
884
- // Ensure we don't schedule in the past
885
- if (targetPlayTime < currentTime + 0.005) {
886
- targetPlayTime = currentTime + 0.005;
797
+ // When the scheduler has fallen behind (gap due to silence or decode delay),
798
+ // reset with a 50ms lookahead so the next chunk starts cleanly.
799
+ if (this.nextAudioPlayTime < currentTime) {
800
+ this.nextAudioPlayTime = currentTime + 0.05;
887
801
  }
888
802
 
889
- // Schedule this buffer
890
803
  const source = this.audioContext.createBufferSource();
891
804
  source.buffer = buffer;
892
805
  source.connect(this.audioContext.destination);
893
- source.start(targetPlayTime);
806
+ source.start(this.nextAudioPlayTime);
894
807
 
895
- // Track scheduled buffer
896
- this.audioBufferQueue.push({ buffer, scheduledTime: targetPlayTime });
897
-
898
- // Update next play time for continuous scheduling (back-to-back)
899
- this.nextAudioPlayTime = targetPlayTime + bufferDuration;
900
-
901
- // Limit queue size to prevent memory buildup
902
- if (this.audioBufferQueue.length > this.maxAudioQueueSize) {
903
- this.audioBufferQueue.shift();
904
- }
905
-
906
- // Cleanup old scheduled buffers
907
- source.onended = () => {
908
- const index = this.audioBufferQueue.findIndex(item => item.buffer === buffer);
909
- if (index !== -1) {
910
- this.audioBufferQueue.splice(index, 1);
911
- }
912
- };
808
+ // Back-to-back: next chunk plays immediately after this one ends
809
+ this.nextAudioPlayTime += buffer.duration;
913
810
  } catch (error) {
914
811
  debug.warn('webcodecs', 'Audio playback error:', error);
915
812
  }
@@ -1106,7 +1003,7 @@ export class BrowserWebCodecsService {
1106
1003
 
1107
1004
  if (this.ctx) {
1108
1005
  this.ctx.imageSmoothingEnabled = true;
1109
- this.ctx.imageSmoothingQuality = 'low';
1006
+ this.ctx.imageSmoothingQuality = 'medium';
1110
1007
  }
1111
1008
 
1112
1009
  try {
@@ -214,10 +214,11 @@ export function isSessionUnread(sessionId: string): boolean {
214
214
 
215
215
  /**
216
216
  * Check if a project has any unread sessions.
217
+ * Optionally exclude a specific session (e.g. the currently viewed one).
217
218
  */
218
- export function hasUnreadSessionsForProject(projectId: string): boolean {
219
- for (const pId of appState.unreadSessions.values()) {
220
- if (pId === projectId) return true;
219
+ export function hasUnreadSessionsForProject(projectId: string, excludeSessionId?: string): boolean {
220
+ for (const [sId, pId] of appState.unreadSessions.entries()) {
221
+ if (pId === projectId && sId !== excludeSessionId) return true;
221
222
  }
222
223
  return false;
223
224
  }
@@ -119,8 +119,9 @@ export function getProjectStatusColor(projectId: string): string {
119
119
  return 'bg-amber-500';
120
120
  }
121
121
 
122
- // Check for unread sessions in this project
123
- if (hasUnreadSessionsForProject(projectId)) return 'bg-blue-500';
122
+ // Check for unread sessions in this project, excluding the currently viewed session
123
+ const currentSessionId = sessionState.currentSession?.id;
124
+ if (hasUnreadSessionsForProject(projectId, currentSessionId)) return 'bg-blue-500';
124
125
 
125
126
  return 'bg-slate-500/30';
126
127
  }
@@ -295,6 +295,8 @@ export async function loadSessions() {
295
295
  // wiping out any stream_event injected by catchup.
296
296
  await loadMessagesForSession(targetSession.id);
297
297
  sessionState.currentSession = targetSession;
298
+ // Clear unread status — user is actively viewing this session
299
+ markSessionRead(targetSession.id);
298
300
  // Join chat session room so we receive session-scoped events
299
301
  // (stream, input sync, edit mode, model sync).
300
302
  // Critical after refresh — without it, connection misses all events.
@@ -6,6 +6,9 @@ export const notificationStore = $state({
6
6
  maxNotifications: 5
7
7
  });
8
8
 
9
+ // Monotonic counter for unique notification IDs
10
+ let _notificationCounter = 0;
11
+
9
12
  // Derived values as functions (cannot export derived state from modules)
10
13
  export function hasNotifications() {
11
14
  return notificationStore.notifications.length > 0;
@@ -18,7 +21,7 @@ export function notificationCount() {
18
21
  // Notification management functions
19
22
  export function addNotification(notification: Omit<ToastNotification, 'id'>) {
20
23
  const newNotification: ToastNotification = {
21
- id: Date.now().toString(),
24
+ id: `${Date.now()}-${++_notificationCounter}`,
22
25
  ...notification,
23
26
  duration: notification.duration || 5000 // Default 5 seconds
24
27
  };