@myrialabs/clopen 0.2.5 → 0.2.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/backend/chat/stream-manager.ts +136 -10
  2. package/backend/database/queries/session-queries.ts +9 -0
  3. package/backend/engine/adapters/claude/error-handler.ts +7 -2
  4. package/backend/engine/adapters/claude/stream.ts +16 -7
  5. package/backend/index.ts +25 -3
  6. package/backend/mcp/servers/browser-automation/browser.ts +23 -6
  7. package/backend/preview/browser/browser-mcp-control.ts +32 -16
  8. package/backend/preview/browser/browser-pool.ts +3 -1
  9. package/backend/preview/browser/browser-preview-service.ts +16 -17
  10. package/backend/preview/browser/browser-tab-manager.ts +1 -1
  11. package/backend/preview/browser/browser-video-capture.ts +199 -156
  12. package/backend/preview/browser/scripts/audio-stream.ts +11 -0
  13. package/backend/preview/browser/scripts/video-stream.ts +3 -5
  14. package/backend/snapshot/helpers.ts +15 -2
  15. package/backend/ws/chat/stream.ts +1 -1
  16. package/backend/ws/preview/browser/tab-info.ts +5 -2
  17. package/backend/ws/snapshot/restore.ts +43 -2
  18. package/frontend/components/chat/input/ChatInput.svelte +6 -4
  19. package/frontend/components/chat/input/components/ChatInputActions.svelte +10 -0
  20. package/frontend/components/chat/input/composables/use-chat-actions.svelte.ts +6 -2
  21. package/frontend/components/chat/message/MessageBubble.svelte +22 -1
  22. package/frontend/components/chat/tools/components/TerminalCommand.svelte +2 -2
  23. package/frontend/components/files/FileViewer.svelte +13 -2
  24. package/frontend/components/history/HistoryModal.svelte +1 -1
  25. package/frontend/components/preview/browser/BrowserPreview.svelte +15 -0
  26. package/frontend/components/preview/browser/components/Canvas.svelte +432 -69
  27. package/frontend/components/preview/browser/components/Container.svelte +23 -1
  28. package/frontend/components/preview/browser/components/Toolbar.svelte +1 -8
  29. package/frontend/components/preview/browser/core/coordinator.svelte.ts +27 -4
  30. package/frontend/components/preview/browser/core/mcp-handlers.svelte.ts +36 -3
  31. package/frontend/components/preview/browser/core/tab-operations.svelte.ts +1 -0
  32. package/frontend/components/terminal/TerminalTabs.svelte +1 -2
  33. package/frontend/components/workspace/PanelHeader.svelte +15 -0
  34. package/frontend/components/workspace/panels/FilesPanel.svelte +1 -0
  35. package/frontend/components/workspace/panels/GitPanel.svelte +27 -13
  36. package/frontend/components/workspace/panels/PreviewPanel.svelte +2 -0
  37. package/frontend/services/chat/chat.service.ts +9 -8
  38. package/frontend/services/preview/browser/browser-webcodecs.service.ts +43 -138
  39. package/frontend/stores/core/app.svelte.ts +4 -3
  40. package/frontend/stores/core/presence.svelte.ts +3 -2
  41. package/frontend/stores/core/sessions.svelte.ts +2 -0
  42. package/frontend/stores/ui/notification.svelte.ts +4 -1
  43. package/package.json +1 -1
@@ -41,6 +41,7 @@ interface VideoStreamSession {
41
41
  headlessReady: boolean;
42
42
  pendingCandidates: RTCIceCandidateInit[];
43
43
  scriptInjected: boolean; // Track if persistent script was injected
44
+ scriptsPreInjected: boolean; // Track if scripts were pre-injected during tab creation
44
45
  stats: {
45
46
  videoBytesSent: number;
46
47
  audioBytesSent: number;
@@ -52,11 +53,120 @@ interface VideoStreamSession {
52
53
 
53
54
  export class BrowserVideoCapture extends EventEmitter {
54
55
  private sessions = new Map<string, VideoStreamSession>();
56
+ private preInjectPromises = new Map<string, Promise<boolean>>();
55
57
 
56
58
  constructor() {
57
59
  super();
58
60
  }
59
61
 
62
+ /**
63
+ * Pre-inject WebCodecs scripts during tab creation.
64
+ * This overlaps script injection with frontend processing,
65
+ * so startStreaming() only needs batched init + CDP setup (~50-80ms).
66
+ */
67
+ preInjectScripts(sessionId: string, session: BrowserTab): Promise<boolean> {
68
+ const promise = this.doPreInject(sessionId, session);
69
+ this.preInjectPromises.set(sessionId, promise);
70
+ return promise;
71
+ }
72
+
73
+ private async doPreInject(sessionId: string, session: BrowserTab): Promise<boolean> {
74
+ if (!session.page || session.page.isClosed()) return false;
75
+
76
+ try {
77
+ const page = session.page;
78
+ const viewport = page.viewport()!;
79
+ const config = DEFAULT_STREAMING_CONFIG;
80
+ const scale = session.scale || 1;
81
+
82
+ const scaledWidth = Math.round(viewport.width * scale);
83
+ const scaledHeight = Math.round(viewport.height * scale);
84
+
85
+ const videoConfig: StreamingConfig['video'] = {
86
+ ...config.video,
87
+ width: scaledWidth,
88
+ height: scaledHeight
89
+ };
90
+
91
+ // Create session tracking
92
+ const videoSession: VideoStreamSession = {
93
+ sessionId,
94
+ isActive: false,
95
+ clientConnected: false,
96
+ headlessReady: false,
97
+ pendingCandidates: [],
98
+ scriptInjected: true,
99
+ scriptsPreInjected: false, // Set to true only after injection completes
100
+ stats: {
101
+ videoBytesSent: 0,
102
+ audioBytesSent: 0,
103
+ videoFramesEncoded: 0,
104
+ audioFramesEncoded: 0,
105
+ connectionState: 'new'
106
+ }
107
+ };
108
+ this.sessions.set(sessionId, videoSession);
109
+
110
+ await this.injectScripts(sessionId, page, videoConfig, config);
111
+
112
+ // Mark as pre-injected only after successful completion
113
+ videoSession.scriptsPreInjected = true;
114
+
115
+ debug.log('webcodecs', `Pre-injected scripts for ${sessionId}`);
116
+ return true;
117
+ } catch (error) {
118
+ debug.warn('webcodecs', `Pre-injection failed for ${sessionId}:`, error);
119
+ // Clean up so startStreaming() will do full injection
120
+ this.sessions.delete(sessionId);
121
+ return false;
122
+ } finally {
123
+ this.preInjectPromises.delete(sessionId);
124
+ }
125
+ }
126
+
127
+ /**
128
+ * Inject signaling bindings + encoder scripts into page
129
+ */
130
+ private async injectScripts(
131
+ sessionId: string,
132
+ page: Page,
133
+ videoConfig: StreamingConfig['video'],
134
+ config: StreamingConfig
135
+ ): Promise<void> {
136
+ // Check if bindings exist
137
+ const bindingsExist = await page.evaluate(() => {
138
+ return typeof (window as any).__sendIceCandidate === 'function';
139
+ });
140
+
141
+ // Expose signaling functions (persists across navigations)
142
+ if (!bindingsExist) {
143
+ await page.exposeFunction('__sendIceCandidate', (candidate: RTCIceCandidateInit) => {
144
+ const activeSession = Array.from(this.sessions.values()).find(s => s.isActive);
145
+ if (!activeSession) return;
146
+ this.emit('ice-candidate', { sessionId: activeSession.sessionId, candidate, from: 'headless' });
147
+ });
148
+
149
+ await page.exposeFunction('__sendConnectionState', (state: string) => {
150
+ const activeSession = Array.from(this.sessions.values()).find(s => s.isActive);
151
+ if (activeSession) {
152
+ activeSession.stats.connectionState = state;
153
+ this.emit('connection-state', { sessionId: activeSession.sessionId, state });
154
+ }
155
+ });
156
+
157
+ await page.exposeFunction('__sendCursorChange', (cursor: string) => {
158
+ const activeSession = Array.from(this.sessions.values()).find(s => s.isActive);
159
+ if (activeSession) {
160
+ this.emit('cursor-change', { sessionId: activeSession.sessionId, cursor });
161
+ }
162
+ });
163
+ }
164
+
165
+ // Inject video encoder + audio capture scripts
166
+ await page.evaluate(videoEncoderScript, videoConfig);
167
+ await page.evaluate(audioCaptureScript, config.audio);
168
+ }
169
+
60
170
  /**
61
171
  * Start video streaming for a session
62
172
  */
@@ -67,11 +177,20 @@ export class BrowserVideoCapture extends EventEmitter {
67
177
  ): Promise<boolean> {
68
178
  debug.log('webcodecs', `Starting streaming for session ${sessionId}`);
69
179
 
70
- // If session exists, stop it first
71
- if (this.sessions.has(sessionId)) {
72
- debug.log('webcodecs', `Session ${sessionId} exists, stopping for restart`);
180
+ // Wait for any pending pre-injection to complete
181
+ const pendingPreInject = this.preInjectPromises.get(sessionId);
182
+ if (pendingPreInject) {
183
+ debug.log('webcodecs', `Waiting for pre-injection to complete for ${sessionId}`);
184
+ await pendingPreInject.catch(() => {});
185
+ }
186
+
187
+ // If session is already actively streaming, stop it for a clean reconnect.
188
+ // This ensures the old PeerConnection + DataChannel are torn down and
189
+ // a fresh one is created, preventing stale connections where no frames flow.
190
+ const existingSession = this.sessions.get(sessionId);
191
+ if (existingSession && existingSession.isActive) {
192
+ debug.log('webcodecs', `Session ${sessionId} already active, stopping for clean reconnect`);
73
193
  await this.stopStreaming(sessionId, session);
74
- await new Promise(resolve => setTimeout(resolve, 100));
75
194
  }
76
195
 
77
196
  if (!session.page || session.page.isClosed()) {
@@ -100,6 +219,7 @@ export class BrowserVideoCapture extends EventEmitter {
100
219
  headlessReady: false,
101
220
  pendingCandidates: [],
102
221
  scriptInjected: false,
222
+ scriptsPreInjected: false,
103
223
  stats: {
104
224
  videoBytesSent: 0,
105
225
  audioBytesSent: 0,
@@ -111,35 +231,6 @@ export class BrowserVideoCapture extends EventEmitter {
111
231
  this.sessions.set(sessionId, videoSession);
112
232
  }
113
233
 
114
- // Check if bindings exist
115
- const bindingsExist = await page.evaluate(() => {
116
- return typeof (window as any).__sendIceCandidate === 'function';
117
- });
118
-
119
- // Expose signaling functions (persists across navigations)
120
- if (!bindingsExist) {
121
- await page.exposeFunction('__sendIceCandidate', (candidate: RTCIceCandidateInit) => {
122
- const activeSession = Array.from(this.sessions.values()).find(s => s.isActive);
123
- if (!activeSession) return;
124
- this.emit('ice-candidate', { sessionId: activeSession.sessionId, candidate, from: 'headless' });
125
- });
126
-
127
- await page.exposeFunction('__sendConnectionState', (state: string) => {
128
- const activeSession = Array.from(this.sessions.values()).find(s => s.isActive);
129
- if (activeSession) {
130
- activeSession.stats.connectionState = state;
131
- this.emit('connection-state', { sessionId: activeSession.sessionId, state });
132
- }
133
- });
134
-
135
- await page.exposeFunction('__sendCursorChange', (cursor: string) => {
136
- const activeSession = Array.from(this.sessions.values()).find(s => s.isActive);
137
- if (activeSession) {
138
- this.emit('cursor-change', { sessionId: activeSession.sessionId, cursor });
139
- }
140
- });
141
- }
142
-
143
234
  // Calculate scaled dimensions
144
235
  const scaledWidth = Math.round(viewport.width * scale);
145
236
  const scaledHeight = Math.round(viewport.height * scale);
@@ -150,91 +241,60 @@ export class BrowserVideoCapture extends EventEmitter {
150
241
  height: scaledHeight
151
242
  };
152
243
 
153
- // Store config globally for persistent script access
154
- await page.evaluate((cfg) => {
155
- (window as any).__videoEncoderConfig = cfg;
156
- }, videoConfig);
157
-
158
- // Inject persistent video encoder script (survives navigation)
159
- // Only inject once per page instance
160
- if (!videoSession.scriptInjected) {
161
- // Temporarily disable evaluateOnNewDocument for evasion test
162
- // await page.evaluateOnNewDocument(videoEncoderScript, videoConfig);
244
+ // Skip script injection if already pre-injected during tab creation
245
+ if (!videoSession.scriptsPreInjected) {
246
+ await this.injectScripts(sessionId, page, videoConfig, config);
163
247
  videoSession.scriptInjected = true;
164
- debug.log('webcodecs', `Persistent video encoder script injected for ${sessionId}`);
248
+ } else {
249
+ debug.log('webcodecs', `Scripts already pre-injected for ${sessionId}, skipping injection`);
165
250
  }
166
251
 
167
- // Also inject immediately for current page context
168
- // (evaluateOnNewDocument only runs on NEXT navigation)
169
- await page.evaluate(videoEncoderScript, videoConfig);
252
+ // Single batched call: verify peer + start streaming + init audio
253
+ // (saves ~60ms of IPC overhead vs 4 separate page.evaluate calls)
254
+ const initResult = await page.evaluate(async () => {
255
+ const peer = (window as any).__webCodecsPeer;
256
+ if (typeof peer?.startStreaming !== 'function') {
257
+ return { peerExists: false, started: false, audioInitialized: false };
258
+ }
170
259
 
171
- // Inject audio capture script post-navigation to avoid CF detection.
172
- // Using page.evaluate() instead of evaluateOnNewDocument() ensures
173
- // AudioContext patching happens AFTER Cloudflare challenges pass,
174
- // preventing fingerprint detection of constructor interception.
175
- await page.evaluate(audioCaptureScript, config.audio);
260
+ const started = await peer.startStreaming();
261
+ if (!started) {
262
+ return { peerExists: true, started: false, audioInitialized: false };
263
+ }
264
+
265
+ // Initialize audio encoder if available
266
+ let audioInitialized = false;
267
+ const encoder = (window as any).__audioEncoder;
268
+ if (typeof encoder?.init === 'function') {
269
+ try {
270
+ const initiated = await encoder.init();
271
+ if (initiated) {
272
+ audioInitialized = !!encoder.start();
273
+ }
274
+ } catch {}
275
+ }
176
276
 
177
- // Verify peer was created
178
- const peerExists = await page.evaluate(() => {
179
- return typeof (window as any).__webCodecsPeer?.startStreaming === 'function';
277
+ return { peerExists: true, started: true, audioInitialized };
180
278
  });
181
279
 
182
- if (!peerExists) {
280
+ if (!initResult.peerExists) {
183
281
  debug.error('webcodecs', `Peer script injected but __webCodecsPeer not available`);
184
282
  this.sessions.delete(sessionId);
185
283
  return false;
186
284
  }
187
285
 
188
- videoSession.isActive = true;
189
-
190
- // Wait for page to be fully loaded
191
- try {
192
- const loadState = await page.evaluate(() => document.readyState);
193
- if (loadState !== 'complete') {
194
- debug.log('webcodecs', `Waiting for page load...`);
195
- await page.waitForFunction(() => document.readyState === 'complete', { timeout: 60000 });
196
- }
197
- } catch (loadError) {
198
- debug.warn('webcodecs', 'Page load wait timed out, proceeding anyway');
199
- }
200
-
201
- // Start video streaming
202
- const started = await page.evaluate(() => {
203
- return (window as any).__webCodecsPeer?.startStreaming();
204
- });
205
-
206
- if (!started) {
286
+ if (!initResult.started) {
207
287
  debug.error('webcodecs', `startStreaming returned false`);
208
288
  this.sessions.delete(sessionId);
209
289
  return false;
210
290
  }
211
291
 
212
- // Initialize and start audio encoder (from AudioContext interception)
213
- const audioEncoderAvailable = await page.evaluate(() => {
214
- return typeof (window as any).__audioEncoder?.init === 'function';
215
- });
216
-
217
- if (audioEncoderAvailable) {
218
- debug.log('webcodecs', 'Initializing audio encoder from AudioContext interception...');
219
-
220
- const audioInitialized = await page.evaluate(async () => {
221
- const encoder = (window as any).__audioEncoder;
222
- if (!encoder) return false;
223
-
224
- const initiated = await encoder.init();
225
- if (initiated) {
226
- return encoder.start();
227
- }
228
- return false;
229
- });
292
+ videoSession.isActive = true;
230
293
 
231
- if (audioInitialized) {
232
- debug.log('webcodecs', 'Audio encoder initialized and started');
233
- } else {
234
- debug.warn('webcodecs', 'Audio encoder initialization failed, continuing with video only');
235
- }
294
+ if (initResult.audioInitialized) {
295
+ debug.log('webcodecs', 'Audio encoder initialized and started');
236
296
  } else {
237
- debug.warn('webcodecs', 'Audio encoder not available (AudioEncoder API may not be supported)');
297
+ debug.warn('webcodecs', 'Audio not available, continuing with video only');
238
298
  }
239
299
 
240
300
  videoSession.headlessReady = true;
@@ -325,25 +385,16 @@ export class BrowserVideoCapture extends EventEmitter {
325
385
  return null;
326
386
  }
327
387
 
328
- const maxRetries = 5;
329
- const retryDelay = 100;
388
+ const maxRetries = 3;
389
+ const retryDelay = 50;
330
390
 
331
391
  for (let attempt = 0; attempt < maxRetries; attempt++) {
332
392
  try {
333
- const peerReady = await session.page.evaluate(() => {
334
- return typeof (window as any).__webCodecsPeer?.createOffer === 'function';
335
- });
336
-
337
- if (!peerReady) {
338
- if (attempt < maxRetries - 1) {
339
- await new Promise(resolve => setTimeout(resolve, retryDelay));
340
- continue;
341
- }
342
- return null;
343
- }
344
-
345
- const offer = await session.page.evaluate(() => {
346
- return (window as any).__webCodecsPeer?.createOffer();
393
+ // Single evaluate: check peer + create offer in one IPC round-trip
394
+ const offer = await session.page.evaluate(async () => {
395
+ const peer = (window as any).__webCodecsPeer;
396
+ if (typeof peer?.createOffer !== 'function') return null;
397
+ return peer.createOffer();
347
398
  });
348
399
 
349
400
  if (offer) return offer;
@@ -570,54 +621,50 @@ export class BrowserVideoCapture extends EventEmitter {
570
621
  height: scaledHeight
571
622
  };
572
623
 
573
- // Re-inject video encoder script to new page context
574
- // (evaluateOnNewDocument doesn't run for the current navigation, only future ones)
575
- await page.evaluate((cfg) => {
576
- (window as any).__videoEncoderConfig = cfg;
577
- }, videoConfig);
578
-
624
+ // Re-inject video encoder and audio capture scripts to new page context
579
625
  await page.evaluate(videoEncoderScript, videoConfig);
580
-
581
- // Re-inject audio capture script for new page context (post-navigation)
582
626
  await page.evaluate(audioCaptureScript, config.audio);
583
627
 
584
- // Verify peer was re-created
585
- const peerExists = await page.evaluate(() => {
586
- return typeof (window as any).__webCodecsPeer?.startStreaming === 'function';
628
+ // Single batched call: verify peer + start streaming + init audio
629
+ const initResult = await page.evaluate(async () => {
630
+ const peer = (window as any).__webCodecsPeer;
631
+ if (typeof peer?.startStreaming !== 'function') {
632
+ return { peerExists: false, started: false, audioInitialized: false };
633
+ }
634
+
635
+ const started = await peer.startStreaming();
636
+ if (!started) {
637
+ return { peerExists: true, started: false, audioInitialized: false };
638
+ }
639
+
640
+ let audioInitialized = false;
641
+ const encoder = (window as any).__audioEncoder;
642
+ if (typeof encoder?.init === 'function') {
643
+ try {
644
+ const initiated = await encoder.init();
645
+ if (initiated) {
646
+ audioInitialized = !!encoder.start();
647
+ }
648
+ } catch {}
649
+ }
650
+
651
+ return { peerExists: true, started: true, audioInitialized };
587
652
  });
588
653
 
589
- if (!peerExists) {
654
+ if (!initResult.peerExists) {
590
655
  debug.error('webcodecs', `Peer script re-injection failed - peer not available`);
591
656
  return false;
592
657
  }
593
658
 
594
- // Start video streaming on new page
595
- const started = await page.evaluate(() => {
596
- return (window as any).__webCodecsPeer?.startStreaming();
597
- });
598
-
599
- if (!started) {
659
+ if (!initResult.started) {
600
660
  debug.error('webcodecs', `Failed to start streaming on new page`);
601
661
  return false;
602
662
  }
603
663
 
604
- // Re-initialize and start audio capture after navigation
605
- try {
606
- const audioReady = await page.evaluate(async () => {
607
- const encoder = (window as any).__audioEncoder;
608
- if (!encoder) return false;
609
- const initiated = await encoder.init();
610
- if (initiated) return encoder.start();
611
- return false;
612
- });
613
-
614
- if (audioReady) {
615
- debug.log('webcodecs', 'Audio re-initialized after navigation');
616
- } else {
617
- debug.warn('webcodecs', 'Audio not available after navigation, continuing with video only');
618
- }
619
- } catch {
620
- debug.warn('webcodecs', 'Audio re-init failed after navigation, continuing with video only');
664
+ if (initResult.audioInitialized) {
665
+ debug.log('webcodecs', 'Audio re-initialized after navigation');
666
+ } else {
667
+ debug.warn('webcodecs', 'Audio not available after navigation, continuing with video only');
621
668
  }
622
669
 
623
670
  // Restart CDP screencast
@@ -661,15 +708,11 @@ export class BrowserVideoCapture extends EventEmitter {
661
708
 
662
709
  if (session?.page && !session.page.isClosed()) {
663
710
  try {
664
- // Stop audio encoder
711
+ // Stop audio + peer in one IPC round-trip
665
712
  await session.page.evaluate(() => {
666
713
  (window as any).__audioEncoder?.stop();
667
- }).catch(() => {});
668
-
669
- // Stop peer
670
- await session.page.evaluate(() => {
671
714
  (window as any).__webCodecsPeer?.stopStreaming();
672
- });
715
+ }).catch(() => {});
673
716
 
674
717
  // Stop CDP screencast
675
718
  const cdp = (session as any).__webCodecsCdp;
@@ -148,6 +148,12 @@ export function audioCaptureScript(config: StreamingConfig['audio']) {
148
148
  if (interceptedContexts.has(ctx)) return;
149
149
  interceptedContexts.add(ctx);
150
150
 
151
+ // Resume AudioContext immediately — in headless Chrome without a user gesture,
152
+ // AudioContext starts in 'suspended' state and onaudioprocess never fires.
153
+ if (ctx.state === 'suspended') {
154
+ ctx.resume().catch(() => {});
155
+ }
156
+
151
157
  // Store original destination
152
158
  const originalDestination = ctx.destination;
153
159
 
@@ -213,6 +219,11 @@ export function audioCaptureScript(config: StreamingConfig['audio']) {
213
219
  const OriginalAudioContext = (window as any).__OriginalAudioContext || window.AudioContext;
214
220
  const ctx = new OriginalAudioContext();
215
221
 
222
+ // Resume context immediately — headless Chrome requires explicit resume
223
+ if (ctx.state === 'suspended') {
224
+ ctx.resume().catch(() => {});
225
+ }
226
+
216
227
  // Create media element source
217
228
  const source = ctx.createMediaElementSource(element);
218
229
 
@@ -34,11 +34,9 @@ export function videoEncoderScript(config: StreamingConfig['video']) {
34
34
  let lastCursor = 'default';
35
35
  let cursorCheckInterval: any = null;
36
36
 
37
- // ICE servers
38
- const iceServers = [
39
- { urls: 'stun:stun.l.google.com:19302' },
40
- { urls: 'stun:stun1.l.google.com:19302' }
41
- ];
37
+ // ICE servers - empty for local connections (both peers on same machine)
38
+ // STUN servers are unnecessary for localhost and add 100-500ms ICE gathering latency
39
+ const iceServers: { urls: string }[] = [];
42
40
 
43
41
  // Check cursor style from page
44
42
  function checkCursor() {
@@ -35,12 +35,25 @@ export interface TimelineResponse {
35
35
  }
36
36
 
37
37
  /**
38
- * Check if a user message is an internal tool confirmation message
39
- * Internal messages contain tool_result blocks, not regular text input
38
+ * Check if a user message is an internal/non-genuine user message.
39
+ *
40
+ * The SDK uses `type: 'user'` for several kinds of messages that are NOT
41
+ * typed by the real human user:
42
+ * 1. Tool-result confirmations — content contains `tool_result` blocks
43
+ * 2. Sub-agent / Task prompts — `parent_tool_use_id` is non-null
44
+ * 3. Post-compaction synthetic summaries — `isSynthetic` is true
45
+ *
46
+ * Only messages that pass none of these checks are genuine user input.
40
47
  */
41
48
  export function isInternalToolMessage(sdkMessage: any): boolean {
42
49
  if (sdkMessage.type !== 'user') return false;
43
50
 
51
+ // Sub-agent or tool-result message (parent_tool_use_id is set)
52
+ if (sdkMessage.parent_tool_use_id != null) return true;
53
+
54
+ // Synthetic message generated after context compaction
55
+ if (sdkMessage.isSynthetic === true) return true;
56
+
44
57
  const content = sdkMessage.message?.content;
45
58
  if (!content) return false;
46
59
 
@@ -470,7 +470,7 @@ export const streamHandler = createRouter()
470
470
  return;
471
471
  }
472
472
 
473
- const cancelled = await streamManager.cancelStream(streamState.streamId);
473
+ await streamManager.cancelStream(streamState.streamId);
474
474
  // Always send cancelled to chat session room to clear UI
475
475
  ws.emit.chatSession(chatSessionId, 'chat:cancelled', {
476
476
  status: 'cancelled',
@@ -7,6 +7,7 @@
7
7
  import { t } from 'elysia';
8
8
  import { createRouter } from '$shared/utils/ws-server';
9
9
  import { browserPreviewServiceManager } from '../../../preview/index';
10
+ import { browserMcpControl } from '../../../preview/browser/browser-mcp-control';
10
11
  import { ws } from '$backend/utils/ws';
11
12
  import { debug } from '$shared/utils/logger';
12
13
 
@@ -62,7 +63,8 @@ export const tabInfoPreviewHandler = createRouter()
62
63
  isStreaming: t.Boolean(),
63
64
  deviceSize: t.String(),
64
65
  rotation: t.String(),
65
- isActive: t.Boolean()
66
+ isActive: t.Boolean(),
67
+ isMcpControlled: t.Boolean()
66
68
  })),
67
69
  activeTabId: t.Union([t.String(), t.Null()]),
68
70
  count: t.Number()
@@ -87,7 +89,8 @@ export const tabInfoPreviewHandler = createRouter()
87
89
  isStreaming: tab.isStreaming,
88
90
  deviceSize: tab.deviceSize,
89
91
  rotation: tab.rotation,
90
- isActive: tab.isActive
92
+ isActive: tab.isActive,
93
+ isMcpControlled: browserMcpControl.isTabControlled(tab.id, projectId)
91
94
  })),
92
95
  activeTabId: activeTab?.id || null,
93
96
  count: allTabsInfo.length
@@ -200,18 +200,56 @@ export const restoreHandler = createRouter()
200
200
  sessionQueries.updateHead(sessionId, sessionEnd.id);
201
201
  debug.log('snapshot', `HEAD updated to: ${sessionEnd.id}`);
202
202
 
203
- // 5b. Update latest_sdk_session_id so resume works correctly
203
+ // 5b. Update latest_sdk_session_id so resume works correctly.
204
+ // Claude Code: skip cancelled fork session_ids (partial messages from cancelStream).
205
+ // OpenCode: simple walk — any session_id is valid (sessions created synchronously).
204
206
  {
205
- let walkId: string | null = sessionEnd.id;
206
207
  let foundSdkSessionId: string | null = null;
207
208
  const msgLookup = new Map(allMessages.map(m => [m.id, m]));
209
+ const sessionRecord = sessionQueries.getById(sessionId);
210
+ const isClaudeCode = sessionRecord?.engine === 'claude-code';
211
+
212
+ // Claude Code only: detect cancelled stream by partialText marker on sessionEnd
213
+ let cancelledSessionId: string | null = null;
214
+ if (isClaudeCode) {
215
+ try {
216
+ const endMsg = msgLookup.get(sessionEnd.id);
217
+ if (endMsg) {
218
+ const endSdk = JSON.parse(endMsg.sdk_message);
219
+ if (endSdk.partialText) {
220
+ cancelledSessionId = endSdk.session_id || null;
221
+ }
222
+ }
223
+ } catch { /* skip */ }
224
+ }
208
225
 
226
+ let walkId: string | null = sessionEnd.id;
209
227
  while (walkId) {
210
228
  const walkMsg = msgLookup.get(walkId);
211
229
  if (!walkMsg) break;
212
230
 
213
231
  try {
214
232
  const sdk = JSON.parse(walkMsg.sdk_message);
233
+
234
+ // Claude Code only: skip partial messages (from cancelled streams)
235
+ if (isClaudeCode && sdk.partialText) {
236
+ walkId = walkMsg.parent_message_id || null;
237
+ continue;
238
+ }
239
+
240
+ // Claude Code only: skip assistant messages from the same cancelled fork
241
+ if (isClaudeCode && cancelledSessionId && sdk.session_id === cancelledSessionId) {
242
+ walkId = walkMsg.parent_message_id || null;
243
+ continue;
244
+ }
245
+
246
+ // Claude Code only: user message's `resume` field records the last valid session_id
247
+ if (isClaudeCode && sdk.type === 'user' && 'resume' in sdk) {
248
+ foundSdkSessionId = sdk.resume || null;
249
+ break;
250
+ }
251
+
252
+ // Any engine: message with session_id
215
253
  if (sdk.session_id) {
216
254
  foundSdkSessionId = sdk.session_id;
217
255
  break;
@@ -224,6 +262,9 @@ export const restoreHandler = createRouter()
224
262
  if (foundSdkSessionId) {
225
263
  sessionQueries.updateLatestSdkSessionId(sessionId, foundSdkSessionId);
226
264
  debug.log('snapshot', `latest_sdk_session_id updated to: ${foundSdkSessionId}`);
265
+ } else {
266
+ sessionQueries.clearLatestSdkSessionId(sessionId);
267
+ debug.log('snapshot', 'latest_sdk_session_id cleared (no valid session found in restored chain)');
227
268
  }
228
269
  }
229
270
 
@@ -66,9 +66,6 @@
66
66
 
67
67
  // Chat actions params
68
68
  const chatActionsParams = {
69
- get messageText() {
70
- return messageText;
71
- },
72
69
  get attachedFiles() {
73
70
  return fileHandling.attachedFiles;
74
71
  },
@@ -229,7 +226,11 @@
229
226
  appState.isLoading = false;
230
227
  lastCatchupProjectId = undefined;
231
228
  } else if (!hasActiveForSession && !appState.isLoading) {
232
- // No active streams for this session — just reset catchup tracking.
229
+ // No active streams for this session — clear cancelling state and reset catchup tracking.
230
+ // This is the authoritative signal that the cancel is fully complete (presence confirmed).
231
+ if (appState.isCancelling) {
232
+ appState.isCancelling = false;
233
+ }
233
234
  lastCatchupProjectId = undefined;
234
235
  }
235
236
  });
@@ -441,6 +442,7 @@
441
442
  <!-- Action buttons -->
442
443
  <ChatInputActions
443
444
  isLoading={appState.isLoading}
445
+ isCancelling={appState.isCancelling}
444
446
  hasActiveProject={hasActiveProject}
445
447
  messageText={messageText}
446
448
  attachedFiles={fileHandling.attachedFiles}