@myrialabs/clopen 0.2.4 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/backend/chat/stream-manager.ts +136 -10
- package/backend/database/queries/session-queries.ts +9 -0
- package/backend/engine/adapters/claude/error-handler.ts +7 -2
- package/backend/engine/adapters/claude/stream.ts +21 -3
- package/backend/engine/adapters/opencode/message-converter.ts +37 -2
- package/backend/index.ts +25 -3
- package/backend/preview/browser/browser-preview-service.ts +16 -17
- package/backend/preview/browser/browser-video-capture.ts +199 -156
- package/backend/preview/browser/scripts/video-stream.ts +3 -5
- package/backend/snapshot/helpers.ts +15 -2
- package/backend/ws/snapshot/restore.ts +43 -2
- package/backend/ws/user/crud.ts +6 -3
- package/frontend/components/chat/input/ChatInput.svelte +6 -1
- package/frontend/components/chat/input/components/ChatInputActions.svelte +10 -0
- package/frontend/components/chat/message/MessageBubble.svelte +22 -1
- package/frontend/components/chat/tools/components/FileHeader.svelte +19 -5
- package/frontend/components/chat/tools/components/TerminalCommand.svelte +2 -2
- package/frontend/components/chat/widgets/FloatingTodoList.svelte +30 -26
- package/frontend/components/common/media/MediaPreview.svelte +187 -0
- package/frontend/components/files/FileViewer.svelte +23 -144
- package/frontend/components/git/DiffViewer.svelte +50 -130
- package/frontend/components/git/FileChangeItem.svelte +22 -0
- package/frontend/components/preview/browser/BrowserPreview.svelte +1 -0
- package/frontend/components/preview/browser/components/Canvas.svelte +110 -21
- package/frontend/components/preview/browser/components/Container.svelte +2 -1
- package/frontend/components/preview/browser/components/Toolbar.svelte +1 -8
- package/frontend/components/preview/browser/core/coordinator.svelte.ts +12 -4
- package/frontend/components/terminal/TerminalTabs.svelte +1 -2
- package/frontend/components/workspace/DesktopNavigator.svelte +27 -1
- package/frontend/components/workspace/WorkspaceLayout.svelte +3 -1
- package/frontend/components/workspace/panels/FilesPanel.svelte +77 -1
- package/frontend/components/workspace/panels/GitPanel.svelte +72 -28
- package/frontend/services/chat/chat.service.ts +6 -1
- package/frontend/services/preview/browser/browser-webcodecs.service.ts +13 -5
- package/frontend/stores/core/files.svelte.ts +15 -1
- package/frontend/stores/ui/todo-panel.svelte.ts +39 -0
- package/frontend/utils/file-type.ts +68 -0
- package/index.html +1 -0
- package/package.json +1 -1
- package/shared/constants/binary-extensions.ts +40 -0
- package/shared/types/messaging/tool.ts +1 -0
- package/shared/utils/file-type-detection.ts +9 -1
- package/static/manifest.json +16 -0
|
@@ -41,6 +41,7 @@ interface VideoStreamSession {
|
|
|
41
41
|
headlessReady: boolean;
|
|
42
42
|
pendingCandidates: RTCIceCandidateInit[];
|
|
43
43
|
scriptInjected: boolean; // Track if persistent script was injected
|
|
44
|
+
scriptsPreInjected: boolean; // Track if scripts were pre-injected during tab creation
|
|
44
45
|
stats: {
|
|
45
46
|
videoBytesSent: number;
|
|
46
47
|
audioBytesSent: number;
|
|
@@ -52,11 +53,120 @@ interface VideoStreamSession {
|
|
|
52
53
|
|
|
53
54
|
export class BrowserVideoCapture extends EventEmitter {
|
|
54
55
|
private sessions = new Map<string, VideoStreamSession>();
|
|
56
|
+
private preInjectPromises = new Map<string, Promise<boolean>>();
|
|
55
57
|
|
|
56
58
|
constructor() {
|
|
57
59
|
super();
|
|
58
60
|
}
|
|
59
61
|
|
|
62
|
+
/**
|
|
63
|
+
* Pre-inject WebCodecs scripts during tab creation.
|
|
64
|
+
* This overlaps script injection with frontend processing,
|
|
65
|
+
* so startStreaming() only needs batched init + CDP setup (~50-80ms).
|
|
66
|
+
*/
|
|
67
|
+
preInjectScripts(sessionId: string, session: BrowserTab): Promise<boolean> {
|
|
68
|
+
const promise = this.doPreInject(sessionId, session);
|
|
69
|
+
this.preInjectPromises.set(sessionId, promise);
|
|
70
|
+
return promise;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
private async doPreInject(sessionId: string, session: BrowserTab): Promise<boolean> {
|
|
74
|
+
if (!session.page || session.page.isClosed()) return false;
|
|
75
|
+
|
|
76
|
+
try {
|
|
77
|
+
const page = session.page;
|
|
78
|
+
const viewport = page.viewport()!;
|
|
79
|
+
const config = DEFAULT_STREAMING_CONFIG;
|
|
80
|
+
const scale = session.scale || 1;
|
|
81
|
+
|
|
82
|
+
const scaledWidth = Math.round(viewport.width * scale);
|
|
83
|
+
const scaledHeight = Math.round(viewport.height * scale);
|
|
84
|
+
|
|
85
|
+
const videoConfig: StreamingConfig['video'] = {
|
|
86
|
+
...config.video,
|
|
87
|
+
width: scaledWidth,
|
|
88
|
+
height: scaledHeight
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
// Create session tracking
|
|
92
|
+
const videoSession: VideoStreamSession = {
|
|
93
|
+
sessionId,
|
|
94
|
+
isActive: false,
|
|
95
|
+
clientConnected: false,
|
|
96
|
+
headlessReady: false,
|
|
97
|
+
pendingCandidates: [],
|
|
98
|
+
scriptInjected: true,
|
|
99
|
+
scriptsPreInjected: false, // Set to true only after injection completes
|
|
100
|
+
stats: {
|
|
101
|
+
videoBytesSent: 0,
|
|
102
|
+
audioBytesSent: 0,
|
|
103
|
+
videoFramesEncoded: 0,
|
|
104
|
+
audioFramesEncoded: 0,
|
|
105
|
+
connectionState: 'new'
|
|
106
|
+
}
|
|
107
|
+
};
|
|
108
|
+
this.sessions.set(sessionId, videoSession);
|
|
109
|
+
|
|
110
|
+
await this.injectScripts(sessionId, page, videoConfig, config);
|
|
111
|
+
|
|
112
|
+
// Mark as pre-injected only after successful completion
|
|
113
|
+
videoSession.scriptsPreInjected = true;
|
|
114
|
+
|
|
115
|
+
debug.log('webcodecs', `Pre-injected scripts for ${sessionId}`);
|
|
116
|
+
return true;
|
|
117
|
+
} catch (error) {
|
|
118
|
+
debug.warn('webcodecs', `Pre-injection failed for ${sessionId}:`, error);
|
|
119
|
+
// Clean up so startStreaming() will do full injection
|
|
120
|
+
this.sessions.delete(sessionId);
|
|
121
|
+
return false;
|
|
122
|
+
} finally {
|
|
123
|
+
this.preInjectPromises.delete(sessionId);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Inject signaling bindings + encoder scripts into page
|
|
129
|
+
*/
|
|
130
|
+
private async injectScripts(
|
|
131
|
+
sessionId: string,
|
|
132
|
+
page: Page,
|
|
133
|
+
videoConfig: StreamingConfig['video'],
|
|
134
|
+
config: StreamingConfig
|
|
135
|
+
): Promise<void> {
|
|
136
|
+
// Check if bindings exist
|
|
137
|
+
const bindingsExist = await page.evaluate(() => {
|
|
138
|
+
return typeof (window as any).__sendIceCandidate === 'function';
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
// Expose signaling functions (persists across navigations)
|
|
142
|
+
if (!bindingsExist) {
|
|
143
|
+
await page.exposeFunction('__sendIceCandidate', (candidate: RTCIceCandidateInit) => {
|
|
144
|
+
const activeSession = Array.from(this.sessions.values()).find(s => s.isActive);
|
|
145
|
+
if (!activeSession) return;
|
|
146
|
+
this.emit('ice-candidate', { sessionId: activeSession.sessionId, candidate, from: 'headless' });
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
await page.exposeFunction('__sendConnectionState', (state: string) => {
|
|
150
|
+
const activeSession = Array.from(this.sessions.values()).find(s => s.isActive);
|
|
151
|
+
if (activeSession) {
|
|
152
|
+
activeSession.stats.connectionState = state;
|
|
153
|
+
this.emit('connection-state', { sessionId: activeSession.sessionId, state });
|
|
154
|
+
}
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
await page.exposeFunction('__sendCursorChange', (cursor: string) => {
|
|
158
|
+
const activeSession = Array.from(this.sessions.values()).find(s => s.isActive);
|
|
159
|
+
if (activeSession) {
|
|
160
|
+
this.emit('cursor-change', { sessionId: activeSession.sessionId, cursor });
|
|
161
|
+
}
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
// Inject video encoder + audio capture scripts
|
|
166
|
+
await page.evaluate(videoEncoderScript, videoConfig);
|
|
167
|
+
await page.evaluate(audioCaptureScript, config.audio);
|
|
168
|
+
}
|
|
169
|
+
|
|
60
170
|
/**
|
|
61
171
|
* Start video streaming for a session
|
|
62
172
|
*/
|
|
@@ -67,11 +177,20 @@ export class BrowserVideoCapture extends EventEmitter {
|
|
|
67
177
|
): Promise<boolean> {
|
|
68
178
|
debug.log('webcodecs', `Starting streaming for session ${sessionId}`);
|
|
69
179
|
|
|
70
|
-
//
|
|
71
|
-
|
|
72
|
-
|
|
180
|
+
// Wait for any pending pre-injection to complete
|
|
181
|
+
const pendingPreInject = this.preInjectPromises.get(sessionId);
|
|
182
|
+
if (pendingPreInject) {
|
|
183
|
+
debug.log('webcodecs', `Waiting for pre-injection to complete for ${sessionId}`);
|
|
184
|
+
await pendingPreInject.catch(() => {});
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
// If session is already actively streaming, stop it for a clean reconnect.
|
|
188
|
+
// This ensures the old PeerConnection + DataChannel are torn down and
|
|
189
|
+
// a fresh one is created, preventing stale connections where no frames flow.
|
|
190
|
+
const existingSession = this.sessions.get(sessionId);
|
|
191
|
+
if (existingSession && existingSession.isActive) {
|
|
192
|
+
debug.log('webcodecs', `Session ${sessionId} already active, stopping for clean reconnect`);
|
|
73
193
|
await this.stopStreaming(sessionId, session);
|
|
74
|
-
await new Promise(resolve => setTimeout(resolve, 100));
|
|
75
194
|
}
|
|
76
195
|
|
|
77
196
|
if (!session.page || session.page.isClosed()) {
|
|
@@ -100,6 +219,7 @@ export class BrowserVideoCapture extends EventEmitter {
|
|
|
100
219
|
headlessReady: false,
|
|
101
220
|
pendingCandidates: [],
|
|
102
221
|
scriptInjected: false,
|
|
222
|
+
scriptsPreInjected: false,
|
|
103
223
|
stats: {
|
|
104
224
|
videoBytesSent: 0,
|
|
105
225
|
audioBytesSent: 0,
|
|
@@ -111,35 +231,6 @@ export class BrowserVideoCapture extends EventEmitter {
|
|
|
111
231
|
this.sessions.set(sessionId, videoSession);
|
|
112
232
|
}
|
|
113
233
|
|
|
114
|
-
// Check if bindings exist
|
|
115
|
-
const bindingsExist = await page.evaluate(() => {
|
|
116
|
-
return typeof (window as any).__sendIceCandidate === 'function';
|
|
117
|
-
});
|
|
118
|
-
|
|
119
|
-
// Expose signaling functions (persists across navigations)
|
|
120
|
-
if (!bindingsExist) {
|
|
121
|
-
await page.exposeFunction('__sendIceCandidate', (candidate: RTCIceCandidateInit) => {
|
|
122
|
-
const activeSession = Array.from(this.sessions.values()).find(s => s.isActive);
|
|
123
|
-
if (!activeSession) return;
|
|
124
|
-
this.emit('ice-candidate', { sessionId: activeSession.sessionId, candidate, from: 'headless' });
|
|
125
|
-
});
|
|
126
|
-
|
|
127
|
-
await page.exposeFunction('__sendConnectionState', (state: string) => {
|
|
128
|
-
const activeSession = Array.from(this.sessions.values()).find(s => s.isActive);
|
|
129
|
-
if (activeSession) {
|
|
130
|
-
activeSession.stats.connectionState = state;
|
|
131
|
-
this.emit('connection-state', { sessionId: activeSession.sessionId, state });
|
|
132
|
-
}
|
|
133
|
-
});
|
|
134
|
-
|
|
135
|
-
await page.exposeFunction('__sendCursorChange', (cursor: string) => {
|
|
136
|
-
const activeSession = Array.from(this.sessions.values()).find(s => s.isActive);
|
|
137
|
-
if (activeSession) {
|
|
138
|
-
this.emit('cursor-change', { sessionId: activeSession.sessionId, cursor });
|
|
139
|
-
}
|
|
140
|
-
});
|
|
141
|
-
}
|
|
142
|
-
|
|
143
234
|
// Calculate scaled dimensions
|
|
144
235
|
const scaledWidth = Math.round(viewport.width * scale);
|
|
145
236
|
const scaledHeight = Math.round(viewport.height * scale);
|
|
@@ -150,91 +241,60 @@ export class BrowserVideoCapture extends EventEmitter {
|
|
|
150
241
|
height: scaledHeight
|
|
151
242
|
};
|
|
152
243
|
|
|
153
|
-
//
|
|
154
|
-
|
|
155
|
-
(
|
|
156
|
-
}, videoConfig);
|
|
157
|
-
|
|
158
|
-
// Inject persistent video encoder script (survives navigation)
|
|
159
|
-
// Only inject once per page instance
|
|
160
|
-
if (!videoSession.scriptInjected) {
|
|
161
|
-
// Temporarily disable evaluateOnNewDocument for evasion test
|
|
162
|
-
// await page.evaluateOnNewDocument(videoEncoderScript, videoConfig);
|
|
244
|
+
// Skip script injection if already pre-injected during tab creation
|
|
245
|
+
if (!videoSession.scriptsPreInjected) {
|
|
246
|
+
await this.injectScripts(sessionId, page, videoConfig, config);
|
|
163
247
|
videoSession.scriptInjected = true;
|
|
164
|
-
|
|
248
|
+
} else {
|
|
249
|
+
debug.log('webcodecs', `Scripts already pre-injected for ${sessionId}, skipping injection`);
|
|
165
250
|
}
|
|
166
251
|
|
|
167
|
-
//
|
|
168
|
-
// (
|
|
169
|
-
await page.evaluate(
|
|
252
|
+
// Single batched call: verify peer + start streaming + init audio
|
|
253
|
+
// (saves ~60ms of IPC overhead vs 4 separate page.evaluate calls)
|
|
254
|
+
const initResult = await page.evaluate(async () => {
|
|
255
|
+
const peer = (window as any).__webCodecsPeer;
|
|
256
|
+
if (typeof peer?.startStreaming !== 'function') {
|
|
257
|
+
return { peerExists: false, started: false, audioInitialized: false };
|
|
258
|
+
}
|
|
170
259
|
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
260
|
+
const started = await peer.startStreaming();
|
|
261
|
+
if (!started) {
|
|
262
|
+
return { peerExists: true, started: false, audioInitialized: false };
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
// Initialize audio encoder if available
|
|
266
|
+
let audioInitialized = false;
|
|
267
|
+
const encoder = (window as any).__audioEncoder;
|
|
268
|
+
if (typeof encoder?.init === 'function') {
|
|
269
|
+
try {
|
|
270
|
+
const initiated = await encoder.init();
|
|
271
|
+
if (initiated) {
|
|
272
|
+
audioInitialized = !!encoder.start();
|
|
273
|
+
}
|
|
274
|
+
} catch {}
|
|
275
|
+
}
|
|
176
276
|
|
|
177
|
-
|
|
178
|
-
const peerExists = await page.evaluate(() => {
|
|
179
|
-
return typeof (window as any).__webCodecsPeer?.startStreaming === 'function';
|
|
277
|
+
return { peerExists: true, started: true, audioInitialized };
|
|
180
278
|
});
|
|
181
279
|
|
|
182
|
-
if (!peerExists) {
|
|
280
|
+
if (!initResult.peerExists) {
|
|
183
281
|
debug.error('webcodecs', `Peer script injected but __webCodecsPeer not available`);
|
|
184
282
|
this.sessions.delete(sessionId);
|
|
185
283
|
return false;
|
|
186
284
|
}
|
|
187
285
|
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
// Wait for page to be fully loaded
|
|
191
|
-
try {
|
|
192
|
-
const loadState = await page.evaluate(() => document.readyState);
|
|
193
|
-
if (loadState !== 'complete') {
|
|
194
|
-
debug.log('webcodecs', `Waiting for page load...`);
|
|
195
|
-
await page.waitForFunction(() => document.readyState === 'complete', { timeout: 60000 });
|
|
196
|
-
}
|
|
197
|
-
} catch (loadError) {
|
|
198
|
-
debug.warn('webcodecs', 'Page load wait timed out, proceeding anyway');
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
// Start video streaming
|
|
202
|
-
const started = await page.evaluate(() => {
|
|
203
|
-
return (window as any).__webCodecsPeer?.startStreaming();
|
|
204
|
-
});
|
|
205
|
-
|
|
206
|
-
if (!started) {
|
|
286
|
+
if (!initResult.started) {
|
|
207
287
|
debug.error('webcodecs', `startStreaming returned false`);
|
|
208
288
|
this.sessions.delete(sessionId);
|
|
209
289
|
return false;
|
|
210
290
|
}
|
|
211
291
|
|
|
212
|
-
|
|
213
|
-
const audioEncoderAvailable = await page.evaluate(() => {
|
|
214
|
-
return typeof (window as any).__audioEncoder?.init === 'function';
|
|
215
|
-
});
|
|
216
|
-
|
|
217
|
-
if (audioEncoderAvailable) {
|
|
218
|
-
debug.log('webcodecs', 'Initializing audio encoder from AudioContext interception...');
|
|
219
|
-
|
|
220
|
-
const audioInitialized = await page.evaluate(async () => {
|
|
221
|
-
const encoder = (window as any).__audioEncoder;
|
|
222
|
-
if (!encoder) return false;
|
|
223
|
-
|
|
224
|
-
const initiated = await encoder.init();
|
|
225
|
-
if (initiated) {
|
|
226
|
-
return encoder.start();
|
|
227
|
-
}
|
|
228
|
-
return false;
|
|
229
|
-
});
|
|
292
|
+
videoSession.isActive = true;
|
|
230
293
|
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
} else {
|
|
234
|
-
debug.warn('webcodecs', 'Audio encoder initialization failed, continuing with video only');
|
|
235
|
-
}
|
|
294
|
+
if (initResult.audioInitialized) {
|
|
295
|
+
debug.log('webcodecs', 'Audio encoder initialized and started');
|
|
236
296
|
} else {
|
|
237
|
-
debug.warn('webcodecs', 'Audio
|
|
297
|
+
debug.warn('webcodecs', 'Audio not available, continuing with video only');
|
|
238
298
|
}
|
|
239
299
|
|
|
240
300
|
videoSession.headlessReady = true;
|
|
@@ -325,25 +385,16 @@ export class BrowserVideoCapture extends EventEmitter {
|
|
|
325
385
|
return null;
|
|
326
386
|
}
|
|
327
387
|
|
|
328
|
-
const maxRetries =
|
|
329
|
-
const retryDelay =
|
|
388
|
+
const maxRetries = 3;
|
|
389
|
+
const retryDelay = 50;
|
|
330
390
|
|
|
331
391
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
332
392
|
try {
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
if (attempt < maxRetries - 1) {
|
|
339
|
-
await new Promise(resolve => setTimeout(resolve, retryDelay));
|
|
340
|
-
continue;
|
|
341
|
-
}
|
|
342
|
-
return null;
|
|
343
|
-
}
|
|
344
|
-
|
|
345
|
-
const offer = await session.page.evaluate(() => {
|
|
346
|
-
return (window as any).__webCodecsPeer?.createOffer();
|
|
393
|
+
// Single evaluate: check peer + create offer in one IPC round-trip
|
|
394
|
+
const offer = await session.page.evaluate(async () => {
|
|
395
|
+
const peer = (window as any).__webCodecsPeer;
|
|
396
|
+
if (typeof peer?.createOffer !== 'function') return null;
|
|
397
|
+
return peer.createOffer();
|
|
347
398
|
});
|
|
348
399
|
|
|
349
400
|
if (offer) return offer;
|
|
@@ -570,54 +621,50 @@ export class BrowserVideoCapture extends EventEmitter {
|
|
|
570
621
|
height: scaledHeight
|
|
571
622
|
};
|
|
572
623
|
|
|
573
|
-
// Re-inject video encoder
|
|
574
|
-
// (evaluateOnNewDocument doesn't run for the current navigation, only future ones)
|
|
575
|
-
await page.evaluate((cfg) => {
|
|
576
|
-
(window as any).__videoEncoderConfig = cfg;
|
|
577
|
-
}, videoConfig);
|
|
578
|
-
|
|
624
|
+
// Re-inject video encoder and audio capture scripts to new page context
|
|
579
625
|
await page.evaluate(videoEncoderScript, videoConfig);
|
|
580
|
-
|
|
581
|
-
// Re-inject audio capture script for new page context (post-navigation)
|
|
582
626
|
await page.evaluate(audioCaptureScript, config.audio);
|
|
583
627
|
|
|
584
|
-
//
|
|
585
|
-
const
|
|
586
|
-
|
|
628
|
+
// Single batched call: verify peer + start streaming + init audio
|
|
629
|
+
const initResult = await page.evaluate(async () => {
|
|
630
|
+
const peer = (window as any).__webCodecsPeer;
|
|
631
|
+
if (typeof peer?.startStreaming !== 'function') {
|
|
632
|
+
return { peerExists: false, started: false, audioInitialized: false };
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
const started = await peer.startStreaming();
|
|
636
|
+
if (!started) {
|
|
637
|
+
return { peerExists: true, started: false, audioInitialized: false };
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
let audioInitialized = false;
|
|
641
|
+
const encoder = (window as any).__audioEncoder;
|
|
642
|
+
if (typeof encoder?.init === 'function') {
|
|
643
|
+
try {
|
|
644
|
+
const initiated = await encoder.init();
|
|
645
|
+
if (initiated) {
|
|
646
|
+
audioInitialized = !!encoder.start();
|
|
647
|
+
}
|
|
648
|
+
} catch {}
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
return { peerExists: true, started: true, audioInitialized };
|
|
587
652
|
});
|
|
588
653
|
|
|
589
|
-
if (!peerExists) {
|
|
654
|
+
if (!initResult.peerExists) {
|
|
590
655
|
debug.error('webcodecs', `Peer script re-injection failed - peer not available`);
|
|
591
656
|
return false;
|
|
592
657
|
}
|
|
593
658
|
|
|
594
|
-
|
|
595
|
-
const started = await page.evaluate(() => {
|
|
596
|
-
return (window as any).__webCodecsPeer?.startStreaming();
|
|
597
|
-
});
|
|
598
|
-
|
|
599
|
-
if (!started) {
|
|
659
|
+
if (!initResult.started) {
|
|
600
660
|
debug.error('webcodecs', `Failed to start streaming on new page`);
|
|
601
661
|
return false;
|
|
602
662
|
}
|
|
603
663
|
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
if (!encoder) return false;
|
|
609
|
-
const initiated = await encoder.init();
|
|
610
|
-
if (initiated) return encoder.start();
|
|
611
|
-
return false;
|
|
612
|
-
});
|
|
613
|
-
|
|
614
|
-
if (audioReady) {
|
|
615
|
-
debug.log('webcodecs', 'Audio re-initialized after navigation');
|
|
616
|
-
} else {
|
|
617
|
-
debug.warn('webcodecs', 'Audio not available after navigation, continuing with video only');
|
|
618
|
-
}
|
|
619
|
-
} catch {
|
|
620
|
-
debug.warn('webcodecs', 'Audio re-init failed after navigation, continuing with video only');
|
|
664
|
+
if (initResult.audioInitialized) {
|
|
665
|
+
debug.log('webcodecs', 'Audio re-initialized after navigation');
|
|
666
|
+
} else {
|
|
667
|
+
debug.warn('webcodecs', 'Audio not available after navigation, continuing with video only');
|
|
621
668
|
}
|
|
622
669
|
|
|
623
670
|
// Restart CDP screencast
|
|
@@ -661,15 +708,11 @@ export class BrowserVideoCapture extends EventEmitter {
|
|
|
661
708
|
|
|
662
709
|
if (session?.page && !session.page.isClosed()) {
|
|
663
710
|
try {
|
|
664
|
-
// Stop audio
|
|
711
|
+
// Stop audio + peer in one IPC round-trip
|
|
665
712
|
await session.page.evaluate(() => {
|
|
666
713
|
(window as any).__audioEncoder?.stop();
|
|
667
|
-
}).catch(() => {});
|
|
668
|
-
|
|
669
|
-
// Stop peer
|
|
670
|
-
await session.page.evaluate(() => {
|
|
671
714
|
(window as any).__webCodecsPeer?.stopStreaming();
|
|
672
|
-
});
|
|
715
|
+
}).catch(() => {});
|
|
673
716
|
|
|
674
717
|
// Stop CDP screencast
|
|
675
718
|
const cdp = (session as any).__webCodecsCdp;
|
|
@@ -34,11 +34,9 @@ export function videoEncoderScript(config: StreamingConfig['video']) {
|
|
|
34
34
|
let lastCursor = 'default';
|
|
35
35
|
let cursorCheckInterval: any = null;
|
|
36
36
|
|
|
37
|
-
// ICE servers
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
{ urls: 'stun:stun1.l.google.com:19302' }
|
|
41
|
-
];
|
|
37
|
+
// ICE servers - empty for local connections (both peers on same machine)
|
|
38
|
+
// STUN servers are unnecessary for localhost and add 100-500ms ICE gathering latency
|
|
39
|
+
const iceServers: { urls: string }[] = [];
|
|
42
40
|
|
|
43
41
|
// Check cursor style from page
|
|
44
42
|
function checkCursor() {
|
|
@@ -35,12 +35,25 @@ export interface TimelineResponse {
|
|
|
35
35
|
}
|
|
36
36
|
|
|
37
37
|
/**
|
|
38
|
-
* Check if a user message is an internal
|
|
39
|
-
*
|
|
38
|
+
* Check if a user message is an internal/non-genuine user message.
|
|
39
|
+
*
|
|
40
|
+
* The SDK uses `type: 'user'` for several kinds of messages that are NOT
|
|
41
|
+
* typed by the real human user:
|
|
42
|
+
* 1. Tool-result confirmations — content contains `tool_result` blocks
|
|
43
|
+
* 2. Sub-agent / Task prompts — `parent_tool_use_id` is non-null
|
|
44
|
+
* 3. Post-compaction synthetic summaries — `isSynthetic` is true
|
|
45
|
+
*
|
|
46
|
+
* Only messages that pass none of these checks are genuine user input.
|
|
40
47
|
*/
|
|
41
48
|
export function isInternalToolMessage(sdkMessage: any): boolean {
|
|
42
49
|
if (sdkMessage.type !== 'user') return false;
|
|
43
50
|
|
|
51
|
+
// Sub-agent or tool-result message (parent_tool_use_id is set)
|
|
52
|
+
if (sdkMessage.parent_tool_use_id != null) return true;
|
|
53
|
+
|
|
54
|
+
// Synthetic message generated after context compaction
|
|
55
|
+
if (sdkMessage.isSynthetic === true) return true;
|
|
56
|
+
|
|
44
57
|
const content = sdkMessage.message?.content;
|
|
45
58
|
if (!content) return false;
|
|
46
59
|
|
|
@@ -200,18 +200,56 @@ export const restoreHandler = createRouter()
|
|
|
200
200
|
sessionQueries.updateHead(sessionId, sessionEnd.id);
|
|
201
201
|
debug.log('snapshot', `HEAD updated to: ${sessionEnd.id}`);
|
|
202
202
|
|
|
203
|
-
// 5b. Update latest_sdk_session_id so resume works correctly
|
|
203
|
+
// 5b. Update latest_sdk_session_id so resume works correctly.
|
|
204
|
+
// Claude Code: skip cancelled fork session_ids (partial messages from cancelStream).
|
|
205
|
+
// OpenCode: simple walk — any session_id is valid (sessions created synchronously).
|
|
204
206
|
{
|
|
205
|
-
let walkId: string | null = sessionEnd.id;
|
|
206
207
|
let foundSdkSessionId: string | null = null;
|
|
207
208
|
const msgLookup = new Map(allMessages.map(m => [m.id, m]));
|
|
209
|
+
const sessionRecord = sessionQueries.getById(sessionId);
|
|
210
|
+
const isClaudeCode = sessionRecord?.engine === 'claude-code';
|
|
211
|
+
|
|
212
|
+
// Claude Code only: detect cancelled stream by partialText marker on sessionEnd
|
|
213
|
+
let cancelledSessionId: string | null = null;
|
|
214
|
+
if (isClaudeCode) {
|
|
215
|
+
try {
|
|
216
|
+
const endMsg = msgLookup.get(sessionEnd.id);
|
|
217
|
+
if (endMsg) {
|
|
218
|
+
const endSdk = JSON.parse(endMsg.sdk_message);
|
|
219
|
+
if (endSdk.partialText) {
|
|
220
|
+
cancelledSessionId = endSdk.session_id || null;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
} catch { /* skip */ }
|
|
224
|
+
}
|
|
208
225
|
|
|
226
|
+
let walkId: string | null = sessionEnd.id;
|
|
209
227
|
while (walkId) {
|
|
210
228
|
const walkMsg = msgLookup.get(walkId);
|
|
211
229
|
if (!walkMsg) break;
|
|
212
230
|
|
|
213
231
|
try {
|
|
214
232
|
const sdk = JSON.parse(walkMsg.sdk_message);
|
|
233
|
+
|
|
234
|
+
// Claude Code only: skip partial messages (from cancelled streams)
|
|
235
|
+
if (isClaudeCode && sdk.partialText) {
|
|
236
|
+
walkId = walkMsg.parent_message_id || null;
|
|
237
|
+
continue;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// Claude Code only: skip assistant messages from the same cancelled fork
|
|
241
|
+
if (isClaudeCode && cancelledSessionId && sdk.session_id === cancelledSessionId) {
|
|
242
|
+
walkId = walkMsg.parent_message_id || null;
|
|
243
|
+
continue;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// Claude Code only: user message's `resume` field records the last valid session_id
|
|
247
|
+
if (isClaudeCode && sdk.type === 'user' && 'resume' in sdk) {
|
|
248
|
+
foundSdkSessionId = sdk.resume || null;
|
|
249
|
+
break;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
// Any engine: message with session_id
|
|
215
253
|
if (sdk.session_id) {
|
|
216
254
|
foundSdkSessionId = sdk.session_id;
|
|
217
255
|
break;
|
|
@@ -224,6 +262,9 @@ export const restoreHandler = createRouter()
|
|
|
224
262
|
if (foundSdkSessionId) {
|
|
225
263
|
sessionQueries.updateLatestSdkSessionId(sessionId, foundSdkSessionId);
|
|
226
264
|
debug.log('snapshot', `latest_sdk_session_id updated to: ${foundSdkSessionId}`);
|
|
265
|
+
} else {
|
|
266
|
+
sessionQueries.clearLatestSdkSessionId(sessionId);
|
|
267
|
+
debug.log('snapshot', 'latest_sdk_session_id cleared (no valid session found in restored chain)');
|
|
227
268
|
}
|
|
228
269
|
}
|
|
229
270
|
|
package/backend/ws/user/crud.ts
CHANGED
|
@@ -109,7 +109,8 @@ export const crudHandler = createRouter()
|
|
|
109
109
|
currentProjectId: t.Union([t.String(), t.Null()]),
|
|
110
110
|
lastView: t.Union([t.String(), t.Null()]),
|
|
111
111
|
settings: t.Union([t.Any(), t.Null()]),
|
|
112
|
-
unreadSessions: t.Union([t.Any(), t.Null()])
|
|
112
|
+
unreadSessions: t.Union([t.Any(), t.Null()]),
|
|
113
|
+
todoPanelState: t.Union([t.Any(), t.Null()])
|
|
113
114
|
})
|
|
114
115
|
}, async ({ conn }) => {
|
|
115
116
|
const userId = ws.getUserId(conn);
|
|
@@ -118,6 +119,7 @@ export const crudHandler = createRouter()
|
|
|
118
119
|
const lastView = getUserState(userId, 'lastView') as string | null;
|
|
119
120
|
const userSettings = getUserState(userId, 'settings');
|
|
120
121
|
const unreadSessions = getUserState(userId, 'unreadSessions');
|
|
122
|
+
const todoPanelState = getUserState(userId, 'todoPanelState');
|
|
121
123
|
|
|
122
124
|
debug.log('user', `Restored state for ${userId}:`, {
|
|
123
125
|
currentProjectId,
|
|
@@ -130,7 +132,8 @@ export const crudHandler = createRouter()
|
|
|
130
132
|
currentProjectId: currentProjectId ?? null,
|
|
131
133
|
lastView: lastView ?? null,
|
|
132
134
|
settings: userSettings ?? null,
|
|
133
|
-
unreadSessions: unreadSessions ?? null
|
|
135
|
+
unreadSessions: unreadSessions ?? null,
|
|
136
|
+
todoPanelState: todoPanelState ?? null
|
|
134
137
|
};
|
|
135
138
|
})
|
|
136
139
|
|
|
@@ -147,7 +150,7 @@ export const crudHandler = createRouter()
|
|
|
147
150
|
const userId = ws.getUserId(conn);
|
|
148
151
|
|
|
149
152
|
// Validate allowed keys to prevent arbitrary data storage
|
|
150
|
-
const allowedKeys = ['currentProjectId', 'lastView', 'settings', 'unreadSessions'];
|
|
153
|
+
const allowedKeys = ['currentProjectId', 'lastView', 'settings', 'unreadSessions', 'todoPanelState'];
|
|
151
154
|
if (!allowedKeys.includes(data.key)) {
|
|
152
155
|
throw new Error(`Invalid state key: ${data.key}. Allowed: ${allowedKeys.join(', ')}`);
|
|
153
156
|
}
|
|
@@ -229,7 +229,11 @@
|
|
|
229
229
|
appState.isLoading = false;
|
|
230
230
|
lastCatchupProjectId = undefined;
|
|
231
231
|
} else if (!hasActiveForSession && !appState.isLoading) {
|
|
232
|
-
// No active streams for this session —
|
|
232
|
+
// No active streams for this session — clear cancelling state and reset catchup tracking.
|
|
233
|
+
// This is the authoritative signal that the cancel is fully complete (presence confirmed).
|
|
234
|
+
if (appState.isCancelling) {
|
|
235
|
+
appState.isCancelling = false;
|
|
236
|
+
}
|
|
233
237
|
lastCatchupProjectId = undefined;
|
|
234
238
|
}
|
|
235
239
|
});
|
|
@@ -441,6 +445,7 @@
|
|
|
441
445
|
<!-- Action buttons -->
|
|
442
446
|
<ChatInputActions
|
|
443
447
|
isLoading={appState.isLoading}
|
|
448
|
+
isCancelling={appState.isCancelling}
|
|
444
449
|
hasActiveProject={hasActiveProject}
|
|
445
450
|
messageText={messageText}
|
|
446
451
|
attachedFiles={fileHandling.attachedFiles}
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
interface Props {
|
|
6
6
|
isLoading: boolean;
|
|
7
|
+
isCancelling: boolean;
|
|
7
8
|
hasActiveProject: boolean;
|
|
8
9
|
messageText: string;
|
|
9
10
|
attachedFiles: FileAttachment[];
|
|
@@ -16,6 +17,7 @@
|
|
|
16
17
|
|
|
17
18
|
const {
|
|
18
19
|
isLoading,
|
|
20
|
+
isCancelling,
|
|
19
21
|
hasActiveProject,
|
|
20
22
|
messageText,
|
|
21
23
|
attachedFiles,
|
|
@@ -65,6 +67,14 @@
|
|
|
65
67
|
>
|
|
66
68
|
<Icon name="lucide:circle-stop" class="text-white w-5 h-5" />
|
|
67
69
|
</button>
|
|
70
|
+
{:else if isCancelling}
|
|
71
|
+
<button
|
|
72
|
+
disabled
|
|
73
|
+
class="w-10 h-10 bg-red-500 opacity-70 rounded-xl flex items-center justify-center transition-all duration-200 cursor-not-allowed"
|
|
74
|
+
aria-label="Cancelling..."
|
|
75
|
+
>
|
|
76
|
+
<Icon name="lucide:loader-circle" class="text-white w-5 h-5 animate-spin" />
|
|
77
|
+
</button>
|
|
68
78
|
{:else}
|
|
69
79
|
<button
|
|
70
80
|
onclick={onSend}
|