@myrialabs/clopen 0.2.9 ā 0.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/backend/index.ts +12 -0
- package/backend/preview/browser/browser-navigation-tracker.ts +188 -31
- package/backend/preview/browser/browser-pool.ts +1 -1
- package/backend/preview/browser/browser-preview-service.ts +23 -0
- package/backend/preview/browser/browser-tab-manager.ts +16 -1
- package/backend/preview/browser/browser-video-capture.ts +2 -2
- package/backend/preview/browser/scripts/video-stream.ts +39 -4
- package/backend/ws/preview/browser/webcodecs.ts +11 -0
- package/backend/ws/preview/index.ts +8 -0
- package/frontend/components/chat/input/ChatInput.svelte +3 -3
- package/frontend/components/preview/browser/BrowserPreview.svelte +10 -3
- package/frontend/components/preview/browser/components/Canvas.svelte +40 -23
- package/frontend/components/preview/browser/components/Container.svelte +8 -5
- package/frontend/components/preview/browser/components/Toolbar.svelte +15 -0
- package/frontend/components/preview/browser/core/coordinator.svelte.ts +13 -0
- package/frontend/components/preview/browser/core/stream-handler.svelte.ts +37 -4
- package/frontend/components/workspace/PanelHeader.svelte +8 -6
- package/frontend/components/workspace/panels/PreviewPanel.svelte +1 -0
- package/frontend/services/preview/browser/browser-webcodecs.service.ts +110 -18
- package/package.json +1 -1
package/backend/index.ts
CHANGED
|
@@ -156,7 +156,12 @@ startServer().catch((error) => {
|
|
|
156
156
|
});
|
|
157
157
|
|
|
158
158
|
// Graceful shutdown - properly close server and database
|
|
159
|
+
let isShuttingDown = false;
|
|
160
|
+
|
|
159
161
|
async function gracefulShutdown() {
|
|
162
|
+
if (isShuttingDown) return;
|
|
163
|
+
isShuttingDown = true;
|
|
164
|
+
|
|
160
165
|
console.log('\nš Shutting down server...');
|
|
161
166
|
try {
|
|
162
167
|
// Close MCP remote server (before engines, as they may still reference it)
|
|
@@ -177,6 +182,13 @@ async function gracefulShutdown() {
|
|
|
177
182
|
process.on('SIGINT', gracefulShutdown);
|
|
178
183
|
process.on('SIGTERM', gracefulShutdown);
|
|
179
184
|
|
|
185
|
+
// Ignore SIGHUP ā sent when the controlling terminal closes or an SSH session
|
|
186
|
+
// disconnects. Without a handler Bun exits immediately; we want the server to
|
|
187
|
+
// keep running (e.g. started in a background tab or remote shell).
|
|
188
|
+
process.on('SIGHUP', () => {
|
|
189
|
+
debug.log('server', 'Received SIGHUP ā ignoring (server stays running)');
|
|
190
|
+
});
|
|
191
|
+
|
|
180
192
|
// Safety net: prevent server crash from unhandled errors.
|
|
181
193
|
// These can occur when AI engine SDKs emit asynchronous errors that bypass
|
|
182
194
|
// the normal try/catch flow (e.g., subprocess killed during initialization).
|
|
@@ -1,22 +1,74 @@
|
|
|
1
1
|
import { EventEmitter } from 'events';
|
|
2
|
-
import type { Page, HTTPRequest, Frame } from 'puppeteer';
|
|
2
|
+
import type { Page, HTTPRequest, Frame, CDPSession } from 'puppeteer';
|
|
3
3
|
import type { BrowserTab } from './types';
|
|
4
|
+
import { debug } from '$shared/utils/logger';
|
|
4
5
|
|
|
5
6
|
export class BrowserNavigationTracker extends EventEmitter {
|
|
7
|
+
private cdpSessions = new Map<string, CDPSession>();
|
|
8
|
+
|
|
6
9
|
constructor() {
|
|
7
10
|
super();
|
|
8
11
|
}
|
|
9
12
|
|
|
13
|
+
/**
|
|
14
|
+
* Check if two URLs differ only by hash/fragment.
|
|
15
|
+
* Hash-only changes are same-document navigations and should NOT trigger
|
|
16
|
+
* full page reload or streaming restart.
|
|
17
|
+
*/
|
|
18
|
+
private isHashOnlyChange(oldUrl: string, newUrl: string): boolean {
|
|
19
|
+
try {
|
|
20
|
+
const oldParsed = new URL(oldUrl);
|
|
21
|
+
const newParsed = new URL(newUrl);
|
|
22
|
+
// Compare URLs without hash ā if identical, it's a hash-only change
|
|
23
|
+
oldParsed.hash = '';
|
|
24
|
+
newParsed.hash = '';
|
|
25
|
+
return oldParsed.href === newParsed.href;
|
|
26
|
+
} catch {
|
|
27
|
+
return false;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Check if two URLs share the same origin (protocol + host + port).
|
|
33
|
+
* Same-origin navigations are likely SPA internal navigations and should
|
|
34
|
+
* NOT show a progress bar ā the streaming restart happens silently while
|
|
35
|
+
* the last rendered frame stays visible.
|
|
36
|
+
*/
|
|
37
|
+
private isSameOrigin(oldUrl: string, newUrl: string): boolean {
|
|
38
|
+
try {
|
|
39
|
+
return new URL(oldUrl).origin === new URL(newUrl).origin;
|
|
40
|
+
} catch {
|
|
41
|
+
return false;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
10
45
|
async setupNavigationTracking(sessionId: string, page: Page, session: BrowserTab) {
|
|
11
46
|
|
|
12
|
-
// Track navigation start (loading begins)
|
|
47
|
+
// Track navigation start (loading begins) ā only for cross-origin document navigations
|
|
13
48
|
page.on('request', (request: HTTPRequest) => {
|
|
14
49
|
// Only track main frame document requests (not resources like images, CSS, etc.)
|
|
15
50
|
// Puppeteer uses resourceType() instead of isNavigationRequest()
|
|
16
51
|
if (request.resourceType() === 'document' && request.frame() === page.mainFrame()) {
|
|
17
52
|
const targetUrl = request.url();
|
|
18
53
|
|
|
19
|
-
//
|
|
54
|
+
// Skip hash-only changes ā they are same-document navigations
|
|
55
|
+
// that don't need loading states or streaming restart
|
|
56
|
+
if (this.isHashOnlyChange(session.url, targetUrl)) {
|
|
57
|
+
debug.log('preview', `āļø Skipping navigation-loading for hash-only change: ${session.url} ā ${targetUrl}`);
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// Skip same-origin navigations ā they are likely SPA internal navigations.
|
|
62
|
+
// No progress bar is shown; the last rendered frame stays visible while
|
|
63
|
+
// streaming restarts silently in the background. This makes SPA navigation
|
|
64
|
+
// feel instant, similar to how a real browser shows the old page until
|
|
65
|
+
// the new one is ready.
|
|
66
|
+
if (this.isSameOrigin(session.url, targetUrl)) {
|
|
67
|
+
debug.log('preview', `āļø Skipping navigation-loading for same-origin navigation: ${session.url} ā ${targetUrl}`);
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// Emit navigation loading event to frontend (cross-origin navigations only)
|
|
20
72
|
this.emit('navigation-loading', {
|
|
21
73
|
sessionId,
|
|
22
74
|
type: 'navigation-loading',
|
|
@@ -26,12 +78,59 @@ export class BrowserNavigationTracker extends EventEmitter {
|
|
|
26
78
|
}
|
|
27
79
|
});
|
|
28
80
|
|
|
29
|
-
// Track
|
|
30
|
-
page.on('framenavigated', (frame: Frame) => {
|
|
81
|
+
// Track full page navigations (actual page loads, not SPA)
|
|
82
|
+
page.on('framenavigated', async (frame: Frame) => {
|
|
31
83
|
// Only track main frame navigation (not iframes)
|
|
32
84
|
if (frame === page.mainFrame()) {
|
|
33
85
|
const newUrl = frame.url();
|
|
34
86
|
|
|
87
|
+
// Skip internal Chrome error/system pages ā they indicate a failed navigation
|
|
88
|
+
// and should not be surfaced to the frontend as a real URL change.
|
|
89
|
+
if (newUrl.startsWith('chrome-error://') || newUrl.startsWith('chrome://')) return;
|
|
90
|
+
|
|
91
|
+
// Skip if URL hasn't changed (already handled by navigatedWithinDocument)
|
|
92
|
+
if (newUrl === session.url) return;
|
|
93
|
+
|
|
94
|
+
// Hash-only changes should be treated as SPA navigations
|
|
95
|
+
// (no streaming restart needed, page context is unchanged)
|
|
96
|
+
if (this.isHashOnlyChange(session.url, newUrl)) {
|
|
97
|
+
debug.log('preview', `š Hash-only change detected, treating as SPA navigation: ${session.url} ā ${newUrl}`);
|
|
98
|
+
session.url = newUrl;
|
|
99
|
+
this.emit('navigation-spa', {
|
|
100
|
+
sessionId,
|
|
101
|
+
type: 'navigation-spa',
|
|
102
|
+
url: newUrl,
|
|
103
|
+
timestamp: Date.now()
|
|
104
|
+
});
|
|
105
|
+
return;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// Same-origin navigation: check if the video encoder script survived.
|
|
109
|
+
// SPA frameworks (SvelteKit, Next.js, etc.) often trigger framenavigated
|
|
110
|
+
// for client-side routing even though the page context is NOT replaced.
|
|
111
|
+
// If __webCodecsPeer still exists, the scripts are alive ā SPA navigation.
|
|
112
|
+
// If it's gone, the page was truly replaced ā full navigation + stream restart.
|
|
113
|
+
if (this.isSameOrigin(session.url, newUrl)) {
|
|
114
|
+
try {
|
|
115
|
+
const scriptAlive = await page.evaluate(() => !!(window as any).__webCodecsPeer);
|
|
116
|
+
if (scriptAlive) {
|
|
117
|
+
debug.log('preview', `š Same-origin navigation with script alive (SPA): ${session.url} ā ${newUrl}`);
|
|
118
|
+
session.url = newUrl;
|
|
119
|
+
this.emit('navigation-spa', {
|
|
120
|
+
sessionId,
|
|
121
|
+
type: 'navigation-spa',
|
|
122
|
+
url: newUrl,
|
|
123
|
+
timestamp: Date.now()
|
|
124
|
+
});
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
127
|
+
debug.log('preview', `š Same-origin navigation with script dead (full reload): ${session.url} ā ${newUrl}`);
|
|
128
|
+
} catch {
|
|
129
|
+
// page.evaluate failed ā page context was replaced, fall through to full navigation
|
|
130
|
+
debug.log('preview', `š Same-origin navigation evaluate failed (full reload): ${session.url} ā ${newUrl}`);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
35
134
|
// Update session URL
|
|
36
135
|
session.url = newUrl;
|
|
37
136
|
|
|
@@ -48,10 +147,43 @@ export class BrowserNavigationTracker extends EventEmitter {
|
|
|
48
147
|
// Also track URL changes via JavaScript (for single page applications)
|
|
49
148
|
page.on('load', async () => {
|
|
50
149
|
const currentUrl = page.url();
|
|
150
|
+
// Skip internal Chrome error/system pages
|
|
151
|
+
if (currentUrl.startsWith('chrome-error://') || currentUrl.startsWith('chrome://')) return;
|
|
51
152
|
if (currentUrl !== session.url) {
|
|
52
|
-
|
|
153
|
+
|
|
154
|
+
// Hash-only changes on load ā treat as SPA navigation
|
|
155
|
+
if (this.isHashOnlyChange(session.url, currentUrl)) {
|
|
156
|
+
session.url = currentUrl;
|
|
157
|
+
this.emit('navigation-spa', {
|
|
158
|
+
sessionId,
|
|
159
|
+
type: 'navigation-spa',
|
|
160
|
+
url: currentUrl,
|
|
161
|
+
timestamp: Date.now()
|
|
162
|
+
});
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
// Same-origin: check if video encoder script survived
|
|
167
|
+
if (this.isSameOrigin(session.url, currentUrl)) {
|
|
168
|
+
try {
|
|
169
|
+
const scriptAlive = await page.evaluate(() => !!(window as any).__webCodecsPeer);
|
|
170
|
+
if (scriptAlive) {
|
|
171
|
+
session.url = currentUrl;
|
|
172
|
+
this.emit('navigation-spa', {
|
|
173
|
+
sessionId,
|
|
174
|
+
type: 'navigation-spa',
|
|
175
|
+
url: currentUrl,
|
|
176
|
+
timestamp: Date.now()
|
|
177
|
+
});
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
} catch {
|
|
181
|
+
// Fall through to full navigation
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
53
185
|
session.url = currentUrl;
|
|
54
|
-
|
|
186
|
+
|
|
55
187
|
this.emit('navigation', {
|
|
56
188
|
sessionId,
|
|
57
189
|
type: 'navigation',
|
|
@@ -61,34 +193,59 @@ export class BrowserNavigationTracker extends EventEmitter {
|
|
|
61
193
|
}
|
|
62
194
|
});
|
|
63
195
|
|
|
64
|
-
// Track
|
|
65
|
-
//
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
196
|
+
// Track SPA navigations (pushState/replaceState) via CDP
|
|
197
|
+
// Uses Page.navigatedWithinDocument which fires for same-document navigations
|
|
198
|
+
// This is purely CDP-level ā no script injection, safe from CloudFlare detection
|
|
199
|
+
try {
|
|
200
|
+
const cdp = await page.createCDPSession();
|
|
201
|
+
this.cdpSessions.set(sessionId, cdp);
|
|
69
202
|
|
|
70
|
-
|
|
71
|
-
const checkUrlChange = () => {
|
|
72
|
-
const currentUrl = window.location.href;
|
|
73
|
-
if (currentUrl !== lastUrl) {
|
|
74
|
-
lastUrl = currentUrl;
|
|
203
|
+
await cdp.send('Page.enable');
|
|
75
204
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
timestamp: Date.now()
|
|
80
|
-
};
|
|
81
|
-
}
|
|
82
|
-
};
|
|
205
|
+
// Get main frame ID via CDP (reliable across Puppeteer versions)
|
|
206
|
+
const frameTree = await cdp.send('Page.getFrameTree');
|
|
207
|
+
const mainFrameId = frameTree.frameTree.frame.id;
|
|
83
208
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
209
|
+
cdp.on('Page.navigatedWithinDocument', (params: { frameId: string; url: string }) => {
|
|
210
|
+
// Only track main frame SPA navigations (ignore iframe pushState)
|
|
211
|
+
if (params.frameId !== mainFrameId) return;
|
|
87
212
|
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
213
|
+
const newUrl = params.url;
|
|
214
|
+
if (newUrl === session.url) return;
|
|
215
|
+
|
|
216
|
+
debug.log('preview', `š SPA navigation detected: ${session.url} ā ${newUrl}`);
|
|
217
|
+
|
|
218
|
+
// Update session URL
|
|
219
|
+
session.url = newUrl;
|
|
220
|
+
|
|
221
|
+
// Emit SPA navigation event ā no loading state, no stream restart
|
|
222
|
+
this.emit('navigation-spa', {
|
|
223
|
+
sessionId,
|
|
224
|
+
type: 'navigation-spa',
|
|
225
|
+
url: newUrl,
|
|
226
|
+
timestamp: Date.now()
|
|
227
|
+
});
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
debug.log('preview', `ā
CDP SPA navigation tracking setup for session: ${sessionId}`);
|
|
231
|
+
} catch (error) {
|
|
232
|
+
debug.warn('preview', `ā ļø Failed to setup CDP SPA tracking for ${sessionId}:`, error);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/**
|
|
237
|
+
* Cleanup CDP session for a tab
|
|
238
|
+
*/
|
|
239
|
+
async cleanupSession(sessionId: string) {
|
|
240
|
+
const cdp = this.cdpSessions.get(sessionId);
|
|
241
|
+
if (cdp) {
|
|
242
|
+
try {
|
|
243
|
+
await cdp.detach();
|
|
244
|
+
} catch {
|
|
245
|
+
// Ignore detach errors
|
|
246
|
+
}
|
|
247
|
+
this.cdpSessions.delete(sessionId);
|
|
248
|
+
}
|
|
92
249
|
}
|
|
93
250
|
|
|
94
251
|
async navigateSession(sessionId: string, session: BrowserTab, url: string): Promise<string> {
|
|
@@ -51,7 +51,7 @@ const CHROMIUM_ARGS = [
|
|
|
51
51
|
'--disable-blink-features=AutomationControlled',
|
|
52
52
|
'--window-size=1366,768',
|
|
53
53
|
'--autoplay-policy=no-user-gesture-required',
|
|
54
|
-
'--disable-features=AudioServiceOutOfProcess'
|
|
54
|
+
'--disable-features=AudioServiceOutOfProcess,WebRtcHideLocalIpsWithMdns'
|
|
55
55
|
];
|
|
56
56
|
|
|
57
57
|
class BrowserPool {
|
|
@@ -94,6 +94,7 @@ export class BrowserPreviewService extends EventEmitter {
|
|
|
94
94
|
});
|
|
95
95
|
|
|
96
96
|
// Forward navigation events and handle video streaming restart
|
|
97
|
+
// Only full navigations (framenavigated) need streaming restart
|
|
97
98
|
this.navigationTracker.on('navigation', async (data) => {
|
|
98
99
|
this.emit('preview:browser-navigation', data);
|
|
99
100
|
|
|
@@ -121,6 +122,12 @@ export class BrowserPreviewService extends EventEmitter {
|
|
|
121
122
|
this.emit('preview:browser-navigation-loading', data);
|
|
122
123
|
});
|
|
123
124
|
|
|
125
|
+
// Forward SPA navigation events (pushState/replaceState)
|
|
126
|
+
// No streaming restart needed ā page context is unchanged
|
|
127
|
+
this.navigationTracker.on('navigation-spa', (data) => {
|
|
128
|
+
this.emit('preview:browser-navigation-spa', data);
|
|
129
|
+
});
|
|
130
|
+
|
|
124
131
|
// Forward new window events
|
|
125
132
|
this.tabManager.on('new-window', (data) => {
|
|
126
133
|
this.emit('preview:browser-new-window', data);
|
|
@@ -140,6 +147,10 @@ export class BrowserPreviewService extends EventEmitter {
|
|
|
140
147
|
this.emit('preview:browser-tab-navigated', data);
|
|
141
148
|
});
|
|
142
149
|
|
|
150
|
+
this.tabManager.on('preview:browser-viewport-changed', (data) => {
|
|
151
|
+
this.emit('preview:browser-viewport-changed', data);
|
|
152
|
+
});
|
|
153
|
+
|
|
143
154
|
// Forward video capture events
|
|
144
155
|
this.videoCapture.on('ice-candidate', (data) => {
|
|
145
156
|
this.emit('preview:browser-webcodecs-ice-candidate', data);
|
|
@@ -249,6 +260,9 @@ export class BrowserPreviewService extends EventEmitter {
|
|
|
249
260
|
// Stop WebCodecs streaming first
|
|
250
261
|
await this.stopWebCodecsStreaming(tabId);
|
|
251
262
|
|
|
263
|
+
// Cleanup navigation tracker CDP session
|
|
264
|
+
await this.navigationTracker.cleanupSession(tabId);
|
|
265
|
+
|
|
252
266
|
// Clear cursor tracking for this tab
|
|
253
267
|
this.interactionHandler.clearSessionCursor(tabId);
|
|
254
268
|
|
|
@@ -650,6 +664,11 @@ class BrowserPreviewServiceManager {
|
|
|
650
664
|
ws.emit.project(projectId, 'preview:browser-navigation', data);
|
|
651
665
|
});
|
|
652
666
|
|
|
667
|
+
// Forward SPA navigation events (pushState/replaceState ā URL-only update)
|
|
668
|
+
service.on('preview:browser-navigation-spa', (data) => {
|
|
669
|
+
ws.emit.project(projectId, 'preview:browser-navigation-spa', data);
|
|
670
|
+
});
|
|
671
|
+
|
|
653
672
|
// Forward tab events
|
|
654
673
|
service.on('preview:browser-tab-opened', (data) => {
|
|
655
674
|
debug.log('preview', `š Forwarding preview:browser-tab-opened to project ${projectId}:`, data);
|
|
@@ -668,6 +687,10 @@ class BrowserPreviewServiceManager {
|
|
|
668
687
|
ws.emit.project(projectId, 'preview:browser-tab-navigated', data);
|
|
669
688
|
});
|
|
670
689
|
|
|
690
|
+
service.on('preview:browser-viewport-changed', (data) => {
|
|
691
|
+
ws.emit.project(projectId, 'preview:browser-viewport-changed', data);
|
|
692
|
+
});
|
|
693
|
+
|
|
671
694
|
// Forward console events
|
|
672
695
|
service.on('preview:browser-console-message', (data) => {
|
|
673
696
|
ws.emit.project(projectId, 'preview:browser-console-message', data);
|
|
@@ -730,6 +730,21 @@ export class BrowserTabManager extends EventEmitter {
|
|
|
730
730
|
// await page.evaluateOnNewDocument(cursorTrackingScript);
|
|
731
731
|
}
|
|
732
732
|
|
|
733
|
+
/**
|
|
734
|
+
* Returns true for errors where retrying is pointless because the page/session is gone.
|
|
735
|
+
*/
|
|
736
|
+
private isNonRetryableError(error: unknown): boolean {
|
|
737
|
+
if (error instanceof Error) {
|
|
738
|
+
const msg = error.message;
|
|
739
|
+
return (
|
|
740
|
+
msg.includes('Session closed') ||
|
|
741
|
+
msg.includes('detached Frame') ||
|
|
742
|
+
error.constructor.name === 'TargetCloseError'
|
|
743
|
+
);
|
|
744
|
+
}
|
|
745
|
+
return false;
|
|
746
|
+
}
|
|
747
|
+
|
|
733
748
|
/**
|
|
734
749
|
* Navigate with retry, including Cloudflare auto-pass detection and CAPTCHA popup dismissal.
|
|
735
750
|
*/
|
|
@@ -750,7 +765,7 @@ export class BrowserTabManager extends EventEmitter {
|
|
|
750
765
|
} catch (error) {
|
|
751
766
|
retries--;
|
|
752
767
|
debug.warn('preview', `ā ļø Navigation failed, ${retries} retries left:`, error);
|
|
753
|
-
if (retries === 0) throw error;
|
|
768
|
+
if (retries === 0 || this.isNonRetryableError(error)) throw error;
|
|
754
769
|
|
|
755
770
|
// Wait before retry
|
|
756
771
|
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
@@ -383,8 +383,8 @@ export class BrowserVideoCapture extends EventEmitter {
|
|
|
383
383
|
return null;
|
|
384
384
|
}
|
|
385
385
|
|
|
386
|
-
const maxRetries =
|
|
387
|
-
const retryDelay =
|
|
386
|
+
const maxRetries = 6;
|
|
387
|
+
const retryDelay = 150;
|
|
388
388
|
|
|
389
389
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
390
390
|
try {
|
|
@@ -38,6 +38,23 @@ export function videoEncoderScript(config: StreamingConfig['video']) {
|
|
|
38
38
|
// STUN servers are unnecessary for localhost and add 100-500ms ICE gathering latency
|
|
39
39
|
const iceServers: { urls: string }[] = [];
|
|
40
40
|
|
|
41
|
+
// Create a loopback (127.0.0.1) copy of a host ICE candidate.
|
|
42
|
+
// Ensures WebRTC connects via loopback when VPN (e.g. Cloudflare WARP)
|
|
43
|
+
// interferes with host candidate connectivity between same-machine peers.
|
|
44
|
+
function createLoopbackCandidate(candidate: { candidate?: string; sdpMid?: string | null; sdpMLineIndex?: number | null }) {
|
|
45
|
+
if (!candidate.candidate) return null;
|
|
46
|
+
if (!candidate.candidate.includes('typ host')) return null;
|
|
47
|
+
|
|
48
|
+
const parts = candidate.candidate.split(' ');
|
|
49
|
+
if (parts.length < 8) return null;
|
|
50
|
+
|
|
51
|
+
const address = parts[4];
|
|
52
|
+
if (address === '127.0.0.1' || address === '::1') return null;
|
|
53
|
+
|
|
54
|
+
parts[4] = '127.0.0.1';
|
|
55
|
+
return { ...candidate, candidate: parts.join(' ') };
|
|
56
|
+
}
|
|
57
|
+
|
|
41
58
|
// Check cursor style from page
|
|
42
59
|
function checkCursor() {
|
|
43
60
|
try {
|
|
@@ -103,11 +120,18 @@ export function videoEncoderScript(config: StreamingConfig['video']) {
|
|
|
103
120
|
// Handle ICE candidates
|
|
104
121
|
peerConnection.onicecandidate = (event) => {
|
|
105
122
|
if (event.candidate && (window as any).__sendIceCandidate) {
|
|
106
|
-
|
|
123
|
+
const candidateInit = {
|
|
107
124
|
candidate: event.candidate.candidate,
|
|
108
125
|
sdpMid: event.candidate.sdpMid,
|
|
109
126
|
sdpMLineIndex: event.candidate.sdpMLineIndex
|
|
110
|
-
}
|
|
127
|
+
};
|
|
128
|
+
(window as any).__sendIceCandidate(candidateInit);
|
|
129
|
+
|
|
130
|
+
// Also send loopback version for VPN compatibility (same-machine peers)
|
|
131
|
+
const loopback = createLoopbackCandidate(candidateInit);
|
|
132
|
+
if (loopback) {
|
|
133
|
+
(window as any).__sendIceCandidate(loopback);
|
|
134
|
+
}
|
|
111
135
|
}
|
|
112
136
|
};
|
|
113
137
|
|
|
@@ -337,16 +361,27 @@ export function videoEncoderScript(config: StreamingConfig['video']) {
|
|
|
337
361
|
}
|
|
338
362
|
}
|
|
339
363
|
|
|
340
|
-
// Add ICE candidate
|
|
364
|
+
// Add ICE candidate (+ loopback variant for VPN compatibility)
|
|
341
365
|
async function addIceCandidate(candidate: RTCIceCandidateInit) {
|
|
342
366
|
if (!peerConnection) return false;
|
|
343
367
|
|
|
344
368
|
try {
|
|
345
369
|
await peerConnection.addIceCandidate(new RTCIceCandidate(candidate));
|
|
346
|
-
return true;
|
|
347
370
|
} catch (error) {
|
|
348
371
|
return false;
|
|
349
372
|
}
|
|
373
|
+
|
|
374
|
+
// Also try loopback version for VPN compatibility (same-machine peers)
|
|
375
|
+
const loopback = createLoopbackCandidate(candidate);
|
|
376
|
+
if (loopback) {
|
|
377
|
+
try {
|
|
378
|
+
await peerConnection.addIceCandidate(new RTCIceCandidate(loopback as RTCIceCandidateInit));
|
|
379
|
+
} catch {
|
|
380
|
+
// Expected to fail if loopback is not applicable
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
return true;
|
|
350
385
|
}
|
|
351
386
|
|
|
352
387
|
// Reconfigure video encoder with new dimensions (hot-swap)
|
|
@@ -252,6 +252,17 @@ export const streamPreviewHandler = createRouter()
|
|
|
252
252
|
url: t.String(),
|
|
253
253
|
timestamp: t.Number()
|
|
254
254
|
})
|
|
255
|
+
)
|
|
256
|
+
|
|
257
|
+
// Server ā Client: SPA navigation (pushState/replaceState ā URL-only update, no page reload)
|
|
258
|
+
.emit(
|
|
259
|
+
'preview:browser-navigation-spa',
|
|
260
|
+
t.Object({
|
|
261
|
+
sessionId: t.String(),
|
|
262
|
+
type: t.String(),
|
|
263
|
+
url: t.String(),
|
|
264
|
+
timestamp: t.Number()
|
|
265
|
+
})
|
|
255
266
|
);
|
|
256
267
|
|
|
257
268
|
// Setup event forwarding from preview service to WebSocket
|
|
@@ -143,4 +143,12 @@ export const previewRouter = createRouter()
|
|
|
143
143
|
sessionId: t.String(),
|
|
144
144
|
timestamp: t.Number(),
|
|
145
145
|
source: t.Literal('mcp')
|
|
146
|
+
}))
|
|
147
|
+
.emit('preview:browser-viewport-changed', t.Object({
|
|
148
|
+
tabId: t.String(),
|
|
149
|
+
deviceSize: t.String(),
|
|
150
|
+
rotation: t.String(),
|
|
151
|
+
width: t.Number(),
|
|
152
|
+
height: t.Number(),
|
|
153
|
+
timestamp: t.Number()
|
|
146
154
|
}));
|
|
@@ -416,12 +416,12 @@
|
|
|
416
416
|
ondrop={fileHandling.handleDrop}
|
|
417
417
|
>
|
|
418
418
|
<div class="flex-1">
|
|
419
|
-
<!-- Engine/Model Picker -->
|
|
420
|
-
<EngineModelPicker />
|
|
421
|
-
|
|
422
419
|
<!-- Edit Mode Indicator -->
|
|
423
420
|
<EditModeIndicator onCancel={handleCancelEdit} />
|
|
424
421
|
|
|
422
|
+
<!-- Engine/Model Picker -->
|
|
423
|
+
<EngineModelPicker />
|
|
424
|
+
|
|
425
425
|
<div class="flex items-end">
|
|
426
426
|
<textarea
|
|
427
427
|
bind:this={textareaElement}
|
|
@@ -223,6 +223,12 @@
|
|
|
223
223
|
let previousUrl = '';
|
|
224
224
|
$effect(() => {
|
|
225
225
|
if (!url || url === previousUrl) return;
|
|
226
|
+
// Ignore browser-internal error pages (e.g. DNS failure) ā they are not real URLs
|
|
227
|
+
// and should never trigger a new navigation attempt.
|
|
228
|
+
if (url.startsWith('chrome-error://') || url.startsWith('chrome://')) {
|
|
229
|
+
previousUrl = url;
|
|
230
|
+
return;
|
|
231
|
+
}
|
|
226
232
|
if (mcpLaunchInProgress) {
|
|
227
233
|
previousUrl = url;
|
|
228
234
|
urlInput = url;
|
|
@@ -306,7 +312,7 @@
|
|
|
306
312
|
|
|
307
313
|
// Initialize URL input
|
|
308
314
|
$effect(() => {
|
|
309
|
-
if (url && !url.startsWith('http://') && !url.startsWith('https://')) {
|
|
315
|
+
if (url && !url.startsWith('http://') && !url.startsWith('https://') && !url.startsWith('file://')) {
|
|
310
316
|
url = 'http://' + url;
|
|
311
317
|
}
|
|
312
318
|
if (url && !urlInput) {
|
|
@@ -319,7 +325,7 @@
|
|
|
319
325
|
if (!urlInput.trim()) return;
|
|
320
326
|
|
|
321
327
|
let processedUrl = urlInput.trim();
|
|
322
|
-
if (!processedUrl.startsWith('http://') && !processedUrl.startsWith('https://')) {
|
|
328
|
+
if (!processedUrl.startsWith('http://') && !processedUrl.startsWith('https://') && !processedUrl.startsWith('file://')) {
|
|
323
329
|
processedUrl = 'http://' + processedUrl;
|
|
324
330
|
}
|
|
325
331
|
|
|
@@ -447,7 +453,8 @@
|
|
|
447
453
|
},
|
|
448
454
|
getSessionInfo: () => sessionInfo,
|
|
449
455
|
getIsStreamReady: () => isStreamReady,
|
|
450
|
-
getErrorMessage: () => errorMessage
|
|
456
|
+
getErrorMessage: () => errorMessage,
|
|
457
|
+
getIsMcpControlled: () => isCurrentTabMcpControlled()
|
|
451
458
|
};
|
|
452
459
|
</script>
|
|
453
460
|
|
|
@@ -26,6 +26,7 @@
|
|
|
26
26
|
onStatsUpdate = $bindable<(stats: BrowserWebCodecsStreamStats | null) => void>(() => {}),
|
|
27
27
|
onRequestScreencastRefresh = $bindable<() => void>(() => {}), // Called when stream is stuck
|
|
28
28
|
touchMode = $bindable<'scroll' | 'cursor'>('scroll'),
|
|
29
|
+
touchTarget = undefined as HTMLElement | undefined, // Container element for touch events
|
|
29
30
|
onTouchCursorUpdate = $bindable<(pos: { x: number; y: number; visible: boolean; clicking?: boolean }) => void>(() => {})
|
|
30
31
|
} = $props();
|
|
31
32
|
|
|
@@ -566,12 +567,16 @@
|
|
|
566
567
|
debug.log('webcodecs', 'Streaming started successfully');
|
|
567
568
|
} else {
|
|
568
569
|
// Service handles errors internally and returns false.
|
|
569
|
-
//
|
|
570
|
-
|
|
570
|
+
// Retry after a delay ā the peer/offer may need more time to initialize.
|
|
571
|
+
retries++;
|
|
572
|
+
if (retries < maxRetries) {
|
|
573
|
+
debug.warn('webcodecs', `Streaming start returned false, retrying in ${retryDelay * retries}ms (${retries}/${maxRetries})`);
|
|
574
|
+
await new Promise(resolve => setTimeout(resolve, retryDelay * retries));
|
|
575
|
+
continue;
|
|
576
|
+
}
|
|
577
|
+
debug.error('webcodecs', 'Streaming start failed after all retries');
|
|
578
|
+
break;
|
|
571
579
|
}
|
|
572
|
-
// Always break after the service returns (success or failure).
|
|
573
|
-
// The service catches all exceptions internally, so the catch block
|
|
574
|
-
// below never runs, making retries/retryDelay dead code anyway.
|
|
575
580
|
break;
|
|
576
581
|
} catch (error: any) {
|
|
577
582
|
// This block only runs if the service unexpectedly throws.
|
|
@@ -1002,20 +1007,6 @@
|
|
|
1002
1007
|
canvas.focus();
|
|
1003
1008
|
});
|
|
1004
1009
|
|
|
1005
|
-
const touchStartHandler = (e: TouchEvent) => handleTouchStart(e, canvas);
|
|
1006
|
-
let lastTouchMoveTime = 0;
|
|
1007
|
-
const touchMoveHandler = (e: TouchEvent) => {
|
|
1008
|
-
const now = Date.now();
|
|
1009
|
-
if (now - lastTouchMoveTime >= 16) {
|
|
1010
|
-
lastTouchMoveTime = now;
|
|
1011
|
-
handleTouchMove(e, canvas);
|
|
1012
|
-
}
|
|
1013
|
-
};
|
|
1014
|
-
const touchEndHandler = (e: TouchEvent) => handleTouchEnd(e, canvas);
|
|
1015
|
-
|
|
1016
|
-
canvas.addEventListener('touchstart', touchStartHandler, { passive: false });
|
|
1017
|
-
canvas.addEventListener('touchmove', touchMoveHandler, { passive: false });
|
|
1018
|
-
canvas.addEventListener('touchend', touchEndHandler, { passive: false });
|
|
1019
1010
|
|
|
1020
1011
|
const handleMouseLeave = () => {
|
|
1021
1012
|
if (isMouseDown) {
|
|
@@ -1045,13 +1036,38 @@
|
|
|
1045
1036
|
canvas.removeEventListener('mousedown', (e) => handleCanvasMouseDown(e, canvas));
|
|
1046
1037
|
canvas.removeEventListener('mouseup', (e) => handleCanvasMouseUp(e, canvas));
|
|
1047
1038
|
canvas.removeEventListener('mousemove', handleMouseMove);
|
|
1048
|
-
canvas.removeEventListener('touchstart', touchStartHandler);
|
|
1049
|
-
canvas.removeEventListener('touchmove', touchMoveHandler);
|
|
1050
|
-
canvas.removeEventListener('touchend', touchEndHandler);
|
|
1051
1039
|
};
|
|
1052
1040
|
}
|
|
1053
1041
|
});
|
|
1054
1042
|
|
|
1043
|
+
// Attach touch events to touchTarget (Container's previewContainer) instead of canvas
|
|
1044
|
+
$effect(() => {
|
|
1045
|
+
if (!touchTarget || !canvasElement) return;
|
|
1046
|
+
|
|
1047
|
+
const canvas = canvasElement;
|
|
1048
|
+
let lastTouchMoveTime = 0;
|
|
1049
|
+
|
|
1050
|
+
const touchStartHandler = (e: TouchEvent) => handleTouchStart(e, canvas);
|
|
1051
|
+
const touchMoveHandler = (e: TouchEvent) => {
|
|
1052
|
+
const now = Date.now();
|
|
1053
|
+
if (now - lastTouchMoveTime >= 16) {
|
|
1054
|
+
lastTouchMoveTime = now;
|
|
1055
|
+
handleTouchMove(e, canvas);
|
|
1056
|
+
}
|
|
1057
|
+
};
|
|
1058
|
+
const touchEndHandler = (e: TouchEvent) => handleTouchEnd(e, canvas);
|
|
1059
|
+
|
|
1060
|
+
touchTarget.addEventListener('touchstart', touchStartHandler, { passive: false });
|
|
1061
|
+
touchTarget.addEventListener('touchmove', touchMoveHandler, { passive: false });
|
|
1062
|
+
touchTarget.addEventListener('touchend', touchEndHandler, { passive: false });
|
|
1063
|
+
|
|
1064
|
+
return () => {
|
|
1065
|
+
touchTarget.removeEventListener('touchstart', touchStartHandler);
|
|
1066
|
+
touchTarget.removeEventListener('touchmove', touchMoveHandler);
|
|
1067
|
+
touchTarget.removeEventListener('touchend', touchEndHandler);
|
|
1068
|
+
};
|
|
1069
|
+
});
|
|
1070
|
+
|
|
1055
1071
|
// Convert canvas coordinates to viewport (screen) coordinates for VirtualCursor display
|
|
1056
1072
|
function canvasToScreen(cx: number, cy: number): { x: number; y: number } {
|
|
1057
1073
|
if (!canvasElement) return { x: 0, y: 0 };
|
|
@@ -1391,7 +1407,8 @@
|
|
|
1391
1407
|
getStats: () => webCodecsService?.getStats() ?? null,
|
|
1392
1408
|
getLatency: () => latencyMs,
|
|
1393
1409
|
// Navigation handling
|
|
1394
|
-
notifyNavigationComplete
|
|
1410
|
+
notifyNavigationComplete,
|
|
1411
|
+
freezeForSpaNavigation: () => webCodecsService?.freezeForSpaNavigation()
|
|
1395
1412
|
};
|
|
1396
1413
|
});
|
|
1397
1414
|
|
|
@@ -70,10 +70,11 @@
|
|
|
70
70
|
let showNavigationOverlay = $state(false);
|
|
71
71
|
let overlayHideTimeout: ReturnType<typeof setTimeout> | null = null;
|
|
72
72
|
|
|
73
|
-
// Debounced navigation overlay -
|
|
74
|
-
//
|
|
73
|
+
// Debounced navigation overlay - only for user-initiated toolbar navigations
|
|
74
|
+
// In-browser navigations (link clicks) only show progress bar, not this overlay
|
|
75
|
+
// This makes the preview behave like a real browser
|
|
75
76
|
$effect(() => {
|
|
76
|
-
const shouldShowOverlay =
|
|
77
|
+
const shouldShowOverlay = isNavigating && isStreamReady;
|
|
77
78
|
|
|
78
79
|
// Cancel any pending hide when overlay should show
|
|
79
80
|
if (shouldShowOverlay && overlayHideTimeout) {
|
|
@@ -385,6 +386,7 @@
|
|
|
385
386
|
bind:isNavigating
|
|
386
387
|
bind:isReconnecting
|
|
387
388
|
bind:touchMode
|
|
389
|
+
touchTarget={previewContainer}
|
|
388
390
|
onInteraction={handleCanvasInteraction}
|
|
389
391
|
onCursorUpdate={handleCursorUpdate}
|
|
390
392
|
onFrameUpdate={handleFrameUpdate}
|
|
@@ -408,7 +410,8 @@
|
|
|
408
410
|
</div>
|
|
409
411
|
{/if}
|
|
410
412
|
|
|
411
|
-
<!-- Navigation Overlay:
|
|
413
|
+
<!-- Navigation Overlay: Only for user-initiated toolbar navigations (Go button/Enter) -->
|
|
414
|
+
<!-- In-browser link clicks only show the progress bar, not this overlay -->
|
|
412
415
|
{#if showNavigationOverlay}
|
|
413
416
|
<div
|
|
414
417
|
class="absolute inset-0 bg-white/60 dark:bg-slate-800/60 backdrop-blur-[2px] flex items-center justify-center z-10"
|
|
@@ -416,7 +419,7 @@
|
|
|
416
419
|
<div class="flex flex-col items-center gap-2">
|
|
417
420
|
<Icon name="lucide:loader-circle" class="w-8 h-8 animate-spin text-violet-600" />
|
|
418
421
|
<div class="text-slate-600 dark:text-slate-300 text-center">
|
|
419
|
-
<div class="text-sm font-medium">
|
|
422
|
+
<div class="text-sm font-medium">Navigating...</div>
|
|
420
423
|
</div>
|
|
421
424
|
</div>
|
|
422
425
|
</div>
|
|
@@ -163,6 +163,21 @@
|
|
|
163
163
|
progressPercent = 0;
|
|
164
164
|
}
|
|
165
165
|
|
|
166
|
+
// Reset progress bar immediately when active tab changes
|
|
167
|
+
// This prevents stale progress from a previous tab leaking into the new tab
|
|
168
|
+
let previousActiveTabId = $state<string | null>(null);
|
|
169
|
+
$effect(() => {
|
|
170
|
+
if (activeTabId !== previousActiveTabId) {
|
|
171
|
+
previousActiveTabId = activeTabId;
|
|
172
|
+
// Immediately stop any running progress animation and clear pending timeouts
|
|
173
|
+
stopProgress();
|
|
174
|
+
if (progressCompleteTimeout) {
|
|
175
|
+
clearTimeout(progressCompleteTimeout);
|
|
176
|
+
progressCompleteTimeout = null;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
});
|
|
180
|
+
|
|
166
181
|
// Watch loading states to control progress bar
|
|
167
182
|
// Progress bar should be active during:
|
|
168
183
|
// 1. isLaunchingBrowser: API call to launch browser
|
|
@@ -771,6 +771,19 @@ export function createBrowserCoordinator(config: BrowserCoordinatorConfig) {
|
|
|
771
771
|
debug.warn('preview', `Tab not found for sessionId: ${data.sessionId}`);
|
|
772
772
|
}
|
|
773
773
|
});
|
|
774
|
+
|
|
775
|
+
// Listen for SPA navigation events (pushState/replaceState)
|
|
776
|
+
ws.on('preview:browser-navigation-spa', (data: { sessionId: string; type: string; url: string; timestamp: number }) => {
|
|
777
|
+
debug.log('preview', `š SPA navigation event received: ${data.sessionId} ā ${data.url}`);
|
|
778
|
+
|
|
779
|
+
const tab = tabManager.tabs.find(t => t.sessionId === data.sessionId);
|
|
780
|
+
if (tab) {
|
|
781
|
+
streamHandler.handleStreamMessage({
|
|
782
|
+
type: 'navigation-spa',
|
|
783
|
+
data: { url: data.url }
|
|
784
|
+
}, tab.id);
|
|
785
|
+
}
|
|
786
|
+
});
|
|
774
787
|
});
|
|
775
788
|
}
|
|
776
789
|
|
|
@@ -73,6 +73,10 @@ export function createStreamMessageHandler(config: StreamMessageHandlerConfig) {
|
|
|
73
73
|
handleNavigation(targetTabId, message.data, tab);
|
|
74
74
|
break;
|
|
75
75
|
|
|
76
|
+
case 'navigation-spa':
|
|
77
|
+
handleNavigationSpa(targetTabId, message.data, tab);
|
|
78
|
+
break;
|
|
79
|
+
|
|
76
80
|
case 'new-window':
|
|
77
81
|
handleNewWindow(message.data);
|
|
78
82
|
break;
|
|
@@ -172,13 +176,14 @@ export function createStreamMessageHandler(config: StreamMessageHandlerConfig) {
|
|
|
172
176
|
function handleNavigationLoading(tabId: string, data: any) {
|
|
173
177
|
if (data && data.url) {
|
|
174
178
|
const tab = tabManager.getTab(tabId);
|
|
175
|
-
// isNavigating: true if session already exists (navigating within same session)
|
|
176
|
-
// isNavigating: false if no session yet (initial load)
|
|
177
|
-
const isNavigating = tab?.sessionId ? true : false;
|
|
178
179
|
|
|
180
|
+
// Only set isLoading (progress bar) for in-browser navigations.
|
|
181
|
+
// Do NOT set isNavigating here ā that flag is reserved for user-initiated
|
|
182
|
+
// toolbar navigations (Go button/Enter), which is set in navigateBrowserForTab().
|
|
183
|
+
// This prevents the "Loading preview..." overlay from showing on link clicks
|
|
184
|
+
// within the browser, making it behave like a real browser.
|
|
179
185
|
tabManager.updateTab(tabId, {
|
|
180
186
|
isLoading: true,
|
|
181
|
-
isNavigating,
|
|
182
187
|
url: data.url,
|
|
183
188
|
title: getTabTitle(data.url)
|
|
184
189
|
});
|
|
@@ -216,6 +221,34 @@ export function createStreamMessageHandler(config: StreamMessageHandlerConfig) {
|
|
|
216
221
|
}
|
|
217
222
|
}
|
|
218
223
|
|
|
224
|
+
function handleNavigationSpa(tabId: string, data: any, tab: PreviewTab) {
|
|
225
|
+
if (data && data.url && data.url !== tab.url) {
|
|
226
|
+
debug.log('preview', `š SPA navigation for tab ${tabId}: ${tab.url} ā ${data.url}`);
|
|
227
|
+
|
|
228
|
+
// Freeze canvas briefly to avoid showing white flash during SPA transition
|
|
229
|
+
// The last rendered frame is held while the DOM settles
|
|
230
|
+
tab.canvasAPI?.freezeForSpaNavigation?.();
|
|
231
|
+
|
|
232
|
+
// SPA navigation: update URL/title and reset any loading states.
|
|
233
|
+
// A preceding navigation-loading event may have set isLoading=true
|
|
234
|
+
// (e.g., if the browser started a document request before the SPA
|
|
235
|
+
// router intercepted it). Reset those states here since the SPA
|
|
236
|
+
// handled the navigation without a full page reload.
|
|
237
|
+
// Video streaming continues uninterrupted since page context is unchanged.
|
|
238
|
+
tabManager.updateTab(tabId, {
|
|
239
|
+
url: data.url,
|
|
240
|
+
title: getTabTitle(data.url),
|
|
241
|
+
isLoading: false,
|
|
242
|
+
isNavigating: false
|
|
243
|
+
});
|
|
244
|
+
|
|
245
|
+
// Update parent if this is the active tab
|
|
246
|
+
if (tabId === tabManager.activeTabId && onNavigationUpdate) {
|
|
247
|
+
onNavigationUpdate(tabId, data.url);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
219
252
|
function handleNewWindow(data: any) {
|
|
220
253
|
if (data && data.url) {
|
|
221
254
|
tabManager.createTab(data.url);
|
|
@@ -390,9 +390,10 @@
|
|
|
390
390
|
<div class="relative">
|
|
391
391
|
<button
|
|
392
392
|
type="button"
|
|
393
|
-
class="flex items-center justify-center gap-1.5 {isMobile ? 'px-2 h-9' : 'px-1 h-6'} bg-transparent border-none rounded-md text-slate-
|
|
394
|
-
onclick={toggleDeviceDropdown}
|
|
395
|
-
|
|
393
|
+
class="flex items-center justify-center gap-1.5 {isMobile ? 'px-2 h-9' : 'px-1 h-6'} bg-transparent border-none rounded-md transition-all duration-150 {previewPanelRef?.panelActions?.getIsMcpControlled() ? 'text-slate-400 dark:text-slate-600 cursor-not-allowed opacity-50' : 'text-slate-500 cursor-pointer hover:bg-violet-500/10 hover:text-slate-900 dark:hover:text-slate-100'}"
|
|
394
|
+
onclick={previewPanelRef?.panelActions?.getIsMcpControlled() ? undefined : toggleDeviceDropdown}
|
|
395
|
+
disabled={previewPanelRef?.panelActions?.getIsMcpControlled()}
|
|
396
|
+
title={previewPanelRef?.panelActions?.getIsMcpControlled() ? 'Controlled by MCP agent' : 'Select device size'}
|
|
396
397
|
>
|
|
397
398
|
{#if previewPanelRef?.panelActions?.getDeviceSize() === 'desktop'}
|
|
398
399
|
<Icon name="lucide:monitor" class={isMobile ? 'w-4.5 h-4.5' : 'w-4 h-4'} />
|
|
@@ -505,9 +506,10 @@
|
|
|
505
506
|
<!-- Rotation toggle -->
|
|
506
507
|
<button
|
|
507
508
|
type="button"
|
|
508
|
-
class="flex items-center justify-center gap-1.5 {isMobile ? 'px-2 h-9' : 'px-1 h-6'} bg-transparent border-none rounded-md text-slate-
|
|
509
|
-
onclick={() => previewPanelRef?.panelActions?.toggleRotation()}
|
|
510
|
-
|
|
509
|
+
class="flex items-center justify-center gap-1.5 {isMobile ? 'px-2 h-9' : 'px-1 h-6'} bg-transparent border-none rounded-md transition-all duration-150 {previewPanelRef?.panelActions?.getIsMcpControlled() ? 'text-slate-400 dark:text-slate-600 cursor-not-allowed opacity-50' : 'text-slate-500 cursor-pointer hover:bg-violet-500/10 hover:text-slate-900 dark:hover:text-slate-100'}"
|
|
510
|
+
onclick={previewPanelRef?.panelActions?.getIsMcpControlled() ? undefined : () => previewPanelRef?.panelActions?.toggleRotation()}
|
|
511
|
+
disabled={previewPanelRef?.panelActions?.getIsMcpControlled()}
|
|
512
|
+
title={previewPanelRef?.panelActions?.getIsMcpControlled() ? 'Controlled by MCP agent' : 'Toggle orientation'}
|
|
511
513
|
>
|
|
512
514
|
<Icon name="lucide:rotate-cw" class={isMobile ? 'w-4.5 h-4.5' : 'w-4 h-4'} />
|
|
513
515
|
<span class="text-xs font-medium">
|
|
@@ -113,6 +113,7 @@
|
|
|
113
113
|
getSessionInfo: () => browserPreviewRef?.browserActions?.getSessionInfo() || null,
|
|
114
114
|
getIsStreamReady: () => browserPreviewRef?.browserActions?.getIsStreamReady() || false,
|
|
115
115
|
getErrorMessage: () => browserPreviewRef?.browserActions?.getErrorMessage() || null,
|
|
116
|
+
getIsMcpControlled: () => browserPreviewRef?.browserActions?.getIsMcpControlled() || false,
|
|
116
117
|
setDeviceSize: (size: DeviceSize) => {
|
|
117
118
|
if (browserPreviewRef?.browserActions) {
|
|
118
119
|
browserPreviewRef.browserActions.changeDeviceSize(size);
|
|
@@ -133,6 +133,9 @@ export class BrowserWebCodecsService {
|
|
|
133
133
|
private isNavigating = false;
|
|
134
134
|
private navigationCleanupFn: (() => void) | null = null;
|
|
135
135
|
|
|
136
|
+
// SPA navigation frame freeze ā holds last frame briefly during SPA transitions
|
|
137
|
+
private spaFreezeUntil = 0;
|
|
138
|
+
|
|
136
139
|
// WebSocket cleanup
|
|
137
140
|
private wsCleanupFunctions: Array<() => void> = [];
|
|
138
141
|
|
|
@@ -237,16 +240,32 @@ export class BrowserWebCodecsService {
|
|
|
237
240
|
sdp: response.offer.sdp
|
|
238
241
|
});
|
|
239
242
|
} else {
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
243
|
+
// Offer not ready yet ā peer may still be initializing. Retry with backoff.
|
|
244
|
+
debug.log('webcodecs', `[DIAG] No offer in stream-start response, retrying stream-offer with backoff`);
|
|
245
|
+
|
|
246
|
+
let offer: { type: string; sdp?: string } | undefined;
|
|
247
|
+
const offerMaxRetries = 5;
|
|
248
|
+
const offerRetryDelay = 200;
|
|
249
|
+
|
|
250
|
+
for (let attempt = 0; attempt < offerMaxRetries; attempt++) {
|
|
251
|
+
if (attempt > 0) {
|
|
252
|
+
await new Promise(resolve => setTimeout(resolve, offerRetryDelay * attempt));
|
|
253
|
+
}
|
|
254
|
+
debug.log('webcodecs', `[DIAG] stream-offer attempt ${attempt + 1}/${offerMaxRetries}`);
|
|
255
|
+
const offerResponse = await ws.http('preview:browser-stream-offer', {}, 10000);
|
|
256
|
+
if (offerResponse.offer) {
|
|
257
|
+
offer = offerResponse.offer;
|
|
258
|
+
break;
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
if (offer) {
|
|
244
263
|
await this.handleOffer({
|
|
245
|
-
type:
|
|
246
|
-
sdp:
|
|
264
|
+
type: offer.type as RTCSdpType,
|
|
265
|
+
sdp: offer.sdp
|
|
247
266
|
});
|
|
248
267
|
} else {
|
|
249
|
-
throw new Error('No offer received from server');
|
|
268
|
+
throw new Error('No offer received from server after retries');
|
|
250
269
|
}
|
|
251
270
|
}
|
|
252
271
|
|
|
@@ -338,16 +357,22 @@ export class BrowserWebCodecsService {
|
|
|
338
357
|
// Handle ICE candidates
|
|
339
358
|
this.peerConnection.onicecandidate = (event) => {
|
|
340
359
|
if (event.candidate && this.sessionId) {
|
|
360
|
+
const candidateInit: RTCIceCandidateInit = {
|
|
361
|
+
candidate: event.candidate.candidate,
|
|
362
|
+
sdpMid: event.candidate.sdpMid,
|
|
363
|
+
sdpMLineIndex: event.candidate.sdpMLineIndex
|
|
364
|
+
};
|
|
365
|
+
|
|
341
366
|
// Backend uses active tab automatically
|
|
342
|
-
ws.http('preview:browser-stream-ice', {
|
|
343
|
-
candidate: {
|
|
344
|
-
candidate: event.candidate.candidate,
|
|
345
|
-
sdpMid: event.candidate.sdpMid,
|
|
346
|
-
sdpMLineIndex: event.candidate.sdpMLineIndex
|
|
347
|
-
}
|
|
348
|
-
}).catch((error) => {
|
|
367
|
+
ws.http('preview:browser-stream-ice', { candidate: candidateInit }).catch((error) => {
|
|
349
368
|
debug.warn('webcodecs', 'Failed to send ICE candidate:', error);
|
|
350
369
|
});
|
|
370
|
+
|
|
371
|
+
// Also send loopback version for VPN compatibility (same-machine peers)
|
|
372
|
+
const loopback = this.createLoopbackCandidate(candidateInit);
|
|
373
|
+
if (loopback) {
|
|
374
|
+
ws.http('preview:browser-stream-ice', { candidate: loopback }).catch(() => {});
|
|
375
|
+
}
|
|
351
376
|
}
|
|
352
377
|
};
|
|
353
378
|
|
|
@@ -624,6 +649,17 @@ export class BrowserWebCodecsService {
|
|
|
624
649
|
return;
|
|
625
650
|
}
|
|
626
651
|
|
|
652
|
+
// During SPA navigation freeze, skip rendering to hold the last frame
|
|
653
|
+
// This prevents brief white flashes during SPA page transitions
|
|
654
|
+
if (this.spaFreezeUntil > 0 && Date.now() < this.spaFreezeUntil) {
|
|
655
|
+
frame.close();
|
|
656
|
+
return;
|
|
657
|
+
}
|
|
658
|
+
// Auto-reset freeze after it expires
|
|
659
|
+
if (this.spaFreezeUntil > 0) {
|
|
660
|
+
this.spaFreezeUntil = 0;
|
|
661
|
+
}
|
|
662
|
+
|
|
627
663
|
try {
|
|
628
664
|
// Update stats
|
|
629
665
|
this.stats.videoFramesDecoded++;
|
|
@@ -849,9 +885,16 @@ export class BrowserWebCodecsService {
|
|
|
849
885
|
|
|
850
886
|
const cleanupNavComplete = ws.on('preview:browser-navigation', (data) => {
|
|
851
887
|
if (data.sessionId === this.sessionId) {
|
|
852
|
-
// Keep isNavigating true for a short period to allow reconnection
|
|
853
|
-
// Will be reset when new frames arrive or reconnection completes
|
|
854
888
|
debug.log('webcodecs', `Navigation completed (direct WS) for session ${data.sessionId}`);
|
|
889
|
+
|
|
890
|
+
// If isNavigating was NOT set by navigation-loading (SPA-like case where
|
|
891
|
+
// framenavigated fires without a document request), set it now so the
|
|
892
|
+
// subsequent DataChannel close triggers fast reconnect instead of full recovery
|
|
893
|
+
if (!this.isNavigating) {
|
|
894
|
+
this.isNavigating = true;
|
|
895
|
+
debug.log('webcodecs', 'ā
Set isNavigating=true on navigation complete (no loading event preceded)');
|
|
896
|
+
}
|
|
897
|
+
|
|
855
898
|
// Signal reconnecting state IMMEDIATELY when navigation completes
|
|
856
899
|
// This eliminates the gap between isNavigating=false and DataChannel close
|
|
857
900
|
// ensuring the overlay stays visible continuously
|
|
@@ -862,11 +905,41 @@ export class BrowserWebCodecsService {
|
|
|
862
905
|
}
|
|
863
906
|
});
|
|
864
907
|
|
|
865
|
-
|
|
908
|
+
// Listen for SPA navigation events (pushState/replaceState/hash changes)
|
|
909
|
+
// Reset isNavigating if it was set by a preceding navigation-loading event
|
|
910
|
+
// that the SPA router intercepted (cancelled the full navigation)
|
|
911
|
+
const cleanupNavSpa = ws.on('preview:browser-navigation-spa', (data) => {
|
|
912
|
+
if (data.sessionId === this.sessionId && this.isNavigating) {
|
|
913
|
+
debug.log('webcodecs', 'š SPA navigation received - resetting isNavigating (no stream restart needed)');
|
|
914
|
+
this.isNavigating = false;
|
|
915
|
+
}
|
|
916
|
+
});
|
|
917
|
+
|
|
918
|
+
this.wsCleanupFunctions = [cleanupIce, cleanupState, cleanupCursor, cleanupNavLoading, cleanupNavComplete, cleanupNavSpa];
|
|
866
919
|
}
|
|
867
920
|
|
|
868
921
|
/**
|
|
869
|
-
*
|
|
922
|
+
* Create a loopback (127.0.0.1) copy of a host ICE candidate.
|
|
923
|
+
* Ensures WebRTC connects via loopback when VPN (e.g. Cloudflare WARP)
|
|
924
|
+
* interferes with host candidate connectivity between same-machine peers.
|
|
925
|
+
*/
|
|
926
|
+
private createLoopbackCandidate(candidate: RTCIceCandidateInit): RTCIceCandidateInit | null {
|
|
927
|
+
if (!candidate.candidate) return null;
|
|
928
|
+
if (!candidate.candidate.includes('typ host')) return null;
|
|
929
|
+
|
|
930
|
+
const parts = candidate.candidate.split(' ');
|
|
931
|
+
if (parts.length < 8) return null;
|
|
932
|
+
|
|
933
|
+
// Index 4 is the address field in ICE candidate format
|
|
934
|
+
const address = parts[4];
|
|
935
|
+
if (address === '127.0.0.1' || address === '::1') return null;
|
|
936
|
+
|
|
937
|
+
parts[4] = '127.0.0.1';
|
|
938
|
+
return { ...candidate, candidate: parts.join(' ') };
|
|
939
|
+
}
|
|
940
|
+
|
|
941
|
+
/**
|
|
942
|
+
* Add ICE candidate (+ loopback variant for VPN compatibility)
|
|
870
943
|
*/
|
|
871
944
|
private async addIceCandidate(candidate: RTCIceCandidateInit): Promise<void> {
|
|
872
945
|
if (!this.peerConnection) return;
|
|
@@ -876,6 +949,16 @@ export class BrowserWebCodecsService {
|
|
|
876
949
|
} catch (error) {
|
|
877
950
|
debug.warn('webcodecs', 'Add ICE candidate error:', error);
|
|
878
951
|
}
|
|
952
|
+
|
|
953
|
+
// Also try loopback version for VPN compatibility (same-machine peers)
|
|
954
|
+
const loopback = this.createLoopbackCandidate(candidate);
|
|
955
|
+
if (loopback) {
|
|
956
|
+
try {
|
|
957
|
+
await this.peerConnection.addIceCandidate(new RTCIceCandidate(loopback));
|
|
958
|
+
} catch {
|
|
959
|
+
// Expected to fail if loopback is not applicable
|
|
960
|
+
}
|
|
961
|
+
}
|
|
879
962
|
}
|
|
880
963
|
|
|
881
964
|
/**
|
|
@@ -1382,6 +1465,15 @@ export class BrowserWebCodecsService {
|
|
|
1382
1465
|
this.onFirstFrame = handler;
|
|
1383
1466
|
}
|
|
1384
1467
|
|
|
1468
|
+
/**
|
|
1469
|
+
* Freeze frame rendering briefly during SPA navigation.
|
|
1470
|
+
* Holds the current canvas content to prevent white flash during
|
|
1471
|
+
* SPA page transitions (pushState/replaceState).
|
|
1472
|
+
*/
|
|
1473
|
+
freezeForSpaNavigation(durationMs = 150): void {
|
|
1474
|
+
this.spaFreezeUntil = Date.now() + durationMs;
|
|
1475
|
+
}
|
|
1476
|
+
|
|
1385
1477
|
setErrorHandler(handler: (error: Error) => void): void {
|
|
1386
1478
|
this.onError = handler;
|
|
1387
1479
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@myrialabs/clopen",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.10",
|
|
4
4
|
"description": "All-in-one web workspace for Claude Code & OpenCode ā chat, terminal, git, browser preview, checkpoints, and real-time collaboration",
|
|
5
5
|
"author": "Myria Labs",
|
|
6
6
|
"license": "MIT",
|