@myrialabs/clopen 0.2.6 → 0.2.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (28) hide show
  1. package/backend/chat/stream-manager.ts +1 -1
  2. package/backend/engine/adapters/claude/stream.ts +10 -19
  3. package/backend/mcp/servers/browser-automation/browser.ts +23 -6
  4. package/backend/preview/browser/browser-mcp-control.ts +32 -16
  5. package/backend/preview/browser/browser-pool.ts +3 -1
  6. package/backend/preview/browser/browser-tab-manager.ts +1 -1
  7. package/backend/preview/browser/scripts/audio-stream.ts +11 -0
  8. package/backend/ws/chat/stream.ts +1 -1
  9. package/backend/ws/preview/browser/tab-info.ts +5 -2
  10. package/frontend/components/chat/input/ChatInput.svelte +0 -3
  11. package/frontend/components/chat/input/composables/use-chat-actions.svelte.ts +6 -2
  12. package/frontend/components/history/HistoryModal.svelte +1 -1
  13. package/frontend/components/preview/browser/BrowserPreview.svelte +14 -0
  14. package/frontend/components/preview/browser/components/Canvas.svelte +322 -48
  15. package/frontend/components/preview/browser/components/Container.svelte +21 -0
  16. package/frontend/components/preview/browser/core/coordinator.svelte.ts +15 -0
  17. package/frontend/components/preview/browser/core/mcp-handlers.svelte.ts +36 -3
  18. package/frontend/components/preview/browser/core/tab-operations.svelte.ts +1 -0
  19. package/frontend/components/workspace/PanelHeader.svelte +15 -0
  20. package/frontend/components/workspace/panels/GitPanel.svelte +27 -13
  21. package/frontend/components/workspace/panels/PreviewPanel.svelte +2 -0
  22. package/frontend/services/chat/chat.service.ts +3 -7
  23. package/frontend/services/preview/browser/browser-webcodecs.service.ts +30 -133
  24. package/frontend/stores/core/app.svelte.ts +4 -3
  25. package/frontend/stores/core/presence.svelte.ts +3 -2
  26. package/frontend/stores/core/sessions.svelte.ts +2 -0
  27. package/frontend/stores/ui/notification.svelte.ts +4 -1
  28. package/package.json +1 -1
@@ -1132,7 +1132,7 @@ class StreamManager extends EventEmitter {
1132
1132
  }
1133
1133
  }
1134
1134
 
1135
- // Cancel the per-project engine FIRST — this sends an interrupt to the
1135
+ // Cancel the per-project engine — this sends an interrupt to the
1136
1136
  // still-alive SDK subprocess, then aborts the controller. If we abort
1137
1137
  // the controller first, the subprocess dies and the SDK's subsequent
1138
1138
  // interrupt write fails with "Operation aborted" (unhandled rejection
@@ -180,38 +180,29 @@ export class ClaudeCodeEngine implements AIEngine {
180
180
  * Cancel active query
181
181
  */
182
182
  async cancel(): Promise<void> {
183
- // Resolve all pending AskUserQuestion promises BEFORE aborting.
184
- // This lets the SDK process the denial responses while the subprocess
185
- // is still alive, preventing "Operation aborted" write errors.
183
+ // Resolve all pending AskUserQuestion promises before terminating.
186
184
  for (const [, pending] of this.pendingUserAnswers) {
187
185
  pending.resolve({ behavior: 'deny', message: 'Cancelled' });
188
186
  }
189
187
  this.pendingUserAnswers.clear();
190
188
 
191
- // Only interrupt if the controller hasn't been aborted yet.
192
- // Interrupting after abort causes the SDK to write to a dead subprocess,
193
- // resulting in "Operation aborted" unhandled rejections that crash Bun.
194
- if (this.activeQuery && typeof this.activeQuery.interrupt === 'function'
195
- && this.activeController && !this.activeController.signal.aborted) {
189
+ // Use close() to forcefully terminate the query process and clean up
190
+ // all resources (docs: "Forcefully ends the query and cleans up all
191
+ // resources"). Unlike interrupt() which can hang indefinitely when the
192
+ // subprocess is unresponsive, close() is synchronous and guaranteed to
193
+ // complete making cancel deterministic.
194
+ if (this.activeQuery && typeof this.activeQuery.close === 'function') {
196
195
  try {
197
- await this.activeQuery.interrupt();
196
+ this.activeQuery.close();
198
197
  } catch {
199
- // Ignore interrupt errors
198
+ // Ignore close errors — process may already be dead
200
199
  }
201
200
  }
202
201
 
203
- // Brief delay between interrupt and abort to let the SDK flush pending
204
- // write operations (e.g., handleControlRequest responses). Without this,
205
- // aborting immediately after interrupt kills the subprocess while the SDK
206
- // still has in-flight writes, causing "Operation aborted" rejections.
207
202
  if (this.activeController && !this.activeController.signal.aborted) {
208
- await new Promise(resolve => setTimeout(resolve, 100));
209
- }
210
-
211
- if (this.activeController) {
212
203
  this.activeController.abort();
213
- this.activeController = null;
214
204
  }
205
+ this.activeController = null;
215
206
  this.activeQuery = null;
216
207
  }
217
208
 
@@ -89,10 +89,26 @@ interface CloseTabResponse {
89
89
  * Internal helper: Get active tab
90
90
  * Throws error if no active tab found
91
91
  * Automatically acquires MCP control for the active tab to ensure UI sync
92
+ *
93
+ * If MCP is already controlling a specific tab, that tab is returned regardless
94
+ * of which tab the user has currently active in the frontend. This prevents MCP
95
+ * from "following" the user when they switch tabs mid-session.
92
96
  */
93
97
  export async function getActiveTabSession(projectId?: string) {
94
- // Get active tab directly from backend tab manager
95
98
  const previewService = getPreviewService(projectId);
99
+
100
+ // If MCP is already controlling a specific tab, stick to that tab.
101
+ // This prevents user tab-switching from hijacking the MCP session.
102
+ const controlState = browserMcpControl.getControlState();
103
+ if (controlState.isControlling && controlState.browserTabId) {
104
+ const controlledTab = previewService.getTab(controlState.browserTabId);
105
+ if (controlledTab) {
106
+ debug.log('mcp', `🎮 Using MCP-controlled tab: ${controlledTab.id} (ignoring active tab)`);
107
+ return { tab: controlledTab, session: controlledTab };
108
+ }
109
+ }
110
+
111
+ // No controlled tab — use the active tab and acquire control
96
112
  const tab = previewService.getActiveTab();
97
113
 
98
114
  if (!tab) {
@@ -101,8 +117,9 @@ export async function getActiveTabSession(projectId?: string) {
101
117
 
102
118
  // Acquire control for active tab (ensures UI sync after idle timeout)
103
119
  // This is idempotent - if already controlling this tab, just updates timestamp
104
- if (!browserMcpControl.isTabControlled(tab.id)) {
105
- const acquired = browserMcpControl.acquireControl(tab.id);
120
+ const resolvedProjectId = previewService.getProjectId();
121
+ if (!browserMcpControl.isTabControlled(tab.id, resolvedProjectId)) {
122
+ const acquired = browserMcpControl.acquireControl(tab.id, undefined, resolvedProjectId);
106
123
  if (acquired) {
107
124
  debug.log('mcp', `🔄 Auto-acquired control for tab ${tab.id} (resumed after idle)`);
108
125
  }
@@ -189,7 +206,7 @@ export async function switchTabHandler(args: { tabId: string; projectId?: string
189
206
  // Update MCP control to the new tab
190
207
  if (tab) {
191
208
  browserMcpControl.releaseControl();
192
- browserMcpControl.acquireControl(tab.id);
209
+ browserMcpControl.acquireControl(tab.id, undefined, previewService.getProjectId());
193
210
  }
194
211
 
195
212
  // Update last action to keep control alive
@@ -244,7 +261,7 @@ export async function openNewTabHandler(args: { url?: string; deviceSize?: 'desk
244
261
 
245
262
  // Auto-acquire control of the new tab
246
263
  browserMcpControl.releaseControl();
247
- browserMcpControl.acquireControl(tab.id);
264
+ browserMcpControl.acquireControl(tab.id, undefined, previewService.getProjectId());
248
265
 
249
266
  // Update last action to keep control alive
250
267
  browserMcpControl.updateLastAction();
@@ -296,7 +313,7 @@ export async function closeTabHandler(args: { tabId: string; projectId?: string
296
313
  if (result.newActiveTabId) {
297
314
  const newActiveTab = previewService.getTab(result.newActiveTabId);
298
315
  if (newActiveTab) {
299
- browserMcpControl.acquireControl(newActiveTab.id);
316
+ browserMcpControl.acquireControl(newActiveTab.id, undefined, previewService.getProjectId());
300
317
  }
301
318
  }
302
319
 
@@ -28,6 +28,7 @@ export interface McpControlState {
28
28
  isControlling: boolean;
29
29
  mcpSessionId: string | null;
30
30
  browserTabId: string | null;
31
+ projectId: string | null;
31
32
  startedAt: number | null;
32
33
  lastActionAt: number | null;
33
34
  }
@@ -60,6 +61,7 @@ export class BrowserMcpControl extends EventEmitter {
60
61
  isControlling: false,
61
62
  mcpSessionId: null,
62
63
  browserTabId: null,
64
+ projectId: null,
63
65
  startedAt: null,
64
66
  lastActionAt: null
65
67
  };
@@ -96,13 +98,16 @@ export class BrowserMcpControl extends EventEmitter {
96
98
 
97
99
  /**
98
100
  * Handle tab destroyed event
99
- * Auto-release control if the destroyed tab was being controlled
101
+ * Auto-release control if the destroyed tab was being controlled.
102
+ * Uses the service's projectId to avoid cross-project false-positives.
100
103
  */
101
104
  private handleTabDestroyed(tabId: string): void {
102
- if (this.controlState.isControlling && this.controlState.browserTabId === tabId) {
103
- debug.warn('mcp', `⚠️ Controlled tab ${tabId} was destroyed - auto-releasing control`);
104
- this.releaseControl();
105
- }
105
+ if (!this.controlState.isControlling || this.controlState.browserTabId !== tabId) return;
106
+ // Validate project to prevent cross-project collisions (tab IDs are not globally unique)
107
+ const serviceProjectId = this.previewService?.getProjectId();
108
+ if (serviceProjectId && this.controlState.projectId && this.controlState.projectId !== serviceProjectId) return;
109
+ debug.warn('mcp', `⚠️ Controlled tab ${tabId} was destroyed - auto-releasing control`);
110
+ this.releaseControl();
106
111
  }
107
112
 
108
113
  /**
@@ -167,18 +172,25 @@ export class BrowserMcpControl extends EventEmitter {
167
172
  }
168
173
 
169
174
  /**
170
- * Check if a specific browser tab is being controlled
175
+ * Check if a specific browser tab is being controlled.
176
+ * When projectId is provided, also validates the project to prevent cross-project
177
+ * false-positives (tab IDs are only unique per project, not globally).
171
178
  */
172
- isTabControlled(browserTabId: string): boolean {
173
- return this.controlState.isControlling &&
174
- this.controlState.browserTabId === browserTabId;
179
+ isTabControlled(browserTabId: string, projectId?: string): boolean {
180
+ if (!this.controlState.isControlling || this.controlState.browserTabId !== browserTabId) {
181
+ return false;
182
+ }
183
+ if (projectId && this.controlState.projectId && this.controlState.projectId !== projectId) {
184
+ return false;
185
+ }
186
+ return true;
175
187
  }
176
188
 
177
189
  /**
178
190
  * Acquire control of a browser tab
179
191
  * Returns true if control was acquired, false if already controlled by another MCP
180
192
  */
181
- acquireControl(browserTabId: string, mcpSessionId?: string): boolean {
193
+ acquireControl(browserTabId: string, mcpSessionId?: string, projectId?: string): boolean {
182
194
  // Validate tab exists before acquiring control
183
195
  if (this.previewService && !this.previewService.getTab(browserTabId)) {
184
196
  debug.warn('mcp', `❌ Cannot acquire control: tab ${browserTabId} does not exist`);
@@ -204,6 +216,7 @@ export class BrowserMcpControl extends EventEmitter {
204
216
  isControlling: true,
205
217
  mcpSessionId: mcpSessionId || null,
206
218
  browserTabId,
219
+ projectId: projectId || null,
207
220
  startedAt: now,
208
221
  lastActionAt: now
209
222
  };
@@ -238,6 +251,7 @@ export class BrowserMcpControl extends EventEmitter {
238
251
  isControlling: false,
239
252
  mcpSessionId: null,
240
253
  browserTabId: null,
254
+ projectId: null,
241
255
  startedAt: null,
242
256
  lastActionAt: null
243
257
  };
@@ -380,13 +394,14 @@ export class BrowserMcpControl extends EventEmitter {
380
394
  }
381
395
 
382
396
  /**
383
- * Auto-release control for a specific browser tab (called when tab closes)
397
+ * Auto-release control for a specific browser tab (called when tab closes).
398
+ * projectId is used to prevent accidental release across projects with same tab IDs.
384
399
  */
385
- autoReleaseForTab(browserTabId: string): void {
386
- if (this.controlState.isControlling && this.controlState.browserTabId === browserTabId) {
387
- debug.log('mcp', `🗑️ Auto-releasing MCP control for closed tab: ${browserTabId}`);
388
- this.releaseControl(browserTabId);
389
- }
400
+ autoReleaseForTab(browserTabId: string, projectId?: string): void {
401
+ if (!this.controlState.isControlling || this.controlState.browserTabId !== browserTabId) return;
402
+ if (projectId && this.controlState.projectId && this.controlState.projectId !== projectId) return;
403
+ debug.log('mcp', `🗑️ Auto-releasing MCP control for closed tab: ${browserTabId}`);
404
+ this.releaseControl(browserTabId);
390
405
  }
391
406
 
392
407
  /**
@@ -403,6 +418,7 @@ export class BrowserMcpControl extends EventEmitter {
403
418
  isControlling: false,
404
419
  mcpSessionId: null,
405
420
  browserTabId: null,
421
+ projectId: null,
406
422
  startedAt: null,
407
423
  lastActionAt: null
408
424
  };
@@ -49,7 +49,9 @@ const DEFAULT_CONFIG: PoolConfig = {
49
49
  const CHROMIUM_ARGS = [
50
50
  '--no-sandbox',
51
51
  '--disable-blink-features=AutomationControlled',
52
- '--window-size=1366,768'
52
+ '--window-size=1366,768',
53
+ '--autoplay-policy=no-user-gesture-required',
54
+ '--disable-features=AudioServiceOutOfProcess'
53
55
  ];
54
56
 
55
57
  class BrowserPool {
@@ -273,7 +273,7 @@ export class BrowserTabManager extends EventEmitter {
273
273
  const wasActive = tab.isActive;
274
274
 
275
275
  // Auto-release MCP control if this tab is being controlled
276
- browserMcpControl.autoReleaseForTab(tabId);
276
+ browserMcpControl.autoReleaseForTab(tabId, this.projectId);
277
277
 
278
278
  // IMMEDIATELY set destroyed flag and stop streaming
279
279
  tab.isDestroyed = true;
@@ -148,6 +148,12 @@ export function audioCaptureScript(config: StreamingConfig['audio']) {
148
148
  if (interceptedContexts.has(ctx)) return;
149
149
  interceptedContexts.add(ctx);
150
150
 
151
+ // Resume AudioContext immediately — in headless Chrome without a user gesture,
152
+ // AudioContext starts in 'suspended' state and onaudioprocess never fires.
153
+ if (ctx.state === 'suspended') {
154
+ ctx.resume().catch(() => {});
155
+ }
156
+
151
157
  // Store original destination
152
158
  const originalDestination = ctx.destination;
153
159
 
@@ -213,6 +219,11 @@ export function audioCaptureScript(config: StreamingConfig['audio']) {
213
219
  const OriginalAudioContext = (window as any).__OriginalAudioContext || window.AudioContext;
214
220
  const ctx = new OriginalAudioContext();
215
221
 
222
+ // Resume context immediately — headless Chrome requires explicit resume
223
+ if (ctx.state === 'suspended') {
224
+ ctx.resume().catch(() => {});
225
+ }
226
+
216
227
  // Create media element source
217
228
  const source = ctx.createMediaElementSource(element);
218
229
 
@@ -470,7 +470,7 @@ export const streamHandler = createRouter()
470
470
  return;
471
471
  }
472
472
 
473
- const cancelled = await streamManager.cancelStream(streamState.streamId);
473
+ await streamManager.cancelStream(streamState.streamId);
474
474
  // Always send cancelled to chat session room to clear UI
475
475
  ws.emit.chatSession(chatSessionId, 'chat:cancelled', {
476
476
  status: 'cancelled',
@@ -7,6 +7,7 @@
7
7
  import { t } from 'elysia';
8
8
  import { createRouter } from '$shared/utils/ws-server';
9
9
  import { browserPreviewServiceManager } from '../../../preview/index';
10
+ import { browserMcpControl } from '../../../preview/browser/browser-mcp-control';
10
11
  import { ws } from '$backend/utils/ws';
11
12
  import { debug } from '$shared/utils/logger';
12
13
 
@@ -62,7 +63,8 @@ export const tabInfoPreviewHandler = createRouter()
62
63
  isStreaming: t.Boolean(),
63
64
  deviceSize: t.String(),
64
65
  rotation: t.String(),
65
- isActive: t.Boolean()
66
+ isActive: t.Boolean(),
67
+ isMcpControlled: t.Boolean()
66
68
  })),
67
69
  activeTabId: t.Union([t.String(), t.Null()]),
68
70
  count: t.Number()
@@ -87,7 +89,8 @@ export const tabInfoPreviewHandler = createRouter()
87
89
  isStreaming: tab.isStreaming,
88
90
  deviceSize: tab.deviceSize,
89
91
  rotation: tab.rotation,
90
- isActive: tab.isActive
92
+ isActive: tab.isActive,
93
+ isMcpControlled: browserMcpControl.isTabControlled(tab.id, projectId)
91
94
  })),
92
95
  activeTabId: activeTab?.id || null,
93
96
  count: allTabsInfo.length
@@ -66,9 +66,6 @@
66
66
 
67
67
  // Chat actions params
68
68
  const chatActionsParams = {
69
- get messageText() {
70
- return messageText;
71
- },
72
69
  get attachedFiles() {
73
70
  return fileHandling.attachedFiles;
74
71
  },
@@ -10,7 +10,6 @@ import { debug } from '$shared/utils/logger';
10
10
  import type { FileAttachment } from './use-file-handling.svelte';
11
11
 
12
12
  interface ChatActionsParams {
13
- messageText: string;
14
13
  attachedFiles: FileAttachment[];
15
14
  clearAllAttachments: () => void;
16
15
  adjustTextareaHeight: () => void;
@@ -27,7 +26,6 @@ export function useChatActions(params: ChatActionsParams) {
27
26
  function handleCancelEdit() {
28
27
  cancelEdit();
29
28
  clearInput();
30
- params.messageText = ''; // This won't work directly, need to pass setter
31
29
  params.clearAllAttachments();
32
30
  params.adjustTextareaHeight();
33
31
  }
@@ -52,6 +50,12 @@ export function useChatActions(params: ChatActionsParams) {
52
50
  await snapshotService.restore(restoreTargetId, sessionState.currentSession.id);
53
51
  }
54
52
 
53
+ // Set skip and clear draft BEFORE reloading messages — editing the first message
54
+ // causes messages to become empty (welcome state), which remounts ChatInput.
55
+ // Without this, the new instance restores stale server state into the input.
56
+ setSkipNextRestore(true);
57
+ params.clearDraft();
58
+
55
59
  // Reload messages from database to update UI
56
60
  if (sessionState.currentSession?.id) {
57
61
  await loadMessagesForSession(sessionState.currentSession.id);
@@ -428,7 +428,7 @@
428
428
  <span
429
429
  class="absolute -bottom-0.5 -right-0.5 w-2.5 h-2.5 rounded-full border-2 border-white dark:border-slate-900 {isSessionWaitingInput(session.id, projectState.currentProject?.id) ? 'bg-amber-500' : 'bg-emerald-500'}"
430
430
  ></span>
431
- {:else if isSessionUnread(session.id)}
431
+ {:else if isSessionUnread(session.id) && session.id !== sessionState.currentSession?.id}
432
432
  <span
433
433
  class="absolute -bottom-0.5 -right-0.5 w-2.5 h-2.5 rounded-full border-2 border-white dark:border-slate-900 bg-blue-500"
434
434
  ></span>
@@ -61,6 +61,9 @@
61
61
  // Flag to prevent URL watcher from double-launching during MCP session creation
62
62
  const mcpLaunchInProgress = $state(false);
63
63
 
64
+ // Touch interaction mode for canvas
65
+ let touchMode = $state<'scroll' | 'cursor'>('scroll');
66
+
64
67
  // Flag to track if sessions were recovered (prevents creating empty tab on mount)
65
68
  let sessionsRecovered = $state(false);
66
69
 
@@ -417,6 +420,14 @@
417
420
  return mcpHandler.isCurrentTabMcpControlled();
418
421
  }
419
422
 
423
+ // Hide MCP virtual cursor when switching to a non-MCP-controlled tab
424
+ $effect(() => {
425
+ void activeTabId; // track activeTabId changes
426
+ if (!isCurrentTabMcpControlled()) {
427
+ mcpVirtualCursor = { x: 0, y: 0, visible: false, clicking: false };
428
+ }
429
+ });
430
+
420
431
  // Stream message handling
421
432
  $effect(() => {
422
433
  if (activeTabId && sessionId) {
@@ -426,6 +437,8 @@
426
437
 
427
438
  // Expose methods for parent (PreviewPanel)
428
439
  export const browserActions = {
440
+ getTouchMode: () => touchMode,
441
+ setTouchMode: (mode: 'scroll' | 'cursor') => { touchMode = mode; },
429
442
  changeDeviceSize: (size: DeviceSize) => {
430
443
  coordinator.changeDeviceSize(size, previewDimensions?.scale);
431
444
  },
@@ -494,6 +507,7 @@
494
507
  bind:canvasAPI
495
508
  bind:previewDimensions
496
509
  bind:lastFrameData={currentTabLastFrameData}
510
+ bind:touchMode
497
511
  isMcpControlled={isCurrentTabMcpControlled()}
498
512
  onInteraction={handleCanvasInteraction}
499
513
  onRetry={handleGoClick}