@myrialabs/clopen 0.2.10 → 0.2.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +61 -27
- package/backend/chat/stream-manager.ts +11 -7
- package/backend/engine/adapters/opencode/stream.ts +37 -19
- package/backend/index.ts +5 -0
- package/backend/mcp/servers/browser-automation/browser.ts +2 -0
- package/backend/preview/browser/browser-mcp-control.ts +16 -0
- package/backend/preview/browser/browser-navigation-tracker.ts +31 -3
- package/backend/preview/browser/browser-preview-service.ts +0 -34
- package/backend/preview/browser/browser-video-capture.ts +13 -1
- package/backend/preview/browser/scripts/audio-stream.ts +5 -0
- package/backend/preview/browser/types.ts +7 -6
- package/backend/ws/preview/browser/interact.ts +46 -50
- package/backend/ws/preview/browser/webcodecs.ts +24 -15
- package/frontend/components/common/feedback/NotificationToast.svelte +26 -11
- package/frontend/components/files/FileNode.svelte +16 -58
- package/frontend/components/git/CommitForm.svelte +1 -1
- package/frontend/components/preview/browser/components/Canvas.svelte +119 -42
- package/frontend/components/preview/browser/components/Container.svelte +18 -3
- package/frontend/components/preview/browser/components/Toolbar.svelte +23 -21
- package/frontend/components/preview/browser/core/coordinator.svelte.ts +13 -1
- package/frontend/components/preview/browser/core/stream-handler.svelte.ts +31 -7
- package/frontend/components/preview/browser/core/tab-operations.svelte.ts +5 -4
- package/frontend/services/chat/chat.service.ts +25 -3
- package/frontend/services/notification/push.service.ts +2 -2
- package/frontend/services/preview/browser/browser-webcodecs.service.ts +170 -46
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -1,26 +1,64 @@
|
|
|
1
|
-
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="https://clopen.myrialabs.dev/favicon.svg" alt="Clopen" width="72" height="72" />
|
|
3
|
+
</p>
|
|
2
4
|
|
|
3
|
-
|
|
4
|
-
[](https://bun.sh)
|
|
5
|
+
<h1 align="center">Clopen</h1>
|
|
5
6
|
|
|
6
|
-
|
|
7
|
+
<p align="center">
|
|
8
|
+
<strong>Build more. Switch less.</strong><br />
|
|
9
|
+
All-in-one workspace for Claude Code & OpenCode
|
|
10
|
+
</p>
|
|
11
|
+
|
|
12
|
+
<p align="center">
|
|
13
|
+
<a href="https://clopen.myrialabs.dev">Website</a> ·
|
|
14
|
+
<a href="https://github.com/myrialabs/clopen/issues">Issues</a> ·
|
|
15
|
+
<a href="https://www.npmjs.com/package/@myrialabs/clopen">npm</a>
|
|
16
|
+
</p>
|
|
17
|
+
|
|
18
|
+
<p align="center">
|
|
19
|
+
<a href="https://www.npmjs.com/package/@myrialabs/clopen"><img src="https://img.shields.io/npm/v/@myrialabs/clopen" alt="npm version" /></a>
|
|
20
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/License-MIT-blue.svg" alt="License: MIT" /></a>
|
|
21
|
+
<a href="https://github.com/myrialabs/clopen/pulls"><img src="https://img.shields.io/badge/PRs-welcome-brightgreen.svg" alt="PRs Welcome" /></a>
|
|
22
|
+
<a href="https://bun.sh"><img src="https://img.shields.io/badge/Built%20with-Bun-black" alt="Built with Bun" /></a>
|
|
23
|
+
</p>
|
|
24
|
+
|
|
25
|
+
<p align="center">
|
|
26
|
+
<img src="https://clopen.myrialabs.dev/images/workspace-overview.webp" alt="Clopen workspace overview" />
|
|
27
|
+
</p>
|
|
28
|
+
|
|
29
|
+
All-in-one workspace for Claude Code & OpenCode. Chat, terminal, git, browser preview, and real-time collaboration, built for multi-project and multi-session workflows.
|
|
30
|
+
|
|
31
|
+
---
|
|
32
|
+
|
|
33
|
+
## Screenshots
|
|
34
|
+
|
|
35
|
+

|
|
36
|
+
|
|
37
|
+

|
|
38
|
+
|
|
39
|
+

|
|
40
|
+
|
|
41
|
+

|
|
7
42
|
|
|
8
43
|
---
|
|
9
44
|
|
|
10
45
|
## Features
|
|
11
46
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
- **
|
|
15
|
-
- **
|
|
16
|
-
- **
|
|
17
|
-
- **
|
|
18
|
-
- **
|
|
19
|
-
- **
|
|
20
|
-
- **
|
|
21
|
-
- **
|
|
22
|
-
- **
|
|
23
|
-
- **
|
|
47
|
+
A complete development environment designed around AI-assisted workflows, built to disappear into the background and just work.
|
|
48
|
+
|
|
49
|
+
- **Multi-Account Claude Code** — Manage multiple Claude Code accounts (personal, work, or team) and switch between them instantly per chat session
|
|
50
|
+
- **Multi-Engine Support** — Switch between Claude Code and OpenCode as your AI engine, per session
|
|
51
|
+
- **Integrated Terminal** — Full PTY emulation with xterm.js UI. Multi-tab terminal sessions with complete ANSI/VT sequence support and full keyboard control
|
|
52
|
+
- **Full Git Management** — Stage, commit, branch, push, pull, stash, log, and resolve conflicts, all from a clean UI. Powered by native git CLI for accuracy
|
|
53
|
+
- **Real Browser Preview** — A live browser preview streams directly into your workspace. Interact with your app manually, or let the AI drive: clicking, typing, and scrolling for autonomous visual testing
|
|
54
|
+
- **Git-Like Checkpoints** — Multi-branch undo/redo with full file snapshots. Roll back to any point in your AI conversation without touching your actual git history
|
|
55
|
+
- **Real-Time Collaboration** — WebSocket-based presence tracking per project. Multiple users can work on the same codebase simultaneously with live awareness
|
|
56
|
+
- **Monaco File Editor** — VS Code's editor embedded in the browser. Full syntax highlighting, autocomplete, and live file watching, right beside your AI chat
|
|
57
|
+
- **Cloudflare Tunnel** — One-click public HTTPS URL for your local dev server. Built-in QR code for instant mobile access. Share your work without deploying
|
|
58
|
+
- **MCP Support** — Full Model Context Protocol integration. Connect AI tools, external APIs, and custom capabilities to your AI agents with zero friction
|
|
59
|
+
- **Flexible Authentication** — No Login or With Login mode with admin/member roles, invite links, rate-limited login, and CLI token recovery
|
|
60
|
+
- **Background Processing** — Chat, terminal, and other processes continue running even when you close the browser — come back later and pick up where you left off
|
|
61
|
+
- **Database Management** — Browse tables, run queries, and inspect your database directly from the workspace *(coming soon)*
|
|
24
62
|
|
|
25
63
|
---
|
|
26
64
|
|
|
@@ -28,8 +66,8 @@
|
|
|
28
66
|
|
|
29
67
|
### Prerequisites
|
|
30
68
|
|
|
31
|
-
- [Bun](https://bun.sh/) v1.2.12+
|
|
32
|
-
- [Claude Code](https://github.com/anthropics/claude-code)
|
|
69
|
+
- [Bun.js](https://bun.sh/) v1.2.12+
|
|
70
|
+
- [Claude Code](https://github.com/anthropics/claude-code) or [OpenCode](https://opencode.ai) — required for AI functionality
|
|
33
71
|
|
|
34
72
|
### Installation
|
|
35
73
|
|
|
@@ -82,7 +120,9 @@ This regenerates and displays a new admin PAT.
|
|
|
82
120
|
|
|
83
121
|
---
|
|
84
122
|
|
|
85
|
-
##
|
|
123
|
+
## Contributing
|
|
124
|
+
|
|
125
|
+
Clopen is open source and contributions are welcome! Whether it's a bug fix, new feature, or improvement to docs — feel free to open an issue or submit a pull request.
|
|
86
126
|
|
|
87
127
|
```bash
|
|
88
128
|
git clone https://github.com/myrialabs/clopen.git
|
|
@@ -94,6 +134,8 @@ bun run check # Type checking
|
|
|
94
134
|
|
|
95
135
|
When running in development mode, Clopen uses `~/.clopen-dev` instead of `~/.clopen`, keeping dev data separate from any production instance.
|
|
96
136
|
|
|
137
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines and [DECISIONS.md](DECISIONS.md) for architectural decisions.
|
|
138
|
+
|
|
97
139
|
---
|
|
98
140
|
|
|
99
141
|
## Architecture
|
|
@@ -112,14 +154,6 @@ Clopen uses an engine-agnostic adapter pattern — both engines normalize output
|
|
|
112
154
|
|
|
113
155
|
---
|
|
114
156
|
|
|
115
|
-
## Documentation
|
|
116
|
-
|
|
117
|
-
- [Technical Decisions](DECISIONS.md) - Architectural and technical decision log
|
|
118
|
-
- [Contributing](CONTRIBUTING.md) - How to contribute to this project
|
|
119
|
-
- [Development Guidelines](CLAUDE.md) - Guidelines for working with Claude Code on this project
|
|
120
|
-
|
|
121
|
-
---
|
|
122
|
-
|
|
123
157
|
## Troubleshooting
|
|
124
158
|
|
|
125
159
|
### Port 9141 Already in Use
|
|
@@ -1141,16 +1141,19 @@ class StreamManager extends EventEmitter {
|
|
|
1141
1141
|
}
|
|
1142
1142
|
}
|
|
1143
1143
|
|
|
1144
|
-
// Cancel the per-project engine
|
|
1145
|
-
//
|
|
1146
|
-
//
|
|
1147
|
-
//
|
|
1148
|
-
//
|
|
1144
|
+
// Cancel the per-project engine with a bounded timeout.
|
|
1145
|
+
// engine.cancel() stops the SDK process (Claude Code: close() kills subprocess,
|
|
1146
|
+
// OpenCode: aborts controller + HTTP abort to server). If cancel() hangs
|
|
1147
|
+
// (e.g. unresponsive SDK), the timeout ensures we always proceed to emit
|
|
1148
|
+
// events and update presence — preventing infinite loader on the frontend.
|
|
1149
1149
|
const projectId = streamState.projectId || 'default';
|
|
1150
1150
|
try {
|
|
1151
1151
|
const engine = getProjectEngine(projectId, streamState.engine);
|
|
1152
1152
|
if (engine.isActive) {
|
|
1153
|
-
await
|
|
1153
|
+
await Promise.race([
|
|
1154
|
+
engine.cancel(),
|
|
1155
|
+
new Promise<void>(resolve => setTimeout(resolve, 5000))
|
|
1156
|
+
]);
|
|
1154
1157
|
}
|
|
1155
1158
|
} catch (error) {
|
|
1156
1159
|
debug.error('chat', 'Error cancelling engine (non-fatal):', error);
|
|
@@ -1158,7 +1161,8 @@ class StreamManager extends EventEmitter {
|
|
|
1158
1161
|
|
|
1159
1162
|
// Abort the stream-manager's controller as a fallback.
|
|
1160
1163
|
// engine.cancel() already aborts the same controller, so this is
|
|
1161
|
-
// typically a no-op but ensures cleanup if the engine
|
|
1164
|
+
// typically a no-op but ensures cleanup if the engine timed out
|
|
1165
|
+
// or wasn't active.
|
|
1162
1166
|
if (!streamState.abortController?.signal.aborted) {
|
|
1163
1167
|
streamState.abortController?.abort();
|
|
1164
1168
|
}
|
|
@@ -770,20 +770,15 @@ export class OpenCodeEngine implements AIEngine {
|
|
|
770
770
|
}
|
|
771
771
|
|
|
772
772
|
async cancel(): Promise<void> {
|
|
773
|
-
//
|
|
774
|
-
const
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
} catch (error) {
|
|
783
|
-
debug.warn('engine', 'Failed to abort Open Code session:', error);
|
|
784
|
-
}
|
|
785
|
-
}
|
|
786
|
-
|
|
773
|
+
// Capture refs before clearing — needed for server-side abort below
|
|
774
|
+
const sessionId = this.activeSessionId;
|
|
775
|
+
const projectPath = this.activeProjectPath;
|
|
776
|
+
|
|
777
|
+
// 1. FIRST: Abort local stream processing immediately.
|
|
778
|
+
// This breaks the SSE event stream and causes the for-await loop
|
|
779
|
+
// in processStream() to throw AbortError, stopping all local processing.
|
|
780
|
+
// Must happen BEFORE the HTTP call because client.session.abort() can
|
|
781
|
+
// hang indefinitely if the OpenCode server is busy/unresponsive.
|
|
787
782
|
if (this.activeAbortController) {
|
|
788
783
|
this.activeAbortController.abort();
|
|
789
784
|
this.activeAbortController = null;
|
|
@@ -792,6 +787,26 @@ export class OpenCodeEngine implements AIEngine {
|
|
|
792
787
|
this.activeSessionId = null;
|
|
793
788
|
this.activeProjectPath = null;
|
|
794
789
|
this.pendingQuestions.clear();
|
|
790
|
+
|
|
791
|
+
// 2. THEN: Tell the OpenCode server to stop processing (with timeout).
|
|
792
|
+
// This is a courtesy cleanup — local processing is already stopped.
|
|
793
|
+
// The server-side session would otherwise keep running (consuming
|
|
794
|
+
// LLM API calls and compute resources) until it naturally completes.
|
|
795
|
+
const client = getClient();
|
|
796
|
+
if (client && sessionId) {
|
|
797
|
+
try {
|
|
798
|
+
await Promise.race([
|
|
799
|
+
client.session.abort({
|
|
800
|
+
path: { id: sessionId },
|
|
801
|
+
...(projectPath && { query: { directory: projectPath } }),
|
|
802
|
+
}),
|
|
803
|
+
new Promise<void>(resolve => setTimeout(resolve, 5000))
|
|
804
|
+
]);
|
|
805
|
+
debug.log('engine', 'Open Code session aborted:', sessionId);
|
|
806
|
+
} catch (error) {
|
|
807
|
+
debug.warn('engine', 'Failed to abort Open Code session (non-fatal):', error);
|
|
808
|
+
}
|
|
809
|
+
}
|
|
795
810
|
}
|
|
796
811
|
|
|
797
812
|
/**
|
|
@@ -802,13 +817,16 @@ export class OpenCodeEngine implements AIEngine {
|
|
|
802
817
|
const client = getClient();
|
|
803
818
|
if (!client || !sessionId) return;
|
|
804
819
|
try {
|
|
805
|
-
await
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
820
|
+
await Promise.race([
|
|
821
|
+
client.session.abort({
|
|
822
|
+
path: { id: sessionId },
|
|
823
|
+
...(projectPath && { query: { directory: projectPath } }),
|
|
824
|
+
}),
|
|
825
|
+
new Promise<void>(resolve => setTimeout(resolve, 5000))
|
|
826
|
+
]);
|
|
809
827
|
debug.log('engine', 'Open Code session aborted (per-stream):', sessionId);
|
|
810
828
|
} catch (error) {
|
|
811
|
-
debug.warn('engine', 'Failed to abort Open Code session:', error);
|
|
829
|
+
debug.warn('engine', 'Failed to abort Open Code session (non-fatal):', error);
|
|
812
830
|
}
|
|
813
831
|
}
|
|
814
832
|
|
package/backend/index.ts
CHANGED
|
@@ -27,6 +27,9 @@ import { statSync } from 'node:fs';
|
|
|
27
27
|
// Import WebSocket router
|
|
28
28
|
import { wsRouter } from './ws';
|
|
29
29
|
|
|
30
|
+
// Import browser preview manager for graceful shutdown
|
|
31
|
+
import { browserPreviewServiceManager } from './preview';
|
|
32
|
+
|
|
30
33
|
// MCP remote server for Open Code custom tools
|
|
31
34
|
import { handleMcpRequest, closeMcpServer } from './mcp/remote-server';
|
|
32
35
|
|
|
@@ -166,6 +169,8 @@ async function gracefulShutdown() {
|
|
|
166
169
|
try {
|
|
167
170
|
// Close MCP remote server (before engines, as they may still reference it)
|
|
168
171
|
await closeMcpServer();
|
|
172
|
+
// Cleanup browser preview sessions
|
|
173
|
+
await browserPreviewServiceManager.cleanup();
|
|
169
174
|
// Dispose all AI engines
|
|
170
175
|
await disposeAllEngines();
|
|
171
176
|
// Stop accepting new connections
|
|
@@ -177,6 +177,8 @@ export async function switchTabHandler(args: { tabId: string; projectId?: string
|
|
|
177
177
|
isError: true
|
|
178
178
|
};
|
|
179
179
|
}
|
|
180
|
+
// Promote tab to end of session's set so getActiveTabSession() targets it next
|
|
181
|
+
browserMcpControl.promoteSessionTab(tab.id, chatSessionId);
|
|
180
182
|
}
|
|
181
183
|
|
|
182
184
|
return {
|
|
@@ -206,6 +206,22 @@ export class BrowserMcpControl extends EventEmitter {
|
|
|
206
206
|
// Control Acquisition
|
|
207
207
|
// ============================================================================
|
|
208
208
|
|
|
209
|
+
/**
|
|
210
|
+
* Promote a tab to the end of the session's controlled set.
|
|
211
|
+
* This ensures getSessionTabs()[last] returns the most recently activated tab,
|
|
212
|
+
* which is used by getActiveTabSession to determine which tab MCP operates on.
|
|
213
|
+
*
|
|
214
|
+
* Must be called after switch_tab to reflect the new active tab.
|
|
215
|
+
*/
|
|
216
|
+
promoteSessionTab(browserTabId: string, chatSessionId: string): void {
|
|
217
|
+
const sessionSet = this.sessionTabs.get(chatSessionId);
|
|
218
|
+
if (sessionSet && sessionSet.has(browserTabId)) {
|
|
219
|
+
sessionSet.delete(browserTabId);
|
|
220
|
+
sessionSet.add(browserTabId);
|
|
221
|
+
debug.log('mcp', `🔀 Promoted tab ${browserTabId} to end of session ${chatSessionId.slice(0, 8)} set`);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
|
|
209
225
|
/**
|
|
210
226
|
* Acquire control of a browser tab for a chat session.
|
|
211
227
|
*
|
|
@@ -6,10 +6,35 @@ import { debug } from '$shared/utils/logger';
|
|
|
6
6
|
export class BrowserNavigationTracker extends EventEmitter {
|
|
7
7
|
private cdpSessions = new Map<string, CDPSession>();
|
|
8
8
|
|
|
9
|
+
// Deduplication: track the last emitted navigation URL+timestamp per session.
|
|
10
|
+
// framenavigated and load often fire for the same navigation;
|
|
11
|
+
// this prevents emitting duplicate 'navigation' events that would cause
|
|
12
|
+
// parallel handleNavigation calls and double streaming restarts.
|
|
13
|
+
private lastNavigationEmit = new Map<string, { url: string; time: number }>();
|
|
14
|
+
private readonly DEDUP_WINDOW_MS = 500; // Ignore duplicate within 500ms
|
|
15
|
+
|
|
9
16
|
constructor() {
|
|
10
17
|
super();
|
|
11
18
|
}
|
|
12
19
|
|
|
20
|
+
/**
|
|
21
|
+
* Emit a navigation event with deduplication.
|
|
22
|
+
* Returns true if the event was emitted, false if it was a duplicate.
|
|
23
|
+
*/
|
|
24
|
+
private emitNavigationDeduped(event: string, sessionId: string, url: string, data: any): boolean {
|
|
25
|
+
const now = Date.now();
|
|
26
|
+
const last = this.lastNavigationEmit.get(sessionId);
|
|
27
|
+
|
|
28
|
+
if (last && last.url === url && (now - last.time) < this.DEDUP_WINDOW_MS) {
|
|
29
|
+
debug.log('preview', `⏭️ Deduped ${event} for ${url} (${now - last.time}ms since last emit)`);
|
|
30
|
+
return false;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
this.lastNavigationEmit.set(sessionId, { url, time: now });
|
|
34
|
+
this.emit(event, data);
|
|
35
|
+
return true;
|
|
36
|
+
}
|
|
37
|
+
|
|
13
38
|
/**
|
|
14
39
|
* Check if two URLs differ only by hash/fragment.
|
|
15
40
|
* Hash-only changes are same-document navigations and should NOT trigger
|
|
@@ -134,8 +159,9 @@ export class BrowserNavigationTracker extends EventEmitter {
|
|
|
134
159
|
// Update session URL
|
|
135
160
|
session.url = newUrl;
|
|
136
161
|
|
|
137
|
-
// Emit navigation completed event to
|
|
138
|
-
|
|
162
|
+
// Emit navigation completed event (deduplicated to prevent double events
|
|
163
|
+
// from framenavigated + load firing for the same navigation)
|
|
164
|
+
this.emitNavigationDeduped('navigation', sessionId, newUrl, {
|
|
139
165
|
sessionId,
|
|
140
166
|
type: 'navigation',
|
|
141
167
|
url: newUrl,
|
|
@@ -184,7 +210,8 @@ export class BrowserNavigationTracker extends EventEmitter {
|
|
|
184
210
|
|
|
185
211
|
session.url = currentUrl;
|
|
186
212
|
|
|
187
|
-
this
|
|
213
|
+
// Deduplicated: framenavigated already emitted for this URL
|
|
214
|
+
this.emitNavigationDeduped('navigation', sessionId, currentUrl, {
|
|
188
215
|
sessionId,
|
|
189
216
|
type: 'navigation',
|
|
190
217
|
url: currentUrl,
|
|
@@ -246,6 +273,7 @@ export class BrowserNavigationTracker extends EventEmitter {
|
|
|
246
273
|
}
|
|
247
274
|
this.cdpSessions.delete(sessionId);
|
|
248
275
|
}
|
|
276
|
+
this.lastNavigationEmit.delete(sessionId);
|
|
249
277
|
}
|
|
250
278
|
|
|
251
279
|
async navigateSession(sessionId: string, session: BrowserTab, url: string): Promise<string> {
|
|
@@ -869,37 +869,3 @@ class BrowserPreviewServiceManager {
|
|
|
869
869
|
// Service manager instance (singleton)
|
|
870
870
|
export const browserPreviewServiceManager = new BrowserPreviewServiceManager();
|
|
871
871
|
|
|
872
|
-
// Graceful shutdown handlers
|
|
873
|
-
const gracefulShutdown = async (signal: string) => {
|
|
874
|
-
try {
|
|
875
|
-
await browserPreviewServiceManager.cleanup();
|
|
876
|
-
process.exit(0);
|
|
877
|
-
} catch (error) {
|
|
878
|
-
process.exit(1);
|
|
879
|
-
}
|
|
880
|
-
};
|
|
881
|
-
|
|
882
|
-
// Handle various termination signals
|
|
883
|
-
process.on('SIGINT', () => gracefulShutdown('SIGINT'));
|
|
884
|
-
process.on('SIGTERM', () => gracefulShutdown('SIGTERM'));
|
|
885
|
-
process.on('SIGHUP', () => gracefulShutdown('SIGHUP'));
|
|
886
|
-
|
|
887
|
-
// Handle Windows-specific signals
|
|
888
|
-
if (process.platform === 'win32') {
|
|
889
|
-
process.on('SIGBREAK', () => gracefulShutdown('SIGBREAK'));
|
|
890
|
-
}
|
|
891
|
-
|
|
892
|
-
// Handle uncaught exceptions and unhandled rejections
|
|
893
|
-
process.on('uncaughtException', async (error) => {
|
|
894
|
-
await browserPreviewServiceManager.cleanup();
|
|
895
|
-
process.exit(1);
|
|
896
|
-
});
|
|
897
|
-
|
|
898
|
-
process.on('unhandledRejection', async (reason, promise) => {
|
|
899
|
-
await browserPreviewServiceManager.cleanup();
|
|
900
|
-
process.exit(1);
|
|
901
|
-
});
|
|
902
|
-
|
|
903
|
-
// Handle process exit
|
|
904
|
-
process.on('exit', (code) => {
|
|
905
|
-
});
|
|
@@ -42,6 +42,7 @@ interface VideoStreamSession {
|
|
|
42
42
|
pendingCandidates: RTCIceCandidateInit[];
|
|
43
43
|
scriptInjected: boolean; // Track if persistent script was injected
|
|
44
44
|
scriptsPreInjected: boolean; // Track if scripts were pre-injected during tab creation
|
|
45
|
+
audioOnNewDocumentInjected: boolean; // Track if evaluateOnNewDocument was registered for audio
|
|
45
46
|
stats: {
|
|
46
47
|
videoBytesSent: number;
|
|
47
48
|
audioBytesSent: number;
|
|
@@ -97,6 +98,7 @@ export class BrowserVideoCapture extends EventEmitter {
|
|
|
97
98
|
pendingCandidates: [],
|
|
98
99
|
scriptInjected: true,
|
|
99
100
|
scriptsPreInjected: false, // Set to true only after injection completes
|
|
101
|
+
audioOnNewDocumentInjected: false,
|
|
100
102
|
stats: {
|
|
101
103
|
videoBytesSent: 0,
|
|
102
104
|
audioBytesSent: 0,
|
|
@@ -162,7 +164,16 @@ export class BrowserVideoCapture extends EventEmitter {
|
|
|
162
164
|
});
|
|
163
165
|
}
|
|
164
166
|
|
|
165
|
-
//
|
|
167
|
+
// Register audio capture as a startup script — runs before page scripts on every new document load.
|
|
168
|
+
// Critical for SPAs that create AudioContext during initialization (before page.evaluate runs).
|
|
169
|
+
// The idempotency guard in audioCaptureScript prevents double-injection.
|
|
170
|
+
const session = this.sessions.get(sessionId);
|
|
171
|
+
if (session && !session.audioOnNewDocumentInjected) {
|
|
172
|
+
await page.evaluateOnNewDocument(audioCaptureScript, config.audio);
|
|
173
|
+
session.audioOnNewDocumentInjected = true;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// Inject video encoder + audio capture scripts into the current page context
|
|
166
177
|
await page.evaluate(videoEncoderScript, videoConfig);
|
|
167
178
|
await page.evaluate(audioCaptureScript, config.audio);
|
|
168
179
|
}
|
|
@@ -220,6 +231,7 @@ export class BrowserVideoCapture extends EventEmitter {
|
|
|
220
231
|
pendingCandidates: [],
|
|
221
232
|
scriptInjected: false,
|
|
222
233
|
scriptsPreInjected: false,
|
|
234
|
+
audioOnNewDocumentInjected: false,
|
|
223
235
|
stats: {
|
|
224
236
|
videoBytesSent: 0,
|
|
225
237
|
audioBytesSent: 0,
|
|
@@ -16,6 +16,11 @@ import type { StreamingConfig } from '../types';
|
|
|
16
16
|
* This script intercepts AudioContext and captures all audio
|
|
17
17
|
*/
|
|
18
18
|
export function audioCaptureScript(config: StreamingConfig['audio']) {
|
|
19
|
+
// Idempotency guard — prevent double-injection when both evaluateOnNewDocument
|
|
20
|
+
// and page.evaluate inject this script into the same page context.
|
|
21
|
+
if ((window as any).__audioCaptureInstalled) return;
|
|
22
|
+
(window as any).__audioCaptureInstalled = true;
|
|
23
|
+
|
|
19
24
|
// Check AudioEncoder support
|
|
20
25
|
if (typeof AudioEncoder === 'undefined') {
|
|
21
26
|
(window as any).__audioEncoderSupported = false;
|
|
@@ -229,10 +229,11 @@ export interface StreamingConfig {
|
|
|
229
229
|
/**
|
|
230
230
|
* Default streaming configuration
|
|
231
231
|
*
|
|
232
|
-
* Optimized for visual quality with
|
|
232
|
+
* Optimized for visual quality with reduced resource usage:
|
|
233
233
|
* - Software encoding (hardwareAcceleration: 'no-preference')
|
|
234
|
-
* - JPEG quality
|
|
235
|
-
* - VP8 at 1.2Mbps
|
|
234
|
+
* - JPEG quality 65: slightly lower than before but still preserves thin borders/text
|
|
235
|
+
* - VP8 at 1.0Mbps: ~17% reduction from 1.2Mbps, sharp edges preserved by VP8 codec
|
|
236
|
+
* - keyframeInterval 5s: less frequent large keyframes, saves bandwidth on static pages
|
|
236
237
|
* - Opus for audio (efficient and widely supported)
|
|
237
238
|
*/
|
|
238
239
|
export const DEFAULT_STREAMING_CONFIG: StreamingConfig = {
|
|
@@ -241,9 +242,9 @@ export const DEFAULT_STREAMING_CONFIG: StreamingConfig = {
|
|
|
241
242
|
width: 0,
|
|
242
243
|
height: 0,
|
|
243
244
|
framerate: 24,
|
|
244
|
-
bitrate:
|
|
245
|
-
keyframeInterval:
|
|
246
|
-
screenshotQuality:
|
|
245
|
+
bitrate: 1_000_000,
|
|
246
|
+
keyframeInterval: 5,
|
|
247
|
+
screenshotQuality: 65,
|
|
247
248
|
hardwareAcceleration: 'no-preference',
|
|
248
249
|
latencyMode: 'realtime'
|
|
249
250
|
},
|
|
@@ -12,6 +12,9 @@ import type { KeyInput } from 'puppeteer';
|
|
|
12
12
|
import { debug } from '$shared/utils/logger';
|
|
13
13
|
import { sleep } from '$shared/utils/async';
|
|
14
14
|
|
|
15
|
+
// Throttle cursor detection evaluate calls per session (100ms = ~10/sec is plenty)
|
|
16
|
+
const lastCursorEvalTime = new Map<string, number>();
|
|
17
|
+
|
|
15
18
|
// Helper function to check if error is navigation-related
|
|
16
19
|
function isNavigationError(error: Error): boolean {
|
|
17
20
|
const msg = error.message.toLowerCase();
|
|
@@ -108,8 +111,9 @@ export const interactPreviewHandler = createRouter()
|
|
|
108
111
|
switch (action.type) {
|
|
109
112
|
case 'mousedown':
|
|
110
113
|
try {
|
|
111
|
-
//
|
|
112
|
-
|
|
114
|
+
// Fire-and-forget reset — CDP processes commands in FIFO order so
|
|
115
|
+
// this completes before move/down even though we skip the await.
|
|
116
|
+
session.page.mouse.up().catch(() => {});
|
|
113
117
|
// Move to position and press button
|
|
114
118
|
await session.page.mouse.move(action.x!, action.y!, { steps: 1 });
|
|
115
119
|
await session.page.mouse.down({ button: action.button === 'right' ? 'right' : 'left' });
|
|
@@ -140,14 +144,10 @@ export const interactPreviewHandler = createRouter()
|
|
|
140
144
|
|
|
141
145
|
case 'click':
|
|
142
146
|
try {
|
|
143
|
-
//
|
|
144
|
-
//
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
} catch { /* Ignore - mouse might not be pressed */ }
|
|
148
|
-
|
|
149
|
-
// IMPORTANT: Check for select element BEFORE clicking
|
|
150
|
-
// If it's a select, we'll emit event to frontend instead of clicking
|
|
147
|
+
// Check for select element BEFORE clicking.
|
|
148
|
+
// Skip the mouse.up() reset: page.mouse.click() is atomic (down+up),
|
|
149
|
+
// and Canvas.svelte always sends mouseup before sending click, so the
|
|
150
|
+
// mouse state is already clean at this point.
|
|
151
151
|
const selectInfo = await previewService.checkForSelectElement(session.id, action.x!, action.y!);
|
|
152
152
|
if (selectInfo) {
|
|
153
153
|
// Select element detected - event emitted by checkForSelectElement
|
|
@@ -243,37 +243,40 @@ export const interactPreviewHandler = createRouter()
|
|
|
243
243
|
await session.page.mouse.move(action.x!, action.y!, {
|
|
244
244
|
steps: action.steps || 1 // Reduced from 5 to 1 for faster response
|
|
245
245
|
});
|
|
246
|
-
//
|
|
247
|
-
//
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
const
|
|
254
|
-
|
|
255
|
-
|
|
246
|
+
// Cursor detection via page.evaluate — throttled to ~10/sec per session.
|
|
247
|
+
// Running it on every mousemove queues extra CDP commands that delay clicks/keypresses.
|
|
248
|
+
const nowMs = Date.now();
|
|
249
|
+
const lastEval = lastCursorEvalTime.get(session.id) ?? 0;
|
|
250
|
+
if (nowMs - lastEval >= 100) {
|
|
251
|
+
lastCursorEvalTime.set(session.id, nowMs);
|
|
252
|
+
session.page.evaluate((data) => {
|
|
253
|
+
const { x, y } = data;
|
|
254
|
+
let cursor = 'default';
|
|
255
|
+
try {
|
|
256
|
+
const el = document.elementFromPoint(x, y);
|
|
257
|
+
if (el) {
|
|
258
|
+
cursor = window.getComputedStyle(el).cursor || 'default';
|
|
259
|
+
}
|
|
260
|
+
} catch {}
|
|
261
|
+
|
|
262
|
+
const existing = (window as any).__cursorInfo;
|
|
263
|
+
if (existing) {
|
|
264
|
+
existing.cursor = cursor;
|
|
265
|
+
existing.x = x;
|
|
266
|
+
existing.y = y;
|
|
267
|
+
existing.timestamp = Date.now();
|
|
268
|
+
existing.hasRecentInteraction = true;
|
|
269
|
+
} else {
|
|
270
|
+
(window as any).__cursorInfo = {
|
|
271
|
+
cursor,
|
|
272
|
+
x,
|
|
273
|
+
y,
|
|
274
|
+
timestamp: Date.now(),
|
|
275
|
+
hasRecentInteraction: true
|
|
276
|
+
};
|
|
256
277
|
}
|
|
257
|
-
} catch {}
|
|
258
|
-
|
|
259
|
-
// Initialize or update __cursorInfo
|
|
260
|
-
const existing = (window as any).__cursorInfo;
|
|
261
|
-
if (existing) {
|
|
262
|
-
existing.cursor = cursor;
|
|
263
|
-
existing.x = x;
|
|
264
|
-
existing.y = y;
|
|
265
|
-
existing.timestamp = Date.now();
|
|
266
|
-
existing.hasRecentInteraction = true;
|
|
267
|
-
} else {
|
|
268
|
-
(window as any).__cursorInfo = {
|
|
269
|
-
cursor,
|
|
270
|
-
x,
|
|
271
|
-
y,
|
|
272
|
-
timestamp: Date.now(),
|
|
273
|
-
hasRecentInteraction: true
|
|
274
|
-
};
|
|
275
|
-
}
|
|
276
|
-
}, { x: action.x!, y: action.y! }).catch(() => { /* Ignore evaluation errors */ });
|
|
278
|
+
}, { x: action.x!, y: action.y! }).catch(() => { /* Ignore evaluation errors */ });
|
|
279
|
+
}
|
|
277
280
|
} catch (error) {
|
|
278
281
|
if (error instanceof Error && isNavigationError(error)) {
|
|
279
282
|
ws.emit.user(userId, 'preview:browser-interacted', { action: action.type, message: 'Action deferred (navigation)', deferred: true });
|
|
@@ -297,8 +300,6 @@ export const interactPreviewHandler = createRouter()
|
|
|
297
300
|
|
|
298
301
|
case 'doubleclick':
|
|
299
302
|
try {
|
|
300
|
-
// Reset mouse state first
|
|
301
|
-
try { await session.page.mouse.up(); } catch { }
|
|
302
303
|
await session.page.mouse.click(action.x!, action.y!, { clickCount: 2 });
|
|
303
304
|
} catch (error) {
|
|
304
305
|
if (error instanceof Error) {
|
|
@@ -314,8 +315,8 @@ export const interactPreviewHandler = createRouter()
|
|
|
314
315
|
|
|
315
316
|
case 'rightclick':
|
|
316
317
|
try {
|
|
317
|
-
//
|
|
318
|
-
|
|
318
|
+
// Fire-and-forget reset (see mousedown comment for rationale)
|
|
319
|
+
session.page.mouse.up().catch(() => {});
|
|
319
320
|
|
|
320
321
|
// IMPORTANT: Check for context menu
|
|
321
322
|
// We'll emit context menu event to frontend for custom overlay
|
|
@@ -385,12 +386,7 @@ export const interactPreviewHandler = createRouter()
|
|
|
385
386
|
await session.page.keyboard.press(action.key as KeyInput);
|
|
386
387
|
}
|
|
387
388
|
|
|
388
|
-
|
|
389
|
-
try {
|
|
390
|
-
await sleep(50);
|
|
391
|
-
} catch { }
|
|
392
|
-
}
|
|
393
|
-
}
|
|
389
|
+
}
|
|
394
390
|
break;
|
|
395
391
|
|
|
396
392
|
case 'checkselectoptions':
|