@myrialabs/clopen 0.0.5 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +5 -5
- package/.github/workflows/ci.yml +86 -86
- package/CONTRIBUTING.md +499 -499
- package/LICENSE +21 -21
- package/README.md +209 -209
- package/backend/index.ts +168 -165
- package/backend/lib/chat/helpers.ts +42 -42
- package/backend/lib/chat/index.ts +1 -1
- package/backend/lib/chat/stream-manager.ts +1126 -1126
- package/backend/lib/database/README.md +76 -76
- package/backend/lib/database/index.ts +118 -118
- package/backend/lib/database/migrations/001_create_projects_table.ts +30 -30
- package/backend/lib/database/migrations/002_create_chat_sessions_table.ts +32 -32
- package/backend/lib/database/migrations/003_create_messages_table.ts +31 -31
- package/backend/lib/database/migrations/004_create_prompt_templates_table.ts +34 -34
- package/backend/lib/database/migrations/005_create_settings_table.ts +23 -23
- package/backend/lib/database/migrations/006_add_user_to_messages.ts +57 -57
- package/backend/lib/database/migrations/007_create_stream_states_table.ts +40 -40
- package/backend/lib/database/migrations/008_create_message_snapshots_table.ts +61 -61
- package/backend/lib/database/migrations/009_add_delta_snapshot_fields.ts +41 -41
- package/backend/lib/database/migrations/010_add_soft_delete_and_branch_support.ts +70 -70
- package/backend/lib/database/migrations/011_git_like_commit_graph.ts +156 -156
- package/backend/lib/database/migrations/012_add_file_change_statistics.ts +41 -41
- package/backend/lib/database/migrations/013_checkpoint_tree_state.ts +118 -118
- package/backend/lib/database/migrations/014_add_engine_to_sessions.ts +18 -18
- package/backend/lib/database/migrations/015_add_model_to_sessions.ts +18 -18
- package/backend/lib/database/migrations/016_create_user_projects_table.ts +34 -34
- package/backend/lib/database/migrations/017_add_current_session_to_user_projects.ts +32 -32
- package/backend/lib/database/migrations/018_create_claude_accounts_table.ts +24 -24
- package/backend/lib/database/migrations/019_add_claude_account_to_sessions.ts +18 -18
- package/backend/lib/database/migrations/020_add_snapshot_tree_hash.ts +32 -32
- package/backend/lib/database/migrations/021_drop_prompt_templates_table.ts +33 -33
- package/backend/lib/database/migrations/index.ts +153 -153
- package/backend/lib/database/queries/checkpoint-queries.ts +87 -87
- package/backend/lib/database/queries/engine-queries.ts +75 -75
- package/backend/lib/database/queries/index.ts +8 -8
- package/backend/lib/database/queries/message-queries.ts +471 -471
- package/backend/lib/database/queries/project-queries.ts +117 -117
- package/backend/lib/database/queries/session-queries.ts +270 -270
- package/backend/lib/database/queries/settings-queries.ts +33 -33
- package/backend/lib/database/queries/snapshot-queries.ts +325 -325
- package/backend/lib/database/queries/utils-queries.ts +58 -58
- package/backend/lib/database/seeders/index.ts +12 -12
- package/backend/lib/database/seeders/settings_seeder.ts +83 -83
- package/backend/lib/database/utils/connection.ts +173 -173
- package/backend/lib/database/utils/index.ts +3 -3
- package/backend/lib/database/utils/migration-runner.ts +117 -117
- package/backend/lib/database/utils/seeder-runner.ts +120 -120
- package/backend/lib/engine/adapters/claude/environment.ts +160 -164
- package/backend/lib/engine/adapters/claude/error-handler.ts +60 -60
- package/backend/lib/engine/adapters/claude/index.ts +1 -1
- package/backend/lib/engine/adapters/claude/path-utils.ts +38 -38
- package/backend/lib/engine/adapters/claude/stream.ts +177 -177
- package/backend/lib/engine/adapters/opencode/index.ts +2 -2
- package/backend/lib/engine/adapters/opencode/message-converter.ts +862 -862
- package/backend/lib/engine/adapters/opencode/server.ts +104 -104
- package/backend/lib/engine/adapters/opencode/stream.ts +755 -755
- package/backend/lib/engine/index.ts +196 -196
- package/backend/lib/engine/types.ts +58 -58
- package/backend/lib/files/file-operations.ts +478 -478
- package/backend/lib/files/file-reading.ts +308 -308
- package/backend/lib/files/file-watcher.ts +383 -383
- package/backend/lib/files/path-browsing.ts +382 -382
- package/backend/lib/git/git-executor.ts +89 -88
- package/backend/lib/git/git-parser.ts +411 -411
- package/backend/lib/git/git-service.ts +505 -505
- package/backend/lib/mcp/README.md +1144 -1144
- package/backend/lib/mcp/config.ts +317 -316
- package/backend/lib/mcp/index.ts +35 -35
- package/backend/lib/mcp/project-context.ts +236 -236
- package/backend/lib/mcp/servers/browser-automation/actions.ts +156 -156
- package/backend/lib/mcp/servers/browser-automation/browser.ts +419 -419
- package/backend/lib/mcp/servers/browser-automation/index.ts +791 -791
- package/backend/lib/mcp/servers/browser-automation/inspection.ts +501 -501
- package/backend/lib/mcp/servers/helper.ts +143 -143
- package/backend/lib/mcp/servers/index.ts +44 -44
- package/backend/lib/mcp/servers/weather/get-temperature.ts +56 -56
- package/backend/lib/mcp/servers/weather/index.ts +31 -31
- package/backend/lib/mcp/stdio-server.ts +103 -103
- package/backend/lib/mcp/types.ts +65 -65
- package/backend/lib/preview/browser/browser-audio-capture.ts +86 -86
- package/backend/lib/preview/browser/browser-console-manager.ts +262 -262
- package/backend/lib/preview/browser/browser-dialog-handler.ts +222 -222
- package/backend/lib/preview/browser/browser-interaction-handler.ts +421 -421
- package/backend/lib/preview/browser/browser-mcp-control.ts +415 -415
- package/backend/lib/preview/browser/browser-native-ui-handler.ts +512 -512
- package/backend/lib/preview/browser/browser-navigation-tracker.ts +103 -103
- package/backend/lib/preview/browser/browser-pool.ts +357 -357
- package/backend/lib/preview/browser/browser-preview-service.ts +882 -882
- package/backend/lib/preview/browser/browser-tab-manager.ts +935 -935
- package/backend/lib/preview/browser/browser-video-capture.ts +695 -695
- package/backend/lib/preview/browser/scripts/audio-stream.ts +292 -292
- package/backend/lib/preview/browser/scripts/cursor-tracking.ts +85 -85
- package/backend/lib/preview/browser/scripts/video-stream.ts +438 -438
- package/backend/lib/preview/browser/types.ts +359 -359
- package/backend/lib/preview/index.ts +23 -23
- package/backend/lib/project/index.ts +1 -1
- package/backend/lib/project/status-manager.ts +181 -181
- package/backend/lib/shared/env.ts +117 -0
- package/backend/lib/shared/index.ts +5 -2
- package/backend/lib/shared/port-utils.ts +25 -25
- package/backend/lib/shared/process-manager.ts +280 -280
- package/backend/lib/snapshot/blob-store.ts +227 -227
- package/backend/lib/snapshot/gitignore.ts +307 -307
- package/backend/lib/snapshot/helpers.ts +397 -397
- package/backend/lib/snapshot/snapshot-service.ts +483 -483
- package/backend/lib/terminal/helpers.ts +14 -14
- package/backend/lib/terminal/index.ts +7 -7
- package/backend/lib/terminal/pty-manager.ts +3 -3
- package/backend/lib/terminal/pty-session-manager.ts +370 -387
- package/backend/lib/terminal/shell-utils.ts +315 -312
- package/backend/lib/terminal/stream-manager.ts +292 -292
- package/backend/lib/tunnel/global-tunnel-manager.ts +266 -243
- package/backend/lib/tunnel/project-tunnel-manager.ts +311 -311
- package/backend/lib/user/helpers.ts +87 -87
- package/backend/lib/utils/ws.ts +944 -944
- package/backend/lib/vite-dev.ts +295 -295
- package/backend/middleware/cors.ts +16 -15
- package/backend/middleware/error-handler.ts +50 -49
- package/backend/middleware/logger.ts +9 -9
- package/backend/types/api.ts +24 -24
- package/backend/ws/README.md +1505 -1505
- package/backend/ws/chat/background.ts +198 -198
- package/backend/ws/chat/index.ts +21 -21
- package/backend/ws/chat/stream.ts +707 -707
- package/backend/ws/engine/claude/accounts.ts +399 -401
- package/backend/ws/engine/claude/index.ts +13 -13
- package/backend/ws/engine/claude/status.ts +43 -43
- package/backend/ws/engine/index.ts +14 -14
- package/backend/ws/engine/opencode/index.ts +11 -11
- package/backend/ws/engine/opencode/status.ts +30 -30
- package/backend/ws/engine/utils.ts +36 -36
- package/backend/ws/files/index.ts +30 -30
- package/backend/ws/files/read.ts +189 -189
- package/backend/ws/files/search.ts +453 -453
- package/backend/ws/files/watch.ts +124 -124
- package/backend/ws/files/write.ts +143 -143
- package/backend/ws/git/branch.ts +106 -106
- package/backend/ws/git/commit.ts +39 -39
- package/backend/ws/git/conflict.ts +68 -68
- package/backend/ws/git/diff.ts +69 -69
- package/backend/ws/git/index.ts +24 -24
- package/backend/ws/git/log.ts +41 -41
- package/backend/ws/git/remote.ts +214 -214
- package/backend/ws/git/staging.ts +84 -84
- package/backend/ws/git/status.ts +90 -90
- package/backend/ws/index.ts +69 -69
- package/backend/ws/mcp/index.ts +61 -61
- package/backend/ws/messages/crud.ts +74 -74
- package/backend/ws/messages/index.ts +14 -14
- package/backend/ws/preview/browser/cleanup.ts +129 -129
- package/backend/ws/preview/browser/console.ts +114 -114
- package/backend/ws/preview/browser/interact.ts +513 -513
- package/backend/ws/preview/browser/mcp.ts +129 -129
- package/backend/ws/preview/browser/native-ui.ts +235 -235
- package/backend/ws/preview/browser/stats.ts +55 -55
- package/backend/ws/preview/browser/tab-info.ts +126 -126
- package/backend/ws/preview/browser/tab.ts +166 -166
- package/backend/ws/preview/browser/webcodecs.ts +293 -293
- package/backend/ws/preview/index.ts +146 -146
- package/backend/ws/projects/crud.ts +113 -113
- package/backend/ws/projects/index.ts +25 -25
- package/backend/ws/projects/presence.ts +46 -46
- package/backend/ws/projects/status.ts +116 -116
- package/backend/ws/sessions/crud.ts +327 -327
- package/backend/ws/sessions/index.ts +33 -33
- package/backend/ws/settings/crud.ts +112 -112
- package/backend/ws/settings/index.ts +14 -14
- package/backend/ws/snapshot/index.ts +17 -17
- package/backend/ws/snapshot/restore.ts +173 -173
- package/backend/ws/snapshot/timeline.ts +141 -141
- package/backend/ws/system/index.ts +14 -14
- package/backend/ws/system/operations.ts +49 -49
- package/backend/ws/terminal/index.ts +40 -40
- package/backend/ws/terminal/persistence.ts +153 -153
- package/backend/ws/terminal/session.ts +382 -382
- package/backend/ws/terminal/stream.ts +79 -79
- package/backend/ws/tunnel/index.ts +14 -14
- package/backend/ws/tunnel/operations.ts +91 -91
- package/backend/ws/types.ts +20 -20
- package/backend/ws/user/crud.ts +156 -156
- package/backend/ws/user/index.ts +14 -14
- package/bin/clopen.ts +307 -307
- package/bun.lock +1353 -1352
- package/frontend/App.svelte +38 -34
- package/frontend/app.css +313 -313
- package/frontend/lib/app-environment.ts +10 -10
- package/frontend/lib/components/chat/ChatInterface.svelte +406 -406
- package/frontend/lib/components/chat/formatters/ErrorMessage.svelte +56 -56
- package/frontend/lib/components/chat/formatters/MessageFormatter.svelte +223 -223
- package/frontend/lib/components/chat/formatters/TextMessage.svelte +394 -394
- package/frontend/lib/components/chat/formatters/Tools.svelte +69 -69
- package/frontend/lib/components/chat/formatters/index.ts +2 -2
- package/frontend/lib/components/chat/input/ChatInput.svelte +421 -421
- package/frontend/lib/components/chat/input/components/ChatInputActions.svelte +78 -78
- package/frontend/lib/components/chat/input/components/DragDropOverlay.svelte +30 -30
- package/frontend/lib/components/chat/input/components/EditModeIndicator.svelte +33 -33
- package/frontend/lib/components/chat/input/components/EngineModelPicker.svelte +619 -619
- package/frontend/lib/components/chat/input/components/FileAttachmentPreview.svelte +48 -48
- package/frontend/lib/components/chat/input/components/LoadingIndicator.svelte +31 -31
- package/frontend/lib/components/chat/input/composables/use-animations.svelte.ts +201 -201
- package/frontend/lib/components/chat/input/composables/use-chat-actions.svelte.ts +148 -148
- package/frontend/lib/components/chat/input/composables/use-file-handling.svelte.ts +216 -216
- package/frontend/lib/components/chat/input/composables/use-input-state.svelte.ts +357 -357
- package/frontend/lib/components/chat/input/composables/use-textarea-resize.svelte.ts +57 -57
- package/frontend/lib/components/chat/message/ChatMessage.svelte +478 -478
- package/frontend/lib/components/chat/message/ChatMessages.svelte +541 -541
- package/frontend/lib/components/chat/message/DateSeparator.svelte +86 -86
- package/frontend/lib/components/chat/message/MessageBubble.svelte +86 -86
- package/frontend/lib/components/chat/message/MessageHeader.svelte +157 -157
- package/frontend/lib/components/chat/modal/DebugModal.svelte +59 -59
- package/frontend/lib/components/chat/modal/TokenUsageModal.svelte +124 -124
- package/frontend/lib/components/chat/shared/index.ts +1 -1
- package/frontend/lib/components/chat/shared/utils.ts +115 -115
- package/frontend/lib/components/chat/tools/BashOutputTool.svelte +35 -35
- package/frontend/lib/components/chat/tools/BashTool.svelte +45 -45
- package/frontend/lib/components/chat/tools/CustomMcpTool.svelte +139 -139
- package/frontend/lib/components/chat/tools/EditTool.svelte +47 -47
- package/frontend/lib/components/chat/tools/ExitPlanModeTool.svelte +31 -31
- package/frontend/lib/components/chat/tools/GlobTool.svelte +50 -50
- package/frontend/lib/components/chat/tools/GrepTool.svelte +89 -89
- package/frontend/lib/components/chat/tools/KillShellTool.svelte +25 -25
- package/frontend/lib/components/chat/tools/ListMcpResourcesTool.svelte +30 -30
- package/frontend/lib/components/chat/tools/NotebookEditTool.svelte +37 -37
- package/frontend/lib/components/chat/tools/ReadMcpResourceTool.svelte +33 -33
- package/frontend/lib/components/chat/tools/ReadTool.svelte +40 -40
- package/frontend/lib/components/chat/tools/TaskTool.svelte +63 -63
- package/frontend/lib/components/chat/tools/TodoWriteTool.svelte +74 -74
- package/frontend/lib/components/chat/tools/WebFetchTool.svelte +34 -34
- package/frontend/lib/components/chat/tools/WebSearchTool.svelte +83 -83
- package/frontend/lib/components/chat/tools/WriteTool.svelte +32 -32
- package/frontend/lib/components/chat/tools/components/CodeBlock.svelte +78 -78
- package/frontend/lib/components/chat/tools/components/DiffBlock.svelte +407 -407
- package/frontend/lib/components/chat/tools/components/FileHeader.svelte +45 -45
- package/frontend/lib/components/chat/tools/components/InfoLine.svelte +18 -18
- package/frontend/lib/components/chat/tools/components/StatsBadges.svelte +26 -26
- package/frontend/lib/components/chat/tools/components/TerminalCommand.svelte +53 -53
- package/frontend/lib/components/chat/tools/components/index.ts +7 -7
- package/frontend/lib/components/chat/tools/index.ts +25 -25
- package/frontend/lib/components/chat/widgets/FloatingTodoList.svelte +248 -248
- package/frontend/lib/components/chat/widgets/TokenUsage.svelte +78 -78
- package/frontend/lib/components/checkpoint/TimelineModal.svelte +391 -391
- package/frontend/lib/components/checkpoint/timeline/TimelineEdge.svelte +26 -26
- package/frontend/lib/components/checkpoint/timeline/TimelineGraph.svelte +86 -86
- package/frontend/lib/components/checkpoint/timeline/TimelineNode.svelte +108 -108
- package/frontend/lib/components/checkpoint/timeline/TimelineVersionGroup.svelte +59 -59
- package/frontend/lib/components/checkpoint/timeline/animation.ts +168 -168
- package/frontend/lib/components/checkpoint/timeline/config.ts +44 -44
- package/frontend/lib/components/checkpoint/timeline/graph-builder.ts +304 -304
- package/frontend/lib/components/checkpoint/timeline/types.ts +65 -65
- package/frontend/lib/components/checkpoint/timeline/utils.ts +53 -53
- package/frontend/lib/components/common/Alert.svelte +138 -138
- package/frontend/lib/components/common/AvatarBubble.svelte +55 -55
- package/frontend/lib/components/common/Button.svelte +71 -71
- package/frontend/lib/components/common/Card.svelte +102 -102
- package/frontend/lib/components/common/Checkbox.svelte +48 -48
- package/frontend/lib/components/common/Dialog.svelte +248 -248
- package/frontend/lib/components/common/FolderBrowser.svelte +842 -842
- package/frontend/lib/components/common/Icon.svelte +57 -57
- package/frontend/lib/components/common/Input.svelte +72 -72
- package/frontend/lib/components/common/Lightbox.svelte +232 -232
- package/frontend/lib/components/common/LoadingScreen.svelte +52 -52
- package/frontend/lib/components/common/LoadingSpinner.svelte +48 -48
- package/frontend/lib/components/common/Modal.svelte +177 -177
- package/frontend/lib/components/common/ModalProvider.svelte +27 -27
- package/frontend/lib/components/common/ModelSelector.svelte +110 -110
- package/frontend/lib/components/common/MonacoEditor.svelte +568 -568
- package/frontend/lib/components/common/NotificationToast.svelte +113 -113
- package/frontend/lib/components/common/PageTemplate.svelte +75 -75
- package/frontend/lib/components/common/ProjectUserAvatars.svelte +79 -79
- package/frontend/lib/components/common/Select.svelte +97 -97
- package/frontend/lib/components/common/Textarea.svelte +79 -79
- package/frontend/lib/components/common/ThemeToggle.svelte +44 -44
- package/frontend/lib/components/common/lucide-icons.ts +1642 -1642
- package/frontend/lib/components/common/material-icons.ts +1082 -1082
- package/frontend/lib/components/common/xterm/XTerm.svelte +809 -795
- package/frontend/lib/components/common/xterm/index.ts +15 -15
- package/frontend/lib/components/common/xterm/terminal-config.ts +67 -67
- package/frontend/lib/components/common/xterm/types.ts +30 -30
- package/frontend/lib/components/common/xterm/xterm-service.ts +379 -353
- package/frontend/lib/components/files/FileNode.svelte +383 -383
- package/frontend/lib/components/files/FileTree.svelte +681 -681
- package/frontend/lib/components/files/FileViewer.svelte +728 -728
- package/frontend/lib/components/files/SearchResults.svelte +303 -303
- package/frontend/lib/components/git/BranchManager.svelte +458 -458
- package/frontend/lib/components/git/ChangesSection.svelte +107 -107
- package/frontend/lib/components/git/CommitForm.svelte +76 -76
- package/frontend/lib/components/git/ConflictResolver.svelte +158 -158
- package/frontend/lib/components/git/DiffViewer.svelte +364 -364
- package/frontend/lib/components/git/FileChangeItem.svelte +97 -97
- package/frontend/lib/components/git/GitButton.svelte +33 -33
- package/frontend/lib/components/git/GitLog.svelte +361 -361
- package/frontend/lib/components/git/GitModal.svelte +80 -80
- package/frontend/lib/components/history/HistoryModal.svelte +563 -563
- package/frontend/lib/components/history/HistoryView.svelte +614 -614
- package/frontend/lib/components/index.ts +34 -34
- package/frontend/lib/components/preview/browser/BrowserPreview.svelte +549 -549
- package/frontend/lib/components/preview/browser/components/Canvas.svelte +1058 -1058
- package/frontend/lib/components/preview/browser/components/ConsolePanel.svelte +756 -756
- package/frontend/lib/components/preview/browser/components/Container.svelte +450 -450
- package/frontend/lib/components/preview/browser/components/ContextMenu.svelte +236 -236
- package/frontend/lib/components/preview/browser/components/SelectDropdown.svelte +224 -224
- package/frontend/lib/components/preview/browser/components/Toolbar.svelte +338 -338
- package/frontend/lib/components/preview/browser/components/VirtualCursor.svelte +35 -35
- package/frontend/lib/components/preview/browser/core/cleanup.svelte.ts +155 -155
- package/frontend/lib/components/preview/browser/core/coordinator.svelte.ts +837 -837
- package/frontend/lib/components/preview/browser/core/interactions.svelte.ts +113 -113
- package/frontend/lib/components/preview/browser/core/mcp-handlers.svelte.ts +296 -296
- package/frontend/lib/components/preview/browser/core/native-ui-handlers.svelte.ts +391 -391
- package/frontend/lib/components/preview/browser/core/stream-handler.svelte.ts +231 -231
- package/frontend/lib/components/preview/browser/core/tab-manager.svelte.ts +210 -210
- package/frontend/lib/components/preview/browser/core/tab-operations.svelte.ts +239 -239
- package/frontend/lib/components/preview/index.ts +1 -1
- package/frontend/lib/components/settings/SettingsModal.svelte +235 -235
- package/frontend/lib/components/settings/SettingsView.svelte +36 -36
- package/frontend/lib/components/settings/appearance/AppearanceSettings.svelte +51 -51
- package/frontend/lib/components/settings/appearance/LayoutPresetSettings.svelte +160 -160
- package/frontend/lib/components/settings/appearance/LayoutPreview.svelte +76 -76
- package/frontend/lib/components/settings/engines/AIEnginesSettings.svelte +917 -917
- package/frontend/lib/components/settings/general/AdvancedSettings.svelte +187 -187
- package/frontend/lib/components/settings/general/DataManagementSettings.svelte +203 -203
- package/frontend/lib/components/settings/general/GeneralSettings.svelte +10 -10
- package/frontend/lib/components/settings/model/ModelSettings.svelte +357 -357
- package/frontend/lib/components/settings/notifications/NotificationSettings.svelte +205 -205
- package/frontend/lib/components/settings/user/UserSettings.svelte +197 -197
- package/frontend/lib/components/terminal/Terminal.svelte +367 -367
- package/frontend/lib/components/terminal/TerminalTabs.svelte +87 -87
- package/frontend/lib/components/terminal/TerminalView.svelte +54 -54
- package/frontend/lib/components/tunnel/TunnelActive.svelte +157 -142
- package/frontend/lib/components/tunnel/TunnelButton.svelte +60 -54
- package/frontend/lib/components/tunnel/TunnelInactive.svelte +285 -284
- package/frontend/lib/components/tunnel/TunnelModal.svelte +48 -47
- package/frontend/lib/components/tunnel/TunnelQRCode.svelte +49 -49
- package/frontend/lib/components/workspace/DesktopNavigator.svelte +382 -382
- package/frontend/lib/components/workspace/MobileNavigator.svelte +394 -403
- package/frontend/lib/components/workspace/PanelContainer.svelte +100 -100
- package/frontend/lib/components/workspace/PanelHeader.svelte +505 -505
- package/frontend/lib/components/workspace/ViewMenu.svelte +162 -162
- package/frontend/lib/components/workspace/WorkspaceLayout.svelte +169 -169
- package/frontend/lib/components/workspace/layout/DesktopLayout.svelte +15 -15
- package/frontend/lib/components/workspace/layout/MobileLayout.svelte +17 -17
- package/frontend/lib/components/workspace/layout/split-pane/Container.svelte +42 -42
- package/frontend/lib/components/workspace/layout/split-pane/Handle.svelte +84 -84
- package/frontend/lib/components/workspace/layout/split-pane/Layout.svelte +37 -37
- package/frontend/lib/components/workspace/panels/ChatPanel.svelte +274 -274
- package/frontend/lib/components/workspace/panels/FilesPanel.svelte +1261 -1261
- package/frontend/lib/components/workspace/panels/GitPanel.svelte +1560 -1560
- package/frontend/lib/components/workspace/panels/PreviewPanel.svelte +150 -150
- package/frontend/lib/components/workspace/panels/TerminalPanel.svelte +73 -73
- package/frontend/lib/constants/preview.ts +44 -44
- package/frontend/lib/services/chat/chat.service.ts +704 -704
- package/frontend/lib/services/chat/index.ts +6 -6
- package/frontend/lib/services/notification/global-stream-monitor.ts +86 -86
- package/frontend/lib/services/notification/index.ts +7 -7
- package/frontend/lib/services/notification/push.service.ts +143 -143
- package/frontend/lib/services/notification/sound.service.ts +126 -126
- package/frontend/lib/services/preview/browser/browser-console.service.ts +61 -61
- package/frontend/lib/services/preview/browser/browser-webcodecs.service.ts +1499 -1499
- package/frontend/lib/services/preview/browser/mcp-integration.svelte.ts +67 -67
- package/frontend/lib/services/preview/index.ts +22 -22
- package/frontend/lib/services/project/index.ts +7 -7
- package/frontend/lib/services/project/status.service.ts +159 -159
- package/frontend/lib/services/snapshot/snapshot.service.ts +47 -47
- package/frontend/lib/services/terminal/background/index.ts +129 -129
- package/frontend/lib/services/terminal/background/session-restore.ts +273 -273
- package/frontend/lib/services/terminal/background/stream-manager.ts +285 -285
- package/frontend/lib/services/terminal/index.ts +13 -13
- package/frontend/lib/services/terminal/persistence.service.ts +260 -260
- package/frontend/lib/services/terminal/project.service.ts +952 -952
- package/frontend/lib/services/terminal/session.service.ts +363 -363
- package/frontend/lib/services/terminal/terminal.service.ts +369 -369
- package/frontend/lib/stores/core/app.svelte.ts +117 -117
- package/frontend/lib/stores/core/files.svelte.ts +72 -72
- package/frontend/lib/stores/core/presence.svelte.ts +48 -48
- package/frontend/lib/stores/core/projects.svelte.ts +317 -317
- package/frontend/lib/stores/core/sessions.svelte.ts +383 -383
- package/frontend/lib/stores/features/claude-accounts.svelte.ts +58 -58
- package/frontend/lib/stores/features/models.svelte.ts +89 -89
- package/frontend/lib/stores/features/settings.svelte.ts +87 -87
- package/frontend/lib/stores/features/terminal.svelte.ts +700 -700
- package/frontend/lib/stores/features/tunnel.svelte.ts +163 -161
- package/frontend/lib/stores/features/user.svelte.ts +95 -95
- package/frontend/lib/stores/ui/chat-input.svelte.ts +56 -56
- package/frontend/lib/stores/ui/chat-model.svelte.ts +61 -61
- package/frontend/lib/stores/ui/dialog.svelte.ts +58 -58
- package/frontend/lib/stores/ui/edit-mode.svelte.ts +214 -214
- package/frontend/lib/stores/ui/notification.svelte.ts +166 -166
- package/frontend/lib/stores/ui/settings-modal.svelte.ts +88 -88
- package/frontend/lib/stores/ui/theme.svelte.ts +179 -179
- package/frontend/lib/stores/ui/workspace.svelte.ts +754 -754
- package/frontend/lib/types/native-ui.ts +73 -73
- package/frontend/lib/utils/chat/date-separator.ts +38 -38
- package/frontend/lib/utils/chat/message-grouper.ts +218 -218
- package/frontend/lib/utils/chat/message-processor.ts +134 -134
- package/frontend/lib/utils/chat/tool-handler.ts +160 -160
- package/frontend/lib/utils/chat/virtual-scroll.svelte.ts +142 -142
- package/frontend/lib/utils/click-outside.ts +20 -20
- package/frontend/lib/utils/context-manager.ts +256 -256
- package/frontend/lib/utils/file-icon-mappings.ts +768 -768
- package/frontend/lib/utils/folder-icon-mappings.ts +1029 -1029
- package/frontend/lib/utils/git-status.ts +68 -68
- package/frontend/lib/utils/platform.ts +112 -112
- package/frontend/lib/utils/port-check.ts +64 -64
- package/frontend/lib/utils/terminalFormatter.ts +206 -206
- package/frontend/lib/utils/theme.ts +6 -6
- package/frontend/lib/utils/tree-visualizer.ts +320 -320
- package/frontend/lib/utils/ws.ts +44 -44
- package/frontend/main.ts +13 -13
- package/index.html +70 -70
- package/package.json +111 -111
- package/scripts/generate-icons.ts +86 -86
- package/scripts/pre-publish-check.sh +142 -142
- package/scripts/setup-hooks.sh +134 -134
- package/scripts/validate-branch-name.sh +47 -47
- package/scripts/validate-commit-msg.sh +42 -42
- package/shared/constants/engines.ts +134 -134
- package/shared/types/database/connection.ts +15 -15
- package/shared/types/database/index.ts +5 -5
- package/shared/types/database/schema.ts +140 -140
- package/shared/types/engine/index.ts +45 -45
- package/shared/types/filesystem/index.ts +21 -21
- package/shared/types/git.ts +171 -171
- package/shared/types/messaging/index.ts +238 -238
- package/shared/types/messaging/tool.ts +525 -525
- package/shared/types/network/api.ts +17 -17
- package/shared/types/network/index.ts +4 -4
- package/shared/types/stores/app.ts +22 -22
- package/shared/types/stores/dialog.ts +20 -20
- package/shared/types/stores/index.ts +2 -2
- package/shared/types/stores/settings.ts +15 -15
- package/shared/types/terminal/index.ts +43 -43
- package/shared/types/ui/components.ts +60 -60
- package/shared/types/ui/icons.ts +22 -22
- package/shared/types/ui/index.ts +21 -21
- package/shared/types/ui/notifications.ts +13 -13
- package/shared/types/ui/theme.ts +11 -11
- package/shared/types/websocket/index.ts +43 -43
- package/shared/types/window.d.ts +12 -12
- package/shared/utils/anonymous-user.ts +167 -167
- package/shared/utils/async.ts +10 -10
- package/shared/utils/diff-calculator.ts +184 -184
- package/shared/utils/file-type-detection.ts +165 -165
- package/shared/utils/logger.ts +144 -144
- package/shared/utils/message-formatter.ts +79 -79
- package/shared/utils/path.ts +47 -47
- package/shared/utils/ws-client.ts +768 -768
- package/shared/utils/ws-server.ts +660 -660
- package/static/favicon.svg +7 -7
- package/static/fonts/dm-sans.css +96 -96
- package/svelte.config.js +20 -20
- package/tsconfig.json +41 -41
- package/vite.config.ts +33 -33
|
@@ -1,438 +1,438 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Video Encoder Client Script
|
|
3
|
-
*
|
|
4
|
-
* This script runs in the headless browser to:
|
|
5
|
-
* 1. Initialize RTCPeerConnection with DataChannel
|
|
6
|
-
* 2. Initialize VideoEncoder for encoding frames
|
|
7
|
-
* 3. Handle WebRTC signaling (offer/answer/ICE)
|
|
8
|
-
* 4. Send encoded video chunks via DataChannel
|
|
9
|
-
*/
|
|
10
|
-
|
|
11
|
-
import type { StreamingConfig } from '../types';
|
|
12
|
-
|
|
13
|
-
/**
|
|
14
|
-
* Generate video encoder script that runs in the browser
|
|
15
|
-
*/
|
|
16
|
-
export function videoEncoderScript(config: StreamingConfig['video']) {
|
|
17
|
-
// WebCodecs Encoder for Headless Browser
|
|
18
|
-
if ((window as any).__webCodecsPeer) {
|
|
19
|
-
try {
|
|
20
|
-
(window as any).__webCodecsPeer.stopStreaming();
|
|
21
|
-
} catch (e) {}
|
|
22
|
-
(window as any).__webCodecsPeer = null;
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
let peerConnection: RTCPeerConnection | null = null;
|
|
26
|
-
let dataChannel: RTCDataChannel | null = null;
|
|
27
|
-
let videoEncoder: VideoEncoder | null = null;
|
|
28
|
-
let isCapturing = false;
|
|
29
|
-
let videoFrameCount = 0;
|
|
30
|
-
let audioFrameCount = 0;
|
|
31
|
-
let lastKeyframeTime = 0;
|
|
32
|
-
|
|
33
|
-
// Cursor tracking
|
|
34
|
-
let lastCursor = 'default';
|
|
35
|
-
let cursorCheckInterval: any = null;
|
|
36
|
-
|
|
37
|
-
// ICE servers
|
|
38
|
-
const iceServers = [
|
|
39
|
-
{ urls: 'stun:stun.l.google.com:19302' },
|
|
40
|
-
{ urls: 'stun:stun1.l.google.com:19302' }
|
|
41
|
-
];
|
|
42
|
-
|
|
43
|
-
// Check cursor style from page
|
|
44
|
-
function checkCursor() {
|
|
45
|
-
try {
|
|
46
|
-
const cursorInfo = (window as any).__cursorInfo;
|
|
47
|
-
if (cursorInfo && cursorInfo.cursor && cursorInfo.cursor !== lastCursor) {
|
|
48
|
-
lastCursor = cursorInfo.cursor;
|
|
49
|
-
// Send cursor change to backend via exposed function
|
|
50
|
-
if ((window as any).__sendCursorChange) {
|
|
51
|
-
(window as any).__sendCursorChange(cursorInfo.cursor);
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
} catch (e) {
|
|
55
|
-
// Ignore errors - cursor tracking is non-critical
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
// Start cursor tracking interval
|
|
60
|
-
function startCursorTracking() {
|
|
61
|
-
if (cursorCheckInterval) return;
|
|
62
|
-
// Check cursor every 100ms (low overhead, responsive enough)
|
|
63
|
-
cursorCheckInterval = setInterval(checkCursor, 100);
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
// Stop cursor tracking interval
|
|
67
|
-
function stopCursorTracking() {
|
|
68
|
-
if (cursorCheckInterval) {
|
|
69
|
-
clearInterval(cursorCheckInterval);
|
|
70
|
-
cursorCheckInterval = null;
|
|
71
|
-
}
|
|
72
|
-
lastCursor = 'default';
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
// Detect supported video codec
|
|
76
|
-
async function detectVideoCodec() {
|
|
77
|
-
const codecConfig: VideoEncoderConfig = {
|
|
78
|
-
codec: config.codec,
|
|
79
|
-
width: config.width,
|
|
80
|
-
height: config.height,
|
|
81
|
-
bitrate: config.bitrate,
|
|
82
|
-
framerate: config.framerate,
|
|
83
|
-
hardwareAcceleration: config.hardwareAcceleration,
|
|
84
|
-
latencyMode: config.latencyMode
|
|
85
|
-
};
|
|
86
|
-
|
|
87
|
-
try {
|
|
88
|
-
const support = await VideoEncoder.isConfigSupported(codecConfig);
|
|
89
|
-
if (support.supported) {
|
|
90
|
-
return codecConfig;
|
|
91
|
-
}
|
|
92
|
-
} catch (e) {}
|
|
93
|
-
|
|
94
|
-
return null;
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
// Initialize RTCPeerConnection
|
|
98
|
-
async function initPeerConnection() {
|
|
99
|
-
if (peerConnection) {
|
|
100
|
-
peerConnection.close();
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
peerConnection = new RTCPeerConnection({ iceServers });
|
|
104
|
-
|
|
105
|
-
// Handle ICE candidates
|
|
106
|
-
peerConnection.onicecandidate = (event) => {
|
|
107
|
-
if (event.candidate && (window as any).__sendIceCandidate) {
|
|
108
|
-
(window as any).__sendIceCandidate({
|
|
109
|
-
candidate: event.candidate.candidate,
|
|
110
|
-
sdpMid: event.candidate.sdpMid,
|
|
111
|
-
sdpMLineIndex: event.candidate.sdpMLineIndex
|
|
112
|
-
});
|
|
113
|
-
}
|
|
114
|
-
};
|
|
115
|
-
|
|
116
|
-
// Handle connection state
|
|
117
|
-
peerConnection.onconnectionstatechange = () => {
|
|
118
|
-
if ((window as any).__sendConnectionState && peerConnection) {
|
|
119
|
-
(window as any).__sendConnectionState(peerConnection.connectionState);
|
|
120
|
-
}
|
|
121
|
-
};
|
|
122
|
-
|
|
123
|
-
peerConnection.oniceconnectionstatechange = () => {};
|
|
124
|
-
|
|
125
|
-
// Create DataChannel for encoded chunks
|
|
126
|
-
dataChannel = peerConnection.createDataChannel('media', {
|
|
127
|
-
ordered: false, // Allow out-of-order delivery for lower latency
|
|
128
|
-
maxRetransmits: 0 // No retransmits = lower latency
|
|
129
|
-
});
|
|
130
|
-
|
|
131
|
-
dataChannel.binaryType = 'arraybuffer';
|
|
132
|
-
|
|
133
|
-
dataChannel.onopen = () => {
|
|
134
|
-
// Force keyframe when DataChannel opens (previous keyframes may have been lost)
|
|
135
|
-
lastKeyframeTime = 0;
|
|
136
|
-
};
|
|
137
|
-
|
|
138
|
-
dataChannel.onclose = () => {};
|
|
139
|
-
|
|
140
|
-
dataChannel.onerror = (error) => {};
|
|
141
|
-
|
|
142
|
-
return peerConnection;
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
// Initialize VideoEncoder
|
|
146
|
-
async function initVideoEncoder() {
|
|
147
|
-
const codecConfig = await detectVideoCodec();
|
|
148
|
-
if (!codecConfig) {
|
|
149
|
-
throw new Error('No supported video codec');
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
videoEncoder = new VideoEncoder({
|
|
153
|
-
output: (chunk, metadata) => {
|
|
154
|
-
handleEncodedVideoChunk(chunk, metadata);
|
|
155
|
-
},
|
|
156
|
-
error: (e) => {}
|
|
157
|
-
});
|
|
158
|
-
|
|
159
|
-
await videoEncoder.configure(codecConfig);
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
// Handle encoded video chunk
|
|
163
|
-
function handleEncodedVideoChunk(chunk: EncodedVideoChunk, metadata: any) {
|
|
164
|
-
if (!dataChannel || dataChannel.readyState !== 'open') return;
|
|
165
|
-
|
|
166
|
-
// Send chunk via DataChannel
|
|
167
|
-
// Format: [type(1)][timestamp(8)][keyframe(1)][size(4)][data]
|
|
168
|
-
const isKeyframe = chunk.type === 'key' ? 1 : 0;
|
|
169
|
-
const timestamp = chunk.timestamp;
|
|
170
|
-
const data = new Uint8Array(chunk.byteLength);
|
|
171
|
-
chunk.copyTo(data);
|
|
172
|
-
|
|
173
|
-
const packet = new ArrayBuffer(1 + 8 + 1 + 4 + data.byteLength);
|
|
174
|
-
const view = new DataView(packet);
|
|
175
|
-
const packetData = new Uint8Array(packet);
|
|
176
|
-
|
|
177
|
-
// Type: 0 = video
|
|
178
|
-
view.setUint8(0, 0);
|
|
179
|
-
// Timestamp (microseconds)
|
|
180
|
-
view.setBigUint64(1, BigInt(timestamp), true);
|
|
181
|
-
// Keyframe flag
|
|
182
|
-
view.setUint8(9, isKeyframe);
|
|
183
|
-
// Data size
|
|
184
|
-
view.setUint32(10, data.byteLength, true);
|
|
185
|
-
// Copy data
|
|
186
|
-
packetData.set(data, 14);
|
|
187
|
-
|
|
188
|
-
try {
|
|
189
|
-
dataChannel.send(packet);
|
|
190
|
-
videoFrameCount++;
|
|
191
|
-
} catch (e) {}
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
// Send audio chunk (called from AudioContext interception)
|
|
195
|
-
function sendAudioChunk(timestamp: number, data: Uint8Array) {
|
|
196
|
-
if (!dataChannel || dataChannel.readyState !== 'open') return;
|
|
197
|
-
|
|
198
|
-
// Format: [type(1)][timestamp(8)][size(4)][data]
|
|
199
|
-
const packet = new ArrayBuffer(1 + 8 + 4 + data.byteLength);
|
|
200
|
-
const view = new DataView(packet);
|
|
201
|
-
const packetData = new Uint8Array(packet);
|
|
202
|
-
|
|
203
|
-
// Type: 1 = audio
|
|
204
|
-
view.setUint8(0, 1);
|
|
205
|
-
// Timestamp (microseconds)
|
|
206
|
-
view.setBigUint64(1, BigInt(timestamp), true);
|
|
207
|
-
// Data size
|
|
208
|
-
view.setUint32(9, data.byteLength, true);
|
|
209
|
-
// Copy data
|
|
210
|
-
packetData.set(data, 13);
|
|
211
|
-
|
|
212
|
-
try {
|
|
213
|
-
dataChannel.send(packet);
|
|
214
|
-
audioFrameCount++;
|
|
215
|
-
} catch (e) {}
|
|
216
|
-
}
|
|
217
|
-
|
|
218
|
-
// Encode video frame from JPEG data
|
|
219
|
-
async function encodeFrame(imageData: string) {
|
|
220
|
-
if (!videoEncoder || !isCapturing) return;
|
|
221
|
-
|
|
222
|
-
try {
|
|
223
|
-
// Base64 to arrayBuffer (more efficient)
|
|
224
|
-
const imageBuffer = await fetch(`data:image/jpeg;base64,${imageData}`);
|
|
225
|
-
const arrayBuffer = await imageBuffer.arrayBuffer();
|
|
226
|
-
|
|
227
|
-
// Decode JPEG to VideoFrame
|
|
228
|
-
const decoder = new ImageDecoder({
|
|
229
|
-
data: arrayBuffer,
|
|
230
|
-
type: 'image/jpeg',
|
|
231
|
-
});
|
|
232
|
-
|
|
233
|
-
const { image } = await decoder.decode();
|
|
234
|
-
|
|
235
|
-
// Get aligned timestamp in microseconds
|
|
236
|
-
const timestamp = performance.now() * 1000;
|
|
237
|
-
|
|
238
|
-
// Create VideoFrame with aligned timestamp
|
|
239
|
-
const frame = new VideoFrame(image, {
|
|
240
|
-
timestamp,
|
|
241
|
-
alpha: 'discard'
|
|
242
|
-
});
|
|
243
|
-
|
|
244
|
-
// Check if keyframe needed
|
|
245
|
-
const now = Date.now();
|
|
246
|
-
const needsKeyframe = (now - lastKeyframeTime) > (config.keyframeInterval * 1000);
|
|
247
|
-
|
|
248
|
-
if (needsKeyframe) {
|
|
249
|
-
lastKeyframeTime = now;
|
|
250
|
-
}
|
|
251
|
-
|
|
252
|
-
// Encode frame
|
|
253
|
-
videoEncoder.encode(frame, { keyFrame: needsKeyframe });
|
|
254
|
-
videoFrameCount++;
|
|
255
|
-
|
|
256
|
-
// Close immediately to prevent memory leaks
|
|
257
|
-
frame.close();
|
|
258
|
-
image.close();
|
|
259
|
-
decoder.close();
|
|
260
|
-
} catch (error) {}
|
|
261
|
-
}
|
|
262
|
-
|
|
263
|
-
// Start streaming
|
|
264
|
-
async function startStreaming() {
|
|
265
|
-
if (isCapturing) return true;
|
|
266
|
-
|
|
267
|
-
try {
|
|
268
|
-
await initPeerConnection();
|
|
269
|
-
await initVideoEncoder();
|
|
270
|
-
|
|
271
|
-
isCapturing = true;
|
|
272
|
-
// Set to 0 to force first frame as keyframe (required for decoder init)
|
|
273
|
-
lastKeyframeTime = 0;
|
|
274
|
-
|
|
275
|
-
// Start tracking cursor changes
|
|
276
|
-
startCursorTracking();
|
|
277
|
-
|
|
278
|
-
return true;
|
|
279
|
-
} catch (error) {
|
|
280
|
-
isCapturing = false;
|
|
281
|
-
return false;
|
|
282
|
-
}
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
// Stop streaming
|
|
286
|
-
function stopStreaming() {
|
|
287
|
-
isCapturing = false;
|
|
288
|
-
|
|
289
|
-
// Stop cursor tracking
|
|
290
|
-
stopCursorTracking();
|
|
291
|
-
|
|
292
|
-
if (videoEncoder) {
|
|
293
|
-
try {
|
|
294
|
-
videoEncoder.flush();
|
|
295
|
-
videoEncoder.close();
|
|
296
|
-
} catch (e) {}
|
|
297
|
-
videoEncoder = null;
|
|
298
|
-
}
|
|
299
|
-
|
|
300
|
-
if (dataChannel) {
|
|
301
|
-
dataChannel.close();
|
|
302
|
-
dataChannel = null;
|
|
303
|
-
}
|
|
304
|
-
|
|
305
|
-
if (peerConnection) {
|
|
306
|
-
peerConnection.close();
|
|
307
|
-
peerConnection = null;
|
|
308
|
-
}
|
|
309
|
-
}
|
|
310
|
-
|
|
311
|
-
// Create and send offer
|
|
312
|
-
async function createOffer() {
|
|
313
|
-
if (!peerConnection) {
|
|
314
|
-
await initPeerConnection();
|
|
315
|
-
}
|
|
316
|
-
|
|
317
|
-
try {
|
|
318
|
-
const offer = await peerConnection!.createOffer();
|
|
319
|
-
await peerConnection!.setLocalDescription(offer);
|
|
320
|
-
|
|
321
|
-
return {
|
|
322
|
-
type: offer.type,
|
|
323
|
-
sdp: offer.sdp
|
|
324
|
-
};
|
|
325
|
-
} catch (error) {
|
|
326
|
-
return null;
|
|
327
|
-
}
|
|
328
|
-
}
|
|
329
|
-
|
|
330
|
-
// Handle answer from client
|
|
331
|
-
async function handleAnswer(answer: RTCSessionDescriptionInit) {
|
|
332
|
-
if (!peerConnection) return false;
|
|
333
|
-
|
|
334
|
-
try {
|
|
335
|
-
await peerConnection.setRemoteDescription(new RTCSessionDescription(answer));
|
|
336
|
-
return true;
|
|
337
|
-
} catch (error) {
|
|
338
|
-
return false;
|
|
339
|
-
}
|
|
340
|
-
}
|
|
341
|
-
|
|
342
|
-
// Add ICE candidate
|
|
343
|
-
async function addIceCandidate(candidate: RTCIceCandidateInit) {
|
|
344
|
-
if (!peerConnection) return false;
|
|
345
|
-
|
|
346
|
-
try {
|
|
347
|
-
await peerConnection.addIceCandidate(new RTCIceCandidate(candidate));
|
|
348
|
-
return true;
|
|
349
|
-
} catch (error) {
|
|
350
|
-
return false;
|
|
351
|
-
}
|
|
352
|
-
}
|
|
353
|
-
|
|
354
|
-
// Reconfigure video encoder with new dimensions (hot-swap)
|
|
355
|
-
async function reconfigureEncoder(newWidth: number, newHeight: number) {
|
|
356
|
-
if (!videoEncoder || !isCapturing) {
|
|
357
|
-
return false;
|
|
358
|
-
}
|
|
359
|
-
|
|
360
|
-
try {
|
|
361
|
-
// Flush pending frames
|
|
362
|
-
await videoEncoder.flush();
|
|
363
|
-
|
|
364
|
-
// Create new codec config with updated dimensions
|
|
365
|
-
const newCodecConfig: VideoEncoderConfig = {
|
|
366
|
-
codec: config.codec,
|
|
367
|
-
width: newWidth,
|
|
368
|
-
height: newHeight,
|
|
369
|
-
bitrate: config.bitrate,
|
|
370
|
-
framerate: config.framerate,
|
|
371
|
-
hardwareAcceleration: config.hardwareAcceleration,
|
|
372
|
-
latencyMode: config.latencyMode
|
|
373
|
-
};
|
|
374
|
-
|
|
375
|
-
// Check if new config is supported
|
|
376
|
-
const support = await VideoEncoder.isConfigSupported(newCodecConfig);
|
|
377
|
-
if (!support.supported) {
|
|
378
|
-
return false;
|
|
379
|
-
}
|
|
380
|
-
|
|
381
|
-
// Reconfigure encoder with new dimensions
|
|
382
|
-
await videoEncoder.configure(newCodecConfig);
|
|
383
|
-
|
|
384
|
-
// Update config reference
|
|
385
|
-
config.width = newWidth;
|
|
386
|
-
config.height = newHeight;
|
|
387
|
-
|
|
388
|
-
// Reset keyframe timer to force keyframe after reconfigure
|
|
389
|
-
lastKeyframeTime = 0;
|
|
390
|
-
|
|
391
|
-
return true;
|
|
392
|
-
} catch (error) {
|
|
393
|
-
return false;
|
|
394
|
-
}
|
|
395
|
-
}
|
|
396
|
-
|
|
397
|
-
// Get stats
|
|
398
|
-
async function getStats() {
|
|
399
|
-
if (!peerConnection) return null;
|
|
400
|
-
|
|
401
|
-
try {
|
|
402
|
-
const stats = await peerConnection.getStats();
|
|
403
|
-
const result = {
|
|
404
|
-
videoBytesSent: 0,
|
|
405
|
-
audioBytesSent: 0,
|
|
406
|
-
videoFramesEncoded: videoFrameCount,
|
|
407
|
-
audioFramesEncoded: audioFrameCount,
|
|
408
|
-
connectionState: peerConnection.connectionState,
|
|
409
|
-
videoCodec: config.codec,
|
|
410
|
-
audioCodec: 'opus' as const
|
|
411
|
-
};
|
|
412
|
-
|
|
413
|
-
stats.forEach(report => {
|
|
414
|
-
if (report.type === 'data-channel') {
|
|
415
|
-
result.videoBytesSent = (report as any).bytesSent || 0;
|
|
416
|
-
}
|
|
417
|
-
});
|
|
418
|
-
|
|
419
|
-
return result;
|
|
420
|
-
} catch (error) {
|
|
421
|
-
return null;
|
|
422
|
-
}
|
|
423
|
-
}
|
|
424
|
-
|
|
425
|
-
// Expose API
|
|
426
|
-
(window as any).__webCodecsPeer = {
|
|
427
|
-
startStreaming,
|
|
428
|
-
stopStreaming,
|
|
429
|
-
createOffer,
|
|
430
|
-
handleAnswer,
|
|
431
|
-
addIceCandidate,
|
|
432
|
-
encodeFrame,
|
|
433
|
-
sendAudioChunk,
|
|
434
|
-
getStats,
|
|
435
|
-
reconfigureEncoder,
|
|
436
|
-
isActive: () => isCapturing
|
|
437
|
-
};
|
|
438
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* Video Encoder Client Script
|
|
3
|
+
*
|
|
4
|
+
* This script runs in the headless browser to:
|
|
5
|
+
* 1. Initialize RTCPeerConnection with DataChannel
|
|
6
|
+
* 2. Initialize VideoEncoder for encoding frames
|
|
7
|
+
* 3. Handle WebRTC signaling (offer/answer/ICE)
|
|
8
|
+
* 4. Send encoded video chunks via DataChannel
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import type { StreamingConfig } from '../types';
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Generate video encoder script that runs in the browser
|
|
15
|
+
*/
|
|
16
|
+
export function videoEncoderScript(config: StreamingConfig['video']) {
|
|
17
|
+
// WebCodecs Encoder for Headless Browser
|
|
18
|
+
if ((window as any).__webCodecsPeer) {
|
|
19
|
+
try {
|
|
20
|
+
(window as any).__webCodecsPeer.stopStreaming();
|
|
21
|
+
} catch (e) {}
|
|
22
|
+
(window as any).__webCodecsPeer = null;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
let peerConnection: RTCPeerConnection | null = null;
|
|
26
|
+
let dataChannel: RTCDataChannel | null = null;
|
|
27
|
+
let videoEncoder: VideoEncoder | null = null;
|
|
28
|
+
let isCapturing = false;
|
|
29
|
+
let videoFrameCount = 0;
|
|
30
|
+
let audioFrameCount = 0;
|
|
31
|
+
let lastKeyframeTime = 0;
|
|
32
|
+
|
|
33
|
+
// Cursor tracking
|
|
34
|
+
let lastCursor = 'default';
|
|
35
|
+
let cursorCheckInterval: any = null;
|
|
36
|
+
|
|
37
|
+
// ICE servers
|
|
38
|
+
const iceServers = [
|
|
39
|
+
{ urls: 'stun:stun.l.google.com:19302' },
|
|
40
|
+
{ urls: 'stun:stun1.l.google.com:19302' }
|
|
41
|
+
];
|
|
42
|
+
|
|
43
|
+
// Check cursor style from page
|
|
44
|
+
function checkCursor() {
|
|
45
|
+
try {
|
|
46
|
+
const cursorInfo = (window as any).__cursorInfo;
|
|
47
|
+
if (cursorInfo && cursorInfo.cursor && cursorInfo.cursor !== lastCursor) {
|
|
48
|
+
lastCursor = cursorInfo.cursor;
|
|
49
|
+
// Send cursor change to backend via exposed function
|
|
50
|
+
if ((window as any).__sendCursorChange) {
|
|
51
|
+
(window as any).__sendCursorChange(cursorInfo.cursor);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
} catch (e) {
|
|
55
|
+
// Ignore errors - cursor tracking is non-critical
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// Start cursor tracking interval
|
|
60
|
+
function startCursorTracking() {
|
|
61
|
+
if (cursorCheckInterval) return;
|
|
62
|
+
// Check cursor every 100ms (low overhead, responsive enough)
|
|
63
|
+
cursorCheckInterval = setInterval(checkCursor, 100);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// Stop cursor tracking interval
|
|
67
|
+
function stopCursorTracking() {
|
|
68
|
+
if (cursorCheckInterval) {
|
|
69
|
+
clearInterval(cursorCheckInterval);
|
|
70
|
+
cursorCheckInterval = null;
|
|
71
|
+
}
|
|
72
|
+
lastCursor = 'default';
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// Detect supported video codec
|
|
76
|
+
async function detectVideoCodec() {
|
|
77
|
+
const codecConfig: VideoEncoderConfig = {
|
|
78
|
+
codec: config.codec,
|
|
79
|
+
width: config.width,
|
|
80
|
+
height: config.height,
|
|
81
|
+
bitrate: config.bitrate,
|
|
82
|
+
framerate: config.framerate,
|
|
83
|
+
hardwareAcceleration: config.hardwareAcceleration,
|
|
84
|
+
latencyMode: config.latencyMode
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
try {
|
|
88
|
+
const support = await VideoEncoder.isConfigSupported(codecConfig);
|
|
89
|
+
if (support.supported) {
|
|
90
|
+
return codecConfig;
|
|
91
|
+
}
|
|
92
|
+
} catch (e) {}
|
|
93
|
+
|
|
94
|
+
return null;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// Initialize RTCPeerConnection
|
|
98
|
+
async function initPeerConnection() {
|
|
99
|
+
if (peerConnection) {
|
|
100
|
+
peerConnection.close();
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
peerConnection = new RTCPeerConnection({ iceServers });
|
|
104
|
+
|
|
105
|
+
// Handle ICE candidates
|
|
106
|
+
peerConnection.onicecandidate = (event) => {
|
|
107
|
+
if (event.candidate && (window as any).__sendIceCandidate) {
|
|
108
|
+
(window as any).__sendIceCandidate({
|
|
109
|
+
candidate: event.candidate.candidate,
|
|
110
|
+
sdpMid: event.candidate.sdpMid,
|
|
111
|
+
sdpMLineIndex: event.candidate.sdpMLineIndex
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
};
|
|
115
|
+
|
|
116
|
+
// Handle connection state
|
|
117
|
+
peerConnection.onconnectionstatechange = () => {
|
|
118
|
+
if ((window as any).__sendConnectionState && peerConnection) {
|
|
119
|
+
(window as any).__sendConnectionState(peerConnection.connectionState);
|
|
120
|
+
}
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
peerConnection.oniceconnectionstatechange = () => {};
|
|
124
|
+
|
|
125
|
+
// Create DataChannel for encoded chunks
|
|
126
|
+
dataChannel = peerConnection.createDataChannel('media', {
|
|
127
|
+
ordered: false, // Allow out-of-order delivery for lower latency
|
|
128
|
+
maxRetransmits: 0 // No retransmits = lower latency
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
dataChannel.binaryType = 'arraybuffer';
|
|
132
|
+
|
|
133
|
+
dataChannel.onopen = () => {
|
|
134
|
+
// Force keyframe when DataChannel opens (previous keyframes may have been lost)
|
|
135
|
+
lastKeyframeTime = 0;
|
|
136
|
+
};
|
|
137
|
+
|
|
138
|
+
dataChannel.onclose = () => {};
|
|
139
|
+
|
|
140
|
+
dataChannel.onerror = (error) => {};
|
|
141
|
+
|
|
142
|
+
return peerConnection;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// Initialize VideoEncoder
|
|
146
|
+
async function initVideoEncoder() {
|
|
147
|
+
const codecConfig = await detectVideoCodec();
|
|
148
|
+
if (!codecConfig) {
|
|
149
|
+
throw new Error('No supported video codec');
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
videoEncoder = new VideoEncoder({
|
|
153
|
+
output: (chunk, metadata) => {
|
|
154
|
+
handleEncodedVideoChunk(chunk, metadata);
|
|
155
|
+
},
|
|
156
|
+
error: (e) => {}
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
await videoEncoder.configure(codecConfig);
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
// Handle encoded video chunk
|
|
163
|
+
function handleEncodedVideoChunk(chunk: EncodedVideoChunk, metadata: any) {
|
|
164
|
+
if (!dataChannel || dataChannel.readyState !== 'open') return;
|
|
165
|
+
|
|
166
|
+
// Send chunk via DataChannel
|
|
167
|
+
// Format: [type(1)][timestamp(8)][keyframe(1)][size(4)][data]
|
|
168
|
+
const isKeyframe = chunk.type === 'key' ? 1 : 0;
|
|
169
|
+
const timestamp = chunk.timestamp;
|
|
170
|
+
const data = new Uint8Array(chunk.byteLength);
|
|
171
|
+
chunk.copyTo(data);
|
|
172
|
+
|
|
173
|
+
const packet = new ArrayBuffer(1 + 8 + 1 + 4 + data.byteLength);
|
|
174
|
+
const view = new DataView(packet);
|
|
175
|
+
const packetData = new Uint8Array(packet);
|
|
176
|
+
|
|
177
|
+
// Type: 0 = video
|
|
178
|
+
view.setUint8(0, 0);
|
|
179
|
+
// Timestamp (microseconds)
|
|
180
|
+
view.setBigUint64(1, BigInt(timestamp), true);
|
|
181
|
+
// Keyframe flag
|
|
182
|
+
view.setUint8(9, isKeyframe);
|
|
183
|
+
// Data size
|
|
184
|
+
view.setUint32(10, data.byteLength, true);
|
|
185
|
+
// Copy data
|
|
186
|
+
packetData.set(data, 14);
|
|
187
|
+
|
|
188
|
+
try {
|
|
189
|
+
dataChannel.send(packet);
|
|
190
|
+
videoFrameCount++;
|
|
191
|
+
} catch (e) {}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// Send audio chunk (called from AudioContext interception)
|
|
195
|
+
function sendAudioChunk(timestamp: number, data: Uint8Array) {
|
|
196
|
+
if (!dataChannel || dataChannel.readyState !== 'open') return;
|
|
197
|
+
|
|
198
|
+
// Format: [type(1)][timestamp(8)][size(4)][data]
|
|
199
|
+
const packet = new ArrayBuffer(1 + 8 + 4 + data.byteLength);
|
|
200
|
+
const view = new DataView(packet);
|
|
201
|
+
const packetData = new Uint8Array(packet);
|
|
202
|
+
|
|
203
|
+
// Type: 1 = audio
|
|
204
|
+
view.setUint8(0, 1);
|
|
205
|
+
// Timestamp (microseconds)
|
|
206
|
+
view.setBigUint64(1, BigInt(timestamp), true);
|
|
207
|
+
// Data size
|
|
208
|
+
view.setUint32(9, data.byteLength, true);
|
|
209
|
+
// Copy data
|
|
210
|
+
packetData.set(data, 13);
|
|
211
|
+
|
|
212
|
+
try {
|
|
213
|
+
dataChannel.send(packet);
|
|
214
|
+
audioFrameCount++;
|
|
215
|
+
} catch (e) {}
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// Encode video frame from JPEG data
|
|
219
|
+
async function encodeFrame(imageData: string) {
|
|
220
|
+
if (!videoEncoder || !isCapturing) return;
|
|
221
|
+
|
|
222
|
+
try {
|
|
223
|
+
// Base64 to arrayBuffer (more efficient)
|
|
224
|
+
const imageBuffer = await fetch(`data:image/jpeg;base64,${imageData}`);
|
|
225
|
+
const arrayBuffer = await imageBuffer.arrayBuffer();
|
|
226
|
+
|
|
227
|
+
// Decode JPEG to VideoFrame
|
|
228
|
+
const decoder = new ImageDecoder({
|
|
229
|
+
data: arrayBuffer,
|
|
230
|
+
type: 'image/jpeg',
|
|
231
|
+
});
|
|
232
|
+
|
|
233
|
+
const { image } = await decoder.decode();
|
|
234
|
+
|
|
235
|
+
// Get aligned timestamp in microseconds
|
|
236
|
+
const timestamp = performance.now() * 1000;
|
|
237
|
+
|
|
238
|
+
// Create VideoFrame with aligned timestamp
|
|
239
|
+
const frame = new VideoFrame(image, {
|
|
240
|
+
timestamp,
|
|
241
|
+
alpha: 'discard'
|
|
242
|
+
});
|
|
243
|
+
|
|
244
|
+
// Check if keyframe needed
|
|
245
|
+
const now = Date.now();
|
|
246
|
+
const needsKeyframe = (now - lastKeyframeTime) > (config.keyframeInterval * 1000);
|
|
247
|
+
|
|
248
|
+
if (needsKeyframe) {
|
|
249
|
+
lastKeyframeTime = now;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
// Encode frame
|
|
253
|
+
videoEncoder.encode(frame, { keyFrame: needsKeyframe });
|
|
254
|
+
videoFrameCount++;
|
|
255
|
+
|
|
256
|
+
// Close immediately to prevent memory leaks
|
|
257
|
+
frame.close();
|
|
258
|
+
image.close();
|
|
259
|
+
decoder.close();
|
|
260
|
+
} catch (error) {}
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
// Start streaming
|
|
264
|
+
async function startStreaming() {
|
|
265
|
+
if (isCapturing) return true;
|
|
266
|
+
|
|
267
|
+
try {
|
|
268
|
+
await initPeerConnection();
|
|
269
|
+
await initVideoEncoder();
|
|
270
|
+
|
|
271
|
+
isCapturing = true;
|
|
272
|
+
// Set to 0 to force first frame as keyframe (required for decoder init)
|
|
273
|
+
lastKeyframeTime = 0;
|
|
274
|
+
|
|
275
|
+
// Start tracking cursor changes
|
|
276
|
+
startCursorTracking();
|
|
277
|
+
|
|
278
|
+
return true;
|
|
279
|
+
} catch (error) {
|
|
280
|
+
isCapturing = false;
|
|
281
|
+
return false;
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
// Stop streaming
|
|
286
|
+
function stopStreaming() {
|
|
287
|
+
isCapturing = false;
|
|
288
|
+
|
|
289
|
+
// Stop cursor tracking
|
|
290
|
+
stopCursorTracking();
|
|
291
|
+
|
|
292
|
+
if (videoEncoder) {
|
|
293
|
+
try {
|
|
294
|
+
videoEncoder.flush();
|
|
295
|
+
videoEncoder.close();
|
|
296
|
+
} catch (e) {}
|
|
297
|
+
videoEncoder = null;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
if (dataChannel) {
|
|
301
|
+
dataChannel.close();
|
|
302
|
+
dataChannel = null;
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
if (peerConnection) {
|
|
306
|
+
peerConnection.close();
|
|
307
|
+
peerConnection = null;
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
// Create and send offer
|
|
312
|
+
async function createOffer() {
|
|
313
|
+
if (!peerConnection) {
|
|
314
|
+
await initPeerConnection();
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
try {
|
|
318
|
+
const offer = await peerConnection!.createOffer();
|
|
319
|
+
await peerConnection!.setLocalDescription(offer);
|
|
320
|
+
|
|
321
|
+
return {
|
|
322
|
+
type: offer.type,
|
|
323
|
+
sdp: offer.sdp
|
|
324
|
+
};
|
|
325
|
+
} catch (error) {
|
|
326
|
+
return null;
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
// Handle answer from client
|
|
331
|
+
async function handleAnswer(answer: RTCSessionDescriptionInit) {
|
|
332
|
+
if (!peerConnection) return false;
|
|
333
|
+
|
|
334
|
+
try {
|
|
335
|
+
await peerConnection.setRemoteDescription(new RTCSessionDescription(answer));
|
|
336
|
+
return true;
|
|
337
|
+
} catch (error) {
|
|
338
|
+
return false;
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
// Add ICE candidate
|
|
343
|
+
async function addIceCandidate(candidate: RTCIceCandidateInit) {
|
|
344
|
+
if (!peerConnection) return false;
|
|
345
|
+
|
|
346
|
+
try {
|
|
347
|
+
await peerConnection.addIceCandidate(new RTCIceCandidate(candidate));
|
|
348
|
+
return true;
|
|
349
|
+
} catch (error) {
|
|
350
|
+
return false;
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
// Reconfigure video encoder with new dimensions (hot-swap)
|
|
355
|
+
async function reconfigureEncoder(newWidth: number, newHeight: number) {
|
|
356
|
+
if (!videoEncoder || !isCapturing) {
|
|
357
|
+
return false;
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
try {
|
|
361
|
+
// Flush pending frames
|
|
362
|
+
await videoEncoder.flush();
|
|
363
|
+
|
|
364
|
+
// Create new codec config with updated dimensions
|
|
365
|
+
const newCodecConfig: VideoEncoderConfig = {
|
|
366
|
+
codec: config.codec,
|
|
367
|
+
width: newWidth,
|
|
368
|
+
height: newHeight,
|
|
369
|
+
bitrate: config.bitrate,
|
|
370
|
+
framerate: config.framerate,
|
|
371
|
+
hardwareAcceleration: config.hardwareAcceleration,
|
|
372
|
+
latencyMode: config.latencyMode
|
|
373
|
+
};
|
|
374
|
+
|
|
375
|
+
// Check if new config is supported
|
|
376
|
+
const support = await VideoEncoder.isConfigSupported(newCodecConfig);
|
|
377
|
+
if (!support.supported) {
|
|
378
|
+
return false;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
// Reconfigure encoder with new dimensions
|
|
382
|
+
await videoEncoder.configure(newCodecConfig);
|
|
383
|
+
|
|
384
|
+
// Update config reference
|
|
385
|
+
config.width = newWidth;
|
|
386
|
+
config.height = newHeight;
|
|
387
|
+
|
|
388
|
+
// Reset keyframe timer to force keyframe after reconfigure
|
|
389
|
+
lastKeyframeTime = 0;
|
|
390
|
+
|
|
391
|
+
return true;
|
|
392
|
+
} catch (error) {
|
|
393
|
+
return false;
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
// Get stats
|
|
398
|
+
async function getStats() {
|
|
399
|
+
if (!peerConnection) return null;
|
|
400
|
+
|
|
401
|
+
try {
|
|
402
|
+
const stats = await peerConnection.getStats();
|
|
403
|
+
const result = {
|
|
404
|
+
videoBytesSent: 0,
|
|
405
|
+
audioBytesSent: 0,
|
|
406
|
+
videoFramesEncoded: videoFrameCount,
|
|
407
|
+
audioFramesEncoded: audioFrameCount,
|
|
408
|
+
connectionState: peerConnection.connectionState,
|
|
409
|
+
videoCodec: config.codec,
|
|
410
|
+
audioCodec: 'opus' as const
|
|
411
|
+
};
|
|
412
|
+
|
|
413
|
+
stats.forEach(report => {
|
|
414
|
+
if (report.type === 'data-channel') {
|
|
415
|
+
result.videoBytesSent = (report as any).bytesSent || 0;
|
|
416
|
+
}
|
|
417
|
+
});
|
|
418
|
+
|
|
419
|
+
return result;
|
|
420
|
+
} catch (error) {
|
|
421
|
+
return null;
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
// Expose API
|
|
426
|
+
(window as any).__webCodecsPeer = {
|
|
427
|
+
startStreaming,
|
|
428
|
+
stopStreaming,
|
|
429
|
+
createOffer,
|
|
430
|
+
handleAnswer,
|
|
431
|
+
addIceCandidate,
|
|
432
|
+
encodeFrame,
|
|
433
|
+
sendAudioChunk,
|
|
434
|
+
getStats,
|
|
435
|
+
reconfigureEncoder,
|
|
436
|
+
isActive: () => isCapturing
|
|
437
|
+
};
|
|
438
|
+
}
|