@myrialabs/clopen 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +6 -0
- package/.github/workflows/release.yml +60 -0
- package/.github/workflows/test.yml +40 -0
- package/CONTRIBUTING.md +499 -0
- package/LICENSE +21 -0
- package/README.md +209 -0
- package/backend/index.ts +156 -0
- package/backend/lib/chat/helpers.ts +42 -0
- package/backend/lib/chat/index.ts +2 -0
- package/backend/lib/chat/stream-manager.ts +1126 -0
- package/backend/lib/database/README.md +77 -0
- package/backend/lib/database/index.ts +119 -0
- package/backend/lib/database/migrations/001_create_projects_table.ts +31 -0
- package/backend/lib/database/migrations/002_create_chat_sessions_table.ts +33 -0
- package/backend/lib/database/migrations/003_create_messages_table.ts +32 -0
- package/backend/lib/database/migrations/004_create_prompt_templates_table.ts +34 -0
- package/backend/lib/database/migrations/005_create_settings_table.ts +24 -0
- package/backend/lib/database/migrations/006_add_user_to_messages.ts +58 -0
- package/backend/lib/database/migrations/007_create_stream_states_table.ts +41 -0
- package/backend/lib/database/migrations/008_create_message_snapshots_table.ts +62 -0
- package/backend/lib/database/migrations/009_add_delta_snapshot_fields.ts +41 -0
- package/backend/lib/database/migrations/010_add_soft_delete_and_branch_support.ts +70 -0
- package/backend/lib/database/migrations/011_git_like_commit_graph.ts +156 -0
- package/backend/lib/database/migrations/012_add_file_change_statistics.ts +41 -0
- package/backend/lib/database/migrations/013_checkpoint_tree_state.ts +118 -0
- package/backend/lib/database/migrations/014_add_engine_to_sessions.ts +18 -0
- package/backend/lib/database/migrations/015_add_model_to_sessions.ts +18 -0
- package/backend/lib/database/migrations/016_create_user_projects_table.ts +34 -0
- package/backend/lib/database/migrations/017_add_current_session_to_user_projects.ts +32 -0
- package/backend/lib/database/migrations/018_create_claude_accounts_table.ts +24 -0
- package/backend/lib/database/migrations/019_add_claude_account_to_sessions.ts +18 -0
- package/backend/lib/database/migrations/020_add_snapshot_tree_hash.ts +32 -0
- package/backend/lib/database/migrations/021_drop_prompt_templates_table.ts +33 -0
- package/backend/lib/database/migrations/index.ts +154 -0
- package/backend/lib/database/queries/checkpoint-queries.ts +87 -0
- package/backend/lib/database/queries/engine-queries.ts +75 -0
- package/backend/lib/database/queries/index.ts +9 -0
- package/backend/lib/database/queries/message-queries.ts +472 -0
- package/backend/lib/database/queries/project-queries.ts +117 -0
- package/backend/lib/database/queries/session-queries.ts +271 -0
- package/backend/lib/database/queries/settings-queries.ts +34 -0
- package/backend/lib/database/queries/snapshot-queries.ts +326 -0
- package/backend/lib/database/queries/utils-queries.ts +59 -0
- package/backend/lib/database/seeders/index.ts +13 -0
- package/backend/lib/database/seeders/settings_seeder.ts +84 -0
- package/backend/lib/database/utils/connection.ts +174 -0
- package/backend/lib/database/utils/index.ts +4 -0
- package/backend/lib/database/utils/migration-runner.ts +118 -0
- package/backend/lib/database/utils/seeder-runner.ts +121 -0
- package/backend/lib/engine/adapters/claude/environment.ts +164 -0
- package/backend/lib/engine/adapters/claude/error-handler.ts +60 -0
- package/backend/lib/engine/adapters/claude/index.ts +1 -0
- package/backend/lib/engine/adapters/claude/path-utils.ts +38 -0
- package/backend/lib/engine/adapters/claude/stream.ts +177 -0
- package/backend/lib/engine/adapters/opencode/index.ts +2 -0
- package/backend/lib/engine/adapters/opencode/message-converter.ts +862 -0
- package/backend/lib/engine/adapters/opencode/server.ts +104 -0
- package/backend/lib/engine/adapters/opencode/stream.ts +755 -0
- package/backend/lib/engine/index.ts +196 -0
- package/backend/lib/engine/types.ts +58 -0
- package/backend/lib/files/file-operations.ts +478 -0
- package/backend/lib/files/file-reading.ts +308 -0
- package/backend/lib/files/file-watcher.ts +383 -0
- package/backend/lib/files/path-browsing.ts +382 -0
- package/backend/lib/git/git-executor.ts +88 -0
- package/backend/lib/git/git-parser.ts +411 -0
- package/backend/lib/git/git-service.ts +505 -0
- package/backend/lib/mcp/README.md +1144 -0
- package/backend/lib/mcp/config.ts +316 -0
- package/backend/lib/mcp/index.ts +35 -0
- package/backend/lib/mcp/project-context.ts +236 -0
- package/backend/lib/mcp/servers/browser-automation/actions.ts +156 -0
- package/backend/lib/mcp/servers/browser-automation/browser.ts +419 -0
- package/backend/lib/mcp/servers/browser-automation/index.ts +791 -0
- package/backend/lib/mcp/servers/browser-automation/inspection.ts +501 -0
- package/backend/lib/mcp/servers/helper.ts +143 -0
- package/backend/lib/mcp/servers/index.ts +45 -0
- package/backend/lib/mcp/servers/weather/get-temperature.ts +56 -0
- package/backend/lib/mcp/servers/weather/index.ts +31 -0
- package/backend/lib/mcp/stdio-server.ts +103 -0
- package/backend/lib/mcp/types.ts +65 -0
- package/backend/lib/preview/browser/browser-audio-capture.ts +86 -0
- package/backend/lib/preview/browser/browser-console-manager.ts +263 -0
- package/backend/lib/preview/browser/browser-dialog-handler.ts +222 -0
- package/backend/lib/preview/browser/browser-interaction-handler.ts +421 -0
- package/backend/lib/preview/browser/browser-mcp-control.ts +415 -0
- package/backend/lib/preview/browser/browser-native-ui-handler.ts +512 -0
- package/backend/lib/preview/browser/browser-navigation-tracker.ts +104 -0
- package/backend/lib/preview/browser/browser-pool.ts +357 -0
- package/backend/lib/preview/browser/browser-preview-service.ts +882 -0
- package/backend/lib/preview/browser/browser-tab-manager.ts +935 -0
- package/backend/lib/preview/browser/browser-video-capture.ts +695 -0
- package/backend/lib/preview/browser/scripts/audio-stream.ts +292 -0
- package/backend/lib/preview/browser/scripts/cursor-tracking.ts +85 -0
- package/backend/lib/preview/browser/scripts/video-stream.ts +438 -0
- package/backend/lib/preview/browser/types.ts +359 -0
- package/backend/lib/preview/index.ts +24 -0
- package/backend/lib/project/index.ts +2 -0
- package/backend/lib/project/status-manager.ts +182 -0
- package/backend/lib/shared/index.ts +2 -0
- package/backend/lib/shared/port-utils.ts +25 -0
- package/backend/lib/shared/process-manager.ts +281 -0
- package/backend/lib/snapshot/blob-store.ts +227 -0
- package/backend/lib/snapshot/gitignore.ts +307 -0
- package/backend/lib/snapshot/helpers.ts +397 -0
- package/backend/lib/snapshot/snapshot-service.ts +483 -0
- package/backend/lib/terminal/helpers.ts +14 -0
- package/backend/lib/terminal/index.ts +8 -0
- package/backend/lib/terminal/pty-manager.ts +4 -0
- package/backend/lib/terminal/pty-session-manager.ts +387 -0
- package/backend/lib/terminal/shell-utils.ts +313 -0
- package/backend/lib/terminal/stream-manager.ts +293 -0
- package/backend/lib/tunnel/global-tunnel-manager.ts +243 -0
- package/backend/lib/tunnel/project-tunnel-manager.ts +311 -0
- package/backend/lib/user/helpers.ts +87 -0
- package/backend/lib/utils/ws.ts +944 -0
- package/backend/lib/vite-dev.ts +353 -0
- package/backend/middleware/cors.ts +15 -0
- package/backend/middleware/error-handler.ts +49 -0
- package/backend/middleware/logger.ts +9 -0
- package/backend/types/api.ts +24 -0
- package/backend/ws/README.md +1505 -0
- package/backend/ws/chat/background.ts +198 -0
- package/backend/ws/chat/index.ts +21 -0
- package/backend/ws/chat/stream.ts +707 -0
- package/backend/ws/engine/claude/accounts.ts +401 -0
- package/backend/ws/engine/claude/index.ts +13 -0
- package/backend/ws/engine/claude/status.ts +43 -0
- package/backend/ws/engine/index.ts +14 -0
- package/backend/ws/engine/opencode/index.ts +11 -0
- package/backend/ws/engine/opencode/status.ts +30 -0
- package/backend/ws/engine/utils.ts +36 -0
- package/backend/ws/files/index.ts +30 -0
- package/backend/ws/files/read.ts +189 -0
- package/backend/ws/files/search.ts +453 -0
- package/backend/ws/files/watch.ts +124 -0
- package/backend/ws/files/write.ts +143 -0
- package/backend/ws/git/branch.ts +106 -0
- package/backend/ws/git/commit.ts +39 -0
- package/backend/ws/git/conflict.ts +68 -0
- package/backend/ws/git/diff.ts +69 -0
- package/backend/ws/git/index.ts +24 -0
- package/backend/ws/git/log.ts +41 -0
- package/backend/ws/git/remote.ts +214 -0
- package/backend/ws/git/staging.ts +84 -0
- package/backend/ws/git/status.ts +90 -0
- package/backend/ws/index.ts +69 -0
- package/backend/ws/mcp/index.ts +61 -0
- package/backend/ws/messages/crud.ts +74 -0
- package/backend/ws/messages/index.ts +14 -0
- package/backend/ws/preview/browser/cleanup.ts +129 -0
- package/backend/ws/preview/browser/console.ts +114 -0
- package/backend/ws/preview/browser/interact.ts +513 -0
- package/backend/ws/preview/browser/mcp.ts +129 -0
- package/backend/ws/preview/browser/native-ui.ts +235 -0
- package/backend/ws/preview/browser/stats.ts +55 -0
- package/backend/ws/preview/browser/tab-info.ts +126 -0
- package/backend/ws/preview/browser/tab.ts +166 -0
- package/backend/ws/preview/browser/webcodecs.ts +293 -0
- package/backend/ws/preview/index.ts +146 -0
- package/backend/ws/projects/crud.ts +113 -0
- package/backend/ws/projects/index.ts +25 -0
- package/backend/ws/projects/presence.ts +46 -0
- package/backend/ws/projects/status.ts +116 -0
- package/backend/ws/sessions/crud.ts +327 -0
- package/backend/ws/sessions/index.ts +33 -0
- package/backend/ws/settings/crud.ts +112 -0
- package/backend/ws/settings/index.ts +14 -0
- package/backend/ws/snapshot/index.ts +17 -0
- package/backend/ws/snapshot/restore.ts +173 -0
- package/backend/ws/snapshot/timeline.ts +141 -0
- package/backend/ws/system/index.ts +14 -0
- package/backend/ws/system/operations.ts +49 -0
- package/backend/ws/terminal/index.ts +40 -0
- package/backend/ws/terminal/persistence.ts +153 -0
- package/backend/ws/terminal/session.ts +382 -0
- package/backend/ws/terminal/stream.ts +79 -0
- package/backend/ws/tunnel/index.ts +14 -0
- package/backend/ws/tunnel/operations.ts +91 -0
- package/backend/ws/types.ts +20 -0
- package/backend/ws/user/crud.ts +156 -0
- package/backend/ws/user/index.ts +14 -0
- package/bin/clopen.ts +307 -0
- package/bun.lock +1352 -0
- package/frontend/App.svelte +34 -0
- package/frontend/app.css +313 -0
- package/frontend/lib/app-environment.ts +10 -0
- package/frontend/lib/components/chat/ChatInterface.svelte +407 -0
- package/frontend/lib/components/chat/formatters/ErrorMessage.svelte +57 -0
- package/frontend/lib/components/chat/formatters/MessageFormatter.svelte +224 -0
- package/frontend/lib/components/chat/formatters/TextMessage.svelte +395 -0
- package/frontend/lib/components/chat/formatters/Tools.svelte +70 -0
- package/frontend/lib/components/chat/formatters/index.ts +3 -0
- package/frontend/lib/components/chat/input/ChatInput.svelte +421 -0
- package/frontend/lib/components/chat/input/components/ChatInputActions.svelte +78 -0
- package/frontend/lib/components/chat/input/components/DragDropOverlay.svelte +30 -0
- package/frontend/lib/components/chat/input/components/EditModeIndicator.svelte +33 -0
- package/frontend/lib/components/chat/input/components/EngineModelPicker.svelte +619 -0
- package/frontend/lib/components/chat/input/components/FileAttachmentPreview.svelte +48 -0
- package/frontend/lib/components/chat/input/components/LoadingIndicator.svelte +31 -0
- package/frontend/lib/components/chat/input/composables/use-animations.svelte.ts +201 -0
- package/frontend/lib/components/chat/input/composables/use-chat-actions.svelte.ts +148 -0
- package/frontend/lib/components/chat/input/composables/use-file-handling.svelte.ts +216 -0
- package/frontend/lib/components/chat/input/composables/use-input-state.svelte.ts +357 -0
- package/frontend/lib/components/chat/input/composables/use-textarea-resize.svelte.ts +57 -0
- package/frontend/lib/components/chat/message/ChatMessage.svelte +478 -0
- package/frontend/lib/components/chat/message/ChatMessages.svelte +541 -0
- package/frontend/lib/components/chat/message/DateSeparator.svelte +86 -0
- package/frontend/lib/components/chat/message/MessageBubble.svelte +86 -0
- package/frontend/lib/components/chat/message/MessageHeader.svelte +157 -0
- package/frontend/lib/components/chat/modal/DebugModal.svelte +59 -0
- package/frontend/lib/components/chat/modal/TokenUsageModal.svelte +124 -0
- package/frontend/lib/components/chat/shared/index.ts +2 -0
- package/frontend/lib/components/chat/shared/utils.ts +116 -0
- package/frontend/lib/components/chat/tools/BashOutputTool.svelte +35 -0
- package/frontend/lib/components/chat/tools/BashTool.svelte +46 -0
- package/frontend/lib/components/chat/tools/CustomMcpTool.svelte +139 -0
- package/frontend/lib/components/chat/tools/EditTool.svelte +48 -0
- package/frontend/lib/components/chat/tools/ExitPlanModeTool.svelte +32 -0
- package/frontend/lib/components/chat/tools/GlobTool.svelte +51 -0
- package/frontend/lib/components/chat/tools/GrepTool.svelte +90 -0
- package/frontend/lib/components/chat/tools/KillShellTool.svelte +26 -0
- package/frontend/lib/components/chat/tools/ListMcpResourcesTool.svelte +31 -0
- package/frontend/lib/components/chat/tools/NotebookEditTool.svelte +38 -0
- package/frontend/lib/components/chat/tools/ReadMcpResourceTool.svelte +34 -0
- package/frontend/lib/components/chat/tools/ReadTool.svelte +41 -0
- package/frontend/lib/components/chat/tools/TaskTool.svelte +64 -0
- package/frontend/lib/components/chat/tools/TodoWriteTool.svelte +75 -0
- package/frontend/lib/components/chat/tools/WebFetchTool.svelte +35 -0
- package/frontend/lib/components/chat/tools/WebSearchTool.svelte +84 -0
- package/frontend/lib/components/chat/tools/WriteTool.svelte +33 -0
- package/frontend/lib/components/chat/tools/components/CodeBlock.svelte +79 -0
- package/frontend/lib/components/chat/tools/components/DiffBlock.svelte +408 -0
- package/frontend/lib/components/chat/tools/components/FileHeader.svelte +45 -0
- package/frontend/lib/components/chat/tools/components/InfoLine.svelte +19 -0
- package/frontend/lib/components/chat/tools/components/StatsBadges.svelte +27 -0
- package/frontend/lib/components/chat/tools/components/TerminalCommand.svelte +54 -0
- package/frontend/lib/components/chat/tools/components/index.ts +7 -0
- package/frontend/lib/components/chat/tools/index.ts +26 -0
- package/frontend/lib/components/chat/widgets/FloatingTodoList.svelte +249 -0
- package/frontend/lib/components/chat/widgets/TokenUsage.svelte +78 -0
- package/frontend/lib/components/checkpoint/TimelineModal.svelte +391 -0
- package/frontend/lib/components/checkpoint/timeline/TimelineEdge.svelte +26 -0
- package/frontend/lib/components/checkpoint/timeline/TimelineGraph.svelte +87 -0
- package/frontend/lib/components/checkpoint/timeline/TimelineNode.svelte +108 -0
- package/frontend/lib/components/checkpoint/timeline/TimelineVersionGroup.svelte +59 -0
- package/frontend/lib/components/checkpoint/timeline/animation.ts +168 -0
- package/frontend/lib/components/checkpoint/timeline/config.ts +44 -0
- package/frontend/lib/components/checkpoint/timeline/graph-builder.ts +304 -0
- package/frontend/lib/components/checkpoint/timeline/types.ts +65 -0
- package/frontend/lib/components/checkpoint/timeline/utils.ts +53 -0
- package/frontend/lib/components/common/Alert.svelte +139 -0
- package/frontend/lib/components/common/AvatarBubble.svelte +56 -0
- package/frontend/lib/components/common/Button.svelte +71 -0
- package/frontend/lib/components/common/Card.svelte +102 -0
- package/frontend/lib/components/common/Checkbox.svelte +48 -0
- package/frontend/lib/components/common/Dialog.svelte +249 -0
- package/frontend/lib/components/common/FolderBrowser.svelte +843 -0
- package/frontend/lib/components/common/Icon.svelte +58 -0
- package/frontend/lib/components/common/Input.svelte +72 -0
- package/frontend/lib/components/common/Lightbox.svelte +233 -0
- package/frontend/lib/components/common/LoadingScreen.svelte +52 -0
- package/frontend/lib/components/common/LoadingSpinner.svelte +48 -0
- package/frontend/lib/components/common/Modal.svelte +177 -0
- package/frontend/lib/components/common/ModalProvider.svelte +28 -0
- package/frontend/lib/components/common/ModelSelector.svelte +110 -0
- package/frontend/lib/components/common/MonacoEditor.svelte +569 -0
- package/frontend/lib/components/common/NotificationToast.svelte +113 -0
- package/frontend/lib/components/common/PageTemplate.svelte +76 -0
- package/frontend/lib/components/common/ProjectUserAvatars.svelte +79 -0
- package/frontend/lib/components/common/Select.svelte +98 -0
- package/frontend/lib/components/common/Textarea.svelte +80 -0
- package/frontend/lib/components/common/ThemeToggle.svelte +44 -0
- package/frontend/lib/components/common/lucide-icons.ts +1642 -0
- package/frontend/lib/components/common/material-icons.ts +1082 -0
- package/frontend/lib/components/common/xterm/XTerm.svelte +796 -0
- package/frontend/lib/components/common/xterm/index.ts +16 -0
- package/frontend/lib/components/common/xterm/terminal-config.ts +68 -0
- package/frontend/lib/components/common/xterm/types.ts +31 -0
- package/frontend/lib/components/common/xterm/xterm-service.ts +353 -0
- package/frontend/lib/components/files/FileNode.svelte +384 -0
- package/frontend/lib/components/files/FileTree.svelte +681 -0
- package/frontend/lib/components/files/FileViewer.svelte +728 -0
- package/frontend/lib/components/files/SearchResults.svelte +303 -0
- package/frontend/lib/components/git/BranchManager.svelte +458 -0
- package/frontend/lib/components/git/ChangesSection.svelte +107 -0
- package/frontend/lib/components/git/CommitForm.svelte +76 -0
- package/frontend/lib/components/git/ConflictResolver.svelte +158 -0
- package/frontend/lib/components/git/DiffViewer.svelte +364 -0
- package/frontend/lib/components/git/FileChangeItem.svelte +97 -0
- package/frontend/lib/components/git/GitButton.svelte +33 -0
- package/frontend/lib/components/git/GitLog.svelte +361 -0
- package/frontend/lib/components/git/GitModal.svelte +80 -0
- package/frontend/lib/components/history/HistoryModal.svelte +563 -0
- package/frontend/lib/components/history/HistoryView.svelte +615 -0
- package/frontend/lib/components/index.ts +34 -0
- package/frontend/lib/components/preview/browser/BrowserPreview.svelte +549 -0
- package/frontend/lib/components/preview/browser/components/Canvas.svelte +1058 -0
- package/frontend/lib/components/preview/browser/components/ConsolePanel.svelte +757 -0
- package/frontend/lib/components/preview/browser/components/Container.svelte +450 -0
- package/frontend/lib/components/preview/browser/components/ContextMenu.svelte +236 -0
- package/frontend/lib/components/preview/browser/components/SelectDropdown.svelte +224 -0
- package/frontend/lib/components/preview/browser/components/Toolbar.svelte +339 -0
- package/frontend/lib/components/preview/browser/components/VirtualCursor.svelte +36 -0
- package/frontend/lib/components/preview/browser/core/cleanup.svelte.ts +155 -0
- package/frontend/lib/components/preview/browser/core/coordinator.svelte.ts +837 -0
- package/frontend/lib/components/preview/browser/core/interactions.svelte.ts +113 -0
- package/frontend/lib/components/preview/browser/core/mcp-handlers.svelte.ts +296 -0
- package/frontend/lib/components/preview/browser/core/native-ui-handlers.svelte.ts +391 -0
- package/frontend/lib/components/preview/browser/core/stream-handler.svelte.ts +231 -0
- package/frontend/lib/components/preview/browser/core/tab-manager.svelte.ts +210 -0
- package/frontend/lib/components/preview/browser/core/tab-operations.svelte.ts +239 -0
- package/frontend/lib/components/preview/index.ts +2 -0
- package/frontend/lib/components/settings/SettingsModal.svelte +235 -0
- package/frontend/lib/components/settings/SettingsView.svelte +36 -0
- package/frontend/lib/components/settings/appearance/AppearanceSettings.svelte +51 -0
- package/frontend/lib/components/settings/appearance/LayoutPresetSettings.svelte +160 -0
- package/frontend/lib/components/settings/appearance/LayoutPreview.svelte +76 -0
- package/frontend/lib/components/settings/engines/AIEnginesSettings.svelte +917 -0
- package/frontend/lib/components/settings/general/AdvancedSettings.svelte +187 -0
- package/frontend/lib/components/settings/general/DataManagementSettings.svelte +203 -0
- package/frontend/lib/components/settings/general/GeneralSettings.svelte +10 -0
- package/frontend/lib/components/settings/model/ModelSettings.svelte +357 -0
- package/frontend/lib/components/settings/notifications/NotificationSettings.svelte +205 -0
- package/frontend/lib/components/settings/user/UserSettings.svelte +197 -0
- package/frontend/lib/components/terminal/Terminal.svelte +368 -0
- package/frontend/lib/components/terminal/TerminalTabs.svelte +87 -0
- package/frontend/lib/components/terminal/TerminalView.svelte +55 -0
- package/frontend/lib/components/tunnel/TunnelActive.svelte +142 -0
- package/frontend/lib/components/tunnel/TunnelButton.svelte +54 -0
- package/frontend/lib/components/tunnel/TunnelInactive.svelte +284 -0
- package/frontend/lib/components/tunnel/TunnelModal.svelte +47 -0
- package/frontend/lib/components/tunnel/TunnelQRCode.svelte +49 -0
- package/frontend/lib/components/workspace/DesktopNavigator.svelte +382 -0
- package/frontend/lib/components/workspace/MobileNavigator.svelte +403 -0
- package/frontend/lib/components/workspace/PanelContainer.svelte +100 -0
- package/frontend/lib/components/workspace/PanelHeader.svelte +505 -0
- package/frontend/lib/components/workspace/ViewMenu.svelte +162 -0
- package/frontend/lib/components/workspace/WorkspaceLayout.svelte +169 -0
- package/frontend/lib/components/workspace/layout/DesktopLayout.svelte +15 -0
- package/frontend/lib/components/workspace/layout/MobileLayout.svelte +17 -0
- package/frontend/lib/components/workspace/layout/split-pane/Container.svelte +42 -0
- package/frontend/lib/components/workspace/layout/split-pane/Handle.svelte +85 -0
- package/frontend/lib/components/workspace/layout/split-pane/Layout.svelte +37 -0
- package/frontend/lib/components/workspace/panels/ChatPanel.svelte +274 -0
- package/frontend/lib/components/workspace/panels/FilesPanel.svelte +1261 -0
- package/frontend/lib/components/workspace/panels/GitPanel.svelte +1560 -0
- package/frontend/lib/components/workspace/panels/PreviewPanel.svelte +150 -0
- package/frontend/lib/components/workspace/panels/TerminalPanel.svelte +73 -0
- package/frontend/lib/constants/preview.ts +45 -0
- package/frontend/lib/services/chat/chat.service.ts +704 -0
- package/frontend/lib/services/chat/index.ts +7 -0
- package/frontend/lib/services/notification/global-stream-monitor.ts +86 -0
- package/frontend/lib/services/notification/index.ts +8 -0
- package/frontend/lib/services/notification/push.service.ts +144 -0
- package/frontend/lib/services/notification/sound.service.ts +127 -0
- package/frontend/lib/services/preview/browser/browser-console.service.ts +61 -0
- package/frontend/lib/services/preview/browser/browser-webcodecs.service.ts +1499 -0
- package/frontend/lib/services/preview/browser/mcp-integration.svelte.ts +67 -0
- package/frontend/lib/services/preview/index.ts +23 -0
- package/frontend/lib/services/project/index.ts +8 -0
- package/frontend/lib/services/project/status.service.ts +159 -0
- package/frontend/lib/services/snapshot/snapshot.service.ts +47 -0
- package/frontend/lib/services/terminal/background/index.ts +130 -0
- package/frontend/lib/services/terminal/background/session-restore.ts +274 -0
- package/frontend/lib/services/terminal/background/stream-manager.ts +286 -0
- package/frontend/lib/services/terminal/index.ts +14 -0
- package/frontend/lib/services/terminal/persistence.service.ts +260 -0
- package/frontend/lib/services/terminal/project.service.ts +953 -0
- package/frontend/lib/services/terminal/session.service.ts +364 -0
- package/frontend/lib/services/terminal/terminal.service.ts +369 -0
- package/frontend/lib/stores/core/app.svelte.ts +117 -0
- package/frontend/lib/stores/core/files.svelte.ts +73 -0
- package/frontend/lib/stores/core/presence.svelte.ts +48 -0
- package/frontend/lib/stores/core/projects.svelte.ts +317 -0
- package/frontend/lib/stores/core/sessions.svelte.ts +383 -0
- package/frontend/lib/stores/features/claude-accounts.svelte.ts +58 -0
- package/frontend/lib/stores/features/models.svelte.ts +89 -0
- package/frontend/lib/stores/features/settings.svelte.ts +87 -0
- package/frontend/lib/stores/features/terminal.svelte.ts +701 -0
- package/frontend/lib/stores/features/tunnel.svelte.ts +161 -0
- package/frontend/lib/stores/features/user.svelte.ts +96 -0
- package/frontend/lib/stores/ui/chat-input.svelte.ts +57 -0
- package/frontend/lib/stores/ui/chat-model.svelte.ts +61 -0
- package/frontend/lib/stores/ui/dialog.svelte.ts +59 -0
- package/frontend/lib/stores/ui/edit-mode.svelte.ts +214 -0
- package/frontend/lib/stores/ui/notification.svelte.ts +166 -0
- package/frontend/lib/stores/ui/settings-modal.svelte.ts +88 -0
- package/frontend/lib/stores/ui/theme.svelte.ts +179 -0
- package/frontend/lib/stores/ui/workspace.svelte.ts +754 -0
- package/frontend/lib/types/native-ui.ts +73 -0
- package/frontend/lib/utils/chat/date-separator.ts +39 -0
- package/frontend/lib/utils/chat/message-grouper.ts +219 -0
- package/frontend/lib/utils/chat/message-processor.ts +135 -0
- package/frontend/lib/utils/chat/tool-handler.ts +161 -0
- package/frontend/lib/utils/chat/virtual-scroll.svelte.ts +142 -0
- package/frontend/lib/utils/click-outside.ts +20 -0
- package/frontend/lib/utils/context-manager.ts +257 -0
- package/frontend/lib/utils/file-icon-mappings.ts +769 -0
- package/frontend/lib/utils/folder-icon-mappings.ts +1030 -0
- package/frontend/lib/utils/git-status.ts +68 -0
- package/frontend/lib/utils/platform.ts +113 -0
- package/frontend/lib/utils/port-check.ts +65 -0
- package/frontend/lib/utils/terminalFormatter.ts +207 -0
- package/frontend/lib/utils/theme.ts +6 -0
- package/frontend/lib/utils/tree-visualizer.ts +320 -0
- package/frontend/lib/utils/ws.ts +44 -0
- package/frontend/main.ts +13 -0
- package/index.html +70 -0
- package/package.json +111 -0
- package/scripts/generate-icons.ts +87 -0
- package/scripts/pre-publish-check.sh +142 -0
- package/scripts/setup-hooks.sh +134 -0
- package/scripts/validate-branch-name.sh +47 -0
- package/scripts/validate-commit-msg.sh +42 -0
- package/shared/constants/engines.ts +134 -0
- package/shared/types/database/connection.ts +16 -0
- package/shared/types/database/index.ts +6 -0
- package/shared/types/database/schema.ts +141 -0
- package/shared/types/engine/index.ts +45 -0
- package/shared/types/filesystem/index.ts +22 -0
- package/shared/types/git.ts +171 -0
- package/shared/types/messaging/index.ts +239 -0
- package/shared/types/messaging/tool.ts +526 -0
- package/shared/types/network/api.ts +18 -0
- package/shared/types/network/index.ts +5 -0
- package/shared/types/stores/app.ts +23 -0
- package/shared/types/stores/dialog.ts +21 -0
- package/shared/types/stores/index.ts +3 -0
- package/shared/types/stores/settings.ts +15 -0
- package/shared/types/terminal/index.ts +44 -0
- package/shared/types/ui/components.ts +61 -0
- package/shared/types/ui/icons.ts +23 -0
- package/shared/types/ui/index.ts +22 -0
- package/shared/types/ui/notifications.ts +14 -0
- package/shared/types/ui/theme.ts +12 -0
- package/shared/types/websocket/index.ts +43 -0
- package/shared/types/window.d.ts +13 -0
- package/shared/utils/anonymous-user.ts +168 -0
- package/shared/utils/async.ts +10 -0
- package/shared/utils/diff-calculator.ts +184 -0
- package/shared/utils/file-type-detection.ts +166 -0
- package/shared/utils/logger.ts +158 -0
- package/shared/utils/message-formatter.ts +79 -0
- package/shared/utils/path.ts +47 -0
- package/shared/utils/ws-client.ts +768 -0
- package/shared/utils/ws-server.ts +660 -0
- package/static/audio/notification.ogg +0 -0
- package/static/favicon.svg +8 -0
- package/static/fonts/dm-sans/dm-sans-italic-latin-ext.woff2 +0 -0
- package/static/fonts/dm-sans/dm-sans-italic-latin.woff2 +0 -0
- package/static/fonts/dm-sans/dm-sans-normal-latin-ext.woff2 +0 -0
- package/static/fonts/dm-sans/dm-sans-normal-latin.woff2 +0 -0
- package/static/fonts/dm-sans.css +96 -0
- package/svelte.config.js +20 -0
- package/vite.config.ts +33 -0
|
@@ -0,0 +1,1499 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Browser WebCodecs Service
|
|
3
|
+
*
|
|
4
|
+
* Client-side service for receiving WebCodecs-encoded video/audio via DataChannel.
|
|
5
|
+
* Achieves ultra low-latency (~20-40ms) with lower bandwidth than traditional WebRTC.
|
|
6
|
+
*
|
|
7
|
+
* Flow:
|
|
8
|
+
* 1. Request offer from server (headless browser creates offer with DataChannel)
|
|
9
|
+
* 2. Create RTCPeerConnection and set remote description (offer)
|
|
10
|
+
* 3. Create and send answer
|
|
11
|
+
* 4. Exchange ICE candidates
|
|
12
|
+
* 5. Receive encoded chunks via DataChannel
|
|
13
|
+
* 6. Decode with VideoDecoder + AudioDecoder
|
|
14
|
+
* 7. Render video to canvas, play audio
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import ws from '$frontend/lib/utils/ws';
|
|
18
|
+
import { debug } from '$shared/utils/logger';
|
|
19
|
+
|
|
20
|
+
export interface BrowserWebCodecsStreamStats {
|
|
21
|
+
isConnected: boolean;
|
|
22
|
+
connectionState: RTCPeerConnectionState;
|
|
23
|
+
iceConnectionState: RTCIceConnectionState;
|
|
24
|
+
videoBytesReceived: number;
|
|
25
|
+
audioBytesReceived: number;
|
|
26
|
+
videoFramesReceived: number;
|
|
27
|
+
audioFramesReceived: number;
|
|
28
|
+
videoFramesDecoded: number;
|
|
29
|
+
audioFramesDecoded: number;
|
|
30
|
+
videoFramesDropped: number;
|
|
31
|
+
audioFramesDropped: number;
|
|
32
|
+
frameWidth: number;
|
|
33
|
+
frameHeight: number;
|
|
34
|
+
// Bandwidth stats
|
|
35
|
+
videoBitrate: number; // bits per second
|
|
36
|
+
audioBitrate: number; // bits per second
|
|
37
|
+
totalBitrate: number; // bits per second
|
|
38
|
+
totalBandwidthMBps: number; // MB per second
|
|
39
|
+
// Codec info
|
|
40
|
+
videoCodec: string;
|
|
41
|
+
audioCodec: string;
|
|
42
|
+
// First frame status
|
|
43
|
+
firstFrameRendered: boolean;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export class BrowserWebCodecsService {
|
|
47
|
+
private projectId: string; // REQUIRED for project isolation
|
|
48
|
+
private sessionId: string | null = null;
|
|
49
|
+
private peerConnection: RTCPeerConnection | null = null;
|
|
50
|
+
private dataChannel: RTCDataChannel | null = null;
|
|
51
|
+
private videoDecoder: VideoDecoder | null = null;
|
|
52
|
+
private audioDecoder: AudioDecoder | null = null;
|
|
53
|
+
private audioContext: AudioContext | null = null;
|
|
54
|
+
private canvas: HTMLCanvasElement | null = null;
|
|
55
|
+
private ctx: CanvasRenderingContext2D | null = null;
|
|
56
|
+
private isConnected = false;
|
|
57
|
+
private isCleaningUp = false;
|
|
58
|
+
|
|
59
|
+
// Frame rendering optimization with timestamp-based scheduling
|
|
60
|
+
private pendingFrame: VideoFrame | null = null;
|
|
61
|
+
private isRenderingFrame = false;
|
|
62
|
+
private renderFrameId: number | null = null;
|
|
63
|
+
private lastFrameTime = 0;
|
|
64
|
+
private startTime = 0;
|
|
65
|
+
private firstFrameTimestamp = 0; // Timestamp of first frame (for relative timing)
|
|
66
|
+
|
|
67
|
+
// AV Sync: Shared timeline reference
|
|
68
|
+
// When first video frame arrives, we establish: realTime = streamTimestamp mapping
|
|
69
|
+
private syncEstablished = false;
|
|
70
|
+
private syncRealTimeOrigin = 0; // performance.now() when first video frame decoded
|
|
71
|
+
private syncStreamTimestamp = 0; // stream timestamp of first video frame (microseconds)
|
|
72
|
+
private lastVideoTimestamp = 0; // Last rendered video timestamp for audio sync
|
|
73
|
+
private lastVideoRealTime = 0; // performance.now() when last video frame was rendered
|
|
74
|
+
|
|
75
|
+
// Audio playback scheduling
|
|
76
|
+
private nextAudioPlayTime = 0; // When the next audio chunk should play
|
|
77
|
+
private audioBufferQueue: Array<{ buffer: AudioBuffer; scheduledTime: number }> = [];
|
|
78
|
+
private maxAudioQueueSize = 10; // Limit queue to prevent audio lag
|
|
79
|
+
private audioSyncInitialized = false; // Track if audio sync is initialized
|
|
80
|
+
private audioCalibrationSamples = 0; // Count calibration samples for initial sync
|
|
81
|
+
private audioOffsetAccumulator = 0; // Accumulate offset during calibration
|
|
82
|
+
private readonly CALIBRATION_SAMPLES = 5; // Number of samples needed for calibration
|
|
83
|
+
private calibratedAudioOffset = 0; // Final calibrated offset in seconds
|
|
84
|
+
|
|
85
|
+
// Codec configuration
|
|
86
|
+
private videoCodecConfig: VideoDecoderConfig | null = null;
|
|
87
|
+
private audioCodecConfig: AudioDecoderConfig | null = null;
|
|
88
|
+
|
|
89
|
+
// Stats tracking
|
|
90
|
+
private stats: BrowserWebCodecsStreamStats = {
|
|
91
|
+
isConnected: false,
|
|
92
|
+
connectionState: 'new',
|
|
93
|
+
iceConnectionState: 'new',
|
|
94
|
+
videoBytesReceived: 0,
|
|
95
|
+
audioBytesReceived: 0,
|
|
96
|
+
videoFramesReceived: 0,
|
|
97
|
+
audioFramesReceived: 0,
|
|
98
|
+
videoFramesDecoded: 0,
|
|
99
|
+
audioFramesDecoded: 0,
|
|
100
|
+
videoFramesDropped: 0,
|
|
101
|
+
audioFramesDropped: 0,
|
|
102
|
+
frameWidth: 0,
|
|
103
|
+
frameHeight: 0,
|
|
104
|
+
videoBitrate: 0,
|
|
105
|
+
audioBitrate: 0,
|
|
106
|
+
totalBitrate: 0,
|
|
107
|
+
totalBandwidthMBps: 0,
|
|
108
|
+
videoCodec: 'unknown',
|
|
109
|
+
audioCodec: 'unknown',
|
|
110
|
+
firstFrameRendered: false
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
// Bandwidth tracking
|
|
114
|
+
private lastVideoBytesReceived = 0;
|
|
115
|
+
private lastAudioBytesReceived = 0;
|
|
116
|
+
private lastStatsTime = 0;
|
|
117
|
+
|
|
118
|
+
// ICE servers
|
|
119
|
+
private readonly iceServers: RTCIceServer[] = [
|
|
120
|
+
{ urls: 'stun:stun.l.google.com:19302' },
|
|
121
|
+
{ urls: 'stun:stun1.l.google.com:19302' }
|
|
122
|
+
];
|
|
123
|
+
|
|
124
|
+
// Callbacks
|
|
125
|
+
private onConnectionChange: ((connected: boolean) => void) | null = null;
|
|
126
|
+
private onConnectionFailed: (() => void) | null = null;
|
|
127
|
+
private onNavigationReconnect: (() => void) | null = null; // Fast reconnection after navigation
|
|
128
|
+
private onReconnectingStart: (() => void) | null = null; // Signals reconnecting state started (for UI)
|
|
129
|
+
private onError: ((error: Error) => void) | null = null;
|
|
130
|
+
private onStats: ((stats: BrowserWebCodecsStreamStats) => void) | null = null;
|
|
131
|
+
private onCursorChange: ((cursor: string) => void) | null = null;
|
|
132
|
+
|
|
133
|
+
// Navigation state - when true, DataChannel close is expected and recovery is suppressed
|
|
134
|
+
private isNavigating = false;
|
|
135
|
+
private navigationCleanupFn: (() => void) | null = null;
|
|
136
|
+
|
|
137
|
+
// WebSocket cleanup
|
|
138
|
+
private wsCleanupFunctions: Array<() => void> = [];
|
|
139
|
+
|
|
140
|
+
// Stats interval
|
|
141
|
+
private statsIntervalId: ReturnType<typeof setInterval> | null = null;
|
|
142
|
+
|
|
143
|
+
// Bandwidth logging interval
|
|
144
|
+
private bandwidthLogIntervalId: ReturnType<typeof setInterval> | null = null;
|
|
145
|
+
|
|
146
|
+
constructor(projectId: string) {
|
|
147
|
+
if (!projectId) {
|
|
148
|
+
throw new Error('projectId is required for BrowserWebCodecsService');
|
|
149
|
+
}
|
|
150
|
+
this.projectId = projectId;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* Check if WebCodecs is supported
|
|
155
|
+
*/
|
|
156
|
+
static isSupported(): boolean {
|
|
157
|
+
return (
|
|
158
|
+
typeof VideoDecoder !== 'undefined' &&
|
|
159
|
+
typeof AudioDecoder !== 'undefined' &&
|
|
160
|
+
typeof RTCPeerConnection !== 'undefined'
|
|
161
|
+
);
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* Start WebCodecs streaming for a preview session
|
|
166
|
+
*/
|
|
167
|
+
async startStreaming(sessionId: string, canvas: HTMLCanvasElement): Promise<boolean> {
|
|
168
|
+
debug.log('webcodecs', `Starting streaming for session: ${sessionId}`);
|
|
169
|
+
|
|
170
|
+
if (!BrowserWebCodecsService.isSupported()) {
|
|
171
|
+
debug.error('webcodecs', 'Not supported in this browser');
|
|
172
|
+
if (this.onError) {
|
|
173
|
+
this.onError(new Error('WebCodecs not supported'));
|
|
174
|
+
}
|
|
175
|
+
return false;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// Clean up any existing connection
|
|
179
|
+
if (this.peerConnection || this.isConnected || this.sessionId) {
|
|
180
|
+
debug.log('webcodecs', 'Cleaning up previous connection');
|
|
181
|
+
await this.cleanupConnection();
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
this.isCleaningUp = false;
|
|
185
|
+
this.stats.firstFrameRendered = false;
|
|
186
|
+
|
|
187
|
+
this.sessionId = sessionId;
|
|
188
|
+
this.canvas = canvas;
|
|
189
|
+
this.ctx = canvas.getContext('2d', {
|
|
190
|
+
alpha: false,
|
|
191
|
+
desynchronized: true,
|
|
192
|
+
willReadFrequently: false
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
if (this.ctx) {
|
|
196
|
+
this.ctx.imageSmoothingEnabled = true;
|
|
197
|
+
this.ctx.imageSmoothingQuality = 'low';
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
this.clearCanvas();
|
|
201
|
+
|
|
202
|
+
try {
|
|
203
|
+
// Setup WebSocket listeners
|
|
204
|
+
this.setupEventListeners();
|
|
205
|
+
|
|
206
|
+
// Request server to start streaming and get offer
|
|
207
|
+
const response = await ws.http('preview:browser-stream-start', {}, 30000);
|
|
208
|
+
|
|
209
|
+
if (!response.success) {
|
|
210
|
+
throw new Error(response.message || 'Failed to start streaming');
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
// Create peer connection
|
|
214
|
+
await this.createPeerConnection();
|
|
215
|
+
|
|
216
|
+
// Set remote description (offer)
|
|
217
|
+
if (response.offer) {
|
|
218
|
+
await this.handleOffer({
|
|
219
|
+
type: response.offer.type as RTCSdpType,
|
|
220
|
+
sdp: response.offer.sdp
|
|
221
|
+
});
|
|
222
|
+
} else {
|
|
223
|
+
const offerResponse = await ws.http('preview:browser-stream-offer', {}, 10000);
|
|
224
|
+
if (offerResponse.offer) {
|
|
225
|
+
await this.handleOffer({
|
|
226
|
+
type: offerResponse.offer.type as RTCSdpType,
|
|
227
|
+
sdp: offerResponse.offer.sdp
|
|
228
|
+
});
|
|
229
|
+
} else {
|
|
230
|
+
throw new Error('No offer received from server');
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
debug.log('webcodecs', 'Streaming setup complete');
|
|
235
|
+
return true;
|
|
236
|
+
} catch (error) {
|
|
237
|
+
debug.error('webcodecs', 'Failed to start streaming:', error);
|
|
238
|
+
|
|
239
|
+
if (this.onError) {
|
|
240
|
+
this.onError(error instanceof Error ? error : new Error(String(error)));
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
await this.cleanup();
|
|
244
|
+
return false;
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* Create RTCPeerConnection
|
|
250
|
+
*/
|
|
251
|
+
private async createPeerConnection(): Promise<void> {
|
|
252
|
+
const config: RTCConfiguration = {
|
|
253
|
+
iceServers: this.iceServers,
|
|
254
|
+
bundlePolicy: 'max-bundle',
|
|
255
|
+
rtcpMuxPolicy: 'require',
|
|
256
|
+
iceCandidatePoolSize: 0
|
|
257
|
+
};
|
|
258
|
+
|
|
259
|
+
this.peerConnection = new RTCPeerConnection(config);
|
|
260
|
+
|
|
261
|
+
// Handle DataChannel
|
|
262
|
+
this.peerConnection.ondatachannel = (event) => {
|
|
263
|
+
debug.log('webcodecs', 'DataChannel received');
|
|
264
|
+
this.dataChannel = event.channel;
|
|
265
|
+
this.dataChannel.binaryType = 'arraybuffer';
|
|
266
|
+
|
|
267
|
+
this.dataChannel.onopen = () => {
|
|
268
|
+
debug.log('webcodecs', 'DataChannel open');
|
|
269
|
+
};
|
|
270
|
+
|
|
271
|
+
this.dataChannel.onclose = () => {
|
|
272
|
+
debug.log('webcodecs', 'DataChannel closed');
|
|
273
|
+
// Trigger recovery if channel closed while we were connected
|
|
274
|
+
// BUT skip recovery if we're navigating (expected behavior during page navigation)
|
|
275
|
+
if (this.isConnected && !this.isCleaningUp && !this.isNavigating && this.onConnectionFailed) {
|
|
276
|
+
debug.warn('webcodecs', 'DataChannel closed unexpectedly - triggering recovery');
|
|
277
|
+
this.onConnectionFailed();
|
|
278
|
+
} else if (this.isNavigating) {
|
|
279
|
+
debug.log('webcodecs', 'DataChannel closed during navigation - scheduling fast reconnect');
|
|
280
|
+
// Signal reconnecting state IMMEDIATELY (for UI - keeps progress bar showing)
|
|
281
|
+
if (this.onReconnectingStart) {
|
|
282
|
+
this.onReconnectingStart();
|
|
283
|
+
}
|
|
284
|
+
// Backend needs ~500ms to restart streaming with new peer
|
|
285
|
+
// Wait for backend to be ready, then trigger FAST reconnection (no delay)
|
|
286
|
+
setTimeout(() => {
|
|
287
|
+
if (this.isCleaningUp) return;
|
|
288
|
+
debug.log('webcodecs', '🔄 Triggering fast reconnection after navigation');
|
|
289
|
+
this.isNavigating = false; // Reset navigation state
|
|
290
|
+
// Use dedicated navigation reconnect handler (fast path, no delay)
|
|
291
|
+
if (this.onNavigationReconnect) {
|
|
292
|
+
this.onNavigationReconnect();
|
|
293
|
+
} else if (this.onConnectionFailed) {
|
|
294
|
+
// Fallback to regular recovery if no navigation handler
|
|
295
|
+
this.onConnectionFailed();
|
|
296
|
+
}
|
|
297
|
+
}, 700); // Wait 700ms for backend to restart (usually takes ~500ms)
|
|
298
|
+
}
|
|
299
|
+
};
|
|
300
|
+
|
|
301
|
+
this.dataChannel.onerror = (error) => {
|
|
302
|
+
debug.error('webcodecs', 'DataChannel error:', error);
|
|
303
|
+
debug.log('webcodecs', `DataChannel error state: isConnected=${this.isConnected}, isCleaningUp=${this.isCleaningUp}, isNavigating=${this.isNavigating}`);
|
|
304
|
+
// Trigger recovery on DataChannel error
|
|
305
|
+
// BUT skip recovery if we're navigating (expected behavior during page navigation)
|
|
306
|
+
if (this.isConnected && !this.isCleaningUp && !this.isNavigating && this.onConnectionFailed) {
|
|
307
|
+
debug.warn('webcodecs', 'DataChannel error - triggering recovery');
|
|
308
|
+
this.onConnectionFailed();
|
|
309
|
+
} else if (this.isNavigating) {
|
|
310
|
+
debug.log('webcodecs', '🛡️ DataChannel error during navigation - recovery suppressed');
|
|
311
|
+
}
|
|
312
|
+
};
|
|
313
|
+
|
|
314
|
+
this.dataChannel.onmessage = (event) => {
|
|
315
|
+
this.handleDataChannelMessage(event.data);
|
|
316
|
+
};
|
|
317
|
+
};
|
|
318
|
+
|
|
319
|
+
// Handle ICE candidates
|
|
320
|
+
this.peerConnection.onicecandidate = (event) => {
|
|
321
|
+
if (event.candidate && this.sessionId) {
|
|
322
|
+
// Backend uses active tab automatically
|
|
323
|
+
ws.http('preview:browser-stream-ice', {
|
|
324
|
+
candidate: {
|
|
325
|
+
candidate: event.candidate.candidate,
|
|
326
|
+
sdpMid: event.candidate.sdpMid,
|
|
327
|
+
sdpMLineIndex: event.candidate.sdpMLineIndex
|
|
328
|
+
}
|
|
329
|
+
}).catch((error) => {
|
|
330
|
+
debug.warn('webcodecs', 'Failed to send ICE candidate:', error);
|
|
331
|
+
});
|
|
332
|
+
}
|
|
333
|
+
};
|
|
334
|
+
|
|
335
|
+
// Handle connection state
|
|
336
|
+
this.peerConnection.onconnectionstatechange = () => {
|
|
337
|
+
const state = this.peerConnection?.connectionState || 'closed';
|
|
338
|
+
debug.log('webcodecs', `Connection state: ${state}`);
|
|
339
|
+
|
|
340
|
+
this.stats.connectionState = state as RTCPeerConnectionState;
|
|
341
|
+
|
|
342
|
+
if (state === 'connected') {
|
|
343
|
+
this.isConnected = true;
|
|
344
|
+
this.stats.isConnected = true;
|
|
345
|
+
this.startStatsCollection();
|
|
346
|
+
// this.startBandwidthLogging();
|
|
347
|
+
if (this.onConnectionChange) {
|
|
348
|
+
this.onConnectionChange(true);
|
|
349
|
+
}
|
|
350
|
+
} else if (state === 'failed') {
|
|
351
|
+
debug.error('webcodecs', 'Connection FAILED');
|
|
352
|
+
this.isConnected = false;
|
|
353
|
+
this.stats.isConnected = false;
|
|
354
|
+
this.stopStatsCollection();
|
|
355
|
+
this.stopBandwidthLogging();
|
|
356
|
+
if (this.onConnectionChange) {
|
|
357
|
+
this.onConnectionChange(false);
|
|
358
|
+
}
|
|
359
|
+
if (this.onConnectionFailed) {
|
|
360
|
+
this.onConnectionFailed();
|
|
361
|
+
}
|
|
362
|
+
} else if (state === 'disconnected' || state === 'closed') {
|
|
363
|
+
this.isConnected = false;
|
|
364
|
+
this.stats.isConnected = false;
|
|
365
|
+
this.stopStatsCollection();
|
|
366
|
+
this.stopBandwidthLogging();
|
|
367
|
+
if (this.onConnectionChange) {
|
|
368
|
+
this.onConnectionChange(false);
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
};
|
|
372
|
+
|
|
373
|
+
// Handle ICE connection state
|
|
374
|
+
this.peerConnection.oniceconnectionstatechange = () => {
|
|
375
|
+
const state = this.peerConnection?.iceConnectionState || 'closed';
|
|
376
|
+
debug.log('webcodecs', `ICE connection state: ${state}`);
|
|
377
|
+
this.stats.iceConnectionState = state as RTCIceConnectionState;
|
|
378
|
+
|
|
379
|
+
if (state === 'failed') {
|
|
380
|
+
debug.error('webcodecs', 'ICE connection FAILED');
|
|
381
|
+
if (this.onConnectionFailed) {
|
|
382
|
+
this.onConnectionFailed();
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
};
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
/**
|
|
389
|
+
* Handle offer from headless browser
|
|
390
|
+
*/
|
|
391
|
+
private async handleOffer(offer: RTCSessionDescriptionInit): Promise<void> {
|
|
392
|
+
if (!this.peerConnection) {
|
|
393
|
+
throw new Error('Peer connection not initialized');
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
await this.peerConnection.setRemoteDescription(new RTCSessionDescription(offer));
|
|
397
|
+
debug.log('webcodecs', 'Remote description set');
|
|
398
|
+
|
|
399
|
+
const answer = await this.peerConnection.createAnswer();
|
|
400
|
+
await this.peerConnection.setLocalDescription(answer);
|
|
401
|
+
debug.log('webcodecs', 'Local description set');
|
|
402
|
+
|
|
403
|
+
if (this.sessionId) {
|
|
404
|
+
// Backend uses active tab automatically
|
|
405
|
+
await ws.http('preview:browser-stream-answer', {
|
|
406
|
+
answer: {
|
|
407
|
+
type: answer.type,
|
|
408
|
+
sdp: answer.sdp
|
|
409
|
+
}
|
|
410
|
+
});
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
/**
|
|
415
|
+
* Handle DataChannel message (encoded video/audio chunks)
|
|
416
|
+
*/
|
|
417
|
+
private handleDataChannelMessage(data: ArrayBuffer): void {
|
|
418
|
+
try {
|
|
419
|
+
const view = new DataView(data);
|
|
420
|
+
|
|
421
|
+
// Parse packet header
|
|
422
|
+
const type = view.getUint8(0); // 0 = video, 1 = audio
|
|
423
|
+
|
|
424
|
+
if (type === 0) {
|
|
425
|
+
// Video packet
|
|
426
|
+
// Format: [type(1)][timestamp(8)][keyframe(1)][size(4)][data]
|
|
427
|
+
const timestamp = Number(view.getBigUint64(1, true));
|
|
428
|
+
const isKeyframe = view.getUint8(9) === 1;
|
|
429
|
+
const size = view.getUint32(10, true);
|
|
430
|
+
const chunkData = new Uint8Array(data, 14, size);
|
|
431
|
+
|
|
432
|
+
this.stats.videoBytesReceived += size;
|
|
433
|
+
this.stats.videoFramesReceived++;
|
|
434
|
+
|
|
435
|
+
this.handleVideoChunk(chunkData, timestamp, isKeyframe);
|
|
436
|
+
} else if (type === 1) {
|
|
437
|
+
// Audio packet
|
|
438
|
+
// Format: [type(1)][timestamp(8)][size(4)][data]
|
|
439
|
+
const timestamp = Number(view.getBigUint64(1, true));
|
|
440
|
+
const size = view.getUint32(9, true);
|
|
441
|
+
const chunkData = new Uint8Array(data, 13, size);
|
|
442
|
+
|
|
443
|
+
this.stats.audioBytesReceived += size;
|
|
444
|
+
this.stats.audioFramesReceived++;
|
|
445
|
+
|
|
446
|
+
this.handleAudioChunk(chunkData, timestamp);
|
|
447
|
+
}
|
|
448
|
+
} catch (error) {
|
|
449
|
+
debug.error('webcodecs', 'DataChannel message parse error:', error);
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
/**
|
|
454
|
+
* Handle video chunk - decode and render
|
|
455
|
+
*/
|
|
456
|
+
private async handleVideoChunk(data: Uint8Array, timestamp: number, isKeyframe: boolean): Promise<void> {
|
|
457
|
+
// Initialize decoder on first keyframe
|
|
458
|
+
if (!this.videoDecoder && isKeyframe) {
|
|
459
|
+
await this.initVideoDecoder(data);
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
if (!this.videoDecoder) {
|
|
463
|
+
this.stats.videoFramesDropped++;
|
|
464
|
+
return;
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
try {
|
|
468
|
+
const chunk = new EncodedVideoChunk({
|
|
469
|
+
type: isKeyframe ? 'key' : 'delta',
|
|
470
|
+
timestamp,
|
|
471
|
+
data
|
|
472
|
+
});
|
|
473
|
+
|
|
474
|
+
this.videoDecoder.decode(chunk);
|
|
475
|
+
} catch (error) {
|
|
476
|
+
debug.error('webcodecs', 'Video decode error:', error);
|
|
477
|
+
this.stats.videoFramesDropped++;
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
/**
|
|
482
|
+
* Handle audio chunk - decode and queue for playback
|
|
483
|
+
*/
|
|
484
|
+
private async handleAudioChunk(data: Uint8Array, timestamp: number): Promise<void> {
|
|
485
|
+
// Initialize decoder on first chunk
|
|
486
|
+
if (!this.audioDecoder) {
|
|
487
|
+
await this.initAudioDecoder();
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
if (!this.audioDecoder) {
|
|
491
|
+
this.stats.audioFramesDropped++;
|
|
492
|
+
return;
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
try {
|
|
496
|
+
const chunk = new EncodedAudioChunk({
|
|
497
|
+
type: 'key', // Opus frames are all keyframes
|
|
498
|
+
timestamp,
|
|
499
|
+
data
|
|
500
|
+
});
|
|
501
|
+
|
|
502
|
+
this.audioDecoder.decode(chunk);
|
|
503
|
+
} catch (error) {
|
|
504
|
+
debug.error('webcodecs', 'Audio decode error:', error);
|
|
505
|
+
this.stats.audioFramesDropped++;
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
/**
|
|
510
|
+
* Initialize VideoDecoder
|
|
511
|
+
*/
|
|
512
|
+
private async initVideoDecoder(firstChunkData: Uint8Array): Promise<void> {
|
|
513
|
+
// Use VP8 codec
|
|
514
|
+
const codec = 'vp8';
|
|
515
|
+
this.stats.videoCodec = 'vp8';
|
|
516
|
+
|
|
517
|
+
this.videoCodecConfig = {
|
|
518
|
+
codec,
|
|
519
|
+
optimizeForLatency: true
|
|
520
|
+
};
|
|
521
|
+
|
|
522
|
+
try {
|
|
523
|
+
const support = await VideoDecoder.isConfigSupported(this.videoCodecConfig);
|
|
524
|
+
if (!support.supported) {
|
|
525
|
+
throw new Error(`Video codec ${codec} not supported`);
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
this.videoDecoder = new VideoDecoder({
|
|
529
|
+
output: (frame) => this.handleDecodedVideoFrame(frame),
|
|
530
|
+
error: (e) => {
|
|
531
|
+
debug.error('webcodecs', 'VideoDecoder error:', e);
|
|
532
|
+
}
|
|
533
|
+
});
|
|
534
|
+
|
|
535
|
+
this.videoDecoder.configure(this.videoCodecConfig);
|
|
536
|
+
debug.log('webcodecs', 'VideoDecoder initialized:', codec);
|
|
537
|
+
} catch (error) {
|
|
538
|
+
debug.error('webcodecs', 'VideoDecoder init error:', error);
|
|
539
|
+
throw error;
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
/**
|
|
544
|
+
* Initialize AudioDecoder
|
|
545
|
+
*/
|
|
546
|
+
private async initAudioDecoder(): Promise<void> {
|
|
547
|
+
this.audioCodecConfig = {
|
|
548
|
+
codec: 'opus',
|
|
549
|
+
sampleRate: 44100,
|
|
550
|
+
numberOfChannels: 2
|
|
551
|
+
};
|
|
552
|
+
|
|
553
|
+
try {
|
|
554
|
+
const support = await AudioDecoder.isConfigSupported(this.audioCodecConfig);
|
|
555
|
+
if (!support.supported) {
|
|
556
|
+
throw new Error('Opus audio codec not supported');
|
|
557
|
+
}
|
|
558
|
+
|
|
559
|
+
this.audioDecoder = new AudioDecoder({
|
|
560
|
+
output: (frame) => this.handleDecodedAudioFrame(frame),
|
|
561
|
+
error: (e) => {
|
|
562
|
+
debug.error('webcodecs', 'AudioDecoder error:', e);
|
|
563
|
+
}
|
|
564
|
+
});
|
|
565
|
+
|
|
566
|
+
this.audioDecoder.configure(this.audioCodecConfig);
|
|
567
|
+
this.stats.audioCodec = 'opus';
|
|
568
|
+
|
|
569
|
+
// Initialize AudioContext for playback
|
|
570
|
+
await this.initAudioContext();
|
|
571
|
+
|
|
572
|
+
debug.log('webcodecs', 'AudioDecoder initialized: opus');
|
|
573
|
+
} catch (error) {
|
|
574
|
+
debug.error('webcodecs', 'AudioDecoder init error:', error);
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
/**
|
|
579
|
+
* Initialize AudioContext for audio playback
|
|
580
|
+
*/
|
|
581
|
+
private async initAudioContext(): Promise<void> {
|
|
582
|
+
try {
|
|
583
|
+
this.audioContext = new AudioContext({ sampleRate: 44100 });
|
|
584
|
+
debug.log('webcodecs', 'AudioContext initialized');
|
|
585
|
+
} catch (error) {
|
|
586
|
+
debug.error('webcodecs', 'AudioContext init error:', error);
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
/**
|
|
591
|
+
* Handle decoded video frame - render to canvas with timestamp-based optimization
|
|
592
|
+
*/
|
|
593
|
+
private handleDecodedVideoFrame(frame: VideoFrame): void {
|
|
594
|
+
if (this.isCleaningUp || !this.canvas || !this.ctx) {
|
|
595
|
+
frame.close();
|
|
596
|
+
return;
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
try {
|
|
600
|
+
// Update stats
|
|
601
|
+
this.stats.videoFramesDecoded++;
|
|
602
|
+
this.stats.frameWidth = frame.displayWidth;
|
|
603
|
+
this.stats.frameHeight = frame.displayHeight;
|
|
604
|
+
|
|
605
|
+
// Establish AV sync reference on first video frame
|
|
606
|
+
if (!this.syncEstablished) {
|
|
607
|
+
this.syncEstablished = true;
|
|
608
|
+
this.syncRealTimeOrigin = performance.now();
|
|
609
|
+
this.syncStreamTimestamp = frame.timestamp;
|
|
610
|
+
debug.log('webcodecs', `AV Sync established: realTime=${this.syncRealTimeOrigin.toFixed(0)}ms, streamTs=${this.syncStreamTimestamp}μs`);
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
// Track last video timestamp and real time for audio sync
|
|
614
|
+
this.lastVideoTimestamp = frame.timestamp;
|
|
615
|
+
this.lastVideoRealTime = performance.now();
|
|
616
|
+
|
|
617
|
+
// Mark first frame rendered
|
|
618
|
+
if (!this.stats.firstFrameRendered) {
|
|
619
|
+
this.stats.firstFrameRendered = true;
|
|
620
|
+
this.startTime = performance.now();
|
|
621
|
+
this.firstFrameTimestamp = frame.timestamp;
|
|
622
|
+
debug.log('webcodecs', 'First video frame rendered');
|
|
623
|
+
|
|
624
|
+
// Reset navigation state - frames are flowing, navigation is complete
|
|
625
|
+
if (this.isNavigating) {
|
|
626
|
+
debug.log('webcodecs', 'Navigation complete - frames received, resetting navigation state');
|
|
627
|
+
this.isNavigating = false;
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
// Drop old pending frame if exists (frame skipping for performance)
|
|
632
|
+
if (this.pendingFrame) {
|
|
633
|
+
this.pendingFrame.close();
|
|
634
|
+
this.stats.videoFramesDropped++;
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
// Store frame for next render cycle
|
|
638
|
+
this.pendingFrame = frame;
|
|
639
|
+
|
|
640
|
+
// Schedule render if not already scheduled
|
|
641
|
+
if (!this.isRenderingFrame) {
|
|
642
|
+
this.scheduleFrameRender();
|
|
643
|
+
}
|
|
644
|
+
} catch (error) {
|
|
645
|
+
debug.error('webcodecs', 'Video frame handle error:', error);
|
|
646
|
+
frame.close();
|
|
647
|
+
}
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
/**
|
|
651
|
+
* Schedule frame rendering using requestAnimationFrame with timestamp awareness
|
|
652
|
+
* This mimics requestVideoFrameCallback behavior for optimal video rendering
|
|
653
|
+
*/
|
|
654
|
+
private scheduleFrameRender(): void {
|
|
655
|
+
if (this.isRenderingFrame) return;
|
|
656
|
+
|
|
657
|
+
this.isRenderingFrame = true;
|
|
658
|
+
this.renderFrameId = requestAnimationFrame((timestamp) => {
|
|
659
|
+
this.renderPendingFrame(timestamp);
|
|
660
|
+
});
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
/**
|
|
664
|
+
* Render pending frame to canvas with timestamp-based scheduling
|
|
665
|
+
* This approach provides similar benefits to requestVideoFrameCallback
|
|
666
|
+
*/
|
|
667
|
+
private renderPendingFrame(timestamp: DOMHighResTimeStamp): void {
|
|
668
|
+
this.isRenderingFrame = false;
|
|
669
|
+
|
|
670
|
+
if (!this.pendingFrame || this.isCleaningUp || !this.canvas || !this.ctx) {
|
|
671
|
+
return;
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
try {
|
|
675
|
+
// Calculate frame timing (similar to requestVideoFrameCallback metadata)
|
|
676
|
+
// Use relative timestamp from first frame to sync with frontend clock
|
|
677
|
+
const frameTimestamp = (this.pendingFrame.timestamp - this.firstFrameTimestamp) / 1000; // convert to ms
|
|
678
|
+
const elapsedTime = timestamp - this.startTime;
|
|
679
|
+
|
|
680
|
+
// Check if we're ahead of schedule (frame came too early)
|
|
681
|
+
// If so, we might want to delay rendering, but for low-latency
|
|
682
|
+
// we render immediately to minimize lag
|
|
683
|
+
const timeDrift = elapsedTime - frameTimestamp;
|
|
684
|
+
|
|
685
|
+
// Only log significant drift (for debugging)
|
|
686
|
+
if (Math.abs(timeDrift) > 100) {
|
|
687
|
+
// Drift more than 100ms is significant
|
|
688
|
+
debug.warn('webcodecs', `Frame timing drift: ${timeDrift.toFixed(0)}ms`);
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
// Render to canvas with optimal settings
|
|
692
|
+
this.ctx.drawImage(this.pendingFrame, 0, 0, this.canvas.width, this.canvas.height);
|
|
693
|
+
|
|
694
|
+
this.lastFrameTime = timestamp;
|
|
695
|
+
} catch (error) {
|
|
696
|
+
debug.error('webcodecs', 'Video frame render error:', error);
|
|
697
|
+
} finally {
|
|
698
|
+
// Close frame immediately to free memory
|
|
699
|
+
this.pendingFrame.close();
|
|
700
|
+
this.pendingFrame = null;
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
/**
|
|
705
|
+
* Handle decoded audio frame - play audio
|
|
706
|
+
*/
|
|
707
|
+
private handleDecodedAudioFrame(frame: AudioData): void {
|
|
708
|
+
if (this.isCleaningUp || !this.audioContext) {
|
|
709
|
+
frame.close();
|
|
710
|
+
return;
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
try {
|
|
714
|
+
this.stats.audioFramesDecoded++;
|
|
715
|
+
|
|
716
|
+
// Simple playback: convert AudioData to AudioBuffer and play
|
|
717
|
+
// For production, use AudioWorklet for better latency
|
|
718
|
+
this.playAudioFrame(frame);
|
|
719
|
+
} catch (error) {
|
|
720
|
+
debug.error('webcodecs', 'Audio frame playback error:', error);
|
|
721
|
+
} finally {
|
|
722
|
+
frame.close();
|
|
723
|
+
}
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
/**
|
|
727
|
+
* Play audio frame with proper AV synchronization
|
|
728
|
+
*
|
|
729
|
+
* The key insight: Audio and video timestamps from the server use the same
|
|
730
|
+
* performance.now() origin. However, audio may start LATER than video if the
|
|
731
|
+
* page has no audio initially (silence is skipped).
|
|
732
|
+
*
|
|
733
|
+
* Solution: Use lastVideoTimestamp (currently rendered video) as the reference
|
|
734
|
+
* point, not the first video frame timestamp. This ensures we synchronize
|
|
735
|
+
* audio to the CURRENT video position, not the initial position.
|
|
736
|
+
*/
|
|
737
|
+
private playAudioFrame(audioData: AudioData): void {
|
|
738
|
+
if (!this.audioContext) return;
|
|
739
|
+
|
|
740
|
+
try {
|
|
741
|
+
// Create AudioBuffer
|
|
742
|
+
const buffer = this.audioContext.createBuffer(
|
|
743
|
+
audioData.numberOfChannels,
|
|
744
|
+
audioData.numberOfFrames,
|
|
745
|
+
audioData.sampleRate
|
|
746
|
+
);
|
|
747
|
+
|
|
748
|
+
// Copy audio data to buffer
|
|
749
|
+
for (let channel = 0; channel < audioData.numberOfChannels; channel++) {
|
|
750
|
+
const options = {
|
|
751
|
+
planeIndex: channel,
|
|
752
|
+
frameOffset: 0,
|
|
753
|
+
frameCount: audioData.numberOfFrames,
|
|
754
|
+
format: 'f32-planar' as AudioSampleFormat
|
|
755
|
+
};
|
|
756
|
+
|
|
757
|
+
const requiredSize = audioData.allocationSize(options);
|
|
758
|
+
const tempBuffer = new ArrayBuffer(requiredSize);
|
|
759
|
+
const tempFloat32 = new Float32Array(tempBuffer);
|
|
760
|
+
audioData.copyTo(tempFloat32, options);
|
|
761
|
+
|
|
762
|
+
const channelData = buffer.getChannelData(channel);
|
|
763
|
+
channelData.set(tempFloat32);
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
const currentTime = this.audioContext.currentTime;
|
|
767
|
+
const audioTimestamp = audioData.timestamp; // microseconds
|
|
768
|
+
const bufferDuration = buffer.duration;
|
|
769
|
+
const now = performance.now();
|
|
770
|
+
|
|
771
|
+
// Wait for video to establish sync before playing audio
|
|
772
|
+
if (!this.syncEstablished || this.lastVideoTimestamp === 0) {
|
|
773
|
+
// No video yet - skip this audio frame to prevent desync
|
|
774
|
+
return;
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
// Phase 1: Calibration - collect samples to determine stable offset
|
|
778
|
+
if (this.audioCalibrationSamples < this.CALIBRATION_SAMPLES) {
|
|
779
|
+
// Calculate the offset between audio timestamp and video timeline
|
|
780
|
+
// audioVideoOffset > 0 means audio is AHEAD of video in stream time
|
|
781
|
+
// audioVideoOffset < 0 means audio is BEHIND video in stream time
|
|
782
|
+
const audioVideoOffset = (audioTimestamp - this.lastVideoTimestamp) / 1000000; // seconds
|
|
783
|
+
|
|
784
|
+
// Also account for the time elapsed since video was rendered
|
|
785
|
+
const timeSinceVideoRender = (now - this.lastVideoRealTime) / 1000; // seconds
|
|
786
|
+
|
|
787
|
+
// Expected audio position relative to current video position
|
|
788
|
+
// If audio and video are in sync, audio should play at:
|
|
789
|
+
// currentTime + audioVideoOffset - timeSinceVideoRender
|
|
790
|
+
const expectedOffset = audioVideoOffset - timeSinceVideoRender;
|
|
791
|
+
|
|
792
|
+
this.audioOffsetAccumulator += expectedOffset;
|
|
793
|
+
this.audioCalibrationSamples++;
|
|
794
|
+
|
|
795
|
+
if (this.audioCalibrationSamples === this.CALIBRATION_SAMPLES) {
|
|
796
|
+
// Calibration complete - calculate average offset
|
|
797
|
+
this.calibratedAudioOffset = this.audioOffsetAccumulator / this.CALIBRATION_SAMPLES;
|
|
798
|
+
|
|
799
|
+
// Clamp the offset to reasonable bounds (-500ms to +500ms)
|
|
800
|
+
// Beyond this, something is wrong and we should just play immediately
|
|
801
|
+
if (this.calibratedAudioOffset < -0.5) {
|
|
802
|
+
debug.warn('webcodecs', `Audio calibration: offset ${(this.calibratedAudioOffset * 1000).toFixed(0)}ms too negative, clamping to -500ms`);
|
|
803
|
+
this.calibratedAudioOffset = -0.5;
|
|
804
|
+
} else if (this.calibratedAudioOffset > 0.5) {
|
|
805
|
+
debug.warn('webcodecs', `Audio calibration: offset ${(this.calibratedAudioOffset * 1000).toFixed(0)}ms too positive, clamping to +500ms`);
|
|
806
|
+
this.calibratedAudioOffset = 0.5;
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
// Initialize nextAudioPlayTime based on calibrated offset
|
|
810
|
+
// Add small buffer (30ms) for smooth playback
|
|
811
|
+
this.nextAudioPlayTime = currentTime + Math.max(0.03, this.calibratedAudioOffset + 0.03);
|
|
812
|
+
this.audioSyncInitialized = true;
|
|
813
|
+
|
|
814
|
+
debug.log('webcodecs', `Audio calibration complete: offset=${(this.calibratedAudioOffset * 1000).toFixed(1)}ms, startTime=${(this.nextAudioPlayTime - currentTime).toFixed(3)}s from now`);
|
|
815
|
+
} else {
|
|
816
|
+
// Still calibrating - skip this audio frame
|
|
817
|
+
return;
|
|
818
|
+
}
|
|
819
|
+
}
|
|
820
|
+
|
|
821
|
+
// Phase 2: Synchronized playback with drift correction
|
|
822
|
+
let targetPlayTime = this.nextAudioPlayTime;
|
|
823
|
+
|
|
824
|
+
// If we've fallen too far behind (buffer underrun), reset
|
|
825
|
+
if (targetPlayTime < currentTime - 0.01) {
|
|
826
|
+
// We're behind by more than 10ms, need to catch up
|
|
827
|
+
targetPlayTime = currentTime + 0.02; // Small buffer to recover
|
|
828
|
+
debug.warn('webcodecs', 'Audio buffer underrun, resetting playback');
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
// Periodic drift check (every ~500ms worth of audio, ~25 frames)
|
|
832
|
+
if (this.stats.audioFramesDecoded % 25 === 0) {
|
|
833
|
+
// Recalculate where audio SHOULD be based on current video position
|
|
834
|
+
const audioVideoOffset = (audioTimestamp - this.lastVideoTimestamp) / 1000000;
|
|
835
|
+
const timeSinceVideoRender = (now - this.lastVideoRealTime) / 1000;
|
|
836
|
+
const expectedOffset = audioVideoOffset - timeSinceVideoRender;
|
|
837
|
+
|
|
838
|
+
// Where audio should play relative to currentTime
|
|
839
|
+
const idealTime = currentTime + expectedOffset + 0.03; // +30ms buffer
|
|
840
|
+
|
|
841
|
+
const drift = targetPlayTime - idealTime;
|
|
842
|
+
|
|
843
|
+
// Apply correction based on drift magnitude
|
|
844
|
+
if (Math.abs(drift) > 0.2) {
|
|
845
|
+
// Large drift (>200ms) - aggressive correction (80%)
|
|
846
|
+
targetPlayTime -= drift * 0.8;
|
|
847
|
+
debug.warn('webcodecs', `Large audio drift: ${(drift * 1000).toFixed(0)}ms, aggressive correction`);
|
|
848
|
+
} else if (Math.abs(drift) > 0.05) {
|
|
849
|
+
// Medium drift (50-200ms) - moderate correction (40%)
|
|
850
|
+
targetPlayTime -= drift * 0.4;
|
|
851
|
+
}
|
|
852
|
+
// Small drift (<50ms) - no correction, continuous playback handles it
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
// Ensure we don't schedule in the past
|
|
856
|
+
if (targetPlayTime < currentTime + 0.005) {
|
|
857
|
+
targetPlayTime = currentTime + 0.005;
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
// Schedule this buffer
|
|
861
|
+
const source = this.audioContext.createBufferSource();
|
|
862
|
+
source.buffer = buffer;
|
|
863
|
+
source.connect(this.audioContext.destination);
|
|
864
|
+
source.start(targetPlayTime);
|
|
865
|
+
|
|
866
|
+
// Track scheduled buffer
|
|
867
|
+
this.audioBufferQueue.push({ buffer, scheduledTime: targetPlayTime });
|
|
868
|
+
|
|
869
|
+
// Update next play time for continuous scheduling (back-to-back)
|
|
870
|
+
this.nextAudioPlayTime = targetPlayTime + bufferDuration;
|
|
871
|
+
|
|
872
|
+
// Limit queue size to prevent memory buildup
|
|
873
|
+
if (this.audioBufferQueue.length > this.maxAudioQueueSize) {
|
|
874
|
+
this.audioBufferQueue.shift();
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
// Cleanup old scheduled buffers
|
|
878
|
+
source.onended = () => {
|
|
879
|
+
const index = this.audioBufferQueue.findIndex(item => item.buffer === buffer);
|
|
880
|
+
if (index !== -1) {
|
|
881
|
+
this.audioBufferQueue.splice(index, 1);
|
|
882
|
+
}
|
|
883
|
+
};
|
|
884
|
+
} catch (error) {
|
|
885
|
+
debug.warn('webcodecs', 'Audio playback error:', error);
|
|
886
|
+
}
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
/**
|
|
890
|
+
* Setup WebSocket event listeners
|
|
891
|
+
*/
|
|
892
|
+
private setupEventListeners(): void {
|
|
893
|
+
const cleanupIce = ws.on('preview:browser-stream-ice', async (data) => {
|
|
894
|
+
if (data.sessionId === this.sessionId && data.from === 'headless') {
|
|
895
|
+
await this.addIceCandidate(data.candidate);
|
|
896
|
+
}
|
|
897
|
+
});
|
|
898
|
+
|
|
899
|
+
const cleanupState = ws.on('preview:browser-stream-state', (data) => {
|
|
900
|
+
if (data.sessionId === this.sessionId) {
|
|
901
|
+
debug.log('webcodecs', `Server connection state: ${data.state}`);
|
|
902
|
+
}
|
|
903
|
+
});
|
|
904
|
+
|
|
905
|
+
const cleanupCursor = ws.on('preview:browser-cursor-change', (data) => {
|
|
906
|
+
if (data.sessionId === this.sessionId && this.onCursorChange) {
|
|
907
|
+
this.onCursorChange(data.cursor);
|
|
908
|
+
}
|
|
909
|
+
});
|
|
910
|
+
|
|
911
|
+
// Listen for navigation events DIRECTLY to set isNavigating flag immediately
|
|
912
|
+
// This bypasses Svelte's reactive chain which can be too slow
|
|
913
|
+
debug.log('webcodecs', `🔧 Registering navigation listener for session: ${this.sessionId}`);
|
|
914
|
+
const cleanupNavLoading = ws.on('preview:browser-navigation-loading', (data) => {
|
|
915
|
+
debug.log('webcodecs', `📡 Navigation-loading WS event: eventSessionId=${data.sessionId}, mySessionId=${this.sessionId}`);
|
|
916
|
+
// Set isNavigating regardless of sessionId match to ensure recovery is suppressed
|
|
917
|
+
// Multiple tabs scenario: only suppress for current tab, but log all
|
|
918
|
+
if (data.sessionId === this.sessionId) {
|
|
919
|
+
this.isNavigating = true;
|
|
920
|
+
debug.log('webcodecs', `✅ Navigation started - isNavigating=true for session ${data.sessionId}`);
|
|
921
|
+
}
|
|
922
|
+
});
|
|
923
|
+
|
|
924
|
+
const cleanupNavComplete = ws.on('preview:browser-navigation', (data) => {
|
|
925
|
+
if (data.sessionId === this.sessionId) {
|
|
926
|
+
// Keep isNavigating true for a short period to allow reconnection
|
|
927
|
+
// Will be reset when new frames arrive or reconnection completes
|
|
928
|
+
debug.log('webcodecs', `Navigation completed (direct WS) for session ${data.sessionId}`);
|
|
929
|
+
// Signal reconnecting state IMMEDIATELY when navigation completes
|
|
930
|
+
// This eliminates the gap between isNavigating=false and DataChannel close
|
|
931
|
+
// ensuring the overlay stays visible continuously
|
|
932
|
+
if (this.onReconnectingStart) {
|
|
933
|
+
debug.log('webcodecs', '🔄 Pre-emptive reconnecting state on navigation complete');
|
|
934
|
+
this.onReconnectingStart();
|
|
935
|
+
}
|
|
936
|
+
}
|
|
937
|
+
});
|
|
938
|
+
|
|
939
|
+
this.wsCleanupFunctions = [cleanupIce, cleanupState, cleanupCursor, cleanupNavLoading, cleanupNavComplete];
|
|
940
|
+
}
|
|
941
|
+
|
|
942
|
+
/**
|
|
943
|
+
* Add ICE candidate
|
|
944
|
+
*/
|
|
945
|
+
private async addIceCandidate(candidate: RTCIceCandidateInit): Promise<void> {
|
|
946
|
+
if (!this.peerConnection) return;
|
|
947
|
+
|
|
948
|
+
try {
|
|
949
|
+
await this.peerConnection.addIceCandidate(new RTCIceCandidate(candidate));
|
|
950
|
+
} catch (error) {
|
|
951
|
+
debug.warn('webcodecs', 'Add ICE candidate error:', error);
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
|
|
955
|
+
/**
|
|
956
|
+
* Start stats collection
|
|
957
|
+
*/
|
|
958
|
+
private startStatsCollection(): void {
|
|
959
|
+
if (this.statsIntervalId) return;
|
|
960
|
+
|
|
961
|
+
this.statsIntervalId = setInterval(async () => {
|
|
962
|
+
await this.collectStats();
|
|
963
|
+
}, 1000);
|
|
964
|
+
}
|
|
965
|
+
|
|
966
|
+
/**
|
|
967
|
+
* Stop stats collection
|
|
968
|
+
*/
|
|
969
|
+
private stopStatsCollection(): void {
|
|
970
|
+
if (this.statsIntervalId) {
|
|
971
|
+
clearInterval(this.statsIntervalId);
|
|
972
|
+
this.statsIntervalId = null;
|
|
973
|
+
}
|
|
974
|
+
}
|
|
975
|
+
|
|
976
|
+
/**
|
|
977
|
+
* Collect stats
|
|
978
|
+
*/
|
|
979
|
+
private async collectStats(): Promise<void> {
|
|
980
|
+
const now = performance.now();
|
|
981
|
+
|
|
982
|
+
// Calculate bandwidth if we have previous measurements
|
|
983
|
+
if (this.lastStatsTime > 0) {
|
|
984
|
+
const timeDelta = (now - this.lastStatsTime) / 1000; // seconds
|
|
985
|
+
|
|
986
|
+
// Video bitrate
|
|
987
|
+
const videoBytesReceived = this.stats.videoBytesReceived - this.lastVideoBytesReceived;
|
|
988
|
+
this.stats.videoBitrate = (videoBytesReceived * 8) / timeDelta;
|
|
989
|
+
|
|
990
|
+
// Audio bitrate
|
|
991
|
+
const audioBytesReceived = this.stats.audioBytesReceived - this.lastAudioBytesReceived;
|
|
992
|
+
this.stats.audioBitrate = (audioBytesReceived * 8) / timeDelta;
|
|
993
|
+
|
|
994
|
+
// Total
|
|
995
|
+
this.stats.totalBitrate = this.stats.videoBitrate + this.stats.audioBitrate;
|
|
996
|
+
this.stats.totalBandwidthMBps = this.stats.totalBitrate / 8 / 1024 / 1024;
|
|
997
|
+
}
|
|
998
|
+
|
|
999
|
+
this.lastVideoBytesReceived = this.stats.videoBytesReceived;
|
|
1000
|
+
this.lastAudioBytesReceived = this.stats.audioBytesReceived;
|
|
1001
|
+
this.lastStatsTime = now;
|
|
1002
|
+
|
|
1003
|
+
if (this.onStats) {
|
|
1004
|
+
this.onStats(this.stats);
|
|
1005
|
+
}
|
|
1006
|
+
}
|
|
1007
|
+
|
|
1008
|
+
/**
|
|
1009
|
+
* Start bandwidth logging
|
|
1010
|
+
*/
|
|
1011
|
+
private startBandwidthLogging(): void {
|
|
1012
|
+
if (this.bandwidthLogIntervalId) return;
|
|
1013
|
+
|
|
1014
|
+
this.bandwidthLogIntervalId = setInterval(() => {
|
|
1015
|
+
this.logBandwidthStats();
|
|
1016
|
+
}, 1000); // Log every 5 seconds
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
/**
|
|
1020
|
+
* Stop bandwidth logging
|
|
1021
|
+
*/
|
|
1022
|
+
private stopBandwidthLogging(): void {
|
|
1023
|
+
if (this.bandwidthLogIntervalId) {
|
|
1024
|
+
clearInterval(this.bandwidthLogIntervalId);
|
|
1025
|
+
this.bandwidthLogIntervalId = null;
|
|
1026
|
+
}
|
|
1027
|
+
}
|
|
1028
|
+
|
|
1029
|
+
/**
|
|
1030
|
+
* Log bandwidth statistics to console
|
|
1031
|
+
*/
|
|
1032
|
+
private logBandwidthStats(): void {
|
|
1033
|
+
const videoKbps = (this.stats.videoBitrate / 1000).toFixed(1);
|
|
1034
|
+
const audioKbps = (this.stats.audioBitrate / 1000).toFixed(1);
|
|
1035
|
+
const totalKbps = (this.stats.totalBitrate / 1000).toFixed(1);
|
|
1036
|
+
const totalMBps = this.stats.totalBandwidthMBps.toFixed(3);
|
|
1037
|
+
|
|
1038
|
+
debug.log('webcodecs',
|
|
1039
|
+
`📊 Bandwidth Usage:\n` +
|
|
1040
|
+
` Video: ${videoKbps} Kbps (${this.stats.videoCodec}) - ${this.stats.frameWidth}x${this.stats.frameHeight}\n` +
|
|
1041
|
+
` Audio: ${audioKbps} Kbps (${this.stats.audioCodec})\n` +
|
|
1042
|
+
` Total: ${totalKbps} Kbps (${totalMBps} MB/s)\n` +
|
|
1043
|
+
` Frames: Video ${this.stats.videoFramesDecoded} (dropped: ${this.stats.videoFramesDropped}), ` +
|
|
1044
|
+
`Audio ${this.stats.audioFramesDecoded} (dropped: ${this.stats.audioFramesDropped})`
|
|
1045
|
+
);
|
|
1046
|
+
}
|
|
1047
|
+
|
|
1048
|
+
/**
|
|
1049
|
+
* Stop streaming
|
|
1050
|
+
*/
|
|
1051
|
+
async stopStreaming(): Promise<void> {
|
|
1052
|
+
debug.log('webcodecs', 'Stopping streaming');
|
|
1053
|
+
await this.cleanup();
|
|
1054
|
+
}
|
|
1055
|
+
|
|
1056
|
+
/**
|
|
1057
|
+
* Reconnect to existing backend stream (after navigation)
|
|
1058
|
+
* This does NOT tell backend to stop - just reconnects WebRTC locally
|
|
1059
|
+
*/
|
|
1060
|
+
async reconnectToExistingStream(sessionId: string, canvas: HTMLCanvasElement): Promise<boolean> {
|
|
1061
|
+
debug.log('webcodecs', `🔄 Reconnecting to existing stream for session: ${sessionId}`);
|
|
1062
|
+
|
|
1063
|
+
// Cleanup local WebRTC state WITHOUT notifying backend
|
|
1064
|
+
await this.cleanupLocalConnection();
|
|
1065
|
+
|
|
1066
|
+
this.isCleaningUp = false;
|
|
1067
|
+
this.stats.firstFrameRendered = false;
|
|
1068
|
+
this.isNavigating = false;
|
|
1069
|
+
|
|
1070
|
+
this.sessionId = sessionId;
|
|
1071
|
+
this.canvas = canvas;
|
|
1072
|
+
this.ctx = canvas.getContext('2d', {
|
|
1073
|
+
alpha: false,
|
|
1074
|
+
desynchronized: true,
|
|
1075
|
+
willReadFrequently: false
|
|
1076
|
+
});
|
|
1077
|
+
|
|
1078
|
+
if (this.ctx) {
|
|
1079
|
+
this.ctx.imageSmoothingEnabled = true;
|
|
1080
|
+
this.ctx.imageSmoothingQuality = 'low';
|
|
1081
|
+
}
|
|
1082
|
+
|
|
1083
|
+
try {
|
|
1084
|
+
// Setup WebSocket listeners
|
|
1085
|
+
this.setupEventListeners();
|
|
1086
|
+
|
|
1087
|
+
// Create peer connection
|
|
1088
|
+
await this.createPeerConnection();
|
|
1089
|
+
|
|
1090
|
+
// Get offer from backend's existing peer (don't start new streaming)
|
|
1091
|
+
const offerResponse = await ws.http('preview:browser-stream-offer', {}, 10000);
|
|
1092
|
+
if (offerResponse.offer) {
|
|
1093
|
+
await this.handleOffer({
|
|
1094
|
+
type: offerResponse.offer.type as RTCSdpType,
|
|
1095
|
+
sdp: offerResponse.offer.sdp
|
|
1096
|
+
});
|
|
1097
|
+
} else {
|
|
1098
|
+
throw new Error('No offer received from backend');
|
|
1099
|
+
}
|
|
1100
|
+
|
|
1101
|
+
debug.log('webcodecs', 'Reconnection setup complete');
|
|
1102
|
+
return true;
|
|
1103
|
+
} catch (error) {
|
|
1104
|
+
debug.error('webcodecs', 'Failed to reconnect:', error);
|
|
1105
|
+
if (this.onError) {
|
|
1106
|
+
this.onError(error instanceof Error ? error : new Error(String(error)));
|
|
1107
|
+
}
|
|
1108
|
+
return false;
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
|
|
1112
|
+
/**
|
|
1113
|
+
* Cleanup local WebRTC connection WITHOUT notifying backend
|
|
1114
|
+
*/
|
|
1115
|
+
private async cleanupLocalConnection(): Promise<void> {
|
|
1116
|
+
this.isCleaningUp = true;
|
|
1117
|
+
|
|
1118
|
+
this.stopStatsCollection();
|
|
1119
|
+
this.stopBandwidthLogging();
|
|
1120
|
+
|
|
1121
|
+
// Cancel pending frame render
|
|
1122
|
+
if (this.renderFrameId !== null) {
|
|
1123
|
+
cancelAnimationFrame(this.renderFrameId);
|
|
1124
|
+
this.renderFrameId = null;
|
|
1125
|
+
}
|
|
1126
|
+
|
|
1127
|
+
// Close pending frame
|
|
1128
|
+
if (this.pendingFrame) {
|
|
1129
|
+
this.pendingFrame.close();
|
|
1130
|
+
this.pendingFrame = null;
|
|
1131
|
+
}
|
|
1132
|
+
|
|
1133
|
+
// Cleanup WebSocket listeners
|
|
1134
|
+
this.wsCleanupFunctions.forEach((cleanup) => cleanup());
|
|
1135
|
+
this.wsCleanupFunctions = [];
|
|
1136
|
+
|
|
1137
|
+
// Close decoders
|
|
1138
|
+
if (this.videoDecoder) {
|
|
1139
|
+
try {
|
|
1140
|
+
await this.videoDecoder.flush();
|
|
1141
|
+
this.videoDecoder.close();
|
|
1142
|
+
} catch (e) {}
|
|
1143
|
+
this.videoDecoder = null;
|
|
1144
|
+
}
|
|
1145
|
+
|
|
1146
|
+
if (this.audioDecoder) {
|
|
1147
|
+
try {
|
|
1148
|
+
await this.audioDecoder.flush();
|
|
1149
|
+
this.audioDecoder.close();
|
|
1150
|
+
} catch (e) {}
|
|
1151
|
+
this.audioDecoder = null;
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1154
|
+
// Close audio context
|
|
1155
|
+
if (this.audioContext && this.audioContext.state !== 'closed') {
|
|
1156
|
+
await this.audioContext.close().catch(() => {});
|
|
1157
|
+
this.audioContext = null;
|
|
1158
|
+
}
|
|
1159
|
+
|
|
1160
|
+
// Close data channel
|
|
1161
|
+
if (this.dataChannel) {
|
|
1162
|
+
this.dataChannel.close();
|
|
1163
|
+
this.dataChannel = null;
|
|
1164
|
+
}
|
|
1165
|
+
|
|
1166
|
+
// Close peer connection
|
|
1167
|
+
if (this.peerConnection) {
|
|
1168
|
+
this.peerConnection.close();
|
|
1169
|
+
this.peerConnection = null;
|
|
1170
|
+
}
|
|
1171
|
+
|
|
1172
|
+
this.isConnected = false;
|
|
1173
|
+
// NOTE: Don't clear sessionId - we're reconnecting to same session
|
|
1174
|
+
|
|
1175
|
+
// Reset playback state
|
|
1176
|
+
this.nextAudioPlayTime = 0;
|
|
1177
|
+
this.audioBufferQueue = [];
|
|
1178
|
+
this.audioSyncInitialized = false;
|
|
1179
|
+
this.audioCalibrationSamples = 0;
|
|
1180
|
+
this.audioOffsetAccumulator = 0;
|
|
1181
|
+
this.calibratedAudioOffset = 0;
|
|
1182
|
+
|
|
1183
|
+
this.isRenderingFrame = false;
|
|
1184
|
+
this.lastFrameTime = 0;
|
|
1185
|
+
this.startTime = 0;
|
|
1186
|
+
this.firstFrameTimestamp = 0;
|
|
1187
|
+
|
|
1188
|
+
this.syncEstablished = false;
|
|
1189
|
+
this.syncRealTimeOrigin = 0;
|
|
1190
|
+
this.syncStreamTimestamp = 0;
|
|
1191
|
+
this.lastVideoTimestamp = 0;
|
|
1192
|
+
this.lastVideoRealTime = 0;
|
|
1193
|
+
|
|
1194
|
+
if (this.onConnectionChange) {
|
|
1195
|
+
this.onConnectionChange(false);
|
|
1196
|
+
}
|
|
1197
|
+
|
|
1198
|
+
this.isCleaningUp = false;
|
|
1199
|
+
}
|
|
1200
|
+
|
|
1201
|
+
/**
|
|
1202
|
+
* Cleanup resources
|
|
1203
|
+
*/
|
|
1204
|
+
private async cleanup(): Promise<void> {
|
|
1205
|
+
this.isCleaningUp = true;
|
|
1206
|
+
|
|
1207
|
+
this.stopStatsCollection();
|
|
1208
|
+
this.stopBandwidthLogging();
|
|
1209
|
+
|
|
1210
|
+
// Cancel pending frame render
|
|
1211
|
+
if (this.renderFrameId !== null) {
|
|
1212
|
+
cancelAnimationFrame(this.renderFrameId);
|
|
1213
|
+
this.renderFrameId = null;
|
|
1214
|
+
}
|
|
1215
|
+
|
|
1216
|
+
// Close pending frame
|
|
1217
|
+
if (this.pendingFrame) {
|
|
1218
|
+
this.pendingFrame.close();
|
|
1219
|
+
this.pendingFrame = null;
|
|
1220
|
+
}
|
|
1221
|
+
|
|
1222
|
+
// Cleanup WebSocket listeners
|
|
1223
|
+
this.wsCleanupFunctions.forEach((cleanup) => cleanup());
|
|
1224
|
+
this.wsCleanupFunctions = [];
|
|
1225
|
+
|
|
1226
|
+
// Notify server
|
|
1227
|
+
if (this.sessionId) {
|
|
1228
|
+
try {
|
|
1229
|
+
await ws.http('preview:browser-stream-stop', {});
|
|
1230
|
+
} catch (error) {
|
|
1231
|
+
debug.warn('webcodecs', 'Failed to notify server:', error);
|
|
1232
|
+
}
|
|
1233
|
+
}
|
|
1234
|
+
|
|
1235
|
+
// Close decoders
|
|
1236
|
+
if (this.videoDecoder) {
|
|
1237
|
+
try {
|
|
1238
|
+
await this.videoDecoder.flush();
|
|
1239
|
+
this.videoDecoder.close();
|
|
1240
|
+
} catch (e) {}
|
|
1241
|
+
this.videoDecoder = null;
|
|
1242
|
+
}
|
|
1243
|
+
|
|
1244
|
+
if (this.audioDecoder) {
|
|
1245
|
+
try {
|
|
1246
|
+
await this.audioDecoder.flush();
|
|
1247
|
+
this.audioDecoder.close();
|
|
1248
|
+
} catch (e) {}
|
|
1249
|
+
this.audioDecoder = null;
|
|
1250
|
+
}
|
|
1251
|
+
|
|
1252
|
+
// Close audio context
|
|
1253
|
+
if (this.audioContext && this.audioContext.state !== 'closed') {
|
|
1254
|
+
await this.audioContext.close().catch(() => {});
|
|
1255
|
+
this.audioContext = null;
|
|
1256
|
+
}
|
|
1257
|
+
|
|
1258
|
+
// Close data channel
|
|
1259
|
+
if (this.dataChannel) {
|
|
1260
|
+
this.dataChannel.close();
|
|
1261
|
+
this.dataChannel = null;
|
|
1262
|
+
}
|
|
1263
|
+
|
|
1264
|
+
// Close peer connection
|
|
1265
|
+
if (this.peerConnection) {
|
|
1266
|
+
this.peerConnection.close();
|
|
1267
|
+
this.peerConnection = null;
|
|
1268
|
+
}
|
|
1269
|
+
|
|
1270
|
+
this.isConnected = false;
|
|
1271
|
+
this.sessionId = null;
|
|
1272
|
+
this.canvas = null;
|
|
1273
|
+
this.ctx = null;
|
|
1274
|
+
|
|
1275
|
+
// Reset stats
|
|
1276
|
+
this.stats = {
|
|
1277
|
+
isConnected: false,
|
|
1278
|
+
connectionState: 'closed',
|
|
1279
|
+
iceConnectionState: 'closed',
|
|
1280
|
+
videoBytesReceived: 0,
|
|
1281
|
+
audioBytesReceived: 0,
|
|
1282
|
+
videoFramesReceived: 0,
|
|
1283
|
+
audioFramesReceived: 0,
|
|
1284
|
+
videoFramesDecoded: 0,
|
|
1285
|
+
audioFramesDecoded: 0,
|
|
1286
|
+
videoFramesDropped: 0,
|
|
1287
|
+
audioFramesDropped: 0,
|
|
1288
|
+
frameWidth: 0,
|
|
1289
|
+
frameHeight: 0,
|
|
1290
|
+
videoBitrate: 0,
|
|
1291
|
+
audioBitrate: 0,
|
|
1292
|
+
totalBitrate: 0,
|
|
1293
|
+
totalBandwidthMBps: 0,
|
|
1294
|
+
videoCodec: 'unknown',
|
|
1295
|
+
audioCodec: 'unknown',
|
|
1296
|
+
firstFrameRendered: false
|
|
1297
|
+
};
|
|
1298
|
+
|
|
1299
|
+
this.lastVideoBytesReceived = 0;
|
|
1300
|
+
this.lastAudioBytesReceived = 0;
|
|
1301
|
+
this.lastStatsTime = 0;
|
|
1302
|
+
|
|
1303
|
+
// Reset audio playback state
|
|
1304
|
+
this.nextAudioPlayTime = 0;
|
|
1305
|
+
this.audioBufferQueue = [];
|
|
1306
|
+
this.audioSyncInitialized = false;
|
|
1307
|
+
this.audioCalibrationSamples = 0;
|
|
1308
|
+
this.audioOffsetAccumulator = 0;
|
|
1309
|
+
this.calibratedAudioOffset = 0;
|
|
1310
|
+
|
|
1311
|
+
// Reset frame rendering state
|
|
1312
|
+
this.isRenderingFrame = false;
|
|
1313
|
+
this.lastFrameTime = 0;
|
|
1314
|
+
this.startTime = 0;
|
|
1315
|
+
this.firstFrameTimestamp = 0;
|
|
1316
|
+
|
|
1317
|
+
// Reset AV sync state
|
|
1318
|
+
this.syncEstablished = false;
|
|
1319
|
+
this.syncRealTimeOrigin = 0;
|
|
1320
|
+
this.syncStreamTimestamp = 0;
|
|
1321
|
+
this.lastVideoTimestamp = 0;
|
|
1322
|
+
this.lastVideoRealTime = 0;
|
|
1323
|
+
|
|
1324
|
+
// Reset navigation state
|
|
1325
|
+
this.isNavigating = false;
|
|
1326
|
+
|
|
1327
|
+
if (this.onConnectionChange) {
|
|
1328
|
+
this.onConnectionChange(false);
|
|
1329
|
+
}
|
|
1330
|
+
|
|
1331
|
+
this.isCleaningUp = false;
|
|
1332
|
+
}
|
|
1333
|
+
|
|
1334
|
+
/**
|
|
1335
|
+
* Cleanup connection (internal)
|
|
1336
|
+
*/
|
|
1337
|
+
private async cleanupConnection(): Promise<void> {
|
|
1338
|
+
this.isCleaningUp = true;
|
|
1339
|
+
|
|
1340
|
+
this.stopStatsCollection();
|
|
1341
|
+
this.stopBandwidthLogging();
|
|
1342
|
+
|
|
1343
|
+
// Cancel pending frame render
|
|
1344
|
+
if (this.renderFrameId !== null) {
|
|
1345
|
+
cancelAnimationFrame(this.renderFrameId);
|
|
1346
|
+
this.renderFrameId = null;
|
|
1347
|
+
}
|
|
1348
|
+
|
|
1349
|
+
// Close pending frame
|
|
1350
|
+
if (this.pendingFrame) {
|
|
1351
|
+
this.pendingFrame.close();
|
|
1352
|
+
this.pendingFrame = null;
|
|
1353
|
+
}
|
|
1354
|
+
|
|
1355
|
+
this.wsCleanupFunctions.forEach((cleanup) => cleanup());
|
|
1356
|
+
this.wsCleanupFunctions = [];
|
|
1357
|
+
|
|
1358
|
+
if (this.videoDecoder) {
|
|
1359
|
+
try {
|
|
1360
|
+
await this.videoDecoder.flush();
|
|
1361
|
+
this.videoDecoder.close();
|
|
1362
|
+
} catch (e) {}
|
|
1363
|
+
this.videoDecoder = null;
|
|
1364
|
+
}
|
|
1365
|
+
|
|
1366
|
+
if (this.audioDecoder) {
|
|
1367
|
+
try {
|
|
1368
|
+
await this.audioDecoder.flush();
|
|
1369
|
+
this.audioDecoder.close();
|
|
1370
|
+
} catch (e) {}
|
|
1371
|
+
this.audioDecoder = null;
|
|
1372
|
+
}
|
|
1373
|
+
|
|
1374
|
+
if (this.audioContext && this.audioContext.state !== 'closed') {
|
|
1375
|
+
await this.audioContext.close().catch(() => {});
|
|
1376
|
+
this.audioContext = null;
|
|
1377
|
+
}
|
|
1378
|
+
|
|
1379
|
+
if (this.dataChannel) {
|
|
1380
|
+
this.dataChannel.close();
|
|
1381
|
+
this.dataChannel = null;
|
|
1382
|
+
}
|
|
1383
|
+
|
|
1384
|
+
if (this.peerConnection) {
|
|
1385
|
+
this.peerConnection.close();
|
|
1386
|
+
this.peerConnection = null;
|
|
1387
|
+
}
|
|
1388
|
+
|
|
1389
|
+
this.isConnected = false;
|
|
1390
|
+
this.sessionId = null;
|
|
1391
|
+
|
|
1392
|
+
// Reset audio playback state
|
|
1393
|
+
this.nextAudioPlayTime = 0;
|
|
1394
|
+
this.audioBufferQueue = [];
|
|
1395
|
+
this.audioSyncInitialized = false;
|
|
1396
|
+
this.audioCalibrationSamples = 0;
|
|
1397
|
+
this.audioOffsetAccumulator = 0;
|
|
1398
|
+
this.calibratedAudioOffset = 0;
|
|
1399
|
+
|
|
1400
|
+
// Reset frame rendering state
|
|
1401
|
+
this.isRenderingFrame = false;
|
|
1402
|
+
this.lastFrameTime = 0;
|
|
1403
|
+
this.startTime = 0;
|
|
1404
|
+
this.firstFrameTimestamp = 0;
|
|
1405
|
+
|
|
1406
|
+
// Reset AV sync state
|
|
1407
|
+
this.syncEstablished = false;
|
|
1408
|
+
this.syncRealTimeOrigin = 0;
|
|
1409
|
+
this.syncStreamTimestamp = 0;
|
|
1410
|
+
this.lastVideoTimestamp = 0;
|
|
1411
|
+
this.lastVideoRealTime = 0;
|
|
1412
|
+
|
|
1413
|
+
this.isCleaningUp = false;
|
|
1414
|
+
}
|
|
1415
|
+
|
|
1416
|
+
/**
|
|
1417
|
+
* Clear canvas
|
|
1418
|
+
*/
|
|
1419
|
+
private clearCanvas(): void {
|
|
1420
|
+
if (this.canvas && this.ctx) {
|
|
1421
|
+
this.ctx.fillStyle = '#f1f5f9';
|
|
1422
|
+
this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);
|
|
1423
|
+
}
|
|
1424
|
+
}
|
|
1425
|
+
|
|
1426
|
+
/**
|
|
1427
|
+
* Get stats
|
|
1428
|
+
*/
|
|
1429
|
+
getStats(): BrowserWebCodecsStreamStats {
|
|
1430
|
+
return { ...this.stats };
|
|
1431
|
+
}
|
|
1432
|
+
|
|
1433
|
+
/**
|
|
1434
|
+
* Get connection status
|
|
1435
|
+
*/
|
|
1436
|
+
getConnectionStatus(): boolean {
|
|
1437
|
+
return this.isConnected;
|
|
1438
|
+
}
|
|
1439
|
+
|
|
1440
|
+
// Event handlers
|
|
1441
|
+
setConnectionChangeHandler(handler: (connected: boolean) => void): void {
|
|
1442
|
+
this.onConnectionChange = handler;
|
|
1443
|
+
}
|
|
1444
|
+
|
|
1445
|
+
setConnectionFailedHandler(handler: () => void): void {
|
|
1446
|
+
this.onConnectionFailed = handler;
|
|
1447
|
+
}
|
|
1448
|
+
|
|
1449
|
+
setNavigationReconnectHandler(handler: () => void): void {
|
|
1450
|
+
this.onNavigationReconnect = handler;
|
|
1451
|
+
}
|
|
1452
|
+
|
|
1453
|
+
setReconnectingStartHandler(handler: () => void): void {
|
|
1454
|
+
this.onReconnectingStart = handler;
|
|
1455
|
+
}
|
|
1456
|
+
|
|
1457
|
+
setErrorHandler(handler: (error: Error) => void): void {
|
|
1458
|
+
this.onError = handler;
|
|
1459
|
+
}
|
|
1460
|
+
|
|
1461
|
+
setStatsHandler(handler: (stats: BrowserWebCodecsStreamStats) => void): void {
|
|
1462
|
+
this.onStats = handler;
|
|
1463
|
+
}
|
|
1464
|
+
|
|
1465
|
+
setOnCursorChange(handler: (cursor: string) => void): void {
|
|
1466
|
+
this.onCursorChange = handler;
|
|
1467
|
+
}
|
|
1468
|
+
|
|
1469
|
+
/**
|
|
1470
|
+
* Set navigation state
|
|
1471
|
+
* When navigating, DataChannel close/error won't trigger recovery
|
|
1472
|
+
* Backend will restart streaming after navigation completes
|
|
1473
|
+
*/
|
|
1474
|
+
setNavigating(navigating: boolean): void {
|
|
1475
|
+
this.isNavigating = navigating;
|
|
1476
|
+
debug.log('webcodecs', `Navigation state set: ${navigating}`);
|
|
1477
|
+
}
|
|
1478
|
+
|
|
1479
|
+
/**
|
|
1480
|
+
* Check if currently navigating
|
|
1481
|
+
*/
|
|
1482
|
+
getNavigating(): boolean {
|
|
1483
|
+
return this.isNavigating;
|
|
1484
|
+
}
|
|
1485
|
+
|
|
1486
|
+
/**
|
|
1487
|
+
* Destroy service
|
|
1488
|
+
*/
|
|
1489
|
+
destroy(): void {
|
|
1490
|
+
this.cleanup();
|
|
1491
|
+
this.onConnectionChange = null;
|
|
1492
|
+
this.onConnectionFailed = null;
|
|
1493
|
+
this.onNavigationReconnect = null;
|
|
1494
|
+
this.onReconnectingStart = null;
|
|
1495
|
+
this.onError = null;
|
|
1496
|
+
this.onStats = null;
|
|
1497
|
+
this.onCursorChange = null;
|
|
1498
|
+
}
|
|
1499
|
+
}
|