@myrialabs/clopen 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (456) hide show
  1. package/.env.example +6 -0
  2. package/.github/workflows/release.yml +60 -0
  3. package/.github/workflows/test.yml +40 -0
  4. package/CONTRIBUTING.md +499 -0
  5. package/LICENSE +21 -0
  6. package/README.md +209 -0
  7. package/backend/index.ts +156 -0
  8. package/backend/lib/chat/helpers.ts +42 -0
  9. package/backend/lib/chat/index.ts +2 -0
  10. package/backend/lib/chat/stream-manager.ts +1126 -0
  11. package/backend/lib/database/README.md +77 -0
  12. package/backend/lib/database/index.ts +119 -0
  13. package/backend/lib/database/migrations/001_create_projects_table.ts +31 -0
  14. package/backend/lib/database/migrations/002_create_chat_sessions_table.ts +33 -0
  15. package/backend/lib/database/migrations/003_create_messages_table.ts +32 -0
  16. package/backend/lib/database/migrations/004_create_prompt_templates_table.ts +34 -0
  17. package/backend/lib/database/migrations/005_create_settings_table.ts +24 -0
  18. package/backend/lib/database/migrations/006_add_user_to_messages.ts +58 -0
  19. package/backend/lib/database/migrations/007_create_stream_states_table.ts +41 -0
  20. package/backend/lib/database/migrations/008_create_message_snapshots_table.ts +62 -0
  21. package/backend/lib/database/migrations/009_add_delta_snapshot_fields.ts +41 -0
  22. package/backend/lib/database/migrations/010_add_soft_delete_and_branch_support.ts +70 -0
  23. package/backend/lib/database/migrations/011_git_like_commit_graph.ts +156 -0
  24. package/backend/lib/database/migrations/012_add_file_change_statistics.ts +41 -0
  25. package/backend/lib/database/migrations/013_checkpoint_tree_state.ts +118 -0
  26. package/backend/lib/database/migrations/014_add_engine_to_sessions.ts +18 -0
  27. package/backend/lib/database/migrations/015_add_model_to_sessions.ts +18 -0
  28. package/backend/lib/database/migrations/016_create_user_projects_table.ts +34 -0
  29. package/backend/lib/database/migrations/017_add_current_session_to_user_projects.ts +32 -0
  30. package/backend/lib/database/migrations/018_create_claude_accounts_table.ts +24 -0
  31. package/backend/lib/database/migrations/019_add_claude_account_to_sessions.ts +18 -0
  32. package/backend/lib/database/migrations/020_add_snapshot_tree_hash.ts +32 -0
  33. package/backend/lib/database/migrations/021_drop_prompt_templates_table.ts +33 -0
  34. package/backend/lib/database/migrations/index.ts +154 -0
  35. package/backend/lib/database/queries/checkpoint-queries.ts +87 -0
  36. package/backend/lib/database/queries/engine-queries.ts +75 -0
  37. package/backend/lib/database/queries/index.ts +9 -0
  38. package/backend/lib/database/queries/message-queries.ts +472 -0
  39. package/backend/lib/database/queries/project-queries.ts +117 -0
  40. package/backend/lib/database/queries/session-queries.ts +271 -0
  41. package/backend/lib/database/queries/settings-queries.ts +34 -0
  42. package/backend/lib/database/queries/snapshot-queries.ts +326 -0
  43. package/backend/lib/database/queries/utils-queries.ts +59 -0
  44. package/backend/lib/database/seeders/index.ts +13 -0
  45. package/backend/lib/database/seeders/settings_seeder.ts +84 -0
  46. package/backend/lib/database/utils/connection.ts +174 -0
  47. package/backend/lib/database/utils/index.ts +4 -0
  48. package/backend/lib/database/utils/migration-runner.ts +118 -0
  49. package/backend/lib/database/utils/seeder-runner.ts +121 -0
  50. package/backend/lib/engine/adapters/claude/environment.ts +164 -0
  51. package/backend/lib/engine/adapters/claude/error-handler.ts +60 -0
  52. package/backend/lib/engine/adapters/claude/index.ts +1 -0
  53. package/backend/lib/engine/adapters/claude/path-utils.ts +38 -0
  54. package/backend/lib/engine/adapters/claude/stream.ts +177 -0
  55. package/backend/lib/engine/adapters/opencode/index.ts +2 -0
  56. package/backend/lib/engine/adapters/opencode/message-converter.ts +862 -0
  57. package/backend/lib/engine/adapters/opencode/server.ts +104 -0
  58. package/backend/lib/engine/adapters/opencode/stream.ts +755 -0
  59. package/backend/lib/engine/index.ts +196 -0
  60. package/backend/lib/engine/types.ts +58 -0
  61. package/backend/lib/files/file-operations.ts +478 -0
  62. package/backend/lib/files/file-reading.ts +308 -0
  63. package/backend/lib/files/file-watcher.ts +383 -0
  64. package/backend/lib/files/path-browsing.ts +382 -0
  65. package/backend/lib/git/git-executor.ts +88 -0
  66. package/backend/lib/git/git-parser.ts +411 -0
  67. package/backend/lib/git/git-service.ts +505 -0
  68. package/backend/lib/mcp/README.md +1144 -0
  69. package/backend/lib/mcp/config.ts +316 -0
  70. package/backend/lib/mcp/index.ts +35 -0
  71. package/backend/lib/mcp/project-context.ts +236 -0
  72. package/backend/lib/mcp/servers/browser-automation/actions.ts +156 -0
  73. package/backend/lib/mcp/servers/browser-automation/browser.ts +419 -0
  74. package/backend/lib/mcp/servers/browser-automation/index.ts +791 -0
  75. package/backend/lib/mcp/servers/browser-automation/inspection.ts +501 -0
  76. package/backend/lib/mcp/servers/helper.ts +143 -0
  77. package/backend/lib/mcp/servers/index.ts +45 -0
  78. package/backend/lib/mcp/servers/weather/get-temperature.ts +56 -0
  79. package/backend/lib/mcp/servers/weather/index.ts +31 -0
  80. package/backend/lib/mcp/stdio-server.ts +103 -0
  81. package/backend/lib/mcp/types.ts +65 -0
  82. package/backend/lib/preview/browser/browser-audio-capture.ts +86 -0
  83. package/backend/lib/preview/browser/browser-console-manager.ts +263 -0
  84. package/backend/lib/preview/browser/browser-dialog-handler.ts +222 -0
  85. package/backend/lib/preview/browser/browser-interaction-handler.ts +421 -0
  86. package/backend/lib/preview/browser/browser-mcp-control.ts +415 -0
  87. package/backend/lib/preview/browser/browser-native-ui-handler.ts +512 -0
  88. package/backend/lib/preview/browser/browser-navigation-tracker.ts +104 -0
  89. package/backend/lib/preview/browser/browser-pool.ts +357 -0
  90. package/backend/lib/preview/browser/browser-preview-service.ts +882 -0
  91. package/backend/lib/preview/browser/browser-tab-manager.ts +935 -0
  92. package/backend/lib/preview/browser/browser-video-capture.ts +695 -0
  93. package/backend/lib/preview/browser/scripts/audio-stream.ts +292 -0
  94. package/backend/lib/preview/browser/scripts/cursor-tracking.ts +85 -0
  95. package/backend/lib/preview/browser/scripts/video-stream.ts +438 -0
  96. package/backend/lib/preview/browser/types.ts +359 -0
  97. package/backend/lib/preview/index.ts +24 -0
  98. package/backend/lib/project/index.ts +2 -0
  99. package/backend/lib/project/status-manager.ts +182 -0
  100. package/backend/lib/shared/index.ts +2 -0
  101. package/backend/lib/shared/port-utils.ts +25 -0
  102. package/backend/lib/shared/process-manager.ts +281 -0
  103. package/backend/lib/snapshot/blob-store.ts +227 -0
  104. package/backend/lib/snapshot/gitignore.ts +307 -0
  105. package/backend/lib/snapshot/helpers.ts +397 -0
  106. package/backend/lib/snapshot/snapshot-service.ts +483 -0
  107. package/backend/lib/terminal/helpers.ts +14 -0
  108. package/backend/lib/terminal/index.ts +8 -0
  109. package/backend/lib/terminal/pty-manager.ts +4 -0
  110. package/backend/lib/terminal/pty-session-manager.ts +387 -0
  111. package/backend/lib/terminal/shell-utils.ts +313 -0
  112. package/backend/lib/terminal/stream-manager.ts +293 -0
  113. package/backend/lib/tunnel/global-tunnel-manager.ts +243 -0
  114. package/backend/lib/tunnel/project-tunnel-manager.ts +311 -0
  115. package/backend/lib/user/helpers.ts +87 -0
  116. package/backend/lib/utils/ws.ts +944 -0
  117. package/backend/lib/vite-dev.ts +353 -0
  118. package/backend/middleware/cors.ts +15 -0
  119. package/backend/middleware/error-handler.ts +49 -0
  120. package/backend/middleware/logger.ts +9 -0
  121. package/backend/types/api.ts +24 -0
  122. package/backend/ws/README.md +1505 -0
  123. package/backend/ws/chat/background.ts +198 -0
  124. package/backend/ws/chat/index.ts +21 -0
  125. package/backend/ws/chat/stream.ts +707 -0
  126. package/backend/ws/engine/claude/accounts.ts +401 -0
  127. package/backend/ws/engine/claude/index.ts +13 -0
  128. package/backend/ws/engine/claude/status.ts +43 -0
  129. package/backend/ws/engine/index.ts +14 -0
  130. package/backend/ws/engine/opencode/index.ts +11 -0
  131. package/backend/ws/engine/opencode/status.ts +30 -0
  132. package/backend/ws/engine/utils.ts +36 -0
  133. package/backend/ws/files/index.ts +30 -0
  134. package/backend/ws/files/read.ts +189 -0
  135. package/backend/ws/files/search.ts +453 -0
  136. package/backend/ws/files/watch.ts +124 -0
  137. package/backend/ws/files/write.ts +143 -0
  138. package/backend/ws/git/branch.ts +106 -0
  139. package/backend/ws/git/commit.ts +39 -0
  140. package/backend/ws/git/conflict.ts +68 -0
  141. package/backend/ws/git/diff.ts +69 -0
  142. package/backend/ws/git/index.ts +24 -0
  143. package/backend/ws/git/log.ts +41 -0
  144. package/backend/ws/git/remote.ts +214 -0
  145. package/backend/ws/git/staging.ts +84 -0
  146. package/backend/ws/git/status.ts +90 -0
  147. package/backend/ws/index.ts +69 -0
  148. package/backend/ws/mcp/index.ts +61 -0
  149. package/backend/ws/messages/crud.ts +74 -0
  150. package/backend/ws/messages/index.ts +14 -0
  151. package/backend/ws/preview/browser/cleanup.ts +129 -0
  152. package/backend/ws/preview/browser/console.ts +114 -0
  153. package/backend/ws/preview/browser/interact.ts +513 -0
  154. package/backend/ws/preview/browser/mcp.ts +129 -0
  155. package/backend/ws/preview/browser/native-ui.ts +235 -0
  156. package/backend/ws/preview/browser/stats.ts +55 -0
  157. package/backend/ws/preview/browser/tab-info.ts +126 -0
  158. package/backend/ws/preview/browser/tab.ts +166 -0
  159. package/backend/ws/preview/browser/webcodecs.ts +293 -0
  160. package/backend/ws/preview/index.ts +146 -0
  161. package/backend/ws/projects/crud.ts +113 -0
  162. package/backend/ws/projects/index.ts +25 -0
  163. package/backend/ws/projects/presence.ts +46 -0
  164. package/backend/ws/projects/status.ts +116 -0
  165. package/backend/ws/sessions/crud.ts +327 -0
  166. package/backend/ws/sessions/index.ts +33 -0
  167. package/backend/ws/settings/crud.ts +112 -0
  168. package/backend/ws/settings/index.ts +14 -0
  169. package/backend/ws/snapshot/index.ts +17 -0
  170. package/backend/ws/snapshot/restore.ts +173 -0
  171. package/backend/ws/snapshot/timeline.ts +141 -0
  172. package/backend/ws/system/index.ts +14 -0
  173. package/backend/ws/system/operations.ts +49 -0
  174. package/backend/ws/terminal/index.ts +40 -0
  175. package/backend/ws/terminal/persistence.ts +153 -0
  176. package/backend/ws/terminal/session.ts +382 -0
  177. package/backend/ws/terminal/stream.ts +79 -0
  178. package/backend/ws/tunnel/index.ts +14 -0
  179. package/backend/ws/tunnel/operations.ts +91 -0
  180. package/backend/ws/types.ts +20 -0
  181. package/backend/ws/user/crud.ts +156 -0
  182. package/backend/ws/user/index.ts +14 -0
  183. package/bin/clopen.ts +307 -0
  184. package/bun.lock +1352 -0
  185. package/frontend/App.svelte +34 -0
  186. package/frontend/app.css +313 -0
  187. package/frontend/lib/app-environment.ts +10 -0
  188. package/frontend/lib/components/chat/ChatInterface.svelte +407 -0
  189. package/frontend/lib/components/chat/formatters/ErrorMessage.svelte +57 -0
  190. package/frontend/lib/components/chat/formatters/MessageFormatter.svelte +224 -0
  191. package/frontend/lib/components/chat/formatters/TextMessage.svelte +395 -0
  192. package/frontend/lib/components/chat/formatters/Tools.svelte +70 -0
  193. package/frontend/lib/components/chat/formatters/index.ts +3 -0
  194. package/frontend/lib/components/chat/input/ChatInput.svelte +421 -0
  195. package/frontend/lib/components/chat/input/components/ChatInputActions.svelte +78 -0
  196. package/frontend/lib/components/chat/input/components/DragDropOverlay.svelte +30 -0
  197. package/frontend/lib/components/chat/input/components/EditModeIndicator.svelte +33 -0
  198. package/frontend/lib/components/chat/input/components/EngineModelPicker.svelte +619 -0
  199. package/frontend/lib/components/chat/input/components/FileAttachmentPreview.svelte +48 -0
  200. package/frontend/lib/components/chat/input/components/LoadingIndicator.svelte +31 -0
  201. package/frontend/lib/components/chat/input/composables/use-animations.svelte.ts +201 -0
  202. package/frontend/lib/components/chat/input/composables/use-chat-actions.svelte.ts +148 -0
  203. package/frontend/lib/components/chat/input/composables/use-file-handling.svelte.ts +216 -0
  204. package/frontend/lib/components/chat/input/composables/use-input-state.svelte.ts +357 -0
  205. package/frontend/lib/components/chat/input/composables/use-textarea-resize.svelte.ts +57 -0
  206. package/frontend/lib/components/chat/message/ChatMessage.svelte +478 -0
  207. package/frontend/lib/components/chat/message/ChatMessages.svelte +541 -0
  208. package/frontend/lib/components/chat/message/DateSeparator.svelte +86 -0
  209. package/frontend/lib/components/chat/message/MessageBubble.svelte +86 -0
  210. package/frontend/lib/components/chat/message/MessageHeader.svelte +157 -0
  211. package/frontend/lib/components/chat/modal/DebugModal.svelte +59 -0
  212. package/frontend/lib/components/chat/modal/TokenUsageModal.svelte +124 -0
  213. package/frontend/lib/components/chat/shared/index.ts +2 -0
  214. package/frontend/lib/components/chat/shared/utils.ts +116 -0
  215. package/frontend/lib/components/chat/tools/BashOutputTool.svelte +35 -0
  216. package/frontend/lib/components/chat/tools/BashTool.svelte +46 -0
  217. package/frontend/lib/components/chat/tools/CustomMcpTool.svelte +139 -0
  218. package/frontend/lib/components/chat/tools/EditTool.svelte +48 -0
  219. package/frontend/lib/components/chat/tools/ExitPlanModeTool.svelte +32 -0
  220. package/frontend/lib/components/chat/tools/GlobTool.svelte +51 -0
  221. package/frontend/lib/components/chat/tools/GrepTool.svelte +90 -0
  222. package/frontend/lib/components/chat/tools/KillShellTool.svelte +26 -0
  223. package/frontend/lib/components/chat/tools/ListMcpResourcesTool.svelte +31 -0
  224. package/frontend/lib/components/chat/tools/NotebookEditTool.svelte +38 -0
  225. package/frontend/lib/components/chat/tools/ReadMcpResourceTool.svelte +34 -0
  226. package/frontend/lib/components/chat/tools/ReadTool.svelte +41 -0
  227. package/frontend/lib/components/chat/tools/TaskTool.svelte +64 -0
  228. package/frontend/lib/components/chat/tools/TodoWriteTool.svelte +75 -0
  229. package/frontend/lib/components/chat/tools/WebFetchTool.svelte +35 -0
  230. package/frontend/lib/components/chat/tools/WebSearchTool.svelte +84 -0
  231. package/frontend/lib/components/chat/tools/WriteTool.svelte +33 -0
  232. package/frontend/lib/components/chat/tools/components/CodeBlock.svelte +79 -0
  233. package/frontend/lib/components/chat/tools/components/DiffBlock.svelte +408 -0
  234. package/frontend/lib/components/chat/tools/components/FileHeader.svelte +45 -0
  235. package/frontend/lib/components/chat/tools/components/InfoLine.svelte +19 -0
  236. package/frontend/lib/components/chat/tools/components/StatsBadges.svelte +27 -0
  237. package/frontend/lib/components/chat/tools/components/TerminalCommand.svelte +54 -0
  238. package/frontend/lib/components/chat/tools/components/index.ts +7 -0
  239. package/frontend/lib/components/chat/tools/index.ts +26 -0
  240. package/frontend/lib/components/chat/widgets/FloatingTodoList.svelte +249 -0
  241. package/frontend/lib/components/chat/widgets/TokenUsage.svelte +78 -0
  242. package/frontend/lib/components/checkpoint/TimelineModal.svelte +391 -0
  243. package/frontend/lib/components/checkpoint/timeline/TimelineEdge.svelte +26 -0
  244. package/frontend/lib/components/checkpoint/timeline/TimelineGraph.svelte +87 -0
  245. package/frontend/lib/components/checkpoint/timeline/TimelineNode.svelte +108 -0
  246. package/frontend/lib/components/checkpoint/timeline/TimelineVersionGroup.svelte +59 -0
  247. package/frontend/lib/components/checkpoint/timeline/animation.ts +168 -0
  248. package/frontend/lib/components/checkpoint/timeline/config.ts +44 -0
  249. package/frontend/lib/components/checkpoint/timeline/graph-builder.ts +304 -0
  250. package/frontend/lib/components/checkpoint/timeline/types.ts +65 -0
  251. package/frontend/lib/components/checkpoint/timeline/utils.ts +53 -0
  252. package/frontend/lib/components/common/Alert.svelte +139 -0
  253. package/frontend/lib/components/common/AvatarBubble.svelte +56 -0
  254. package/frontend/lib/components/common/Button.svelte +71 -0
  255. package/frontend/lib/components/common/Card.svelte +102 -0
  256. package/frontend/lib/components/common/Checkbox.svelte +48 -0
  257. package/frontend/lib/components/common/Dialog.svelte +249 -0
  258. package/frontend/lib/components/common/FolderBrowser.svelte +843 -0
  259. package/frontend/lib/components/common/Icon.svelte +58 -0
  260. package/frontend/lib/components/common/Input.svelte +72 -0
  261. package/frontend/lib/components/common/Lightbox.svelte +233 -0
  262. package/frontend/lib/components/common/LoadingScreen.svelte +52 -0
  263. package/frontend/lib/components/common/LoadingSpinner.svelte +48 -0
  264. package/frontend/lib/components/common/Modal.svelte +177 -0
  265. package/frontend/lib/components/common/ModalProvider.svelte +28 -0
  266. package/frontend/lib/components/common/ModelSelector.svelte +110 -0
  267. package/frontend/lib/components/common/MonacoEditor.svelte +569 -0
  268. package/frontend/lib/components/common/NotificationToast.svelte +113 -0
  269. package/frontend/lib/components/common/PageTemplate.svelte +76 -0
  270. package/frontend/lib/components/common/ProjectUserAvatars.svelte +79 -0
  271. package/frontend/lib/components/common/Select.svelte +98 -0
  272. package/frontend/lib/components/common/Textarea.svelte +80 -0
  273. package/frontend/lib/components/common/ThemeToggle.svelte +44 -0
  274. package/frontend/lib/components/common/lucide-icons.ts +1642 -0
  275. package/frontend/lib/components/common/material-icons.ts +1082 -0
  276. package/frontend/lib/components/common/xterm/XTerm.svelte +796 -0
  277. package/frontend/lib/components/common/xterm/index.ts +16 -0
  278. package/frontend/lib/components/common/xterm/terminal-config.ts +68 -0
  279. package/frontend/lib/components/common/xterm/types.ts +31 -0
  280. package/frontend/lib/components/common/xterm/xterm-service.ts +353 -0
  281. package/frontend/lib/components/files/FileNode.svelte +384 -0
  282. package/frontend/lib/components/files/FileTree.svelte +681 -0
  283. package/frontend/lib/components/files/FileViewer.svelte +728 -0
  284. package/frontend/lib/components/files/SearchResults.svelte +303 -0
  285. package/frontend/lib/components/git/BranchManager.svelte +458 -0
  286. package/frontend/lib/components/git/ChangesSection.svelte +107 -0
  287. package/frontend/lib/components/git/CommitForm.svelte +76 -0
  288. package/frontend/lib/components/git/ConflictResolver.svelte +158 -0
  289. package/frontend/lib/components/git/DiffViewer.svelte +364 -0
  290. package/frontend/lib/components/git/FileChangeItem.svelte +97 -0
  291. package/frontend/lib/components/git/GitButton.svelte +33 -0
  292. package/frontend/lib/components/git/GitLog.svelte +361 -0
  293. package/frontend/lib/components/git/GitModal.svelte +80 -0
  294. package/frontend/lib/components/history/HistoryModal.svelte +563 -0
  295. package/frontend/lib/components/history/HistoryView.svelte +615 -0
  296. package/frontend/lib/components/index.ts +34 -0
  297. package/frontend/lib/components/preview/browser/BrowserPreview.svelte +549 -0
  298. package/frontend/lib/components/preview/browser/components/Canvas.svelte +1058 -0
  299. package/frontend/lib/components/preview/browser/components/ConsolePanel.svelte +757 -0
  300. package/frontend/lib/components/preview/browser/components/Container.svelte +450 -0
  301. package/frontend/lib/components/preview/browser/components/ContextMenu.svelte +236 -0
  302. package/frontend/lib/components/preview/browser/components/SelectDropdown.svelte +224 -0
  303. package/frontend/lib/components/preview/browser/components/Toolbar.svelte +339 -0
  304. package/frontend/lib/components/preview/browser/components/VirtualCursor.svelte +36 -0
  305. package/frontend/lib/components/preview/browser/core/cleanup.svelte.ts +155 -0
  306. package/frontend/lib/components/preview/browser/core/coordinator.svelte.ts +837 -0
  307. package/frontend/lib/components/preview/browser/core/interactions.svelte.ts +113 -0
  308. package/frontend/lib/components/preview/browser/core/mcp-handlers.svelte.ts +296 -0
  309. package/frontend/lib/components/preview/browser/core/native-ui-handlers.svelte.ts +391 -0
  310. package/frontend/lib/components/preview/browser/core/stream-handler.svelte.ts +231 -0
  311. package/frontend/lib/components/preview/browser/core/tab-manager.svelte.ts +210 -0
  312. package/frontend/lib/components/preview/browser/core/tab-operations.svelte.ts +239 -0
  313. package/frontend/lib/components/preview/index.ts +2 -0
  314. package/frontend/lib/components/settings/SettingsModal.svelte +235 -0
  315. package/frontend/lib/components/settings/SettingsView.svelte +36 -0
  316. package/frontend/lib/components/settings/appearance/AppearanceSettings.svelte +51 -0
  317. package/frontend/lib/components/settings/appearance/LayoutPresetSettings.svelte +160 -0
  318. package/frontend/lib/components/settings/appearance/LayoutPreview.svelte +76 -0
  319. package/frontend/lib/components/settings/engines/AIEnginesSettings.svelte +917 -0
  320. package/frontend/lib/components/settings/general/AdvancedSettings.svelte +187 -0
  321. package/frontend/lib/components/settings/general/DataManagementSettings.svelte +203 -0
  322. package/frontend/lib/components/settings/general/GeneralSettings.svelte +10 -0
  323. package/frontend/lib/components/settings/model/ModelSettings.svelte +357 -0
  324. package/frontend/lib/components/settings/notifications/NotificationSettings.svelte +205 -0
  325. package/frontend/lib/components/settings/user/UserSettings.svelte +197 -0
  326. package/frontend/lib/components/terminal/Terminal.svelte +368 -0
  327. package/frontend/lib/components/terminal/TerminalTabs.svelte +87 -0
  328. package/frontend/lib/components/terminal/TerminalView.svelte +55 -0
  329. package/frontend/lib/components/tunnel/TunnelActive.svelte +142 -0
  330. package/frontend/lib/components/tunnel/TunnelButton.svelte +54 -0
  331. package/frontend/lib/components/tunnel/TunnelInactive.svelte +284 -0
  332. package/frontend/lib/components/tunnel/TunnelModal.svelte +47 -0
  333. package/frontend/lib/components/tunnel/TunnelQRCode.svelte +49 -0
  334. package/frontend/lib/components/workspace/DesktopNavigator.svelte +382 -0
  335. package/frontend/lib/components/workspace/MobileNavigator.svelte +403 -0
  336. package/frontend/lib/components/workspace/PanelContainer.svelte +100 -0
  337. package/frontend/lib/components/workspace/PanelHeader.svelte +505 -0
  338. package/frontend/lib/components/workspace/ViewMenu.svelte +162 -0
  339. package/frontend/lib/components/workspace/WorkspaceLayout.svelte +169 -0
  340. package/frontend/lib/components/workspace/layout/DesktopLayout.svelte +15 -0
  341. package/frontend/lib/components/workspace/layout/MobileLayout.svelte +17 -0
  342. package/frontend/lib/components/workspace/layout/split-pane/Container.svelte +42 -0
  343. package/frontend/lib/components/workspace/layout/split-pane/Handle.svelte +85 -0
  344. package/frontend/lib/components/workspace/layout/split-pane/Layout.svelte +37 -0
  345. package/frontend/lib/components/workspace/panels/ChatPanel.svelte +274 -0
  346. package/frontend/lib/components/workspace/panels/FilesPanel.svelte +1261 -0
  347. package/frontend/lib/components/workspace/panels/GitPanel.svelte +1560 -0
  348. package/frontend/lib/components/workspace/panels/PreviewPanel.svelte +150 -0
  349. package/frontend/lib/components/workspace/panels/TerminalPanel.svelte +73 -0
  350. package/frontend/lib/constants/preview.ts +45 -0
  351. package/frontend/lib/services/chat/chat.service.ts +704 -0
  352. package/frontend/lib/services/chat/index.ts +7 -0
  353. package/frontend/lib/services/notification/global-stream-monitor.ts +86 -0
  354. package/frontend/lib/services/notification/index.ts +8 -0
  355. package/frontend/lib/services/notification/push.service.ts +144 -0
  356. package/frontend/lib/services/notification/sound.service.ts +127 -0
  357. package/frontend/lib/services/preview/browser/browser-console.service.ts +61 -0
  358. package/frontend/lib/services/preview/browser/browser-webcodecs.service.ts +1499 -0
  359. package/frontend/lib/services/preview/browser/mcp-integration.svelte.ts +67 -0
  360. package/frontend/lib/services/preview/index.ts +23 -0
  361. package/frontend/lib/services/project/index.ts +8 -0
  362. package/frontend/lib/services/project/status.service.ts +159 -0
  363. package/frontend/lib/services/snapshot/snapshot.service.ts +47 -0
  364. package/frontend/lib/services/terminal/background/index.ts +130 -0
  365. package/frontend/lib/services/terminal/background/session-restore.ts +274 -0
  366. package/frontend/lib/services/terminal/background/stream-manager.ts +286 -0
  367. package/frontend/lib/services/terminal/index.ts +14 -0
  368. package/frontend/lib/services/terminal/persistence.service.ts +260 -0
  369. package/frontend/lib/services/terminal/project.service.ts +953 -0
  370. package/frontend/lib/services/terminal/session.service.ts +364 -0
  371. package/frontend/lib/services/terminal/terminal.service.ts +369 -0
  372. package/frontend/lib/stores/core/app.svelte.ts +117 -0
  373. package/frontend/lib/stores/core/files.svelte.ts +73 -0
  374. package/frontend/lib/stores/core/presence.svelte.ts +48 -0
  375. package/frontend/lib/stores/core/projects.svelte.ts +317 -0
  376. package/frontend/lib/stores/core/sessions.svelte.ts +383 -0
  377. package/frontend/lib/stores/features/claude-accounts.svelte.ts +58 -0
  378. package/frontend/lib/stores/features/models.svelte.ts +89 -0
  379. package/frontend/lib/stores/features/settings.svelte.ts +87 -0
  380. package/frontend/lib/stores/features/terminal.svelte.ts +701 -0
  381. package/frontend/lib/stores/features/tunnel.svelte.ts +161 -0
  382. package/frontend/lib/stores/features/user.svelte.ts +96 -0
  383. package/frontend/lib/stores/ui/chat-input.svelte.ts +57 -0
  384. package/frontend/lib/stores/ui/chat-model.svelte.ts +61 -0
  385. package/frontend/lib/stores/ui/dialog.svelte.ts +59 -0
  386. package/frontend/lib/stores/ui/edit-mode.svelte.ts +214 -0
  387. package/frontend/lib/stores/ui/notification.svelte.ts +166 -0
  388. package/frontend/lib/stores/ui/settings-modal.svelte.ts +88 -0
  389. package/frontend/lib/stores/ui/theme.svelte.ts +179 -0
  390. package/frontend/lib/stores/ui/workspace.svelte.ts +754 -0
  391. package/frontend/lib/types/native-ui.ts +73 -0
  392. package/frontend/lib/utils/chat/date-separator.ts +39 -0
  393. package/frontend/lib/utils/chat/message-grouper.ts +219 -0
  394. package/frontend/lib/utils/chat/message-processor.ts +135 -0
  395. package/frontend/lib/utils/chat/tool-handler.ts +161 -0
  396. package/frontend/lib/utils/chat/virtual-scroll.svelte.ts +142 -0
  397. package/frontend/lib/utils/click-outside.ts +20 -0
  398. package/frontend/lib/utils/context-manager.ts +257 -0
  399. package/frontend/lib/utils/file-icon-mappings.ts +769 -0
  400. package/frontend/lib/utils/folder-icon-mappings.ts +1030 -0
  401. package/frontend/lib/utils/git-status.ts +68 -0
  402. package/frontend/lib/utils/platform.ts +113 -0
  403. package/frontend/lib/utils/port-check.ts +65 -0
  404. package/frontend/lib/utils/terminalFormatter.ts +207 -0
  405. package/frontend/lib/utils/theme.ts +6 -0
  406. package/frontend/lib/utils/tree-visualizer.ts +320 -0
  407. package/frontend/lib/utils/ws.ts +44 -0
  408. package/frontend/main.ts +13 -0
  409. package/index.html +70 -0
  410. package/package.json +111 -0
  411. package/scripts/generate-icons.ts +87 -0
  412. package/scripts/pre-publish-check.sh +142 -0
  413. package/scripts/setup-hooks.sh +134 -0
  414. package/scripts/validate-branch-name.sh +47 -0
  415. package/scripts/validate-commit-msg.sh +42 -0
  416. package/shared/constants/engines.ts +134 -0
  417. package/shared/types/database/connection.ts +16 -0
  418. package/shared/types/database/index.ts +6 -0
  419. package/shared/types/database/schema.ts +141 -0
  420. package/shared/types/engine/index.ts +45 -0
  421. package/shared/types/filesystem/index.ts +22 -0
  422. package/shared/types/git.ts +171 -0
  423. package/shared/types/messaging/index.ts +239 -0
  424. package/shared/types/messaging/tool.ts +526 -0
  425. package/shared/types/network/api.ts +18 -0
  426. package/shared/types/network/index.ts +5 -0
  427. package/shared/types/stores/app.ts +23 -0
  428. package/shared/types/stores/dialog.ts +21 -0
  429. package/shared/types/stores/index.ts +3 -0
  430. package/shared/types/stores/settings.ts +15 -0
  431. package/shared/types/terminal/index.ts +44 -0
  432. package/shared/types/ui/components.ts +61 -0
  433. package/shared/types/ui/icons.ts +23 -0
  434. package/shared/types/ui/index.ts +22 -0
  435. package/shared/types/ui/notifications.ts +14 -0
  436. package/shared/types/ui/theme.ts +12 -0
  437. package/shared/types/websocket/index.ts +43 -0
  438. package/shared/types/window.d.ts +13 -0
  439. package/shared/utils/anonymous-user.ts +168 -0
  440. package/shared/utils/async.ts +10 -0
  441. package/shared/utils/diff-calculator.ts +184 -0
  442. package/shared/utils/file-type-detection.ts +166 -0
  443. package/shared/utils/logger.ts +158 -0
  444. package/shared/utils/message-formatter.ts +79 -0
  445. package/shared/utils/path.ts +47 -0
  446. package/shared/utils/ws-client.ts +768 -0
  447. package/shared/utils/ws-server.ts +660 -0
  448. package/static/audio/notification.ogg +0 -0
  449. package/static/favicon.svg +8 -0
  450. package/static/fonts/dm-sans/dm-sans-italic-latin-ext.woff2 +0 -0
  451. package/static/fonts/dm-sans/dm-sans-italic-latin.woff2 +0 -0
  452. package/static/fonts/dm-sans/dm-sans-normal-latin-ext.woff2 +0 -0
  453. package/static/fonts/dm-sans/dm-sans-normal-latin.woff2 +0 -0
  454. package/static/fonts/dm-sans.css +96 -0
  455. package/svelte.config.js +20 -0
  456. package/vite.config.ts +33 -0
@@ -0,0 +1,1499 @@
1
+ /**
2
+ * Browser WebCodecs Service
3
+ *
4
+ * Client-side service for receiving WebCodecs-encoded video/audio via DataChannel.
5
+ * Achieves ultra low-latency (~20-40ms) with lower bandwidth than traditional WebRTC.
6
+ *
7
+ * Flow:
8
+ * 1. Request offer from server (headless browser creates offer with DataChannel)
9
+ * 2. Create RTCPeerConnection and set remote description (offer)
10
+ * 3. Create and send answer
11
+ * 4. Exchange ICE candidates
12
+ * 5. Receive encoded chunks via DataChannel
13
+ * 6. Decode with VideoDecoder + AudioDecoder
14
+ * 7. Render video to canvas, play audio
15
+ */
16
+
17
+ import ws from '$frontend/lib/utils/ws';
18
+ import { debug } from '$shared/utils/logger';
19
+
20
+ export interface BrowserWebCodecsStreamStats {
21
+ isConnected: boolean;
22
+ connectionState: RTCPeerConnectionState;
23
+ iceConnectionState: RTCIceConnectionState;
24
+ videoBytesReceived: number;
25
+ audioBytesReceived: number;
26
+ videoFramesReceived: number;
27
+ audioFramesReceived: number;
28
+ videoFramesDecoded: number;
29
+ audioFramesDecoded: number;
30
+ videoFramesDropped: number;
31
+ audioFramesDropped: number;
32
+ frameWidth: number;
33
+ frameHeight: number;
34
+ // Bandwidth stats
35
+ videoBitrate: number; // bits per second
36
+ audioBitrate: number; // bits per second
37
+ totalBitrate: number; // bits per second
38
+ totalBandwidthMBps: number; // MB per second
39
+ // Codec info
40
+ videoCodec: string;
41
+ audioCodec: string;
42
+ // First frame status
43
+ firstFrameRendered: boolean;
44
+ }
45
+
46
+ export class BrowserWebCodecsService {
47
+ private projectId: string; // REQUIRED for project isolation
48
+ private sessionId: string | null = null;
49
+ private peerConnection: RTCPeerConnection | null = null;
50
+ private dataChannel: RTCDataChannel | null = null;
51
+ private videoDecoder: VideoDecoder | null = null;
52
+ private audioDecoder: AudioDecoder | null = null;
53
+ private audioContext: AudioContext | null = null;
54
+ private canvas: HTMLCanvasElement | null = null;
55
+ private ctx: CanvasRenderingContext2D | null = null;
56
+ private isConnected = false;
57
+ private isCleaningUp = false;
58
+
59
+ // Frame rendering optimization with timestamp-based scheduling
60
+ private pendingFrame: VideoFrame | null = null;
61
+ private isRenderingFrame = false;
62
+ private renderFrameId: number | null = null;
63
+ private lastFrameTime = 0;
64
+ private startTime = 0;
65
+ private firstFrameTimestamp = 0; // Timestamp of first frame (for relative timing)
66
+
67
+ // AV Sync: Shared timeline reference
68
+ // When first video frame arrives, we establish: realTime = streamTimestamp mapping
69
+ private syncEstablished = false;
70
+ private syncRealTimeOrigin = 0; // performance.now() when first video frame decoded
71
+ private syncStreamTimestamp = 0; // stream timestamp of first video frame (microseconds)
72
+ private lastVideoTimestamp = 0; // Last rendered video timestamp for audio sync
73
+ private lastVideoRealTime = 0; // performance.now() when last video frame was rendered
74
+
75
+ // Audio playback scheduling
76
+ private nextAudioPlayTime = 0; // When the next audio chunk should play
77
+ private audioBufferQueue: Array<{ buffer: AudioBuffer; scheduledTime: number }> = [];
78
+ private maxAudioQueueSize = 10; // Limit queue to prevent audio lag
79
+ private audioSyncInitialized = false; // Track if audio sync is initialized
80
+ private audioCalibrationSamples = 0; // Count calibration samples for initial sync
81
+ private audioOffsetAccumulator = 0; // Accumulate offset during calibration
82
+ private readonly CALIBRATION_SAMPLES = 5; // Number of samples needed for calibration
83
+ private calibratedAudioOffset = 0; // Final calibrated offset in seconds
84
+
85
+ // Codec configuration
86
+ private videoCodecConfig: VideoDecoderConfig | null = null;
87
+ private audioCodecConfig: AudioDecoderConfig | null = null;
88
+
89
+ // Stats tracking
90
+ private stats: BrowserWebCodecsStreamStats = {
91
+ isConnected: false,
92
+ connectionState: 'new',
93
+ iceConnectionState: 'new',
94
+ videoBytesReceived: 0,
95
+ audioBytesReceived: 0,
96
+ videoFramesReceived: 0,
97
+ audioFramesReceived: 0,
98
+ videoFramesDecoded: 0,
99
+ audioFramesDecoded: 0,
100
+ videoFramesDropped: 0,
101
+ audioFramesDropped: 0,
102
+ frameWidth: 0,
103
+ frameHeight: 0,
104
+ videoBitrate: 0,
105
+ audioBitrate: 0,
106
+ totalBitrate: 0,
107
+ totalBandwidthMBps: 0,
108
+ videoCodec: 'unknown',
109
+ audioCodec: 'unknown',
110
+ firstFrameRendered: false
111
+ };
112
+
113
+ // Bandwidth tracking
114
+ private lastVideoBytesReceived = 0;
115
+ private lastAudioBytesReceived = 0;
116
+ private lastStatsTime = 0;
117
+
118
+ // ICE servers
119
+ private readonly iceServers: RTCIceServer[] = [
120
+ { urls: 'stun:stun.l.google.com:19302' },
121
+ { urls: 'stun:stun1.l.google.com:19302' }
122
+ ];
123
+
124
+ // Callbacks
125
+ private onConnectionChange: ((connected: boolean) => void) | null = null;
126
+ private onConnectionFailed: (() => void) | null = null;
127
+ private onNavigationReconnect: (() => void) | null = null; // Fast reconnection after navigation
128
+ private onReconnectingStart: (() => void) | null = null; // Signals reconnecting state started (for UI)
129
+ private onError: ((error: Error) => void) | null = null;
130
+ private onStats: ((stats: BrowserWebCodecsStreamStats) => void) | null = null;
131
+ private onCursorChange: ((cursor: string) => void) | null = null;
132
+
133
+ // Navigation state - when true, DataChannel close is expected and recovery is suppressed
134
+ private isNavigating = false;
135
+ private navigationCleanupFn: (() => void) | null = null;
136
+
137
+ // WebSocket cleanup
138
+ private wsCleanupFunctions: Array<() => void> = [];
139
+
140
+ // Stats interval
141
+ private statsIntervalId: ReturnType<typeof setInterval> | null = null;
142
+
143
+ // Bandwidth logging interval
144
+ private bandwidthLogIntervalId: ReturnType<typeof setInterval> | null = null;
145
+
146
+ constructor(projectId: string) {
147
+ if (!projectId) {
148
+ throw new Error('projectId is required for BrowserWebCodecsService');
149
+ }
150
+ this.projectId = projectId;
151
+ }
152
+
153
+ /**
154
+ * Check if WebCodecs is supported
155
+ */
156
+ static isSupported(): boolean {
157
+ return (
158
+ typeof VideoDecoder !== 'undefined' &&
159
+ typeof AudioDecoder !== 'undefined' &&
160
+ typeof RTCPeerConnection !== 'undefined'
161
+ );
162
+ }
163
+
164
+ /**
165
+ * Start WebCodecs streaming for a preview session
166
+ */
167
+ async startStreaming(sessionId: string, canvas: HTMLCanvasElement): Promise<boolean> {
168
+ debug.log('webcodecs', `Starting streaming for session: ${sessionId}`);
169
+
170
+ if (!BrowserWebCodecsService.isSupported()) {
171
+ debug.error('webcodecs', 'Not supported in this browser');
172
+ if (this.onError) {
173
+ this.onError(new Error('WebCodecs not supported'));
174
+ }
175
+ return false;
176
+ }
177
+
178
+ // Clean up any existing connection
179
+ if (this.peerConnection || this.isConnected || this.sessionId) {
180
+ debug.log('webcodecs', 'Cleaning up previous connection');
181
+ await this.cleanupConnection();
182
+ }
183
+
184
+ this.isCleaningUp = false;
185
+ this.stats.firstFrameRendered = false;
186
+
187
+ this.sessionId = sessionId;
188
+ this.canvas = canvas;
189
+ this.ctx = canvas.getContext('2d', {
190
+ alpha: false,
191
+ desynchronized: true,
192
+ willReadFrequently: false
193
+ });
194
+
195
+ if (this.ctx) {
196
+ this.ctx.imageSmoothingEnabled = true;
197
+ this.ctx.imageSmoothingQuality = 'low';
198
+ }
199
+
200
+ this.clearCanvas();
201
+
202
+ try {
203
+ // Setup WebSocket listeners
204
+ this.setupEventListeners();
205
+
206
+ // Request server to start streaming and get offer
207
+ const response = await ws.http('preview:browser-stream-start', {}, 30000);
208
+
209
+ if (!response.success) {
210
+ throw new Error(response.message || 'Failed to start streaming');
211
+ }
212
+
213
+ // Create peer connection
214
+ await this.createPeerConnection();
215
+
216
+ // Set remote description (offer)
217
+ if (response.offer) {
218
+ await this.handleOffer({
219
+ type: response.offer.type as RTCSdpType,
220
+ sdp: response.offer.sdp
221
+ });
222
+ } else {
223
+ const offerResponse = await ws.http('preview:browser-stream-offer', {}, 10000);
224
+ if (offerResponse.offer) {
225
+ await this.handleOffer({
226
+ type: offerResponse.offer.type as RTCSdpType,
227
+ sdp: offerResponse.offer.sdp
228
+ });
229
+ } else {
230
+ throw new Error('No offer received from server');
231
+ }
232
+ }
233
+
234
+ debug.log('webcodecs', 'Streaming setup complete');
235
+ return true;
236
+ } catch (error) {
237
+ debug.error('webcodecs', 'Failed to start streaming:', error);
238
+
239
+ if (this.onError) {
240
+ this.onError(error instanceof Error ? error : new Error(String(error)));
241
+ }
242
+
243
+ await this.cleanup();
244
+ return false;
245
+ }
246
+ }
247
+
248
+ /**
249
+ * Create RTCPeerConnection
250
+ */
251
+ private async createPeerConnection(): Promise<void> {
252
+ const config: RTCConfiguration = {
253
+ iceServers: this.iceServers,
254
+ bundlePolicy: 'max-bundle',
255
+ rtcpMuxPolicy: 'require',
256
+ iceCandidatePoolSize: 0
257
+ };
258
+
259
+ this.peerConnection = new RTCPeerConnection(config);
260
+
261
+ // Handle DataChannel
262
+ this.peerConnection.ondatachannel = (event) => {
263
+ debug.log('webcodecs', 'DataChannel received');
264
+ this.dataChannel = event.channel;
265
+ this.dataChannel.binaryType = 'arraybuffer';
266
+
267
+ this.dataChannel.onopen = () => {
268
+ debug.log('webcodecs', 'DataChannel open');
269
+ };
270
+
271
+ this.dataChannel.onclose = () => {
272
+ debug.log('webcodecs', 'DataChannel closed');
273
+ // Trigger recovery if channel closed while we were connected
274
+ // BUT skip recovery if we're navigating (expected behavior during page navigation)
275
+ if (this.isConnected && !this.isCleaningUp && !this.isNavigating && this.onConnectionFailed) {
276
+ debug.warn('webcodecs', 'DataChannel closed unexpectedly - triggering recovery');
277
+ this.onConnectionFailed();
278
+ } else if (this.isNavigating) {
279
+ debug.log('webcodecs', 'DataChannel closed during navigation - scheduling fast reconnect');
280
+ // Signal reconnecting state IMMEDIATELY (for UI - keeps progress bar showing)
281
+ if (this.onReconnectingStart) {
282
+ this.onReconnectingStart();
283
+ }
284
+ // Backend needs ~500ms to restart streaming with new peer
285
+ // Wait for backend to be ready, then trigger FAST reconnection (no delay)
286
+ setTimeout(() => {
287
+ if (this.isCleaningUp) return;
288
+ debug.log('webcodecs', '🔄 Triggering fast reconnection after navigation');
289
+ this.isNavigating = false; // Reset navigation state
290
+ // Use dedicated navigation reconnect handler (fast path, no delay)
291
+ if (this.onNavigationReconnect) {
292
+ this.onNavigationReconnect();
293
+ } else if (this.onConnectionFailed) {
294
+ // Fallback to regular recovery if no navigation handler
295
+ this.onConnectionFailed();
296
+ }
297
+ }, 700); // Wait 700ms for backend to restart (usually takes ~500ms)
298
+ }
299
+ };
300
+
301
+ this.dataChannel.onerror = (error) => {
302
+ debug.error('webcodecs', 'DataChannel error:', error);
303
+ debug.log('webcodecs', `DataChannel error state: isConnected=${this.isConnected}, isCleaningUp=${this.isCleaningUp}, isNavigating=${this.isNavigating}`);
304
+ // Trigger recovery on DataChannel error
305
+ // BUT skip recovery if we're navigating (expected behavior during page navigation)
306
+ if (this.isConnected && !this.isCleaningUp && !this.isNavigating && this.onConnectionFailed) {
307
+ debug.warn('webcodecs', 'DataChannel error - triggering recovery');
308
+ this.onConnectionFailed();
309
+ } else if (this.isNavigating) {
310
+ debug.log('webcodecs', '🛡️ DataChannel error during navigation - recovery suppressed');
311
+ }
312
+ };
313
+
314
+ this.dataChannel.onmessage = (event) => {
315
+ this.handleDataChannelMessage(event.data);
316
+ };
317
+ };
318
+
319
+ // Handle ICE candidates
320
+ this.peerConnection.onicecandidate = (event) => {
321
+ if (event.candidate && this.sessionId) {
322
+ // Backend uses active tab automatically
323
+ ws.http('preview:browser-stream-ice', {
324
+ candidate: {
325
+ candidate: event.candidate.candidate,
326
+ sdpMid: event.candidate.sdpMid,
327
+ sdpMLineIndex: event.candidate.sdpMLineIndex
328
+ }
329
+ }).catch((error) => {
330
+ debug.warn('webcodecs', 'Failed to send ICE candidate:', error);
331
+ });
332
+ }
333
+ };
334
+
335
+ // Handle connection state
336
+ this.peerConnection.onconnectionstatechange = () => {
337
+ const state = this.peerConnection?.connectionState || 'closed';
338
+ debug.log('webcodecs', `Connection state: ${state}`);
339
+
340
+ this.stats.connectionState = state as RTCPeerConnectionState;
341
+
342
+ if (state === 'connected') {
343
+ this.isConnected = true;
344
+ this.stats.isConnected = true;
345
+ this.startStatsCollection();
346
+ // this.startBandwidthLogging();
347
+ if (this.onConnectionChange) {
348
+ this.onConnectionChange(true);
349
+ }
350
+ } else if (state === 'failed') {
351
+ debug.error('webcodecs', 'Connection FAILED');
352
+ this.isConnected = false;
353
+ this.stats.isConnected = false;
354
+ this.stopStatsCollection();
355
+ this.stopBandwidthLogging();
356
+ if (this.onConnectionChange) {
357
+ this.onConnectionChange(false);
358
+ }
359
+ if (this.onConnectionFailed) {
360
+ this.onConnectionFailed();
361
+ }
362
+ } else if (state === 'disconnected' || state === 'closed') {
363
+ this.isConnected = false;
364
+ this.stats.isConnected = false;
365
+ this.stopStatsCollection();
366
+ this.stopBandwidthLogging();
367
+ if (this.onConnectionChange) {
368
+ this.onConnectionChange(false);
369
+ }
370
+ }
371
+ };
372
+
373
+ // Handle ICE connection state
374
+ this.peerConnection.oniceconnectionstatechange = () => {
375
+ const state = this.peerConnection?.iceConnectionState || 'closed';
376
+ debug.log('webcodecs', `ICE connection state: ${state}`);
377
+ this.stats.iceConnectionState = state as RTCIceConnectionState;
378
+
379
+ if (state === 'failed') {
380
+ debug.error('webcodecs', 'ICE connection FAILED');
381
+ if (this.onConnectionFailed) {
382
+ this.onConnectionFailed();
383
+ }
384
+ }
385
+ };
386
+ }
387
+
388
+ /**
389
+ * Handle offer from headless browser
390
+ */
391
+ private async handleOffer(offer: RTCSessionDescriptionInit): Promise<void> {
392
+ if (!this.peerConnection) {
393
+ throw new Error('Peer connection not initialized');
394
+ }
395
+
396
+ await this.peerConnection.setRemoteDescription(new RTCSessionDescription(offer));
397
+ debug.log('webcodecs', 'Remote description set');
398
+
399
+ const answer = await this.peerConnection.createAnswer();
400
+ await this.peerConnection.setLocalDescription(answer);
401
+ debug.log('webcodecs', 'Local description set');
402
+
403
+ if (this.sessionId) {
404
+ // Backend uses active tab automatically
405
+ await ws.http('preview:browser-stream-answer', {
406
+ answer: {
407
+ type: answer.type,
408
+ sdp: answer.sdp
409
+ }
410
+ });
411
+ }
412
+ }
413
+
414
+ /**
415
+ * Handle DataChannel message (encoded video/audio chunks)
416
+ */
417
+ private handleDataChannelMessage(data: ArrayBuffer): void {
418
+ try {
419
+ const view = new DataView(data);
420
+
421
+ // Parse packet header
422
+ const type = view.getUint8(0); // 0 = video, 1 = audio
423
+
424
+ if (type === 0) {
425
+ // Video packet
426
+ // Format: [type(1)][timestamp(8)][keyframe(1)][size(4)][data]
427
+ const timestamp = Number(view.getBigUint64(1, true));
428
+ const isKeyframe = view.getUint8(9) === 1;
429
+ const size = view.getUint32(10, true);
430
+ const chunkData = new Uint8Array(data, 14, size);
431
+
432
+ this.stats.videoBytesReceived += size;
433
+ this.stats.videoFramesReceived++;
434
+
435
+ this.handleVideoChunk(chunkData, timestamp, isKeyframe);
436
+ } else if (type === 1) {
437
+ // Audio packet
438
+ // Format: [type(1)][timestamp(8)][size(4)][data]
439
+ const timestamp = Number(view.getBigUint64(1, true));
440
+ const size = view.getUint32(9, true);
441
+ const chunkData = new Uint8Array(data, 13, size);
442
+
443
+ this.stats.audioBytesReceived += size;
444
+ this.stats.audioFramesReceived++;
445
+
446
+ this.handleAudioChunk(chunkData, timestamp);
447
+ }
448
+ } catch (error) {
449
+ debug.error('webcodecs', 'DataChannel message parse error:', error);
450
+ }
451
+ }
452
+
453
+ /**
454
+ * Handle video chunk - decode and render
455
+ */
456
+ private async handleVideoChunk(data: Uint8Array, timestamp: number, isKeyframe: boolean): Promise<void> {
457
+ // Initialize decoder on first keyframe
458
+ if (!this.videoDecoder && isKeyframe) {
459
+ await this.initVideoDecoder(data);
460
+ }
461
+
462
+ if (!this.videoDecoder) {
463
+ this.stats.videoFramesDropped++;
464
+ return;
465
+ }
466
+
467
+ try {
468
+ const chunk = new EncodedVideoChunk({
469
+ type: isKeyframe ? 'key' : 'delta',
470
+ timestamp,
471
+ data
472
+ });
473
+
474
+ this.videoDecoder.decode(chunk);
475
+ } catch (error) {
476
+ debug.error('webcodecs', 'Video decode error:', error);
477
+ this.stats.videoFramesDropped++;
478
+ }
479
+ }
480
+
481
+ /**
482
+ * Handle audio chunk - decode and queue for playback
483
+ */
484
+ private async handleAudioChunk(data: Uint8Array, timestamp: number): Promise<void> {
485
+ // Initialize decoder on first chunk
486
+ if (!this.audioDecoder) {
487
+ await this.initAudioDecoder();
488
+ }
489
+
490
+ if (!this.audioDecoder) {
491
+ this.stats.audioFramesDropped++;
492
+ return;
493
+ }
494
+
495
+ try {
496
+ const chunk = new EncodedAudioChunk({
497
+ type: 'key', // Opus frames are all keyframes
498
+ timestamp,
499
+ data
500
+ });
501
+
502
+ this.audioDecoder.decode(chunk);
503
+ } catch (error) {
504
+ debug.error('webcodecs', 'Audio decode error:', error);
505
+ this.stats.audioFramesDropped++;
506
+ }
507
+ }
508
+
509
+ /**
510
+ * Initialize VideoDecoder
511
+ */
512
+ private async initVideoDecoder(firstChunkData: Uint8Array): Promise<void> {
513
+ // Use VP8 codec
514
+ const codec = 'vp8';
515
+ this.stats.videoCodec = 'vp8';
516
+
517
+ this.videoCodecConfig = {
518
+ codec,
519
+ optimizeForLatency: true
520
+ };
521
+
522
+ try {
523
+ const support = await VideoDecoder.isConfigSupported(this.videoCodecConfig);
524
+ if (!support.supported) {
525
+ throw new Error(`Video codec ${codec} not supported`);
526
+ }
527
+
528
+ this.videoDecoder = new VideoDecoder({
529
+ output: (frame) => this.handleDecodedVideoFrame(frame),
530
+ error: (e) => {
531
+ debug.error('webcodecs', 'VideoDecoder error:', e);
532
+ }
533
+ });
534
+
535
+ this.videoDecoder.configure(this.videoCodecConfig);
536
+ debug.log('webcodecs', 'VideoDecoder initialized:', codec);
537
+ } catch (error) {
538
+ debug.error('webcodecs', 'VideoDecoder init error:', error);
539
+ throw error;
540
+ }
541
+ }
542
+
543
+ /**
544
+ * Initialize AudioDecoder
545
+ */
546
+ private async initAudioDecoder(): Promise<void> {
547
+ this.audioCodecConfig = {
548
+ codec: 'opus',
549
+ sampleRate: 44100,
550
+ numberOfChannels: 2
551
+ };
552
+
553
+ try {
554
+ const support = await AudioDecoder.isConfigSupported(this.audioCodecConfig);
555
+ if (!support.supported) {
556
+ throw new Error('Opus audio codec not supported');
557
+ }
558
+
559
+ this.audioDecoder = new AudioDecoder({
560
+ output: (frame) => this.handleDecodedAudioFrame(frame),
561
+ error: (e) => {
562
+ debug.error('webcodecs', 'AudioDecoder error:', e);
563
+ }
564
+ });
565
+
566
+ this.audioDecoder.configure(this.audioCodecConfig);
567
+ this.stats.audioCodec = 'opus';
568
+
569
+ // Initialize AudioContext for playback
570
+ await this.initAudioContext();
571
+
572
+ debug.log('webcodecs', 'AudioDecoder initialized: opus');
573
+ } catch (error) {
574
+ debug.error('webcodecs', 'AudioDecoder init error:', error);
575
+ }
576
+ }
577
+
578
+ /**
579
+ * Initialize AudioContext for audio playback
580
+ */
581
+ private async initAudioContext(): Promise<void> {
582
+ try {
583
+ this.audioContext = new AudioContext({ sampleRate: 44100 });
584
+ debug.log('webcodecs', 'AudioContext initialized');
585
+ } catch (error) {
586
+ debug.error('webcodecs', 'AudioContext init error:', error);
587
+ }
588
+ }
589
+
590
+ /**
591
+ * Handle decoded video frame - render to canvas with timestamp-based optimization
592
+ */
593
+ private handleDecodedVideoFrame(frame: VideoFrame): void {
594
+ if (this.isCleaningUp || !this.canvas || !this.ctx) {
595
+ frame.close();
596
+ return;
597
+ }
598
+
599
+ try {
600
+ // Update stats
601
+ this.stats.videoFramesDecoded++;
602
+ this.stats.frameWidth = frame.displayWidth;
603
+ this.stats.frameHeight = frame.displayHeight;
604
+
605
+ // Establish AV sync reference on first video frame
606
+ if (!this.syncEstablished) {
607
+ this.syncEstablished = true;
608
+ this.syncRealTimeOrigin = performance.now();
609
+ this.syncStreamTimestamp = frame.timestamp;
610
+ debug.log('webcodecs', `AV Sync established: realTime=${this.syncRealTimeOrigin.toFixed(0)}ms, streamTs=${this.syncStreamTimestamp}μs`);
611
+ }
612
+
613
+ // Track last video timestamp and real time for audio sync
614
+ this.lastVideoTimestamp = frame.timestamp;
615
+ this.lastVideoRealTime = performance.now();
616
+
617
+ // Mark first frame rendered
618
+ if (!this.stats.firstFrameRendered) {
619
+ this.stats.firstFrameRendered = true;
620
+ this.startTime = performance.now();
621
+ this.firstFrameTimestamp = frame.timestamp;
622
+ debug.log('webcodecs', 'First video frame rendered');
623
+
624
+ // Reset navigation state - frames are flowing, navigation is complete
625
+ if (this.isNavigating) {
626
+ debug.log('webcodecs', 'Navigation complete - frames received, resetting navigation state');
627
+ this.isNavigating = false;
628
+ }
629
+ }
630
+
631
+ // Drop old pending frame if exists (frame skipping for performance)
632
+ if (this.pendingFrame) {
633
+ this.pendingFrame.close();
634
+ this.stats.videoFramesDropped++;
635
+ }
636
+
637
+ // Store frame for next render cycle
638
+ this.pendingFrame = frame;
639
+
640
+ // Schedule render if not already scheduled
641
+ if (!this.isRenderingFrame) {
642
+ this.scheduleFrameRender();
643
+ }
644
+ } catch (error) {
645
+ debug.error('webcodecs', 'Video frame handle error:', error);
646
+ frame.close();
647
+ }
648
+ }
649
+
650
+ /**
651
+ * Schedule frame rendering using requestAnimationFrame with timestamp awareness
652
+ * This mimics requestVideoFrameCallback behavior for optimal video rendering
653
+ */
654
+ private scheduleFrameRender(): void {
655
+ if (this.isRenderingFrame) return;
656
+
657
+ this.isRenderingFrame = true;
658
+ this.renderFrameId = requestAnimationFrame((timestamp) => {
659
+ this.renderPendingFrame(timestamp);
660
+ });
661
+ }
662
+
663
+ /**
664
+ * Render pending frame to canvas with timestamp-based scheduling
665
+ * This approach provides similar benefits to requestVideoFrameCallback
666
+ */
667
+ private renderPendingFrame(timestamp: DOMHighResTimeStamp): void {
668
+ this.isRenderingFrame = false;
669
+
670
+ if (!this.pendingFrame || this.isCleaningUp || !this.canvas || !this.ctx) {
671
+ return;
672
+ }
673
+
674
+ try {
675
+ // Calculate frame timing (similar to requestVideoFrameCallback metadata)
676
+ // Use relative timestamp from first frame to sync with frontend clock
677
+ const frameTimestamp = (this.pendingFrame.timestamp - this.firstFrameTimestamp) / 1000; // convert to ms
678
+ const elapsedTime = timestamp - this.startTime;
679
+
680
+ // Check if we're ahead of schedule (frame came too early)
681
+ // If so, we might want to delay rendering, but for low-latency
682
+ // we render immediately to minimize lag
683
+ const timeDrift = elapsedTime - frameTimestamp;
684
+
685
+ // Only log significant drift (for debugging)
686
+ if (Math.abs(timeDrift) > 100) {
687
+ // Drift more than 100ms is significant
688
+ debug.warn('webcodecs', `Frame timing drift: ${timeDrift.toFixed(0)}ms`);
689
+ }
690
+
691
+ // Render to canvas with optimal settings
692
+ this.ctx.drawImage(this.pendingFrame, 0, 0, this.canvas.width, this.canvas.height);
693
+
694
+ this.lastFrameTime = timestamp;
695
+ } catch (error) {
696
+ debug.error('webcodecs', 'Video frame render error:', error);
697
+ } finally {
698
+ // Close frame immediately to free memory
699
+ this.pendingFrame.close();
700
+ this.pendingFrame = null;
701
+ }
702
+ }
703
+
704
+ /**
705
+ * Handle decoded audio frame - play audio
706
+ */
707
+ private handleDecodedAudioFrame(frame: AudioData): void {
708
+ if (this.isCleaningUp || !this.audioContext) {
709
+ frame.close();
710
+ return;
711
+ }
712
+
713
+ try {
714
+ this.stats.audioFramesDecoded++;
715
+
716
+ // Simple playback: convert AudioData to AudioBuffer and play
717
+ // For production, use AudioWorklet for better latency
718
+ this.playAudioFrame(frame);
719
+ } catch (error) {
720
+ debug.error('webcodecs', 'Audio frame playback error:', error);
721
+ } finally {
722
+ frame.close();
723
+ }
724
+ }
725
+
726
+ /**
727
+ * Play audio frame with proper AV synchronization
728
+ *
729
+ * The key insight: Audio and video timestamps from the server use the same
730
+ * performance.now() origin. However, audio may start LATER than video if the
731
+ * page has no audio initially (silence is skipped).
732
+ *
733
+ * Solution: Use lastVideoTimestamp (currently rendered video) as the reference
734
+ * point, not the first video frame timestamp. This ensures we synchronize
735
+ * audio to the CURRENT video position, not the initial position.
736
+ */
737
+ private playAudioFrame(audioData: AudioData): void {
738
+ if (!this.audioContext) return;
739
+
740
+ try {
741
+ // Create AudioBuffer
742
+ const buffer = this.audioContext.createBuffer(
743
+ audioData.numberOfChannels,
744
+ audioData.numberOfFrames,
745
+ audioData.sampleRate
746
+ );
747
+
748
+ // Copy audio data to buffer
749
+ for (let channel = 0; channel < audioData.numberOfChannels; channel++) {
750
+ const options = {
751
+ planeIndex: channel,
752
+ frameOffset: 0,
753
+ frameCount: audioData.numberOfFrames,
754
+ format: 'f32-planar' as AudioSampleFormat
755
+ };
756
+
757
+ const requiredSize = audioData.allocationSize(options);
758
+ const tempBuffer = new ArrayBuffer(requiredSize);
759
+ const tempFloat32 = new Float32Array(tempBuffer);
760
+ audioData.copyTo(tempFloat32, options);
761
+
762
+ const channelData = buffer.getChannelData(channel);
763
+ channelData.set(tempFloat32);
764
+ }
765
+
766
+ const currentTime = this.audioContext.currentTime;
767
+ const audioTimestamp = audioData.timestamp; // microseconds
768
+ const bufferDuration = buffer.duration;
769
+ const now = performance.now();
770
+
771
+ // Wait for video to establish sync before playing audio
772
+ if (!this.syncEstablished || this.lastVideoTimestamp === 0) {
773
+ // No video yet - skip this audio frame to prevent desync
774
+ return;
775
+ }
776
+
777
+ // Phase 1: Calibration - collect samples to determine stable offset
778
+ if (this.audioCalibrationSamples < this.CALIBRATION_SAMPLES) {
779
+ // Calculate the offset between audio timestamp and video timeline
780
+ // audioVideoOffset > 0 means audio is AHEAD of video in stream time
781
+ // audioVideoOffset < 0 means audio is BEHIND video in stream time
782
+ const audioVideoOffset = (audioTimestamp - this.lastVideoTimestamp) / 1000000; // seconds
783
+
784
+ // Also account for the time elapsed since video was rendered
785
+ const timeSinceVideoRender = (now - this.lastVideoRealTime) / 1000; // seconds
786
+
787
+ // Expected audio position relative to current video position
788
+ // If audio and video are in sync, audio should play at:
789
+ // currentTime + audioVideoOffset - timeSinceVideoRender
790
+ const expectedOffset = audioVideoOffset - timeSinceVideoRender;
791
+
792
+ this.audioOffsetAccumulator += expectedOffset;
793
+ this.audioCalibrationSamples++;
794
+
795
+ if (this.audioCalibrationSamples === this.CALIBRATION_SAMPLES) {
796
+ // Calibration complete - calculate average offset
797
+ this.calibratedAudioOffset = this.audioOffsetAccumulator / this.CALIBRATION_SAMPLES;
798
+
799
+ // Clamp the offset to reasonable bounds (-500ms to +500ms)
800
+ // Beyond this, something is wrong and we should just play immediately
801
+ if (this.calibratedAudioOffset < -0.5) {
802
+ debug.warn('webcodecs', `Audio calibration: offset ${(this.calibratedAudioOffset * 1000).toFixed(0)}ms too negative, clamping to -500ms`);
803
+ this.calibratedAudioOffset = -0.5;
804
+ } else if (this.calibratedAudioOffset > 0.5) {
805
+ debug.warn('webcodecs', `Audio calibration: offset ${(this.calibratedAudioOffset * 1000).toFixed(0)}ms too positive, clamping to +500ms`);
806
+ this.calibratedAudioOffset = 0.5;
807
+ }
808
+
809
+ // Initialize nextAudioPlayTime based on calibrated offset
810
+ // Add small buffer (30ms) for smooth playback
811
+ this.nextAudioPlayTime = currentTime + Math.max(0.03, this.calibratedAudioOffset + 0.03);
812
+ this.audioSyncInitialized = true;
813
+
814
+ debug.log('webcodecs', `Audio calibration complete: offset=${(this.calibratedAudioOffset * 1000).toFixed(1)}ms, startTime=${(this.nextAudioPlayTime - currentTime).toFixed(3)}s from now`);
815
+ } else {
816
+ // Still calibrating - skip this audio frame
817
+ return;
818
+ }
819
+ }
820
+
821
+ // Phase 2: Synchronized playback with drift correction
822
+ let targetPlayTime = this.nextAudioPlayTime;
823
+
824
+ // If we've fallen too far behind (buffer underrun), reset
825
+ if (targetPlayTime < currentTime - 0.01) {
826
+ // We're behind by more than 10ms, need to catch up
827
+ targetPlayTime = currentTime + 0.02; // Small buffer to recover
828
+ debug.warn('webcodecs', 'Audio buffer underrun, resetting playback');
829
+ }
830
+
831
+ // Periodic drift check (every ~500ms worth of audio, ~25 frames)
832
+ if (this.stats.audioFramesDecoded % 25 === 0) {
833
+ // Recalculate where audio SHOULD be based on current video position
834
+ const audioVideoOffset = (audioTimestamp - this.lastVideoTimestamp) / 1000000;
835
+ const timeSinceVideoRender = (now - this.lastVideoRealTime) / 1000;
836
+ const expectedOffset = audioVideoOffset - timeSinceVideoRender;
837
+
838
+ // Where audio should play relative to currentTime
839
+ const idealTime = currentTime + expectedOffset + 0.03; // +30ms buffer
840
+
841
+ const drift = targetPlayTime - idealTime;
842
+
843
+ // Apply correction based on drift magnitude
844
+ if (Math.abs(drift) > 0.2) {
845
+ // Large drift (>200ms) - aggressive correction (80%)
846
+ targetPlayTime -= drift * 0.8;
847
+ debug.warn('webcodecs', `Large audio drift: ${(drift * 1000).toFixed(0)}ms, aggressive correction`);
848
+ } else if (Math.abs(drift) > 0.05) {
849
+ // Medium drift (50-200ms) - moderate correction (40%)
850
+ targetPlayTime -= drift * 0.4;
851
+ }
852
+ // Small drift (<50ms) - no correction, continuous playback handles it
853
+ }
854
+
855
+ // Ensure we don't schedule in the past
856
+ if (targetPlayTime < currentTime + 0.005) {
857
+ targetPlayTime = currentTime + 0.005;
858
+ }
859
+
860
+ // Schedule this buffer
861
+ const source = this.audioContext.createBufferSource();
862
+ source.buffer = buffer;
863
+ source.connect(this.audioContext.destination);
864
+ source.start(targetPlayTime);
865
+
866
+ // Track scheduled buffer
867
+ this.audioBufferQueue.push({ buffer, scheduledTime: targetPlayTime });
868
+
869
+ // Update next play time for continuous scheduling (back-to-back)
870
+ this.nextAudioPlayTime = targetPlayTime + bufferDuration;
871
+
872
+ // Limit queue size to prevent memory buildup
873
+ if (this.audioBufferQueue.length > this.maxAudioQueueSize) {
874
+ this.audioBufferQueue.shift();
875
+ }
876
+
877
+ // Cleanup old scheduled buffers
878
+ source.onended = () => {
879
+ const index = this.audioBufferQueue.findIndex(item => item.buffer === buffer);
880
+ if (index !== -1) {
881
+ this.audioBufferQueue.splice(index, 1);
882
+ }
883
+ };
884
+ } catch (error) {
885
+ debug.warn('webcodecs', 'Audio playback error:', error);
886
+ }
887
+ }
888
+
889
+ /**
890
+ * Setup WebSocket event listeners
891
+ */
892
+ private setupEventListeners(): void {
893
+ const cleanupIce = ws.on('preview:browser-stream-ice', async (data) => {
894
+ if (data.sessionId === this.sessionId && data.from === 'headless') {
895
+ await this.addIceCandidate(data.candidate);
896
+ }
897
+ });
898
+
899
+ const cleanupState = ws.on('preview:browser-stream-state', (data) => {
900
+ if (data.sessionId === this.sessionId) {
901
+ debug.log('webcodecs', `Server connection state: ${data.state}`);
902
+ }
903
+ });
904
+
905
+ const cleanupCursor = ws.on('preview:browser-cursor-change', (data) => {
906
+ if (data.sessionId === this.sessionId && this.onCursorChange) {
907
+ this.onCursorChange(data.cursor);
908
+ }
909
+ });
910
+
911
+ // Listen for navigation events DIRECTLY to set isNavigating flag immediately
912
+ // This bypasses Svelte's reactive chain which can be too slow
913
+ debug.log('webcodecs', `🔧 Registering navigation listener for session: ${this.sessionId}`);
914
+ const cleanupNavLoading = ws.on('preview:browser-navigation-loading', (data) => {
915
+ debug.log('webcodecs', `📡 Navigation-loading WS event: eventSessionId=${data.sessionId}, mySessionId=${this.sessionId}`);
916
+ // Set isNavigating regardless of sessionId match to ensure recovery is suppressed
917
+ // Multiple tabs scenario: only suppress for current tab, but log all
918
+ if (data.sessionId === this.sessionId) {
919
+ this.isNavigating = true;
920
+ debug.log('webcodecs', `✅ Navigation started - isNavigating=true for session ${data.sessionId}`);
921
+ }
922
+ });
923
+
924
+ const cleanupNavComplete = ws.on('preview:browser-navigation', (data) => {
925
+ if (data.sessionId === this.sessionId) {
926
+ // Keep isNavigating true for a short period to allow reconnection
927
+ // Will be reset when new frames arrive or reconnection completes
928
+ debug.log('webcodecs', `Navigation completed (direct WS) for session ${data.sessionId}`);
929
+ // Signal reconnecting state IMMEDIATELY when navigation completes
930
+ // This eliminates the gap between isNavigating=false and DataChannel close
931
+ // ensuring the overlay stays visible continuously
932
+ if (this.onReconnectingStart) {
933
+ debug.log('webcodecs', '🔄 Pre-emptive reconnecting state on navigation complete');
934
+ this.onReconnectingStart();
935
+ }
936
+ }
937
+ });
938
+
939
+ this.wsCleanupFunctions = [cleanupIce, cleanupState, cleanupCursor, cleanupNavLoading, cleanupNavComplete];
940
+ }
941
+
942
+ /**
943
+ * Add ICE candidate
944
+ */
945
+ private async addIceCandidate(candidate: RTCIceCandidateInit): Promise<void> {
946
+ if (!this.peerConnection) return;
947
+
948
+ try {
949
+ await this.peerConnection.addIceCandidate(new RTCIceCandidate(candidate));
950
+ } catch (error) {
951
+ debug.warn('webcodecs', 'Add ICE candidate error:', error);
952
+ }
953
+ }
954
+
955
+ /**
956
+ * Start stats collection
957
+ */
958
+ private startStatsCollection(): void {
959
+ if (this.statsIntervalId) return;
960
+
961
+ this.statsIntervalId = setInterval(async () => {
962
+ await this.collectStats();
963
+ }, 1000);
964
+ }
965
+
966
+ /**
967
+ * Stop stats collection
968
+ */
969
+ private stopStatsCollection(): void {
970
+ if (this.statsIntervalId) {
971
+ clearInterval(this.statsIntervalId);
972
+ this.statsIntervalId = null;
973
+ }
974
+ }
975
+
976
+ /**
977
+ * Collect stats
978
+ */
979
+ private async collectStats(): Promise<void> {
980
+ const now = performance.now();
981
+
982
+ // Calculate bandwidth if we have previous measurements
983
+ if (this.lastStatsTime > 0) {
984
+ const timeDelta = (now - this.lastStatsTime) / 1000; // seconds
985
+
986
+ // Video bitrate
987
+ const videoBytesReceived = this.stats.videoBytesReceived - this.lastVideoBytesReceived;
988
+ this.stats.videoBitrate = (videoBytesReceived * 8) / timeDelta;
989
+
990
+ // Audio bitrate
991
+ const audioBytesReceived = this.stats.audioBytesReceived - this.lastAudioBytesReceived;
992
+ this.stats.audioBitrate = (audioBytesReceived * 8) / timeDelta;
993
+
994
+ // Total
995
+ this.stats.totalBitrate = this.stats.videoBitrate + this.stats.audioBitrate;
996
+ this.stats.totalBandwidthMBps = this.stats.totalBitrate / 8 / 1024 / 1024;
997
+ }
998
+
999
+ this.lastVideoBytesReceived = this.stats.videoBytesReceived;
1000
+ this.lastAudioBytesReceived = this.stats.audioBytesReceived;
1001
+ this.lastStatsTime = now;
1002
+
1003
+ if (this.onStats) {
1004
+ this.onStats(this.stats);
1005
+ }
1006
+ }
1007
+
1008
+ /**
1009
+ * Start bandwidth logging
1010
+ */
1011
+ private startBandwidthLogging(): void {
1012
+ if (this.bandwidthLogIntervalId) return;
1013
+
1014
+ this.bandwidthLogIntervalId = setInterval(() => {
1015
+ this.logBandwidthStats();
1016
+ }, 1000); // Log every 5 seconds
1017
+ }
1018
+
1019
+ /**
1020
+ * Stop bandwidth logging
1021
+ */
1022
+ private stopBandwidthLogging(): void {
1023
+ if (this.bandwidthLogIntervalId) {
1024
+ clearInterval(this.bandwidthLogIntervalId);
1025
+ this.bandwidthLogIntervalId = null;
1026
+ }
1027
+ }
1028
+
1029
+ /**
1030
+ * Log bandwidth statistics to console
1031
+ */
1032
+ private logBandwidthStats(): void {
1033
+ const videoKbps = (this.stats.videoBitrate / 1000).toFixed(1);
1034
+ const audioKbps = (this.stats.audioBitrate / 1000).toFixed(1);
1035
+ const totalKbps = (this.stats.totalBitrate / 1000).toFixed(1);
1036
+ const totalMBps = this.stats.totalBandwidthMBps.toFixed(3);
1037
+
1038
+ debug.log('webcodecs',
1039
+ `📊 Bandwidth Usage:\n` +
1040
+ ` Video: ${videoKbps} Kbps (${this.stats.videoCodec}) - ${this.stats.frameWidth}x${this.stats.frameHeight}\n` +
1041
+ ` Audio: ${audioKbps} Kbps (${this.stats.audioCodec})\n` +
1042
+ ` Total: ${totalKbps} Kbps (${totalMBps} MB/s)\n` +
1043
+ ` Frames: Video ${this.stats.videoFramesDecoded} (dropped: ${this.stats.videoFramesDropped}), ` +
1044
+ `Audio ${this.stats.audioFramesDecoded} (dropped: ${this.stats.audioFramesDropped})`
1045
+ );
1046
+ }
1047
+
1048
+ /**
1049
+ * Stop streaming
1050
+ */
1051
+ async stopStreaming(): Promise<void> {
1052
+ debug.log('webcodecs', 'Stopping streaming');
1053
+ await this.cleanup();
1054
+ }
1055
+
1056
+ /**
1057
+ * Reconnect to existing backend stream (after navigation)
1058
+ * This does NOT tell backend to stop - just reconnects WebRTC locally
1059
+ */
1060
+ async reconnectToExistingStream(sessionId: string, canvas: HTMLCanvasElement): Promise<boolean> {
1061
+ debug.log('webcodecs', `🔄 Reconnecting to existing stream for session: ${sessionId}`);
1062
+
1063
+ // Cleanup local WebRTC state WITHOUT notifying backend
1064
+ await this.cleanupLocalConnection();
1065
+
1066
+ this.isCleaningUp = false;
1067
+ this.stats.firstFrameRendered = false;
1068
+ this.isNavigating = false;
1069
+
1070
+ this.sessionId = sessionId;
1071
+ this.canvas = canvas;
1072
+ this.ctx = canvas.getContext('2d', {
1073
+ alpha: false,
1074
+ desynchronized: true,
1075
+ willReadFrequently: false
1076
+ });
1077
+
1078
+ if (this.ctx) {
1079
+ this.ctx.imageSmoothingEnabled = true;
1080
+ this.ctx.imageSmoothingQuality = 'low';
1081
+ }
1082
+
1083
+ try {
1084
+ // Setup WebSocket listeners
1085
+ this.setupEventListeners();
1086
+
1087
+ // Create peer connection
1088
+ await this.createPeerConnection();
1089
+
1090
+ // Get offer from backend's existing peer (don't start new streaming)
1091
+ const offerResponse = await ws.http('preview:browser-stream-offer', {}, 10000);
1092
+ if (offerResponse.offer) {
1093
+ await this.handleOffer({
1094
+ type: offerResponse.offer.type as RTCSdpType,
1095
+ sdp: offerResponse.offer.sdp
1096
+ });
1097
+ } else {
1098
+ throw new Error('No offer received from backend');
1099
+ }
1100
+
1101
+ debug.log('webcodecs', 'Reconnection setup complete');
1102
+ return true;
1103
+ } catch (error) {
1104
+ debug.error('webcodecs', 'Failed to reconnect:', error);
1105
+ if (this.onError) {
1106
+ this.onError(error instanceof Error ? error : new Error(String(error)));
1107
+ }
1108
+ return false;
1109
+ }
1110
+ }
1111
+
1112
+ /**
1113
+ * Cleanup local WebRTC connection WITHOUT notifying backend
1114
+ */
1115
+ private async cleanupLocalConnection(): Promise<void> {
1116
+ this.isCleaningUp = true;
1117
+
1118
+ this.stopStatsCollection();
1119
+ this.stopBandwidthLogging();
1120
+
1121
+ // Cancel pending frame render
1122
+ if (this.renderFrameId !== null) {
1123
+ cancelAnimationFrame(this.renderFrameId);
1124
+ this.renderFrameId = null;
1125
+ }
1126
+
1127
+ // Close pending frame
1128
+ if (this.pendingFrame) {
1129
+ this.pendingFrame.close();
1130
+ this.pendingFrame = null;
1131
+ }
1132
+
1133
+ // Cleanup WebSocket listeners
1134
+ this.wsCleanupFunctions.forEach((cleanup) => cleanup());
1135
+ this.wsCleanupFunctions = [];
1136
+
1137
+ // Close decoders
1138
+ if (this.videoDecoder) {
1139
+ try {
1140
+ await this.videoDecoder.flush();
1141
+ this.videoDecoder.close();
1142
+ } catch (e) {}
1143
+ this.videoDecoder = null;
1144
+ }
1145
+
1146
+ if (this.audioDecoder) {
1147
+ try {
1148
+ await this.audioDecoder.flush();
1149
+ this.audioDecoder.close();
1150
+ } catch (e) {}
1151
+ this.audioDecoder = null;
1152
+ }
1153
+
1154
+ // Close audio context
1155
+ if (this.audioContext && this.audioContext.state !== 'closed') {
1156
+ await this.audioContext.close().catch(() => {});
1157
+ this.audioContext = null;
1158
+ }
1159
+
1160
+ // Close data channel
1161
+ if (this.dataChannel) {
1162
+ this.dataChannel.close();
1163
+ this.dataChannel = null;
1164
+ }
1165
+
1166
+ // Close peer connection
1167
+ if (this.peerConnection) {
1168
+ this.peerConnection.close();
1169
+ this.peerConnection = null;
1170
+ }
1171
+
1172
+ this.isConnected = false;
1173
+ // NOTE: Don't clear sessionId - we're reconnecting to same session
1174
+
1175
+ // Reset playback state
1176
+ this.nextAudioPlayTime = 0;
1177
+ this.audioBufferQueue = [];
1178
+ this.audioSyncInitialized = false;
1179
+ this.audioCalibrationSamples = 0;
1180
+ this.audioOffsetAccumulator = 0;
1181
+ this.calibratedAudioOffset = 0;
1182
+
1183
+ this.isRenderingFrame = false;
1184
+ this.lastFrameTime = 0;
1185
+ this.startTime = 0;
1186
+ this.firstFrameTimestamp = 0;
1187
+
1188
+ this.syncEstablished = false;
1189
+ this.syncRealTimeOrigin = 0;
1190
+ this.syncStreamTimestamp = 0;
1191
+ this.lastVideoTimestamp = 0;
1192
+ this.lastVideoRealTime = 0;
1193
+
1194
+ if (this.onConnectionChange) {
1195
+ this.onConnectionChange(false);
1196
+ }
1197
+
1198
+ this.isCleaningUp = false;
1199
+ }
1200
+
1201
+ /**
1202
+ * Cleanup resources
1203
+ */
1204
+ private async cleanup(): Promise<void> {
1205
+ this.isCleaningUp = true;
1206
+
1207
+ this.stopStatsCollection();
1208
+ this.stopBandwidthLogging();
1209
+
1210
+ // Cancel pending frame render
1211
+ if (this.renderFrameId !== null) {
1212
+ cancelAnimationFrame(this.renderFrameId);
1213
+ this.renderFrameId = null;
1214
+ }
1215
+
1216
+ // Close pending frame
1217
+ if (this.pendingFrame) {
1218
+ this.pendingFrame.close();
1219
+ this.pendingFrame = null;
1220
+ }
1221
+
1222
+ // Cleanup WebSocket listeners
1223
+ this.wsCleanupFunctions.forEach((cleanup) => cleanup());
1224
+ this.wsCleanupFunctions = [];
1225
+
1226
+ // Notify server
1227
+ if (this.sessionId) {
1228
+ try {
1229
+ await ws.http('preview:browser-stream-stop', {});
1230
+ } catch (error) {
1231
+ debug.warn('webcodecs', 'Failed to notify server:', error);
1232
+ }
1233
+ }
1234
+
1235
+ // Close decoders
1236
+ if (this.videoDecoder) {
1237
+ try {
1238
+ await this.videoDecoder.flush();
1239
+ this.videoDecoder.close();
1240
+ } catch (e) {}
1241
+ this.videoDecoder = null;
1242
+ }
1243
+
1244
+ if (this.audioDecoder) {
1245
+ try {
1246
+ await this.audioDecoder.flush();
1247
+ this.audioDecoder.close();
1248
+ } catch (e) {}
1249
+ this.audioDecoder = null;
1250
+ }
1251
+
1252
+ // Close audio context
1253
+ if (this.audioContext && this.audioContext.state !== 'closed') {
1254
+ await this.audioContext.close().catch(() => {});
1255
+ this.audioContext = null;
1256
+ }
1257
+
1258
+ // Close data channel
1259
+ if (this.dataChannel) {
1260
+ this.dataChannel.close();
1261
+ this.dataChannel = null;
1262
+ }
1263
+
1264
+ // Close peer connection
1265
+ if (this.peerConnection) {
1266
+ this.peerConnection.close();
1267
+ this.peerConnection = null;
1268
+ }
1269
+
1270
+ this.isConnected = false;
1271
+ this.sessionId = null;
1272
+ this.canvas = null;
1273
+ this.ctx = null;
1274
+
1275
+ // Reset stats
1276
+ this.stats = {
1277
+ isConnected: false,
1278
+ connectionState: 'closed',
1279
+ iceConnectionState: 'closed',
1280
+ videoBytesReceived: 0,
1281
+ audioBytesReceived: 0,
1282
+ videoFramesReceived: 0,
1283
+ audioFramesReceived: 0,
1284
+ videoFramesDecoded: 0,
1285
+ audioFramesDecoded: 0,
1286
+ videoFramesDropped: 0,
1287
+ audioFramesDropped: 0,
1288
+ frameWidth: 0,
1289
+ frameHeight: 0,
1290
+ videoBitrate: 0,
1291
+ audioBitrate: 0,
1292
+ totalBitrate: 0,
1293
+ totalBandwidthMBps: 0,
1294
+ videoCodec: 'unknown',
1295
+ audioCodec: 'unknown',
1296
+ firstFrameRendered: false
1297
+ };
1298
+
1299
+ this.lastVideoBytesReceived = 0;
1300
+ this.lastAudioBytesReceived = 0;
1301
+ this.lastStatsTime = 0;
1302
+
1303
+ // Reset audio playback state
1304
+ this.nextAudioPlayTime = 0;
1305
+ this.audioBufferQueue = [];
1306
+ this.audioSyncInitialized = false;
1307
+ this.audioCalibrationSamples = 0;
1308
+ this.audioOffsetAccumulator = 0;
1309
+ this.calibratedAudioOffset = 0;
1310
+
1311
+ // Reset frame rendering state
1312
+ this.isRenderingFrame = false;
1313
+ this.lastFrameTime = 0;
1314
+ this.startTime = 0;
1315
+ this.firstFrameTimestamp = 0;
1316
+
1317
+ // Reset AV sync state
1318
+ this.syncEstablished = false;
1319
+ this.syncRealTimeOrigin = 0;
1320
+ this.syncStreamTimestamp = 0;
1321
+ this.lastVideoTimestamp = 0;
1322
+ this.lastVideoRealTime = 0;
1323
+
1324
+ // Reset navigation state
1325
+ this.isNavigating = false;
1326
+
1327
+ if (this.onConnectionChange) {
1328
+ this.onConnectionChange(false);
1329
+ }
1330
+
1331
+ this.isCleaningUp = false;
1332
+ }
1333
+
1334
+ /**
1335
+ * Cleanup connection (internal)
1336
+ */
1337
+ private async cleanupConnection(): Promise<void> {
1338
+ this.isCleaningUp = true;
1339
+
1340
+ this.stopStatsCollection();
1341
+ this.stopBandwidthLogging();
1342
+
1343
+ // Cancel pending frame render
1344
+ if (this.renderFrameId !== null) {
1345
+ cancelAnimationFrame(this.renderFrameId);
1346
+ this.renderFrameId = null;
1347
+ }
1348
+
1349
+ // Close pending frame
1350
+ if (this.pendingFrame) {
1351
+ this.pendingFrame.close();
1352
+ this.pendingFrame = null;
1353
+ }
1354
+
1355
+ this.wsCleanupFunctions.forEach((cleanup) => cleanup());
1356
+ this.wsCleanupFunctions = [];
1357
+
1358
+ if (this.videoDecoder) {
1359
+ try {
1360
+ await this.videoDecoder.flush();
1361
+ this.videoDecoder.close();
1362
+ } catch (e) {}
1363
+ this.videoDecoder = null;
1364
+ }
1365
+
1366
+ if (this.audioDecoder) {
1367
+ try {
1368
+ await this.audioDecoder.flush();
1369
+ this.audioDecoder.close();
1370
+ } catch (e) {}
1371
+ this.audioDecoder = null;
1372
+ }
1373
+
1374
+ if (this.audioContext && this.audioContext.state !== 'closed') {
1375
+ await this.audioContext.close().catch(() => {});
1376
+ this.audioContext = null;
1377
+ }
1378
+
1379
+ if (this.dataChannel) {
1380
+ this.dataChannel.close();
1381
+ this.dataChannel = null;
1382
+ }
1383
+
1384
+ if (this.peerConnection) {
1385
+ this.peerConnection.close();
1386
+ this.peerConnection = null;
1387
+ }
1388
+
1389
+ this.isConnected = false;
1390
+ this.sessionId = null;
1391
+
1392
+ // Reset audio playback state
1393
+ this.nextAudioPlayTime = 0;
1394
+ this.audioBufferQueue = [];
1395
+ this.audioSyncInitialized = false;
1396
+ this.audioCalibrationSamples = 0;
1397
+ this.audioOffsetAccumulator = 0;
1398
+ this.calibratedAudioOffset = 0;
1399
+
1400
+ // Reset frame rendering state
1401
+ this.isRenderingFrame = false;
1402
+ this.lastFrameTime = 0;
1403
+ this.startTime = 0;
1404
+ this.firstFrameTimestamp = 0;
1405
+
1406
+ // Reset AV sync state
1407
+ this.syncEstablished = false;
1408
+ this.syncRealTimeOrigin = 0;
1409
+ this.syncStreamTimestamp = 0;
1410
+ this.lastVideoTimestamp = 0;
1411
+ this.lastVideoRealTime = 0;
1412
+
1413
+ this.isCleaningUp = false;
1414
+ }
1415
+
1416
+ /**
1417
+ * Clear canvas
1418
+ */
1419
+ private clearCanvas(): void {
1420
+ if (this.canvas && this.ctx) {
1421
+ this.ctx.fillStyle = '#f1f5f9';
1422
+ this.ctx.fillRect(0, 0, this.canvas.width, this.canvas.height);
1423
+ }
1424
+ }
1425
+
1426
+ /**
1427
+ * Get stats
1428
+ */
1429
+ getStats(): BrowserWebCodecsStreamStats {
1430
+ return { ...this.stats };
1431
+ }
1432
+
1433
+ /**
1434
+ * Get connection status
1435
+ */
1436
+ getConnectionStatus(): boolean {
1437
+ return this.isConnected;
1438
+ }
1439
+
1440
+ // Event handlers
1441
+ setConnectionChangeHandler(handler: (connected: boolean) => void): void {
1442
+ this.onConnectionChange = handler;
1443
+ }
1444
+
1445
+ setConnectionFailedHandler(handler: () => void): void {
1446
+ this.onConnectionFailed = handler;
1447
+ }
1448
+
1449
+ setNavigationReconnectHandler(handler: () => void): void {
1450
+ this.onNavigationReconnect = handler;
1451
+ }
1452
+
1453
+ setReconnectingStartHandler(handler: () => void): void {
1454
+ this.onReconnectingStart = handler;
1455
+ }
1456
+
1457
+ setErrorHandler(handler: (error: Error) => void): void {
1458
+ this.onError = handler;
1459
+ }
1460
+
1461
+ setStatsHandler(handler: (stats: BrowserWebCodecsStreamStats) => void): void {
1462
+ this.onStats = handler;
1463
+ }
1464
+
1465
+ setOnCursorChange(handler: (cursor: string) => void): void {
1466
+ this.onCursorChange = handler;
1467
+ }
1468
+
1469
+ /**
1470
+ * Set navigation state
1471
+ * When navigating, DataChannel close/error won't trigger recovery
1472
+ * Backend will restart streaming after navigation completes
1473
+ */
1474
+ setNavigating(navigating: boolean): void {
1475
+ this.isNavigating = navigating;
1476
+ debug.log('webcodecs', `Navigation state set: ${navigating}`);
1477
+ }
1478
+
1479
+ /**
1480
+ * Check if currently navigating
1481
+ */
1482
+ getNavigating(): boolean {
1483
+ return this.isNavigating;
1484
+ }
1485
+
1486
+ /**
1487
+ * Destroy service
1488
+ */
1489
+ destroy(): void {
1490
+ this.cleanup();
1491
+ this.onConnectionChange = null;
1492
+ this.onConnectionFailed = null;
1493
+ this.onNavigationReconnect = null;
1494
+ this.onReconnectingStart = null;
1495
+ this.onError = null;
1496
+ this.onStats = null;
1497
+ this.onCursorChange = null;
1498
+ }
1499
+ }