@vellumai/assistant 0.12.3-staging.1 → 0.12.3-staging.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (560) hide show
  1. package/ARCHITECTURE.md +43 -29
  2. package/Dockerfile +63 -0
  3. package/docs/architecture/security.md +10 -5
  4. package/docs/architecture/turn-actor.md +2 -0
  5. package/docs/desktop-browser-cli.md +45 -4
  6. package/docs/live-voice-task-outcomes.md +100 -0
  7. package/docs/mode-sessions.md +183 -0
  8. package/docs/voice-input-diagnostics.md +58 -0
  9. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  10. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  11. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  12. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  13. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  14. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  15. package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +20 -0
  16. package/node_modules/@vellumai/gateway-client/src/index.ts +4 -1
  17. package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +38 -7
  18. package/node_modules/@vellumai/gateway-client/src/trust-verdict-contract.ts +5 -6
  19. package/node_modules/@vellumai/service-contracts/package.json +2 -1
  20. package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  21. package/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  22. package/node_modules/@vellumai/slack-text/src/index.ts +4 -8
  23. package/openapi.yaml +665 -14
  24. package/package.json +1 -1
  25. package/scripts/smoke-desktop-browser-cli.ts +10 -26
  26. package/scripts/smoke-desktop-window-theme.ts +2 -3
  27. package/src/__tests__/agent-loop-output-hooks.test.ts +150 -1
  28. package/src/__tests__/agent-loop-provider-error-recording.test.ts +87 -19
  29. package/src/__tests__/annotate-activity-metadata.test.ts +3 -1
  30. package/src/__tests__/annotate-risk-options.test.ts +2 -0
  31. package/src/__tests__/answered-question-persistence.test.ts +2 -0
  32. package/src/__tests__/anthropic-gateway-shared.test.ts +22 -0
  33. package/src/__tests__/approval-message-composer.test.ts +1 -118
  34. package/src/__tests__/approval-routes-http.test.ts +3 -0
  35. package/src/__tests__/assistant-event-hub-connection-history.test.ts +111 -0
  36. package/src/__tests__/assistant-event-hub-targeted.test.ts +50 -0
  37. package/src/__tests__/assistant-event-targeting.test.ts +186 -0
  38. package/src/__tests__/assistant-stream-state.test.ts +29 -9
  39. package/src/__tests__/browser-fill-credential.test.ts +11 -22
  40. package/src/__tests__/btw-routes.test.ts +4 -1
  41. package/src/__tests__/build-persisted-content.test.ts +1 -1
  42. package/src/__tests__/call-conversation-messages.test.ts +5 -0
  43. package/src/__tests__/call-recovery.test.ts +7 -2
  44. package/src/__tests__/call-routes-http.test.ts +12 -0
  45. package/src/__tests__/call-store.test.ts +7 -2
  46. package/src/__tests__/channel-retry-sweep.test.ts +9 -0
  47. package/src/__tests__/chat-reveal-guard-priming.test.ts +2 -0
  48. package/src/__tests__/checker.test.ts +11 -3
  49. package/src/__tests__/client-os-metadata-persistence.test.ts +35 -1
  50. package/src/__tests__/computer-use-skill-manifest-regression.test.ts +0 -25
  51. package/src/__tests__/computer-use-tools.test.ts +1 -233
  52. package/src/__tests__/config-model-image-provider.test.ts +39 -0
  53. package/src/__tests__/config-schema.test.ts +16 -0
  54. package/src/__tests__/config-watcher.test.ts +24 -0
  55. package/src/__tests__/context-search-agent-runner.test.ts +260 -1
  56. package/src/__tests__/context-search-fanout.test.ts +1 -1
  57. package/src/__tests__/context-search-types.test.ts +7 -7
  58. package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +8 -2
  59. package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +2 -0
  60. package/src/__tests__/conversation-agent-loop.test.ts +546 -16
  61. package/src/__tests__/conversation-app-control-lifecycle.test.ts +53 -1
  62. package/src/__tests__/conversation-evictor.test.ts +62 -37
  63. package/src/__tests__/conversation-history-boundary-repair-failure.test.ts +174 -0
  64. package/src/__tests__/conversation-loop-db-lock-resilience.test.ts +2 -0
  65. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +2 -0
  66. package/src/__tests__/conversation-retry-route.test.ts +22 -0
  67. package/src/__tests__/conversation-routes-disk-view.test.ts +2 -0
  68. package/src/__tests__/conversation-routes-enabled-plugins.test.ts +2 -0
  69. package/src/__tests__/conversation-routes-guardian-reply.test.ts +10 -0
  70. package/src/__tests__/conversation-routes-hidden-queue.test.ts +2 -0
  71. package/src/__tests__/conversation-routes-slash-commands.test.ts +2 -0
  72. package/src/__tests__/conversation-runtime-assembly.test.ts +29 -0
  73. package/src/__tests__/conversation-surface-snapshots.test.ts +543 -0
  74. package/src/__tests__/conversation-surfaces-action-delivery.test.ts +151 -0
  75. package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +63 -1
  76. package/src/__tests__/conversation-tool-setup-attribution.test.ts +23 -0
  77. package/src/__tests__/conversation-undo.test.ts +61 -0
  78. package/src/__tests__/credential-execution-client.test.ts +6 -6
  79. package/src/__tests__/credential-prompt-route.test.ts +35 -3
  80. package/src/__tests__/cu-unified-flow.test.ts +82 -1
  81. package/src/__tests__/db-mode-sessions-migration.test.ts +175 -0
  82. package/src/__tests__/deterministic-verification-control-plane.test.ts +2 -0
  83. package/src/__tests__/dm-backfill.test.ts +91 -1
  84. package/src/__tests__/document-create-route.test.ts +88 -0
  85. package/src/__tests__/document-sync-tags.test.ts +27 -0
  86. package/src/__tests__/emit-signal-routing-intent.test.ts +142 -2
  87. package/src/__tests__/events-tail-route.test.ts +25 -0
  88. package/src/__tests__/evict-conversations-for-reload.test.ts +57 -8
  89. package/src/__tests__/guardian-action-sweep.test.ts +16 -0
  90. package/src/__tests__/guardian-dispatch.test.ts +13 -0
  91. package/src/__tests__/headless-browser-interactions.test.ts +64 -20
  92. package/src/__tests__/heartbeat-service.test.ts +2 -59
  93. package/src/__tests__/helpers/mock-conversation.ts +21 -0
  94. package/src/__tests__/helpers/native-web-search-harness.ts +2 -0
  95. package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +5 -0
  96. package/src/__tests__/host-cu-proxy.test.ts +46 -2
  97. package/src/__tests__/http-user-message-parity.test.ts +2 -0
  98. package/src/__tests__/image-credentials.test.ts +49 -0
  99. package/src/__tests__/image-service-dispatcher.test.ts +45 -0
  100. package/src/__tests__/inference-profile-session-handler.test.ts +1 -1
  101. package/src/__tests__/interrupt-on-send.test.ts +57 -43
  102. package/src/__tests__/interrupt-turn-note.test.ts +5 -6
  103. package/src/__tests__/llm-catalog-parity.test.ts +2 -2
  104. package/src/__tests__/media-generate-image.test.ts +49 -1
  105. package/src/__tests__/message-provenance-contact-id.test.ts +69 -0
  106. package/src/__tests__/migration-export-to-gcs.test.ts +142 -0
  107. package/src/__tests__/mode-session-history.test.ts +245 -0
  108. package/src/__tests__/mode-session-recovery.test.ts +222 -0
  109. package/src/__tests__/mode-session-wire.test.ts +141 -0
  110. package/src/__tests__/notification-guardian-path.test.ts +7 -0
  111. package/src/__tests__/notification-telegram-adapter.test.ts +2 -2
  112. package/src/__tests__/notification-vellum-adapter.test.ts +34 -0
  113. package/src/__tests__/oauth-provider-serializer.test.ts +12 -0
  114. package/src/__tests__/oauth-providers-routes.test.ts +1 -0
  115. package/src/__tests__/outbound-slack-persistence.test.ts +2 -0
  116. package/src/__tests__/persistence-secret-redaction.test.ts +2 -0
  117. package/src/__tests__/plugin-api-provider.test.ts +5 -0
  118. package/src/__tests__/plugin-api-resolve-credential.test.ts +97 -0
  119. package/src/__tests__/plugin-import-boundary-guard.test.ts +9 -0
  120. package/src/__tests__/process-message-display-content.test.ts +56 -0
  121. package/src/__tests__/reaction-persistence.test.ts +5 -0
  122. package/src/__tests__/request-secret-standalone-channel-gate.test.ts +25 -9
  123. package/src/__tests__/runtime-events-sse-reconnect.test.ts +1 -1
  124. package/src/__tests__/scripted-turn-metadata-persistence.test.ts +73 -0
  125. package/src/__tests__/secret-ingress-http.test.ts +2 -0
  126. package/src/__tests__/secure-keys.test.ts +43 -1
  127. package/src/__tests__/send-endpoint-busy.test.ts +163 -1
  128. package/src/__tests__/server-history-render.test.ts +91 -1
  129. package/src/__tests__/settings-routes.test.ts +50 -1
  130. package/src/__tests__/skill-script-runner-sandbox.test.ts +32 -0
  131. package/src/__tests__/subagent-fork-notifications.test.ts +1 -2
  132. package/src/__tests__/subagent-manager-notify.test.ts +1 -2
  133. package/src/__tests__/subagent-notify-parent.test.ts +146 -39
  134. package/src/__tests__/subagent-notify-stale-rebuild.test.ts +2 -4
  135. package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +2 -0
  136. package/src/__tests__/thread-backfill.test.ts +53 -0
  137. package/src/__tests__/tool-preview-lifecycle.test.ts +2 -0
  138. package/src/__tests__/tool-result-metadata-plumbing.test.ts +3 -1
  139. package/src/__tests__/tool-start-timestamp.test.ts +2 -0
  140. package/src/__tests__/twilio-routes.test.ts +1 -0
  141. package/src/__tests__/web-search-backend-failure.test.ts +1 -1
  142. package/src/activation/progress-store.test.ts +12 -7
  143. package/src/agent/loop.ts +99 -41
  144. package/src/api/events/assistant-turn-start.ts +3 -0
  145. package/src/api/events/generation-handoff.ts +2 -0
  146. package/src/api/events/message-complete.ts +2 -0
  147. package/src/api/events/notification-conversation-created.ts +14 -16
  148. package/src/api/events/notification-intent.ts +4 -2
  149. package/src/api/events/tool-result.ts +85 -0
  150. package/src/api/events/tool-use-start.ts +3 -0
  151. package/src/api/events/user-message-echo.ts +3 -0
  152. package/src/api/index.ts +31 -6
  153. package/src/api/mode-session.test.ts +131 -0
  154. package/src/api/mode-session.ts +148 -0
  155. package/src/api/remember-tool.ts +15 -0
  156. package/src/api/responses/conversation-message.ts +6 -0
  157. package/src/approvals/AGENTS.md +4 -2
  158. package/src/browser/__tests__/operations.test.ts +29 -0
  159. package/src/browser/operation-meta.ts +2 -1
  160. package/src/browser/operations.ts +29 -0
  161. package/src/browser/virtual-desktop-target.ts +2 -8
  162. package/src/bundler/bundle-signer.ts +1 -3
  163. package/src/calls/__tests__/barge-in-guard.test.ts +3 -0
  164. package/src/calls/__tests__/call-controller.test.ts +94 -0
  165. package/src/calls/__tests__/call-funnel-telemetry.test.ts +204 -0
  166. package/src/calls/__tests__/voice-control-protocol.test.ts +15 -0
  167. package/src/calls/__tests__/voice-escalation-judge.test.ts +209 -0
  168. package/src/calls/__tests__/voice-leg-coordinator.test.ts +168 -0
  169. package/src/{live-voice/__tests__/live-voice-metrics.test.ts → calls/__tests__/voice-metrics.test.ts} +32 -38
  170. package/src/calls/__tests__/voice-session-bridge.test.ts +576 -122
  171. package/src/calls/barge-in-guard.ts +5 -0
  172. package/src/calls/call-controller.ts +196 -1
  173. package/src/calls/call-domain.ts +4 -0
  174. package/src/calls/call-store.ts +87 -2
  175. package/src/calls/guardian-action-sweep.ts +22 -8
  176. package/src/calls/media-stream-audio-transcode.ts +1 -3
  177. package/src/calls/typesafe-noul.ts +156 -0
  178. package/src/calls/voice-control-protocol.ts +6 -0
  179. package/src/calls/voice-escalation-judge.ts +145 -0
  180. package/src/calls/voice-escalation-target.ts +10 -0
  181. package/src/calls/voice-leg-coordinator.ts +121 -3
  182. package/src/{live-voice/live-voice-metrics.ts → calls/voice-metrics.ts} +94 -87
  183. package/src/calls/voice-session-bridge.ts +256 -112
  184. package/src/calls/voice-triage-escalate.ts +7 -0
  185. package/src/cli/__tests__/roadmap-cli.test.ts +4 -41
  186. package/src/cli/commands/__tests__/notifications.test.ts +35 -10
  187. package/src/cli/commands/browser.help.ts +4 -5
  188. package/src/cli/commands/browser.ts +0 -1
  189. package/src/cli/commands/clients.help.ts +46 -0
  190. package/src/cli/commands/clients.ts +116 -0
  191. package/src/cli/commands/credentials.help.ts +6 -1
  192. package/src/cli/commands/notifications.help.ts +16 -2
  193. package/src/cli/commands/notifications.ts +30 -8
  194. package/src/cli/commands/roadmap.help.ts +18 -21
  195. package/src/cli/commands/roadmap.ts +4 -21
  196. package/src/cli/lib/__tests__/list-installed-plugins.test.ts +5 -2
  197. package/src/cli/lib/bundled-marketplace.json +84 -34
  198. package/src/cli/lib/plugin-marketplace.ts +8 -0
  199. package/src/cli/lib/publish-plugin.ts +1 -2
  200. package/src/config/__tests__/default-provider.test.ts +3 -1
  201. package/src/config/__tests__/memory-retrospective-schema.test.ts +14 -1
  202. package/src/config/bundled-skills/computer-use/SKILL.md +1 -1
  203. package/src/config/bundled-skills/computer-use/TOOLS.json +89 -1
  204. package/src/config/bundled-skills/image-studio/SKILL.md +17 -2
  205. package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +7 -11
  206. package/src/config/bundled-skills/sequences/TOOLS.json +1 -5
  207. package/src/config/call-site-defaults.ts +9 -0
  208. package/src/config/feature-flag-registry.json +16 -0
  209. package/src/config/profile-text-generation.test.ts +2 -2
  210. package/src/config/schemas/__tests__/tools.test.ts +31 -0
  211. package/src/config/schemas/call-site-catalog.ts +14 -0
  212. package/src/config/schemas/channels.ts +0 -18
  213. package/src/config/schemas/llm.ts +3 -1
  214. package/src/config/schemas/mcp.ts +2 -1
  215. package/src/config/schemas/memory-retrospective.ts +11 -2
  216. package/src/config/schemas/services.ts +15 -6
  217. package/src/config/schemas/tools.ts +10 -0
  218. package/src/config/session-groups-gate.ts +21 -0
  219. package/src/config/skills.ts +2 -6
  220. package/src/conversations/__tests__/message-consolidation.test.ts +149 -0
  221. package/src/conversations/message-consolidation.ts +102 -5
  222. package/src/credential-execution/ces-connect.test.ts +69 -0
  223. package/src/credential-execution/ces-connect.ts +115 -0
  224. package/src/credential-execution/ces-runtime.ts +55 -93
  225. package/src/credential-execution/executable-discovery.ts +1 -11
  226. package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +10 -1
  227. package/src/daemon/__tests__/inference-profile-notification.test.ts +2 -0
  228. package/src/daemon/browser-mode-session.test.ts +402 -0
  229. package/src/daemon/browser-mode-session.ts +186 -0
  230. package/src/daemon/computer-use-mode-session.test.ts +210 -0
  231. package/src/daemon/computer-use-mode-session.ts +105 -0
  232. package/src/daemon/config-watcher.ts +10 -0
  233. package/src/daemon/conversation-agent-loop-handlers.ts +58 -10
  234. package/src/daemon/conversation-agent-loop.ts +219 -25
  235. package/src/daemon/conversation-evictor.ts +5 -24
  236. package/src/daemon/conversation-history.ts +10 -0
  237. package/src/daemon/conversation-interrupt-repair.ts +4 -43
  238. package/src/daemon/conversation-interrupt.ts +7 -11
  239. package/src/daemon/conversation-messaging.ts +58 -5
  240. package/src/daemon/conversation-mode-session.test.ts +1667 -0
  241. package/src/daemon/conversation-mode-session.ts +1008 -0
  242. package/src/daemon/conversation-process.ts +5 -0
  243. package/src/daemon/conversation-runtime-assembly.ts +10 -20
  244. package/src/daemon/conversation-skill-tools.ts +1 -0
  245. package/src/daemon/conversation-store.ts +4 -4
  246. package/src/daemon/conversation-surface-snapshots.ts +135 -0
  247. package/src/daemon/conversation-surfaces.ts +113 -5
  248. package/src/daemon/conversation-tool-setup.ts +8 -3
  249. package/src/daemon/conversation.ts +143 -41
  250. package/src/daemon/handlers/config-model.ts +7 -5
  251. package/src/daemon/handlers/shared.ts +55 -16
  252. package/src/daemon/host-cu-proxy.ts +51 -22
  253. package/src/daemon/lifecycle.ts +30 -13
  254. package/src/daemon/message-protocol.ts +0 -1
  255. package/src/daemon/message-provenance.ts +28 -4
  256. package/src/daemon/mode-session-startup-recovery.ts +23 -0
  257. package/src/daemon/mode-session-tracking.ts +54 -0
  258. package/src/daemon/process-message.ts +13 -4
  259. package/src/daemon/virtual-desktop-context.ts +21 -0
  260. package/src/desktop/__tests__/fake-desktop.ts +1 -0
  261. package/src/desktop/desktop-accessibility-script.ts +88 -0
  262. package/src/desktop/desktop-accessibility.test.ts +160 -0
  263. package/src/desktop/desktop-accessibility.ts +169 -0
  264. package/src/desktop/desktop-automation-lease.test.ts +76 -68
  265. package/src/desktop/desktop-automation-lease.ts +41 -50
  266. package/src/desktop/desktop-browser-endpoint.ts +4 -0
  267. package/src/desktop/desktop-computer-use-routing.test.ts +330 -0
  268. package/src/desktop/desktop-computer-use.test.ts +449 -0
  269. package/src/desktop/desktop-computer-use.ts +502 -0
  270. package/src/desktop/desktop-dependencies.ts +47 -293
  271. package/src/desktop/desktop-display.ts +2 -2
  272. package/src/desktop/desktop-panel-config.test.ts +13 -3
  273. package/src/desktop/desktop-panel-config.ts +109 -15
  274. package/src/desktop/desktop-session-manager.test.ts +69 -21
  275. package/src/desktop/desktop-session-manager.ts +76 -22
  276. package/src/desktop/desktop-stream-bridge.test.ts +11 -122
  277. package/src/desktop/desktop-stream-bridge.ts +6 -8
  278. package/src/desktop/desktop-window-theme.ts +5 -25
  279. package/src/desktop/virtual-desktop-feature.ts +52 -2
  280. package/src/desktop/virtual-desktop-platform.test.ts +5 -9
  281. package/src/documents/document-store.ts +23 -0
  282. package/src/heartbeat/__tests__/heartbeat-service.test.ts +22 -26
  283. package/src/heartbeat/heartbeat-service.ts +5 -33
  284. package/src/ipc/__tests__/browser-ipc.test.ts +215 -6
  285. package/src/ipc/assistant-server.ts +2 -0
  286. package/src/ipc/routes/__tests__/channel-reply-ipc-routes.test.ts +111 -0
  287. package/src/ipc/routes/channel-reply-ipc-routes.ts +73 -0
  288. package/src/live-voice/__tests__/continuation-judge.test.ts +151 -0
  289. package/src/live-voice/__tests__/live-activity-reporter.test.ts +154 -15
  290. package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +743 -31
  291. package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +191 -10
  292. package/src/live-voice/__tests__/live-voice-session-groups-gate.test.ts +87 -0
  293. package/src/live-voice/__tests__/live-voice-session-manager.test.ts +5 -1
  294. package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +41 -0
  295. package/src/live-voice/__tests__/live-voice-sight-frame.test.ts +1017 -9
  296. package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +224 -1
  297. package/src/live-voice/__tests__/live-voice-tts-session.test.ts +101 -0
  298. package/src/live-voice/__tests__/live-voice-tts.test.ts +44 -0
  299. package/src/live-voice/__tests__/live-voice-vad.test.ts +939 -75
  300. package/src/live-voice/__tests__/microphone-idle-gate.test.ts +87 -0
  301. package/src/live-voice/__tests__/session-controls.test.ts +23 -0
  302. package/src/live-voice/__tests__/task-outcomes.test.ts +117 -0
  303. package/src/live-voice/__tests__/voice-input-diagnostics.test.ts +117 -0
  304. package/src/live-voice/camera-mode-session.test.ts +265 -0
  305. package/src/live-voice/camera-mode-session.ts +277 -0
  306. package/src/live-voice/continuation-judge.ts +123 -0
  307. package/src/live-voice/live-activity-reporter.ts +115 -6
  308. package/src/live-voice/live-voice-manager.ts +36 -1
  309. package/src/live-voice/live-voice-photo.ts +88 -7
  310. package/src/live-voice/live-voice-session-manager.ts +15 -0
  311. package/src/live-voice/live-voice-session.ts +1443 -747
  312. package/src/live-voice/live-voice-tts.ts +22 -4
  313. package/src/live-voice/microphone-idle-gate.ts +56 -0
  314. package/src/live-voice/protocol.ts +171 -4
  315. package/src/live-voice/session-controls.ts +11 -3
  316. package/src/live-voice/task-outcomes.ts +131 -0
  317. package/src/live-voice/voice-input-diagnostics.ts +117 -0
  318. package/src/mcp/__tests__/manager-tool-caps.test.ts +63 -18
  319. package/src/mcp/__tests__/tool-caps.test.ts +28 -1
  320. package/src/mcp/manager.ts +3 -0
  321. package/src/mcp/tool-caps.ts +12 -4
  322. package/src/media/__tests__/image-models.test.ts +58 -0
  323. package/src/media/image-credentials.ts +5 -0
  324. package/src/media/image-models.ts +52 -0
  325. package/src/media/image-service.ts +8 -0
  326. package/src/media/openrouter-image-service.test.ts +133 -0
  327. package/src/media/openrouter-image-service.ts +153 -0
  328. package/src/media/types.ts +1 -1
  329. package/src/messaging/providers/__tests__/transport-dispatch.test.ts +65 -11
  330. package/src/messaging/providers/discord/send.ts +4 -17
  331. package/src/messaging/providers/discord/transport.ts +2 -2
  332. package/src/messaging/providers/outlook/client.ts +16 -22
  333. package/src/messaging/providers/telegram-bot/send.test.ts +236 -1
  334. package/src/messaging/providers/telegram-bot/send.ts +10 -7
  335. package/src/messaging/providers/telegram-bot/transport.ts +3 -10
  336. package/src/messaging/providers/whatsapp/send.test.ts +169 -4
  337. package/src/monitoring/__tests__/control.test.ts +17 -5
  338. package/src/monitoring/control.ts +15 -5
  339. package/src/notifications/AGENTS.md +1 -1
  340. package/src/notifications/README.md +83 -99
  341. package/src/notifications/__tests__/broadcaster.test.ts +1 -1
  342. package/src/notifications/__tests__/channel-allowlist.test.ts +39 -0
  343. package/src/notifications/__tests__/copy-composer.test.ts +1 -1
  344. package/src/notifications/__tests__/decision-engine.test.ts +79 -2
  345. package/src/notifications/__tests__/deterministic-checks.test.ts +10 -10
  346. package/src/notifications/__tests__/home-feed-side-effect.test.ts +93 -4
  347. package/src/notifications/__tests__/notification-utils.test.ts +18 -0
  348. package/src/notifications/adapters/macos.ts +37 -46
  349. package/src/notifications/broadcaster.ts +5 -6
  350. package/src/notifications/channel-allowlist.ts +44 -0
  351. package/src/notifications/copy-composer.ts +0 -23
  352. package/src/notifications/decision-engine.ts +24 -14
  353. package/src/notifications/deterministic-checks.ts +1 -1
  354. package/src/notifications/emit-signal.ts +123 -37
  355. package/src/notifications/home-feed-side-effect.ts +9 -7
  356. package/src/notifications/notification-utils.ts +19 -0
  357. package/src/notifications/signal.ts +4 -19
  358. package/src/notifications/types.ts +15 -0
  359. package/src/oauth/connection-resolver.ts +1 -2
  360. package/src/oauth/oauth-store.ts +1 -3
  361. package/src/oauth/provider-serializer.ts +7 -0
  362. package/src/oauth/seed-providers.ts +30 -0
  363. package/src/onboarding/onboarding-events-store.ts +99 -12
  364. package/src/persistence/__tests__/client-connection-events-store.test.ts +322 -0
  365. package/src/persistence/client-connection-events-store.ts +377 -0
  366. package/src/persistence/conversation-crud.ts +55 -6
  367. package/src/persistence/conversation-mode-sessions.test.ts +424 -0
  368. package/src/persistence/conversation-mode-sessions.ts +460 -0
  369. package/src/persistence/conversation-queries.ts +6 -19
  370. package/src/persistence/llm-usage-store.ts +3 -9
  371. package/src/persistence/message-metadata.ts +25 -0
  372. package/src/persistence/migrations/381-create-conversation-mode-sessions.ts +41 -0
  373. package/src/persistence/migrations/382-create-client-connection-events.ts +40 -0
  374. package/src/persistence/migrations/__tests__/382-create-client-connection-events.test.ts +56 -0
  375. package/src/persistence/schema/conversations.ts +67 -0
  376. package/src/persistence/schema/infrastructure.ts +29 -0
  377. package/src/persistence/steps.ts +17 -0
  378. package/src/plugin-api/__tests__/plugin-name-env.test.ts +249 -0
  379. package/src/plugin-api/effective-context-window.ts +33 -0
  380. package/src/plugin-api/index.ts +24 -5
  381. package/src/plugin-api/plugin-name-env.ts +125 -0
  382. package/src/plugin-api/resolve-credential.ts +79 -20
  383. package/src/plugin-api/vision-support.test.ts +22 -0
  384. package/src/plugin-api/vision-support.ts +17 -5
  385. package/src/plugins/__tests__/active-plugin-env.test.ts +136 -0
  386. package/src/plugins/active-plugin-env.ts +51 -0
  387. package/src/plugins/defaults/index.ts +24 -0
  388. package/src/plugins/defaults/memory/__tests__/archive-removal-framing.test.ts +64 -0
  389. package/src/plugins/defaults/memory/__tests__/worker-control.test.ts +19 -6
  390. package/src/plugins/defaults/memory/context-search/agent-protocol.ts +9 -12
  391. package/src/plugins/defaults/memory/context-search/agent-runner.ts +78 -22
  392. package/src/plugins/defaults/memory/context-search/format.ts +51 -7
  393. package/src/plugins/defaults/memory/context-search/limits.ts +10 -12
  394. package/src/plugins/defaults/memory/context-search/recall-input.ts +38 -0
  395. package/src/plugins/defaults/memory/context-search/search.ts +8 -14
  396. package/src/plugins/defaults/memory/context-search/sources/conversations.ts +1 -0
  397. package/src/plugins/defaults/memory/context-search/types.ts +1 -11
  398. package/src/plugins/defaults/memory/graph/__tests__/handle-remember-v2.test.ts +11 -4
  399. package/src/plugins/defaults/memory/graph/tool-handlers.ts +12 -21
  400. package/src/plugins/defaults/memory/graph/tools.ts +35 -64
  401. package/src/plugins/defaults/memory/src/memory-eval-routes.ts +2 -3
  402. package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +5 -0
  403. package/src/plugins/defaults/memory/src/memory-item-routes.ts +2 -1
  404. package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +2 -2
  405. package/src/plugins/defaults/memory/tools.test.ts +124 -9
  406. package/src/plugins/defaults/memory/tools.ts +38 -11
  407. package/src/plugins/defaults/memory/v3/__tests__/pool-log-store.test.ts +165 -27
  408. package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +626 -27
  409. package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +46 -27
  410. package/src/plugins/defaults/memory/v3/orchestrate.ts +78 -38
  411. package/src/plugins/defaults/memory/v3/pool-log-store.ts +65 -79
  412. package/src/plugins/defaults/memory/v3/pool-select.test.ts +14 -5
  413. package/src/plugins/defaults/memory/v3/pool-select.ts +683 -44
  414. package/src/plugins/defaults/memory/worker-control.ts +15 -5
  415. package/src/plugins/defaults/task-progress/__tests__/registration.test.ts +70 -0
  416. package/src/plugins/defaults/task-progress/__tests__/task-progress-context.test.ts +380 -0
  417. package/src/plugins/defaults/task-progress/hooks/post-compact.ts +19 -0
  418. package/src/plugins/defaults/task-progress/hooks/user-prompt-submit.ts +24 -0
  419. package/src/plugins/defaults/task-progress/package.json +14 -0
  420. package/src/plugins/defaults/task-progress/src/apply-task-progress-context.ts +93 -0
  421. package/src/plugins/defaults/task-progress/src/format-task-progress-context.ts +67 -0
  422. package/src/providers/__tests__/context-overflow-error.test.ts +10 -0
  423. package/src/providers/__tests__/search-provider-catalog.test.ts +18 -0
  424. package/src/providers/anthropic/client.ts +10 -1
  425. package/src/providers/anthropic-gateway-shared.ts +6 -0
  426. package/src/providers/fetch-provider-catalog.ts +11 -0
  427. package/src/providers/gemini/client.ts +25 -8
  428. package/src/providers/inference/adapter-factory.ts +1 -1
  429. package/src/providers/jev/client.test.ts +10 -0
  430. package/src/providers/jev/client.ts +20 -4
  431. package/src/providers/model-catalog.ts +4 -3
  432. package/src/providers/openai/__tests__/chat-completions-extra-body.test.ts +94 -1
  433. package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +106 -1
  434. package/src/providers/openai/__tests__/chat-template-flatten.test.ts +24 -0
  435. package/src/providers/openai/__tests__/google-thought-signature.test.ts +1 -2
  436. package/src/providers/openai/chat-completions-provider.ts +100 -29
  437. package/src/providers/openai/google-thought-signature.ts +0 -6
  438. package/src/providers/openai/responses-provider.ts +23 -1
  439. package/src/providers/ratelimit.ts +1 -3
  440. package/src/providers/search-provider-catalog.ts +24 -0
  441. package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +24 -0
  442. package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +7 -1
  443. package/src/providers/speech-to-text/__tests__/vellum-managed-realtime.test.ts +50 -1
  444. package/src/providers/speech-to-text/deepgram-flux-frames.ts +11 -0
  445. package/src/providers/speech-to-text/vellum-managed-flux-realtime.ts +13 -0
  446. package/src/providers/types.ts +5 -0
  447. package/src/routes/control.ts +7 -5
  448. package/src/runtime/AGENTS.md +4 -2
  449. package/src/runtime/actor-trust-resolver.ts +3 -3
  450. package/src/runtime/agent-wake.ts +1 -1
  451. package/src/runtime/approval-message-composer.ts +3 -69
  452. package/src/runtime/assistant-event-hub.ts +73 -75
  453. package/src/runtime/assistant-event-publish-options.ts +7 -0
  454. package/src/runtime/assistant-event-targeting.ts +79 -0
  455. package/src/runtime/assistant-stream-state.ts +13 -100
  456. package/src/runtime/background-job-runner.ts +5 -5
  457. package/src/runtime/channel-retry-sweep.ts +1 -0
  458. package/src/runtime/guardian-action-message-composer.ts +8 -161
  459. package/src/runtime/guardian-reply-router.ts +28 -32
  460. package/src/runtime/http-types.ts +6 -0
  461. package/src/runtime/message-composer-types.ts +3 -61
  462. package/src/runtime/migrations/__tests__/vbundle-import-parity.test.ts +67 -1
  463. package/src/runtime/migrations/vbundle-builder.ts +24 -1
  464. package/src/runtime/migrations/vbundle-import-analyzer.ts +13 -4
  465. package/src/runtime/migrations/vbundle-import-policy.ts +10 -0
  466. package/src/runtime/migrations/vbundle-importer.ts +4 -1
  467. package/src/runtime/migrations/vbundle-streaming-importer.ts +4 -1
  468. package/src/runtime/migrations/vbundle-validator.ts +1 -0
  469. package/src/runtime/pending-interactions-mode-session.test.ts +62 -0
  470. package/src/runtime/pending-interactions.ts +29 -0
  471. package/src/runtime/routes/__tests__/client-history-routes.test.ts +104 -0
  472. package/src/runtime/routes/__tests__/image-generation-routes.test.ts +97 -0
  473. package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +1 -1
  474. package/src/runtime/routes/__tests__/roadmap-routes.test.ts +29 -6
  475. package/src/runtime/routes/browser-context.ts +31 -11
  476. package/src/runtime/routes/browser-routes.ts +43 -4
  477. package/src/runtime/routes/btw-routes.ts +9 -4
  478. package/src/runtime/routes/client-routes.ts +108 -0
  479. package/src/runtime/routes/conversation-management-routes.ts +4 -1
  480. package/src/runtime/routes/conversation-routes.ts +113 -27
  481. package/src/runtime/routes/credential-prompt-routes.ts +4 -3
  482. package/src/runtime/routes/credential-routes.ts +1 -2
  483. package/src/runtime/routes/desktop-setup-routes.test.ts +9 -18
  484. package/src/runtime/routes/desktop-setup-routes.ts +4 -6
  485. package/src/runtime/routes/documents-routes.ts +51 -0
  486. package/src/runtime/routes/email-routes.ts +3 -7
  487. package/src/runtime/routes/events-routes.ts +75 -54
  488. package/src/runtime/routes/identity-routes.ts +1 -5
  489. package/src/runtime/routes/image-generation-routes.ts +8 -14
  490. package/src/runtime/routes/inbound-message-handler.ts +35 -38
  491. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.test.ts +168 -0
  492. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.ts +96 -0
  493. package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +38 -9
  494. package/src/runtime/routes/inbound-stages/background-dispatch.ts +2 -5
  495. package/src/runtime/routes/inbound-stages/reaction-intercept.ts +2 -0
  496. package/src/runtime/routes/llm-context-normalization.ts +1 -7
  497. package/src/runtime/routes/migration-routes.ts +73 -2
  498. package/src/runtime/routes/notification-routes.ts +10 -0
  499. package/src/runtime/routes/oauth-providers.ts +7 -0
  500. package/src/runtime/routes/plugins-routes.ts +6 -0
  501. package/src/runtime/routes/roadmap-routes.ts +47 -8
  502. package/src/runtime/routes/secret-routes.ts +1 -1
  503. package/src/runtime/routes/settings-routes.ts +10 -4
  504. package/src/runtime/routes/watcher-routes.ts +1 -3
  505. package/src/runtime/trust-class.test.ts +7 -6
  506. package/src/runtime/trust-class.ts +9 -17
  507. package/src/schedule/__tests__/worker-control.test.ts +36 -5
  508. package/src/schedule/worker-control.ts +15 -5
  509. package/src/security/secure-keys.ts +53 -103
  510. package/src/skills/clawhub.ts +3 -8
  511. package/src/skills/managed-store.ts +1 -5
  512. package/src/stt/types.ts +4 -0
  513. package/src/subagent/notify.ts +54 -34
  514. package/src/subagent/parent-notification.ts +6 -0
  515. package/src/telemetry/__tests__/phone-call-funnel.test.ts +72 -0
  516. package/src/telemetry/phone-call-funnel.ts +144 -0
  517. package/src/tools/__tests__/tool-schema-root-combinator-guard.test.ts +81 -0
  518. package/src/tools/ask-question/ask-question-tool.ts +6 -4
  519. package/src/tools/browser/browser-execution.ts +22 -59
  520. package/src/tools/browser/cdp-client/__tests__/cdp-dom-helpers.test.ts +561 -39
  521. package/src/tools/browser/cdp-client/cdp-dom-helpers.ts +435 -39
  522. package/src/tools/browser/cdp-client/cdp-inspect-client.ts +1 -8
  523. package/src/tools/client-os.ts +15 -0
  524. package/src/tools/computer-use/skill-proxy-bridge.ts +1 -1
  525. package/src/tools/computer-use/target.ts +42 -0
  526. package/src/tools/document/document-tool.ts +13 -18
  527. package/src/tools/execution-target.ts +14 -16
  528. package/src/tools/host-terminal/host-shell.ts +4 -1
  529. package/src/tools/network/__tests__/web-fetch-tinyfish.test.ts +250 -0
  530. package/src/tools/network/__tests__/web-search.test.ts +203 -0
  531. package/src/tools/network/web-fetch.ts +274 -7
  532. package/src/tools/network/web-search.ts +468 -6
  533. package/src/tools/permission-checker.ts +3 -3
  534. package/src/tools/policy-context.ts +3 -1
  535. package/src/tools/skills/load.ts +2 -1
  536. package/src/tools/skills/sandbox-runner.ts +11 -2
  537. package/src/tools/skills/skill-script-runner.ts +7 -0
  538. package/src/tools/skills/skill-tool-factory.ts +15 -1
  539. package/src/tools/terminal/__tests__/safe-env.test.ts +11 -0
  540. package/src/tools/terminal/safe-env.ts +4 -0
  541. package/src/tools/terminal/shell.ts +2 -0
  542. package/src/tools/tool-approval-handler.ts +5 -1
  543. package/src/tools/tool-types.ts +3 -3
  544. package/src/tools/types.ts +8 -2
  545. package/src/util/__tests__/worker-process-command.test.ts +5 -4
  546. package/src/util/abort-reasons.ts +18 -26
  547. package/src/util/canonicalize-identity.test.ts +21 -0
  548. package/src/util/canonicalize-identity.ts +16 -0
  549. package/src/util/errors.ts +13 -3
  550. package/src/util/provider-error-patterns.ts +3 -0
  551. package/src/util/worker-process.ts +77 -35
  552. package/src/workspace/migrations/157-add-desktop-files-launcher.ts +94 -0
  553. package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +4 -10
  554. package/src/workspace/migrations/__tests__/157-add-desktop-files-launcher.test.ts +113 -0
  555. package/src/workspace/migrations/registry.ts +2 -0
  556. package/src/__tests__/computer-use-window-schema.test.ts +0 -24
  557. package/src/api/events/heartbeat-alert.ts +0 -20
  558. package/src/daemon/message-types/web-activity.ts +0 -65
  559. package/src/desktop/desktop-dependencies.test.ts +0 -131
  560. package/src/tools/computer-use/definitions.ts +0 -607
@@ -14,7 +14,7 @@
14
14
  * echo, which is the keep stream's alone.
15
15
  */
16
16
 
17
- import { describe, expect, mock, test } from "bun:test";
17
+ import { describe, expect, mock, spyOn, test } from "bun:test";
18
18
 
19
19
  import {
20
20
  createMockProvider,
@@ -25,13 +25,22 @@ import { waitFor } from "../../__tests__/helpers/wait-for.js";
25
25
 
26
26
  setConfig("memory", { enabled: false });
27
27
 
28
- import type { VoiceTurnOptions } from "../../calls/voice-session-bridge.js";
28
+ import type {
29
+ VoiceTurnHandle,
30
+ VoiceTurnOptions,
31
+ } from "../../calls/voice-session-bridge.js";
29
32
  import { Conversation } from "../../daemon/conversation.js";
30
33
  import {
31
34
  deleteConversation,
35
+ findConversation,
32
36
  setConversation,
33
37
  } from "../../daemon/conversation-registry.js";
34
38
  import {
39
+ evictConversationsForReload,
40
+ getOrCreateConversation,
41
+ } from "../../daemon/conversation-store.js";
42
+ import {
43
+ attachmentExists,
35
44
  getAttachmentsForMessage,
36
45
  uploadAttachment,
37
46
  } from "../../persistence/attachments-store.js";
@@ -40,6 +49,7 @@ import {
40
49
  getMessages,
41
50
  selectSightFrameCaptureTimes,
42
51
  } from "../../persistence/conversation-crud.js";
52
+ import { getConversationModeSession } from "../../persistence/conversation-mode-sessions.js";
43
53
  import { sightFrameAttachmentIdsFromMetadata } from "../../persistence/conversation-types.js";
44
54
  import { initializeDb } from "../../persistence/db-init.js";
45
55
  import type {
@@ -48,6 +58,7 @@ import type {
48
58
  } from "../../stt/types.js";
49
59
  import {
50
60
  LiveVoiceSession,
61
+ type LiveVoiceSessionOptions,
51
62
  type LiveVoiceTurnStarter,
52
63
  } from "../live-voice-session.js";
53
64
  import type { LiveVoiceSessionFactoryContext } from "../live-voice-session-manager.js";
@@ -92,7 +103,16 @@ async function uploadFrame(): Promise<string> {
92
103
  * A session bound to a live conversation, so a kept frame has somewhere real
93
104
  * to land. Returns a `dispose` that unregisters it.
94
105
  */
95
- function createSessionHarness(title: string) {
106
+ function createSessionHarness(
107
+ title: string,
108
+ options?: {
109
+ acquireResidency?: boolean;
110
+ acquireModeSessionResidency?: NonNullable<
111
+ LiveVoiceSessionOptions["acquireModeSessionResidency"]
112
+ >;
113
+ startVoiceTurn?: LiveVoiceTurnStarter;
114
+ },
115
+ ) {
96
116
  const conversation = createConversation(title);
97
117
  const { provider } = createMockProvider([textResponse("")]);
98
118
  const activeConversation = new Conversation(
@@ -125,18 +145,32 @@ function createSessionHarness(title: string) {
125
145
  }),
126
146
  };
127
147
 
128
- const startVoiceTurn: LiveVoiceTurnStarter = mock(
129
- async (_options: VoiceTurnOptions) => ({
130
- turnId: "bridge-turn-1",
131
- abort: mock(),
132
- }),
133
- );
148
+ const startVoiceTurn = mock(
149
+ options?.startVoiceTurn ??
150
+ (async (_options: VoiceTurnOptions) => ({
151
+ turnId: "bridge-turn-1",
152
+ abort: mock(() => {
153
+ const requestId = _options.preacceptedModeSession?.requestId;
154
+ if (requestId) {
155
+ activeConversation.modeSessions.releaseTurn(requestId);
156
+ }
157
+ }),
158
+ })),
159
+ ) satisfies LiveVoiceTurnStarter;
134
160
 
135
161
  const session = new LiveVoiceSession(context, {
136
162
  resolveTranscriber: mock(async () => new MockStreamingTranscriber()),
137
163
  startVoiceTurn,
138
164
  createTurnId: () => "live-turn-1",
139
165
  emitMetrics: false,
166
+ acquireModeSessionResidency:
167
+ options?.acquireModeSessionResidency ??
168
+ (async () => ({
169
+ coordinator: activeConversation.modeSessions,
170
+ release: options?.acquireResidency
171
+ ? activeConversation.acquireLiveVoiceResidency()
172
+ : () => {},
173
+ })),
140
174
  });
141
175
 
142
176
  return {
@@ -153,6 +187,53 @@ function createSessionHarness(title: string) {
153
187
  }
154
188
 
155
189
  describe("live-voice sight_frame frame", () => {
190
+ test("accepts camera lifecycle start, frame, and end", () => {
191
+ expect(
192
+ validateLiveVoiceClientFrame({
193
+ type: "sight_start",
194
+ cameraEpoch: 7,
195
+ source: "ambient",
196
+ }),
197
+ ).toEqual({
198
+ ok: true,
199
+ frame: { type: "sight_start", cameraEpoch: 7, source: "ambient" },
200
+ });
201
+ expect(
202
+ validateLiveVoiceClientFrame({
203
+ type: "sight_frame",
204
+ attachmentId: "att-1",
205
+ cameraEpoch: 7,
206
+ source: "ambient",
207
+ }),
208
+ ).toEqual({
209
+ ok: true,
210
+ frame: {
211
+ type: "sight_frame",
212
+ attachmentId: "att-1",
213
+ cameraEpoch: 7,
214
+ source: "ambient",
215
+ },
216
+ });
217
+ expect(
218
+ validateLiveVoiceClientFrame({ type: "sight_end", cameraEpoch: 7 }),
219
+ ).toEqual({
220
+ ok: true,
221
+ frame: { type: "sight_end", cameraEpoch: 7 },
222
+ });
223
+ });
224
+
225
+ test("rejects invalid epochs, sources, and source without an epoch", () => {
226
+ for (const frame of [
227
+ { type: "sight_start", cameraEpoch: 0 },
228
+ { type: "sight_start", cameraEpoch: 1.5 },
229
+ { type: "sight_start", cameraEpoch: 1, source: "screen" },
230
+ { type: "sight_end", cameraEpoch: Number.MAX_SAFE_INTEGER + 1 },
231
+ { type: "sight_frame", attachmentId: "att-1", source: "live" },
232
+ ]) {
233
+ expect(validateLiveVoiceClientFrame(frame).ok).toBe(false);
234
+ }
235
+ });
236
+
156
237
  test("accepts a well-formed frame", () => {
157
238
  const result = validateLiveVoiceClientFrame({
158
239
  type: "sight_frame",
@@ -296,6 +377,933 @@ describe("live-voice sight_frame frame", () => {
296
377
  });
297
378
 
298
379
  describe("live-voice camera frames kept mid-call", () => {
380
+ test("keeps one canonical conversation across reload and camera restarts", async () => {
381
+ const harness = createSessionHarness("Sight residency across reload", {
382
+ acquireResidency: true,
383
+ });
384
+ try {
385
+ await harness.session.start();
386
+ await harness.session.handleClientFrame({
387
+ type: "sight_start",
388
+ cameraEpoch: 41,
389
+ source: "live",
390
+ });
391
+ const firstAttachmentId = await uploadFrame();
392
+ await harness.session.handleClientFrame({
393
+ type: "sight_frame",
394
+ attachmentId: firstAttachmentId,
395
+ cameraEpoch: 41,
396
+ source: "live",
397
+ });
398
+ await waitFor(() => getMessages(harness.conversationId).length === 1);
399
+ const firstMetadata = JSON.parse(
400
+ getMessages(harness.conversationId)[0]!.metadata ?? "{}",
401
+ ) as { modeSession?: { id: string } };
402
+
403
+ evictConversationsForReload();
404
+ await harness.session.handleClientFrame({
405
+ type: "sight_end",
406
+ cameraEpoch: 41,
407
+ });
408
+ await waitFor(
409
+ () =>
410
+ getConversationModeSession(
411
+ harness.conversationId,
412
+ firstMetadata.modeSession!.id,
413
+ )?.status === "completed",
414
+ );
415
+
416
+ expect(await getOrCreateConversation(harness.conversationId)).toBe(
417
+ harness.activeConversation,
418
+ );
419
+
420
+ await harness.session.handleClientFrame({
421
+ type: "sight_start",
422
+ cameraEpoch: 42,
423
+ source: "ambient",
424
+ });
425
+ const secondAttachmentId = await uploadFrame();
426
+ await harness.session.handleClientFrame({
427
+ type: "sight_frame",
428
+ attachmentId: secondAttachmentId,
429
+ cameraEpoch: 42,
430
+ source: "ambient",
431
+ });
432
+ await waitFor(() => getMessages(harness.conversationId).length === 2);
433
+ const secondMetadata = JSON.parse(
434
+ getMessages(harness.conversationId)[1]!.metadata ?? "{}",
435
+ ) as { modeSession?: { id: string; mode: string } };
436
+ expect(secondMetadata.modeSession).toMatchObject({ mode: "ambient" });
437
+ expect(secondMetadata.modeSession?.id).not.toBe(
438
+ firstMetadata.modeSession?.id,
439
+ );
440
+
441
+ await harness.session.handleClientFrame({
442
+ type: "text",
443
+ text: "Describe the current view.",
444
+ });
445
+ await waitFor(() => harness.startVoiceTurn.mock.calls.length === 1);
446
+ expect(
447
+ harness.startVoiceTurn.mock.calls[0]![0].preacceptedModeSession?.source
448
+ .id,
449
+ ).toBe(secondMetadata.modeSession?.id);
450
+
451
+ await harness.session.handleClientFrame({
452
+ type: "sight_end",
453
+ cameraEpoch: 42,
454
+ });
455
+ await harness.session.close("client_end");
456
+ evictConversationsForReload();
457
+ expect(findConversation(harness.conversationId)).toBeUndefined();
458
+ } finally {
459
+ harness.dispose();
460
+ }
461
+ });
462
+
463
+ test("keeps an accepted voice owner when the camera stops during bridge admission", async () => {
464
+ let resolveTurn!: (handle: VoiceTurnHandle) => void;
465
+ const bridgeTurn = new Promise<VoiceTurnHandle>((resolve) => {
466
+ resolveTurn = resolve;
467
+ });
468
+ const harness = createSessionHarness("Sight stop during admission", {
469
+ startVoiceTurn: async () => bridgeTurn,
470
+ });
471
+ try {
472
+ await harness.session.start();
473
+ await harness.session.handleClientFrame({
474
+ type: "sight_start",
475
+ cameraEpoch: 31,
476
+ source: "live",
477
+ });
478
+ const speaking = harness.session.handleClientFrame({
479
+ type: "text",
480
+ text: "What is in view?",
481
+ });
482
+ await waitFor(() => harness.startVoiceTurn.mock.calls.length > 0);
483
+ const accepted =
484
+ harness.startVoiceTurn.mock.calls[0]![0].preacceptedModeSession!;
485
+
486
+ await harness.session.handleClientFrame({
487
+ type: "sight_end",
488
+ cameraEpoch: 31,
489
+ });
490
+
491
+ expect(
492
+ harness.activeConversation.modeSessions.getTurnOwner(
493
+ accepted.requestId,
494
+ ),
495
+ ).toEqual({ id: accepted.source.id, mode: accepted.source.mode });
496
+ resolveTurn({ turnId: "bridge-turn-1", abort: mock() });
497
+ await speaking;
498
+ harness.activeConversation.modeSessions.releaseTurn(accepted.requestId);
499
+ } finally {
500
+ await harness.session.close("client_end");
501
+ harness.dispose();
502
+ }
503
+ });
504
+
505
+ test("keeps an accepted voice owner when a new camera run replaces it during admission", async () => {
506
+ let resolveTurn!: (handle: VoiceTurnHandle) => void;
507
+ const bridgeTurn = new Promise<VoiceTurnHandle>((resolve) => {
508
+ resolveTurn = resolve;
509
+ });
510
+ const harness = createSessionHarness("Sight replacement during admission", {
511
+ startVoiceTurn: async () => bridgeTurn,
512
+ });
513
+ try {
514
+ await harness.session.start();
515
+ await harness.session.handleClientFrame({
516
+ type: "sight_start",
517
+ cameraEpoch: 32,
518
+ source: "live",
519
+ });
520
+ const speaking = harness.session.handleClientFrame({
521
+ type: "text",
522
+ text: "What is in view?",
523
+ });
524
+ await waitFor(() => harness.startVoiceTurn.mock.calls.length > 0);
525
+ const accepted =
526
+ harness.startVoiceTurn.mock.calls[0]![0].preacceptedModeSession!;
527
+
528
+ await harness.session.handleClientFrame({
529
+ type: "sight_start",
530
+ cameraEpoch: 33,
531
+ source: "ambient",
532
+ });
533
+
534
+ expect(accepted.source.generation).toBe(32);
535
+ expect(
536
+ harness.activeConversation.modeSessions.getTurnOwner(
537
+ accepted.requestId,
538
+ ),
539
+ ).toEqual({ id: accepted.source.id, mode: accepted.source.mode });
540
+ resolveTurn({ turnId: "bridge-turn-1", abort: mock() });
541
+ await speaking;
542
+ harness.activeConversation.modeSessions.releaseTurn(accepted.requestId);
543
+ } finally {
544
+ await harness.session.close("client_end");
545
+ harness.dispose();
546
+ }
547
+ });
548
+
549
+ test("orders sight_end behind deferred camera-session preparation", async () => {
550
+ let resolvePreparation!: (value: Conversation["modeSessions"]) => void;
551
+ const preparation = new Promise<Conversation["modeSessions"]>((resolve) => {
552
+ resolvePreparation = resolve;
553
+ });
554
+ const harness = createSessionHarness("Deferred sight end", {
555
+ acquireModeSessionResidency: async () => ({
556
+ coordinator: await preparation,
557
+ release: () => {},
558
+ }),
559
+ });
560
+ try {
561
+ await harness.session.start();
562
+ const starting = harness.session.handleClientFrame({
563
+ type: "sight_start",
564
+ cameraEpoch: 21,
565
+ source: "live",
566
+ });
567
+ const ending = harness.session.handleClientFrame({
568
+ type: "sight_end",
569
+ cameraEpoch: 21,
570
+ });
571
+
572
+ resolvePreparation(harness.activeConversation.modeSessions);
573
+ await Promise.all([starting, ending]);
574
+ await harness.session.handleClientFrame({
575
+ type: "text",
576
+ text: "Is the camera still active?",
577
+ });
578
+ await waitFor(() => harness.startVoiceTurn.mock.calls.length > 0);
579
+
580
+ expect(
581
+ harness.startVoiceTurn.mock.calls[0]![0].modeSessionSource,
582
+ ).toBeUndefined();
583
+ } finally {
584
+ await harness.session.close("client_end");
585
+ harness.dispose();
586
+ }
587
+ });
588
+
589
+ test("orders a kept frame behind deferred camera-session preparation", async () => {
590
+ let resolvePreparation!: (value: Conversation["modeSessions"]) => void;
591
+ const preparation = new Promise<Conversation["modeSessions"]>((resolve) => {
592
+ resolvePreparation = resolve;
593
+ });
594
+ const harness = createSessionHarness("Deferred sight frame", {
595
+ acquireModeSessionResidency: async () => ({
596
+ coordinator: await preparation,
597
+ release: () => {},
598
+ }),
599
+ });
600
+ try {
601
+ await harness.session.start();
602
+ const attachmentId = await uploadFrame();
603
+ const starting = harness.session.handleClientFrame({
604
+ type: "sight_start",
605
+ cameraEpoch: 22,
606
+ source: "ambient",
607
+ });
608
+ const keeping = harness.session.handleClientFrame({
609
+ type: "sight_frame",
610
+ attachmentId,
611
+ cameraEpoch: 22,
612
+ source: "ambient",
613
+ });
614
+
615
+ resolvePreparation(harness.activeConversation.modeSessions);
616
+ await Promise.all([starting, keeping]);
617
+ await waitFor(() => getMessages(harness.conversationId).length > 0);
618
+
619
+ const row = getMessages(harness.conversationId)[0]!;
620
+ const metadata = JSON.parse(row.metadata ?? "{}") as {
621
+ modeSession?: { id: string; mode: string };
622
+ };
623
+ expect(metadata.modeSession?.mode).toBe("ambient");
624
+ expect(
625
+ harness.frames.filter((frame) => frame.type === "error"),
626
+ ).toHaveLength(0);
627
+ } finally {
628
+ await harness.session.close("client_end");
629
+ harness.dispose();
630
+ }
631
+ });
632
+
633
+ test("does not activate a deferred camera start after session close", async () => {
634
+ let resolvePreparation!: (value: Conversation["modeSessions"]) => void;
635
+ const preparation = new Promise<Conversation["modeSessions"]>((resolve) => {
636
+ resolvePreparation = resolve;
637
+ });
638
+ const release = mock(() => {});
639
+ const acquire = mock(async () => ({
640
+ coordinator: await preparation,
641
+ release,
642
+ }));
643
+ const harness = createSessionHarness("Deferred sight close", {
644
+ acquireModeSessionResidency: acquire,
645
+ });
646
+ const activateSource = spyOn(
647
+ harness.activeConversation.modeSessions,
648
+ "activateSource",
649
+ );
650
+ try {
651
+ await harness.session.start();
652
+ const starting = harness.session.handleClientFrame({
653
+ type: "sight_start",
654
+ cameraEpoch: 23,
655
+ source: "live",
656
+ });
657
+ await waitFor(() => acquire.mock.calls.length === 1);
658
+ const closing = harness.session.close("websocket_close");
659
+
660
+ resolvePreparation(harness.activeConversation.modeSessions);
661
+ await Promise.all([starting, closing]);
662
+
663
+ expect(activateSource).not.toHaveBeenCalled();
664
+ expect(release).toHaveBeenCalledTimes(1);
665
+ await harness.session.close("client_end");
666
+ expect(release).toHaveBeenCalledTimes(1);
667
+ } finally {
668
+ activateSource.mockRestore();
669
+ harness.dispose();
670
+ }
671
+ });
672
+
673
+ test("captures the active camera owner for a typed voice turn", async () => {
674
+ const harness = createSessionHarness("Sight-owned voice turn");
675
+ try {
676
+ await harness.session.start();
677
+ await harness.session.handleClientFrame({
678
+ type: "sight_start",
679
+ cameraEpoch: 11,
680
+ source: "ambient",
681
+ });
682
+ await harness.session.handleClientFrame({
683
+ type: "text",
684
+ text: "What do you see?",
685
+ });
686
+ await waitFor(() => harness.startVoiceTurn.mock.calls.length > 0, {
687
+ message: "Timed out waiting for typed voice turn",
688
+ });
689
+
690
+ const options = harness.startVoiceTurn.mock.calls[0]![0];
691
+ expect(options.preacceptedModeSession?.source).toMatchObject({
692
+ generation: 11,
693
+ mode: "ambient",
694
+ });
695
+ } finally {
696
+ await harness.session.close("client_end");
697
+ harness.dispose();
698
+ }
699
+ });
700
+
701
+ test("dispatches voice ungrouped when the first delivery claim throws", async () => {
702
+ const harness = createSessionHarness("Sight voice first claim failure");
703
+ const coordinator = harness.activeConversation.modeSessions;
704
+ const originalClaimTurn = coordinator.claimTurn.bind(coordinator);
705
+ let attemptedRequestId: string | undefined;
706
+ const claimTurn = spyOn(coordinator, "claimTurn").mockImplementation(
707
+ (turnId, source, at) => {
708
+ const owner = originalClaimTurn(turnId, source, at);
709
+ if (!turnId.startsWith("live-voice-camera:")) {
710
+ attemptedRequestId = turnId;
711
+ throw new Error("session tracking unavailable");
712
+ }
713
+ return owner;
714
+ },
715
+ );
716
+ try {
717
+ await harness.session.start();
718
+ await harness.session.handleClientFrame({
719
+ type: "sight_start",
720
+ cameraEpoch: 54,
721
+ source: "live",
722
+ });
723
+ await harness.session.handleClientFrame({
724
+ type: "text",
725
+ text: "What is here?",
726
+ });
727
+ await waitFor(() => harness.startVoiceTurn.mock.calls.length === 1);
728
+
729
+ expect(
730
+ harness.startVoiceTurn.mock.calls[0]![0].preacceptedModeSession,
731
+ ).toBeUndefined();
732
+ expect(attemptedRequestId).toBeDefined();
733
+ expect(coordinator.getTurnOwner(attemptedRequestId!)).toBeUndefined();
734
+ } finally {
735
+ claimTurn.mockRestore();
736
+ await harness.session.close("client_end");
737
+ harness.dispose();
738
+ }
739
+ });
740
+
741
+ test("releases a mismatched first delivery claim before ungrouped dispatch", async () => {
742
+ const harness = createSessionHarness("Sight voice first claim mismatch");
743
+ const coordinator = harness.activeConversation.modeSessions;
744
+ const originalClaimTurn = coordinator.claimTurn.bind(coordinator);
745
+ let attemptedRequestId: string | undefined;
746
+ const claimTurn = spyOn(coordinator, "claimTurn").mockImplementation(
747
+ (turnId, source, at) => {
748
+ const owner = originalClaimTurn(turnId, source, at);
749
+ if (!turnId.startsWith("live-voice-camera:") && owner) {
750
+ attemptedRequestId = turnId;
751
+ return { id: "different-session", mode: owner.mode };
752
+ }
753
+ return owner;
754
+ },
755
+ );
756
+ try {
757
+ await harness.session.start();
758
+ await harness.session.handleClientFrame({
759
+ type: "sight_start",
760
+ cameraEpoch: 57,
761
+ source: "ambient",
762
+ });
763
+ await harness.session.handleClientFrame({
764
+ type: "text",
765
+ text: "What is here?",
766
+ });
767
+ await waitFor(() => harness.startVoiceTurn.mock.calls.length === 1);
768
+
769
+ expect(
770
+ harness.startVoiceTurn.mock.calls[0]![0].preacceptedModeSession,
771
+ ).toBeUndefined();
772
+ expect(attemptedRequestId).toBeDefined();
773
+ expect(coordinator.getTurnOwner(attemptedRequestId!)).toBeUndefined();
774
+ } finally {
775
+ claimTurn.mockRestore();
776
+ await harness.session.close("client_end");
777
+ harness.dispose();
778
+ }
779
+ });
780
+
781
+ test("releases every delivery claim when a later admission throws", async () => {
782
+ const harness = createSessionHarness("Sight voice later claim failure");
783
+ const coordinator = harness.activeConversation.modeSessions;
784
+ const originalClaimTurn = coordinator.claimTurn.bind(coordinator);
785
+ const attemptedRequestIds: string[] = [];
786
+ const claimTurn = spyOn(coordinator, "claimTurn").mockImplementation(
787
+ (turnId, source, at) => {
788
+ const owner = originalClaimTurn(turnId, source, at);
789
+ if (!turnId.startsWith("live-voice-camera:")) {
790
+ attemptedRequestIds.push(turnId);
791
+ if (attemptedRequestIds.length === 3) {
792
+ throw new Error("session tracking unavailable");
793
+ }
794
+ }
795
+ return owner;
796
+ },
797
+ );
798
+ try {
799
+ await harness.session.start();
800
+ await harness.session.handleClientFrame({
801
+ type: "sight_start",
802
+ cameraEpoch: 55,
803
+ source: "ambient",
804
+ });
805
+ await harness.session.handleClientFrame({
806
+ type: "text",
807
+ text: "What is here?",
808
+ });
809
+ await waitFor(() => harness.startVoiceTurn.mock.calls.length === 1);
810
+
811
+ expect(attemptedRequestIds).toHaveLength(3);
812
+ expect(
813
+ harness.startVoiceTurn.mock.calls[0]![0].preacceptedModeSession,
814
+ ).toBeUndefined();
815
+ expect(
816
+ attemptedRequestIds.map((id) => coordinator.getTurnOwner(id)),
817
+ ).toEqual([undefined, undefined, undefined]);
818
+ } finally {
819
+ claimTurn.mockRestore();
820
+ await harness.session.close("client_end");
821
+ harness.dispose();
822
+ }
823
+ });
824
+
825
+ test("keeps a stopped camera session active until voice delivery settles", async () => {
826
+ const harness = createSessionHarness("Sight voice delivery barrier");
827
+ try {
828
+ await harness.session.start();
829
+ await harness.session.handleClientFrame({
830
+ type: "sight_start",
831
+ cameraEpoch: 12,
832
+ source: "live",
833
+ });
834
+ await harness.session.handleClientFrame({
835
+ type: "text",
836
+ text: "Describe this.",
837
+ });
838
+ await waitFor(() => harness.startVoiceTurn.mock.calls.length > 0);
839
+
840
+ const options = harness.startVoiceTurn.mock.calls[0]![0];
841
+ const sessionId = options.preacceptedModeSession!.source.id;
842
+ await harness.session.handleClientFrame({
843
+ type: "sight_end",
844
+ cameraEpoch: 12,
845
+ });
846
+ expect(
847
+ getConversationModeSession(harness.conversationId, sessionId)?.status,
848
+ ).toBe("active");
849
+
850
+ options.callbacks?.message_complete?.({
851
+ type: "message_complete",
852
+ conversationId: harness.conversationId,
853
+ messageId: "assistant-message-1",
854
+ });
855
+ harness.activeConversation.modeSessions.releaseTurn(
856
+ options.preacceptedModeSession!.requestId,
857
+ );
858
+ await waitFor(
859
+ () =>
860
+ getConversationModeSession(harness.conversationId, sessionId)
861
+ ?.status === "completed",
862
+ { message: "Timed out waiting for voice delivery settlement" },
863
+ );
864
+ expect(harness.frames.some((frame) => frame.type === "tts_done")).toBe(
865
+ true,
866
+ );
867
+ } finally {
868
+ await harness.session.close("client_end");
869
+ harness.dispose();
870
+ }
871
+ });
872
+
873
+ test("releases remaining delivery claims when one release throws", async () => {
874
+ const harness = createSessionHarness("Sight voice release failure");
875
+ const coordinator = harness.activeConversation.modeSessions;
876
+ const originalClaimTurn = coordinator.claimTurn.bind(coordinator);
877
+ const claimedRequestIds: string[] = [];
878
+ const claimTurn = spyOn(coordinator, "claimTurn").mockImplementation(
879
+ (turnId, source, at) => {
880
+ const owner = originalClaimTurn(turnId, source, at);
881
+ if (!turnId.startsWith("live-voice-camera:")) {
882
+ claimedRequestIds.push(turnId);
883
+ }
884
+ return owner;
885
+ },
886
+ );
887
+ const originalReleaseTurn = coordinator.releaseTurn.bind(coordinator);
888
+ const releasedRequestIds: string[] = [];
889
+ let injectedFailure = false;
890
+ const releaseTurn = spyOn(coordinator, "releaseTurn").mockImplementation(
891
+ (turnId) => {
892
+ const released = originalReleaseTurn(turnId);
893
+ if (!turnId.startsWith("live-voice-camera:")) {
894
+ releasedRequestIds.push(turnId);
895
+ if (!injectedFailure) {
896
+ injectedFailure = true;
897
+ throw new Error("session tracking unavailable");
898
+ }
899
+ }
900
+ return released;
901
+ },
902
+ );
903
+ try {
904
+ await harness.session.start();
905
+ await harness.session.handleClientFrame({
906
+ type: "sight_start",
907
+ cameraEpoch: 56,
908
+ source: "live",
909
+ });
910
+ await harness.session.handleClientFrame({
911
+ type: "text",
912
+ text: "Describe this.",
913
+ });
914
+ await waitFor(() => harness.startVoiceTurn.mock.calls.length === 1);
915
+ const options = harness.startVoiceTurn.mock.calls[0]![0];
916
+
917
+ options.callbacks?.message_complete?.({
918
+ type: "message_complete",
919
+ conversationId: harness.conversationId,
920
+ messageId: "assistant-message-release-failure",
921
+ });
922
+ await waitFor(() =>
923
+ harness.frames.some((frame) => frame.type === "tts_done"),
924
+ );
925
+
926
+ expect(claimedRequestIds).toHaveLength(3);
927
+ expect(releasedRequestIds).toContain(claimedRequestIds[1]!);
928
+ expect(releasedRequestIds).toContain(claimedRequestIds[2]!);
929
+ expect(coordinator.getTurnOwner(claimedRequestIds[1]!)).toBeUndefined();
930
+ expect(coordinator.getTurnOwner(claimedRequestIds[2]!)).toBeUndefined();
931
+ } finally {
932
+ releaseTurn.mockRestore();
933
+ claimTurn.mockRestore();
934
+ await harness.session.close("client_end");
935
+ harness.dispose();
936
+ }
937
+ });
938
+
939
+ test("socket loss interrupts a camera run after cancelling its voice hold", async () => {
940
+ const harness = createSessionHarness("Sight socket loss");
941
+ try {
942
+ await harness.session.start();
943
+ await harness.session.handleClientFrame({
944
+ type: "sight_start",
945
+ cameraEpoch: 13,
946
+ source: "ambient",
947
+ });
948
+ await harness.session.handleClientFrame({
949
+ type: "text",
950
+ text: "What is here?",
951
+ });
952
+ await waitFor(() => harness.startVoiceTurn.mock.calls.length > 0);
953
+ const sessionId =
954
+ harness.startVoiceTurn.mock.calls[0]![0].preacceptedModeSession!.source
955
+ .id;
956
+
957
+ await harness.session.close("websocket_close");
958
+
959
+ await waitFor(
960
+ () =>
961
+ getConversationModeSession(harness.conversationId, sessionId)
962
+ ?.status === "interrupted",
963
+ { message: "Timed out waiting for interrupted camera session" },
964
+ );
965
+ expect(
966
+ getConversationModeSession(harness.conversationId, sessionId)
967
+ ?.endReason,
968
+ ).toBe("camera_websocket_close");
969
+ } finally {
970
+ harness.dispose();
971
+ }
972
+ });
973
+
974
+ test("releases a reserved voice owner when close rejects deferred admission", async () => {
975
+ const startVoiceTurn: LiveVoiceTurnStarter = async (options) => {
976
+ await new Promise<void>((_resolve, reject) => {
977
+ const rejectAdmission = () => reject(new Error("admission cancelled"));
978
+ if (options.signal?.aborted) {
979
+ rejectAdmission();
980
+ return;
981
+ }
982
+ options.signal?.addEventListener("abort", rejectAdmission, {
983
+ once: true,
984
+ });
985
+ });
986
+ return { turnId: "unreachable", abort: mock() };
987
+ };
988
+ const harness = createSessionHarness("Sight deferred admission close", {
989
+ startVoiceTurn,
990
+ });
991
+ try {
992
+ await harness.session.start();
993
+ await harness.session.handleClientFrame({
994
+ type: "sight_start",
995
+ cameraEpoch: 14,
996
+ source: "live",
997
+ });
998
+ const speaking = harness.session.handleClientFrame({
999
+ type: "text",
1000
+ text: "What is here?",
1001
+ });
1002
+ await waitFor(() => harness.startVoiceTurn.mock.calls.length === 1);
1003
+ const accepted =
1004
+ harness.startVoiceTurn.mock.calls[0]![0].preacceptedModeSession!;
1005
+
1006
+ await harness.session.close("websocket_close");
1007
+ await speaking;
1008
+
1009
+ await waitFor(
1010
+ () =>
1011
+ getConversationModeSession(harness.conversationId, accepted.source.id)
1012
+ ?.status === "interrupted",
1013
+ );
1014
+ expect(
1015
+ harness.activeConversation.modeSessions.getTurnOwner(
1016
+ accepted.requestId,
1017
+ ),
1018
+ ).toBeUndefined();
1019
+ } finally {
1020
+ harness.dispose();
1021
+ }
1022
+ });
1023
+
1024
+ test("stamps negotiated keeps and finalizes the run on sight_end", async () => {
1025
+ const harness = createSessionHarness("Sight session lifecycle");
1026
+ try {
1027
+ await harness.session.start();
1028
+ await harness.session.handleClientFrame({
1029
+ type: "sight_start",
1030
+ cameraEpoch: 1,
1031
+ source: "live",
1032
+ });
1033
+ const attachmentId = await uploadFrame();
1034
+ await harness.session.handleClientFrame({
1035
+ type: "sight_frame",
1036
+ attachmentId,
1037
+ cameraEpoch: 1,
1038
+ source: "live",
1039
+ });
1040
+ await waitFor(() => getMessages(harness.conversationId).length > 0, {
1041
+ message: "Timed out waiting for grouped camera frame",
1042
+ });
1043
+
1044
+ const row = getMessages(harness.conversationId)[0]!;
1045
+ const metadata = JSON.parse(row.metadata ?? "{}") as {
1046
+ modeSession?: { id: string; mode: string };
1047
+ };
1048
+ expect(metadata.modeSession?.mode).toBe("live_vision");
1049
+ await harness.session.handleClientFrame({
1050
+ type: "sight_end",
1051
+ cameraEpoch: 1,
1052
+ });
1053
+
1054
+ await waitFor(
1055
+ () =>
1056
+ getConversationModeSession(
1057
+ harness.conversationId,
1058
+ metadata.modeSession!.id,
1059
+ )?.status === "completed",
1060
+ { message: "Timed out waiting for camera session finalization" },
1061
+ );
1062
+ } finally {
1063
+ harness.dispose();
1064
+ }
1065
+ });
1066
+
1067
+ test("saves an admitted frame after stop while rejecting a frame delivered after stop", async () => {
1068
+ const harness = createSessionHarness("Sight admitted frame drain", {
1069
+ acquireResidency: true,
1070
+ });
1071
+ const processing = harness.activeConversation.acquireProcessing();
1072
+ expect(processing).not.toBeNull();
1073
+ try {
1074
+ await harness.session.start();
1075
+ await harness.session.handleClientFrame({
1076
+ type: "sight_start",
1077
+ cameraEpoch: 61,
1078
+ source: "live",
1079
+ });
1080
+ const accepted = await uploadFrame();
1081
+ await harness.session.handleClientFrame({
1082
+ type: "sight_frame",
1083
+ attachmentId: accepted,
1084
+ cameraEpoch: 61,
1085
+ source: "live",
1086
+ });
1087
+ await harness.session.handleClientFrame({
1088
+ type: "sight_end",
1089
+ cameraEpoch: 61,
1090
+ });
1091
+ const late = await uploadFrame();
1092
+ await harness.session.handleClientFrame({
1093
+ type: "sight_frame",
1094
+ attachmentId: late,
1095
+ cameraEpoch: 61,
1096
+ source: "live",
1097
+ });
1098
+ expect(attachmentExists(late)).toBe(false);
1099
+ expect(getMessages(harness.conversationId)).toHaveLength(0);
1100
+ expect(harness.activeConversation.modeSessions.hasResidentWork()).toBe(
1101
+ true,
1102
+ );
1103
+ await harness.session.close("client_end");
1104
+ expect(harness.activeConversation.hasInFlightWork()).toBe(true);
1105
+ harness.activeConversation.releaseProcessing(processing!);
1106
+ await waitFor(
1107
+ () =>
1108
+ getMessages(harness.conversationId).length === 1 &&
1109
+ !harness.activeConversation.hasInFlightWork(),
1110
+ );
1111
+ const row = getMessages(harness.conversationId)[0]!;
1112
+ const stamp = JSON.parse(row.metadata ?? "{}").modeSession;
1113
+ expect(
1114
+ getConversationModeSession(harness.conversationId, stamp.id)?.status,
1115
+ ).toBe("completed");
1116
+ expect(getAttachmentsForMessage(row.id)).toHaveLength(1);
1117
+ expect(attachmentExists(accepted)).toBe(true);
1118
+ expect(
1119
+ harness.frames.filter((frame) => frame.type === "error"),
1120
+ ).toMatchObject([{ frameType: "sight_frame", attachmentId: late }]);
1121
+ } finally {
1122
+ harness.activeConversation.releaseProcessing(processing!);
1123
+ await harness.session.close("client_end");
1124
+ harness.dispose();
1125
+ }
1126
+ });
1127
+
1128
+ test.each([false, true])(
1129
+ "persists an accepted camera frame when its claim throws (partial: %s)",
1130
+ async (partial) => {
1131
+ const harness = createSessionHarness("Sight frame ownership failure");
1132
+ const coordinator = harness.activeConversation.modeSessions;
1133
+ const originalClaimTurn = coordinator.claimTurn.bind(coordinator);
1134
+ let failedSessionId: string | undefined;
1135
+ const claimTurn = spyOn(coordinator, "claimTurn").mockImplementation(
1136
+ (turnId, source, at) => {
1137
+ if (!turnId.startsWith("live-voice-camera:")) {
1138
+ if (partial) {
1139
+ originalClaimTurn(turnId, source, at);
1140
+ }
1141
+ throw new Error("session tracking unavailable");
1142
+ }
1143
+ const owner = originalClaimTurn(turnId, source, at);
1144
+ failedSessionId = owner?.id;
1145
+ return owner;
1146
+ },
1147
+ );
1148
+ try {
1149
+ await harness.session.start();
1150
+ await harness.session.handleClientFrame({
1151
+ type: "sight_start",
1152
+ cameraEpoch: 52,
1153
+ source: "ambient",
1154
+ });
1155
+ const attachmentId = await uploadFrame();
1156
+ const keeping = harness.session.handleClientFrame({
1157
+ type: "sight_frame",
1158
+ attachmentId,
1159
+ cameraEpoch: 52,
1160
+ source: "ambient",
1161
+ });
1162
+ const ending = harness.session.handleClientFrame({
1163
+ type: "sight_end",
1164
+ cameraEpoch: 52,
1165
+ });
1166
+ await Promise.all([keeping, ending]);
1167
+
1168
+ await waitFor(
1169
+ () =>
1170
+ failedSessionId !== undefined &&
1171
+ getConversationModeSession(harness.conversationId, failedSessionId)
1172
+ ?.status === "completed" &&
1173
+ getMessages(harness.conversationId).length === 1 &&
1174
+ !coordinator.hasResidentWork(),
1175
+ { message: "Timed out waiting for the accepted frame to settle" },
1176
+ );
1177
+ expect(attachmentExists(attachmentId)).toBe(true);
1178
+ const row = getMessages(harness.conversationId)[0]!;
1179
+ expect(getAttachmentsForMessage(row.id)).toHaveLength(1);
1180
+ expect(
1181
+ harness.frames.filter((frame) => frame.type === "error"),
1182
+ ).toHaveLength(0);
1183
+ } finally {
1184
+ claimTurn.mockRestore();
1185
+ await harness.session.close("client_end");
1186
+ harness.dispose();
1187
+ }
1188
+ },
1189
+ );
1190
+
1191
+ test("releases acquired residency after a refused camera start and accepts ordinary frames", async () => {
1192
+ const harness = createSessionHarness("Sight start tracking failure", {
1193
+ acquireResidency: true,
1194
+ });
1195
+ const claim = spyOn(
1196
+ harness.activeConversation.modeSessions,
1197
+ "claimTurn",
1198
+ ).mockImplementation(() => {
1199
+ throw new Error("tracking unavailable");
1200
+ });
1201
+ try {
1202
+ await harness.session.start();
1203
+ await harness.session.handleClientFrame({
1204
+ type: "sight_start",
1205
+ cameraEpoch: 1,
1206
+ source: "live",
1207
+ });
1208
+ expect(
1209
+ harness.frames.find((frame) => frame.type === "error"),
1210
+ ).toMatchObject({ frameType: "sight_start" });
1211
+ expect(harness.activeConversation.modeSessions.hasResidentWork()).toBe(
1212
+ false,
1213
+ );
1214
+ expect(harness.activeConversation.hasInFlightWork()).toBe(false);
1215
+ const attachmentId = await uploadFrame();
1216
+ await harness.session.handleClientFrame({
1217
+ type: "sight_frame",
1218
+ attachmentId,
1219
+ });
1220
+ await waitFor(() => getMessages(harness.conversationId).length === 1);
1221
+ expect(
1222
+ JSON.parse(getMessages(harness.conversationId)[0]!.metadata ?? "{}")
1223
+ .modeSession,
1224
+ ).toBeUndefined();
1225
+ expect(attachmentExists(attachmentId)).toBe(true);
1226
+ } finally {
1227
+ claim.mockRestore();
1228
+ await harness.session.close("client_end");
1229
+ harness.dispose();
1230
+ }
1231
+ });
1232
+
1233
+ test("keeps a persisted frame successful when bookkeeping and ownership release throw", async () => {
1234
+ const harness = createSessionHarness("Sight frame release failure");
1235
+ const coordinator = harness.activeConversation.modeSessions;
1236
+ const originalReleaseTurn = coordinator.releaseTurn.bind(coordinator);
1237
+ const releaseTurn = spyOn(coordinator, "releaseTurn").mockImplementation(
1238
+ (turnId) => {
1239
+ originalReleaseTurn(turnId);
1240
+ if (!turnId.startsWith("live-voice-camera:")) {
1241
+ throw new Error("session tracking unavailable");
1242
+ }
1243
+ },
1244
+ );
1245
+ const originalTrackRow = coordinator.trackPersistedRow.bind(coordinator);
1246
+ const trackPersistedRow = spyOn(
1247
+ coordinator,
1248
+ "trackPersistedRow",
1249
+ ).mockImplementation((turnId, messageId, at, options) => {
1250
+ originalTrackRow(turnId, messageId, at, options);
1251
+ if (turnId.startsWith("live-voice-camera:")) {
1252
+ throw new Error("tracking unavailable after persistence");
1253
+ }
1254
+ });
1255
+ try {
1256
+ await harness.session.start();
1257
+ await harness.session.handleClientFrame({
1258
+ type: "sight_start",
1259
+ cameraEpoch: 53,
1260
+ source: "live",
1261
+ });
1262
+ const attachmentId = await uploadFrame();
1263
+ const before = harness.frames.length;
1264
+ await harness.session.handleClientFrame({
1265
+ type: "sight_frame",
1266
+ attachmentId,
1267
+ cameraEpoch: 53,
1268
+ source: "live",
1269
+ });
1270
+ await waitFor(() => getMessages(harness.conversationId).length === 1);
1271
+ const row = getMessages(harness.conversationId)[0]!;
1272
+
1273
+ await harness.session.handleClientFrame({
1274
+ type: "sight_end",
1275
+ cameraEpoch: 53,
1276
+ });
1277
+ const metadata = JSON.parse(row.metadata ?? "{}") as {
1278
+ modeSession?: { id: string };
1279
+ };
1280
+ await waitFor(
1281
+ () =>
1282
+ getConversationModeSession(
1283
+ harness.conversationId,
1284
+ metadata.modeSession!.id,
1285
+ )?.status === "completed",
1286
+ );
1287
+
1288
+ expect(attachmentExists(attachmentId)).toBe(true);
1289
+ expect(getAttachmentsForMessage(row.id)).toHaveLength(1);
1290
+ expect(
1291
+ trackPersistedRow.mock.calls.some(
1292
+ ([turnId, messageId]) =>
1293
+ turnId.startsWith("live-voice-camera:") && messageId === row.id,
1294
+ ),
1295
+ ).toBe(true);
1296
+ expect(
1297
+ harness.frames.slice(before).filter((frame) => frame.type === "error"),
1298
+ ).toHaveLength(0);
1299
+ } finally {
1300
+ trackPersistedRow.mockRestore();
1301
+ releaseTurn.mockRestore();
1302
+ await harness.session.close("client_end");
1303
+ harness.dispose();
1304
+ }
1305
+ });
1306
+
299
1307
  test("a kept frame persists as its own tagged message and runs no turn", async () => {
300
1308
  const harness = createSessionHarness("Sight keep stream");
301
1309
  try {