@vellumai/assistant 0.12.2 → 0.12.3-staging.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (772) hide show
  1. package/.env.example +2 -0
  2. package/AGENTS.md +1 -1
  3. package/ARCHITECTURE.md +43 -29
  4. package/Dockerfile +63 -0
  5. package/docs/architecture/integrations.md +7 -0
  6. package/docs/architecture/memory.md +17 -2
  7. package/docs/architecture/security.md +10 -5
  8. package/docs/architecture/turn-actor.md +2 -0
  9. package/docs/credential-execution-service.md +1 -1
  10. package/docs/desktop-browser-cli.md +50 -5
  11. package/docs/live-voice-task-outcomes.md +100 -0
  12. package/docs/mode-sessions.md +183 -0
  13. package/docs/voice-input-diagnostics.md +58 -0
  14. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  15. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
  16. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  17. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  18. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
  19. package/node_modules/@vellumai/environments/src/shell.test.ts +21 -0
  20. package/node_modules/@vellumai/environments/src/shell.ts +24 -0
  21. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  22. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
  23. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  24. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  25. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
  26. package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +20 -0
  27. package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +8 -2
  28. package/node_modules/@vellumai/gateway-client/src/index.ts +4 -1
  29. package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +38 -7
  30. package/node_modules/@vellumai/gateway-client/src/trust-verdict-contract.ts +5 -6
  31. package/node_modules/@vellumai/service-contracts/package.json +2 -1
  32. package/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
  33. package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  34. package/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  35. package/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
  36. package/node_modules/@vellumai/slack-text/src/index.ts +4 -8
  37. package/openapi.yaml +716 -16
  38. package/package.json +1 -1
  39. package/scripts/smoke-desktop-browser-cli.ts +11 -26
  40. package/scripts/smoke-desktop-window-theme.ts +2 -3
  41. package/scripts/sync-llm-catalog.ts +3 -0
  42. package/scripts/voice-ttft-spike.ts +2 -2
  43. package/src/__tests__/agent-loop-output-hooks.test.ts +150 -1
  44. package/src/__tests__/agent-loop-provider-error-recording.test.ts +87 -19
  45. package/src/__tests__/agent-loop.test.ts +259 -0
  46. package/src/__tests__/agent-wake-delegation-prompt.test.ts +64 -2
  47. package/src/__tests__/annotate-activity-metadata.test.ts +3 -1
  48. package/src/__tests__/annotate-risk-options.test.ts +2 -0
  49. package/src/__tests__/answered-question-persistence.test.ts +2 -0
  50. package/src/__tests__/anthropic-gateway-shared.test.ts +22 -0
  51. package/src/__tests__/anthropic-provider.test.ts +55 -0
  52. package/src/__tests__/approval-interception-trust-gates.test.ts +40 -0
  53. package/src/__tests__/approval-message-composer.test.ts +1 -118
  54. package/src/__tests__/approval-routes-http.test.ts +3 -0
  55. package/src/__tests__/assistant-event-hub-connection-history.test.ts +111 -0
  56. package/src/__tests__/assistant-event-hub-targeted.test.ts +50 -0
  57. package/src/__tests__/assistant-event-targeting.test.ts +186 -0
  58. package/src/__tests__/assistant-stream-state.test.ts +29 -9
  59. package/src/__tests__/attachments-store.test.ts +22 -3
  60. package/src/__tests__/browser-fill-credential.test.ts +11 -22
  61. package/src/__tests__/btw-routes.test.ts +4 -1
  62. package/src/__tests__/build-persisted-content.test.ts +1 -1
  63. package/src/__tests__/call-conversation-messages.test.ts +5 -0
  64. package/src/__tests__/call-recovery.test.ts +7 -2
  65. package/src/__tests__/call-routes-http.test.ts +12 -0
  66. package/src/__tests__/call-store.test.ts +7 -2
  67. package/src/__tests__/channel-approval.test.ts +9 -14
  68. package/src/__tests__/channel-reply-delivery.test.ts +56 -0
  69. package/src/__tests__/channel-retry-sweep.test.ts +9 -0
  70. package/src/__tests__/chat-credential-redaction.test.ts +23 -0
  71. package/src/__tests__/chat-reveal-guard-priming.test.ts +2 -0
  72. package/src/__tests__/checker.test.ts +11 -3
  73. package/src/__tests__/client-os-metadata-persistence.test.ts +35 -1
  74. package/src/__tests__/computer-use-screenshot-attachments.test.ts +411 -0
  75. package/src/__tests__/computer-use-screenshot-selection.test.ts +50 -0
  76. package/src/__tests__/computer-use-skill-manifest-regression.test.ts +0 -25
  77. package/src/__tests__/computer-use-tools.test.ts +1 -233
  78. package/src/__tests__/config-model-image-provider.test.ts +39 -0
  79. package/src/__tests__/config-schema.test.ts +16 -0
  80. package/src/__tests__/config-watcher.test.ts +24 -0
  81. package/src/__tests__/context-search-agent-runner.test.ts +260 -1
  82. package/src/__tests__/context-search-fanout.test.ts +1 -1
  83. package/src/__tests__/context-search-types.test.ts +7 -7
  84. package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +8 -2
  85. package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +2 -0
  86. package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +2 -0
  87. package/src/__tests__/conversation-agent-loop-overflow.test.ts +2 -0
  88. package/src/__tests__/conversation-agent-loop.test.ts +797 -16
  89. package/src/__tests__/conversation-app-control-lifecycle.test.ts +53 -1
  90. package/src/__tests__/conversation-attachments.test.ts +106 -0
  91. package/src/__tests__/conversation-evictor.test.ts +62 -37
  92. package/src/__tests__/conversation-fork-crud.test.ts +191 -0
  93. package/src/__tests__/conversation-history-boundary-repair-failure.test.ts +174 -0
  94. package/src/__tests__/conversation-loop-db-lock-resilience.test.ts +2 -0
  95. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +2 -0
  96. package/src/__tests__/conversation-rendered-delegation-state.test.ts +276 -0
  97. package/src/__tests__/conversation-retry-route.test.ts +22 -0
  98. package/src/__tests__/conversation-routes-disk-view.test.ts +2 -0
  99. package/src/__tests__/conversation-routes-enabled-plugins.test.ts +2 -0
  100. package/src/__tests__/conversation-routes-guardian-reply.test.ts +10 -0
  101. package/src/__tests__/conversation-routes-hidden-queue.test.ts +2 -0
  102. package/src/__tests__/conversation-routes-slash-commands.test.ts +2 -0
  103. package/src/__tests__/conversation-runtime-assembly.test.ts +29 -0
  104. package/src/__tests__/conversation-surface-snapshots.test.ts +543 -0
  105. package/src/__tests__/conversation-surfaces-action-delivery.test.ts +151 -0
  106. package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +63 -1
  107. package/src/__tests__/conversation-tool-setup-attribution.test.ts +23 -0
  108. package/src/__tests__/conversation-undo.test.ts +61 -0
  109. package/src/__tests__/credential-execution-client.test.ts +72 -55
  110. package/src/__tests__/credential-prompt-route.test.ts +35 -3
  111. package/src/__tests__/cu-unified-flow.test.ts +82 -1
  112. package/src/__tests__/db-conversation-tool-surface.test.ts +221 -0
  113. package/src/__tests__/db-mode-sessions-migration.test.ts +175 -0
  114. package/src/__tests__/deterministic-verification-control-plane.test.ts +2 -0
  115. package/src/__tests__/dm-backfill.test.ts +91 -1
  116. package/src/__tests__/document-create-route.test.ts +88 -0
  117. package/src/__tests__/document-sync-tags.test.ts +27 -0
  118. package/src/__tests__/emit-signal-routing-intent.test.ts +142 -2
  119. package/src/__tests__/events-tail-route.test.ts +58 -0
  120. package/src/__tests__/evict-conversations-for-reload.test.ts +57 -8
  121. package/src/__tests__/guardian-action-sweep.test.ts +16 -0
  122. package/src/__tests__/guardian-dispatch.test.ts +13 -0
  123. package/src/__tests__/headless-browser-interactions.test.ts +64 -20
  124. package/src/__tests__/heartbeat-service.test.ts +2 -59
  125. package/src/__tests__/helpers/mock-conversation.ts +21 -0
  126. package/src/__tests__/helpers/native-web-search-harness.ts +2 -0
  127. package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +5 -0
  128. package/src/__tests__/history-repair.test.ts +53 -0
  129. package/src/__tests__/host-cu-proxy.test.ts +46 -2
  130. package/src/__tests__/http-user-message-parity.test.ts +2 -0
  131. package/src/__tests__/image-credentials.test.ts +49 -0
  132. package/src/__tests__/image-service-dispatcher.test.ts +45 -0
  133. package/src/__tests__/inference-profile-session-handler.test.ts +26 -0
  134. package/src/__tests__/interrupt-on-send.test.ts +57 -43
  135. package/src/__tests__/interrupt-turn-note.test.ts +5 -6
  136. package/src/__tests__/list-messages-tool-merge.test.ts +85 -1
  137. package/src/__tests__/llm-catalog-parity.test.ts +27 -4
  138. package/src/__tests__/media-generate-image.test.ts +49 -1
  139. package/src/__tests__/message-provenance-contact-id.test.ts +69 -0
  140. package/src/__tests__/migration-export-to-gcs.test.ts +142 -0
  141. package/src/__tests__/mode-session-history.test.ts +245 -0
  142. package/src/__tests__/mode-session-recovery.test.ts +222 -0
  143. package/src/__tests__/mode-session-wire.test.ts +141 -0
  144. package/src/__tests__/notification-guardian-path.test.ts +7 -0
  145. package/src/__tests__/notification-telegram-adapter.test.ts +2 -2
  146. package/src/__tests__/notification-vellum-adapter.test.ts +34 -0
  147. package/src/__tests__/oauth-apps-routes.test.ts +1 -0
  148. package/src/__tests__/oauth-commands-routes.test.ts +114 -101
  149. package/src/__tests__/oauth-connect-orchestrator.test.ts +2 -0
  150. package/src/__tests__/oauth-provider-serializer.test.ts +13 -0
  151. package/src/__tests__/oauth-providers-routes.test.ts +3 -0
  152. package/src/__tests__/outbound-slack-persistence.test.ts +2 -0
  153. package/src/__tests__/persist-media-references.test.ts +50 -0
  154. package/src/__tests__/persistence-secret-redaction.test.ts +2 -0
  155. package/src/__tests__/plugin-api-provider.test.ts +5 -0
  156. package/src/__tests__/plugin-api-resolve-credential.test.ts +97 -0
  157. package/src/__tests__/plugin-import-boundary-guard.test.ts +9 -1
  158. package/src/__tests__/process-message-display-content.test.ts +56 -0
  159. package/src/__tests__/reaction-persistence.test.ts +5 -0
  160. package/src/__tests__/request-secret-standalone-channel-gate.test.ts +25 -9
  161. package/src/__tests__/run-conversation-turn-persistence.test.ts +138 -1
  162. package/src/__tests__/runtime-events-sse-reconnect.test.ts +1 -1
  163. package/src/__tests__/schedule-routes.test.ts +20 -0
  164. package/src/__tests__/scheduler-result-notification.test.ts +23 -4
  165. package/src/__tests__/script-proxy-certs.test.ts +1 -1
  166. package/src/__tests__/scripted-turn-metadata-persistence.test.ts +73 -0
  167. package/src/__tests__/secret-ingress-http.test.ts +2 -0
  168. package/src/__tests__/secret-routes-platform-proxy.test.ts +86 -31
  169. package/src/__tests__/secret-routes-scrub.test.ts +22 -27
  170. package/src/__tests__/secret-scanner.test.ts +20 -0
  171. package/src/__tests__/secure-keys-managed-failover.test.ts +8 -0
  172. package/src/__tests__/secure-keys.test.ts +50 -4
  173. package/src/__tests__/send-endpoint-busy.test.ts +163 -1
  174. package/src/__tests__/server-history-render.test.ts +91 -1
  175. package/src/__tests__/server-tool-pairing.test.ts +107 -0
  176. package/src/__tests__/settings-routes.test.ts +50 -1
  177. package/src/__tests__/skill-script-runner-sandbox.test.ts +32 -0
  178. package/src/__tests__/skills.test.ts +5 -4
  179. package/src/__tests__/subagent-fork-notifications.test.ts +1 -2
  180. package/src/__tests__/subagent-manager-notify.test.ts +1 -2
  181. package/src/__tests__/subagent-notify-parent.test.ts +146 -39
  182. package/src/__tests__/subagent-notify-stale-rebuild.test.ts +2 -4
  183. package/src/__tests__/subagent-tool-gate-mode.test.ts +226 -0
  184. package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +2 -0
  185. package/src/__tests__/terminal-tools.test.ts +8 -0
  186. package/src/__tests__/thread-backfill.test.ts +53 -0
  187. package/src/__tests__/tool-preview-lifecycle.test.ts +2 -0
  188. package/src/__tests__/tool-result-follow-up.test.ts +99 -0
  189. package/src/__tests__/tool-result-metadata-plumbing.test.ts +66 -1
  190. package/src/__tests__/tool-start-timestamp.test.ts +2 -0
  191. package/src/__tests__/twilio-routes.test.ts +1 -0
  192. package/src/__tests__/unicode.test.ts +36 -0
  193. package/src/__tests__/web-search-backend-failure.test.ts +1 -1
  194. package/src/activation/progress-store.test.ts +12 -7
  195. package/src/agent/loop.ts +136 -53
  196. package/src/agent/tool-result-follow-up.ts +70 -0
  197. package/src/api/attachment-provenance.test.ts +68 -0
  198. package/src/api/computer-use-tool.test.ts +55 -0
  199. package/src/api/computer-use-tool.ts +27 -0
  200. package/src/api/events/assistant-outbound-attachment.ts +3 -0
  201. package/src/api/events/assistant-turn-start.ts +3 -0
  202. package/src/api/events/desktop-activity-changed.ts +10 -0
  203. package/src/api/events/generation-handoff.ts +2 -0
  204. package/src/api/events/message-complete.ts +2 -0
  205. package/src/api/events/notification-conversation-created.ts +14 -16
  206. package/src/api/events/notification-intent.ts +4 -2
  207. package/src/api/events/question-request.ts +1 -0
  208. package/src/api/events/tool-result.ts +85 -0
  209. package/src/api/events/tool-use-start.ts +3 -0
  210. package/src/api/events/user-message-echo.ts +3 -0
  211. package/src/api/index.ts +42 -6
  212. package/src/api/mode-session.test.ts +131 -0
  213. package/src/api/mode-session.ts +148 -0
  214. package/src/api/remember-tool.ts +15 -0
  215. package/src/api/responses/conversation-message.ts +9 -0
  216. package/src/approvals/AGENTS.md +4 -2
  217. package/src/approvals/approval-primitive.ts +5 -2
  218. package/src/approvals/scoped-approval-grants.ts +6 -2
  219. package/src/browser/__tests__/operations.test.ts +29 -0
  220. package/src/browser/operation-meta.ts +2 -1
  221. package/src/browser/operations.ts +29 -0
  222. package/src/browser/virtual-desktop-target.ts +2 -9
  223. package/src/bundler/bundle-signer.ts +1 -3
  224. package/src/calls/__tests__/barge-in-guard.test.ts +3 -0
  225. package/src/calls/__tests__/call-controller.test.ts +94 -0
  226. package/src/calls/__tests__/call-funnel-telemetry.test.ts +204 -0
  227. package/src/calls/__tests__/voice-control-protocol.test.ts +15 -0
  228. package/src/calls/__tests__/voice-escalation-judge.test.ts +209 -0
  229. package/src/calls/__tests__/voice-leg-coordinator.test.ts +168 -0
  230. package/src/{live-voice/__tests__/live-voice-metrics.test.ts → calls/__tests__/voice-metrics.test.ts} +32 -38
  231. package/src/calls/__tests__/voice-session-bridge.test.ts +576 -122
  232. package/src/calls/barge-in-guard.ts +5 -0
  233. package/src/calls/call-controller.ts +196 -1
  234. package/src/calls/call-domain.ts +4 -0
  235. package/src/calls/call-store.ts +87 -2
  236. package/src/calls/guardian-action-sweep.ts +22 -8
  237. package/src/calls/media-stream-audio-transcode.ts +1 -3
  238. package/src/calls/typesafe-noul.ts +156 -0
  239. package/src/calls/voice-control-protocol.ts +6 -0
  240. package/src/calls/voice-escalation-judge.ts +145 -0
  241. package/src/calls/voice-escalation-target.ts +10 -0
  242. package/src/calls/voice-leg-coordinator.ts +121 -3
  243. package/src/{live-voice/live-voice-metrics.ts → calls/voice-metrics.ts} +94 -87
  244. package/src/calls/voice-session-bridge.ts +256 -112
  245. package/src/calls/voice-triage-escalate.ts +7 -0
  246. package/src/cli/__tests__/roadmap-cli.test.ts +4 -41
  247. package/src/cli/commands/__tests__/cli-test-harness.ts +21 -3
  248. package/src/cli/commands/__tests__/notifications.test.ts +35 -10
  249. package/src/cli/commands/__tests__/plugins.test.ts +20 -1
  250. package/src/cli/commands/__tests__/schedules.test.ts +14 -0
  251. package/src/cli/commands/bash.help.ts +4 -3
  252. package/src/cli/commands/browser.help.ts +9 -6
  253. package/src/cli/commands/browser.ts +0 -1
  254. package/src/cli/commands/clients.help.ts +46 -0
  255. package/src/cli/commands/clients.ts +116 -0
  256. package/src/cli/commands/credentials.help.ts +9 -4
  257. package/src/cli/commands/notifications.help.ts +16 -2
  258. package/src/cli/commands/notifications.ts +30 -8
  259. package/src/cli/commands/oauth/index.help.ts +10 -0
  260. package/src/cli/commands/oauth/providers.ts +12 -0
  261. package/src/cli/commands/oauth/request.test.ts +179 -2
  262. package/src/cli/commands/oauth/request.ts +41 -10
  263. package/src/cli/commands/plugins.ts +12 -5
  264. package/src/cli/commands/roadmap.help.ts +18 -21
  265. package/src/cli/commands/roadmap.ts +4 -21
  266. package/src/cli/commands/schedules.ts +2 -0
  267. package/src/cli/lib/__tests__/inspect-plugin.test.ts +54 -0
  268. package/src/cli/lib/__tests__/install-from-github.test.ts +41 -0
  269. package/src/cli/lib/__tests__/list-installed-plugins.test.ts +5 -2
  270. package/src/cli/lib/__tests__/local-plugin-upgrade.test.ts +133 -51
  271. package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +8 -2
  272. package/src/cli/lib/bundled-marketplace.json +906 -1
  273. package/src/cli/lib/bundled-plugin-packages.json +783 -1
  274. package/src/cli/lib/inspect-plugin.ts +11 -4
  275. package/src/cli/lib/plugin-marketplace.ts +8 -0
  276. package/src/cli/lib/publish-plugin.ts +1 -2
  277. package/src/cli/lib/upgrade-plugin.ts +19 -11
  278. package/src/cli/output.ts +18 -4
  279. package/src/config/__tests__/default-provider.test.ts +6 -0
  280. package/src/config/__tests__/memory-retrospective-schema.test.ts +14 -1
  281. package/src/config/bundled-skills/computer-use/SKILL.md +1 -1
  282. package/src/config/bundled-skills/computer-use/TOOLS.json +89 -1
  283. package/src/config/bundled-skills/image-studio/SKILL.md +17 -2
  284. package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +7 -11
  285. package/src/config/bundled-skills/schedule/SKILL.md +6 -6
  286. package/src/config/bundled-skills/sequences/TOOLS.json +1 -5
  287. package/src/config/call-site-defaults.ts +9 -0
  288. package/src/config/feature-flag-registry.json +16 -0
  289. package/src/config/loader.ts +9 -17
  290. package/src/config/platform-identity.ts +8 -8
  291. package/src/config/profile-text-generation.test.ts +51 -0
  292. package/src/config/profile-text-generation.ts +51 -0
  293. package/src/config/schemas/__tests__/memory-v3.test.ts +12 -0
  294. package/src/config/schemas/__tests__/tools.test.ts +31 -0
  295. package/src/config/schemas/call-site-catalog.ts +14 -0
  296. package/src/config/schemas/channels.ts +0 -18
  297. package/src/config/schemas/llm.ts +9 -2
  298. package/src/config/schemas/mcp.ts +6 -1
  299. package/src/config/schemas/memory-retrospective.ts +11 -2
  300. package/src/config/schemas/memory-v3.ts +12 -0
  301. package/src/config/schemas/services.ts +15 -6
  302. package/src/config/schemas/tools.ts +10 -0
  303. package/src/config/session-groups-gate.ts +21 -0
  304. package/src/config/skills.ts +2 -6
  305. package/src/conversations/__tests__/message-consolidation.test.ts +149 -0
  306. package/src/conversations/message-consolidation.ts +102 -5
  307. package/src/credential-execution/ces-connect.test.ts +69 -0
  308. package/src/credential-execution/ces-connect.ts +115 -0
  309. package/src/credential-execution/ces-runtime.ts +55 -93
  310. package/src/credential-execution/executable-discovery.ts +34 -142
  311. package/src/credential-execution/process-manager.test.ts +20 -24
  312. package/src/credential-execution/process-manager.ts +24 -19
  313. package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +10 -1
  314. package/src/daemon/__tests__/conversation-tool-setup.test.ts +43 -0
  315. package/src/daemon/__tests__/inference-profile-notification.test.ts +2 -0
  316. package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +36 -0
  317. package/src/daemon/assistant-attachments.ts +20 -12
  318. package/src/daemon/browser-mode-session.test.ts +402 -0
  319. package/src/daemon/browser-mode-session.ts +186 -0
  320. package/src/daemon/chat-credential-redaction.ts +16 -1
  321. package/src/daemon/computer-use-mode-session.test.ts +210 -0
  322. package/src/daemon/computer-use-mode-session.ts +105 -0
  323. package/src/daemon/config-watcher.ts +10 -0
  324. package/src/daemon/conversation-agent-loop-handlers.ts +132 -13
  325. package/src/daemon/conversation-agent-loop.ts +231 -25
  326. package/src/daemon/conversation-attachments.ts +93 -12
  327. package/src/daemon/conversation-evictor.ts +5 -24
  328. package/src/daemon/conversation-history.ts +10 -0
  329. package/src/daemon/conversation-interrupt-repair.ts +4 -43
  330. package/src/daemon/conversation-interrupt.ts +7 -11
  331. package/src/daemon/conversation-messaging.ts +58 -5
  332. package/src/daemon/conversation-mode-session.test.ts +1667 -0
  333. package/src/daemon/conversation-mode-session.ts +1008 -0
  334. package/src/daemon/conversation-process.ts +5 -0
  335. package/src/daemon/conversation-runtime-assembly.ts +10 -20
  336. package/src/daemon/conversation-skill-tools.ts +1 -0
  337. package/src/daemon/conversation-store.ts +4 -4
  338. package/src/daemon/conversation-surface-snapshots.ts +135 -0
  339. package/src/daemon/conversation-surfaces.ts +113 -5
  340. package/src/daemon/conversation-tool-setup.ts +79 -6
  341. package/src/daemon/conversation-turn-finalize.ts +26 -13
  342. package/src/daemon/conversation.ts +215 -58
  343. package/src/daemon/daemon-control.ts +3 -7
  344. package/src/daemon/handlers/config-model.ts +7 -5
  345. package/src/daemon/handlers/shared.ts +55 -16
  346. package/src/daemon/host-cu-proxy.ts +51 -22
  347. package/src/daemon/lifecycle.ts +30 -13
  348. package/src/daemon/mcp-reload-service.ts +1 -1
  349. package/src/daemon/message-protocol.ts +0 -1
  350. package/src/daemon/message-provenance.ts +28 -4
  351. package/src/daemon/message-types/shared.ts +1 -0
  352. package/src/daemon/mode-session-startup-recovery.ts +23 -0
  353. package/src/daemon/mode-session-tracking.ts +54 -0
  354. package/src/daemon/orphan-reaper.ts +4 -3
  355. package/src/daemon/persist-media-references.ts +20 -3
  356. package/src/daemon/process-message.ts +13 -4
  357. package/src/daemon/tool-setup-types.ts +6 -0
  358. package/src/daemon/virtual-desktop-context.ts +21 -0
  359. package/src/daemon/wake-conversation-ops.ts +50 -15
  360. package/src/desktop/__tests__/fake-desktop.ts +1 -0
  361. package/src/desktop/desktop-accessibility-script.ts +88 -0
  362. package/src/desktop/desktop-accessibility.test.ts +160 -0
  363. package/src/desktop/desktop-accessibility.ts +169 -0
  364. package/src/desktop/desktop-automation-lease.test.ts +302 -64
  365. package/src/desktop/desktop-automation-lease.ts +129 -54
  366. package/src/desktop/desktop-browser-endpoint.ts +4 -0
  367. package/src/desktop/desktop-computer-use-routing.test.ts +330 -0
  368. package/src/desktop/desktop-computer-use.test.ts +449 -0
  369. package/src/desktop/desktop-computer-use.ts +502 -0
  370. package/src/desktop/desktop-dependencies.ts +47 -293
  371. package/src/desktop/desktop-display.ts +2 -2
  372. package/src/desktop/desktop-help.ts +15 -0
  373. package/src/desktop/desktop-panel-config.test.ts +13 -3
  374. package/src/desktop/desktop-panel-config.ts +109 -15
  375. package/src/desktop/desktop-session-manager.test.ts +69 -21
  376. package/src/desktop/desktop-session-manager.ts +76 -22
  377. package/src/desktop/desktop-stream-bridge.test.ts +76 -132
  378. package/src/desktop/desktop-stream-bridge.ts +9 -11
  379. package/src/desktop/desktop-wallpaper-renderer.ts +132 -0
  380. package/src/desktop/desktop-wallpaper-worker.ts +17 -0
  381. package/src/desktop/desktop-wallpaper.test.ts +48 -7
  382. package/src/desktop/desktop-wallpaper.ts +34 -102
  383. package/src/desktop/desktop-window-theme.ts +5 -25
  384. package/src/desktop/virtual-desktop-feature.ts +53 -3
  385. package/src/desktop/virtual-desktop-platform.test.ts +5 -9
  386. package/src/documents/document-store.ts +23 -0
  387. package/src/heartbeat/__tests__/heartbeat-service.test.ts +22 -26
  388. package/src/heartbeat/heartbeat-service.ts +5 -33
  389. package/src/ipc/__tests__/browser-ipc.test.ts +215 -6
  390. package/src/ipc/assistant-server.ts +2 -0
  391. package/src/ipc/routes/__tests__/channel-reply-ipc-routes.test.ts +111 -0
  392. package/src/ipc/routes/channel-reply-ipc-routes.ts +73 -0
  393. package/src/live-voice/__tests__/continuation-judge.test.ts +151 -0
  394. package/src/live-voice/__tests__/live-activity-reporter.test.ts +154 -15
  395. package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +743 -31
  396. package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +191 -10
  397. package/src/live-voice/__tests__/live-voice-session-groups-gate.test.ts +87 -0
  398. package/src/live-voice/__tests__/live-voice-session-manager.test.ts +5 -1
  399. package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +41 -0
  400. package/src/live-voice/__tests__/live-voice-sight-frame.test.ts +1017 -9
  401. package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +224 -1
  402. package/src/live-voice/__tests__/live-voice-tts-session.test.ts +101 -0
  403. package/src/live-voice/__tests__/live-voice-tts.test.ts +44 -0
  404. package/src/live-voice/__tests__/live-voice-vad.test.ts +939 -75
  405. package/src/live-voice/__tests__/microphone-idle-gate.test.ts +87 -0
  406. package/src/live-voice/__tests__/session-controls.test.ts +23 -0
  407. package/src/live-voice/__tests__/task-outcomes.test.ts +117 -0
  408. package/src/live-voice/__tests__/voice-input-diagnostics.test.ts +117 -0
  409. package/src/live-voice/camera-mode-session.test.ts +265 -0
  410. package/src/live-voice/camera-mode-session.ts +277 -0
  411. package/src/live-voice/continuation-judge.ts +123 -0
  412. package/src/live-voice/live-activity-reporter.ts +115 -6
  413. package/src/live-voice/live-voice-manager.ts +36 -1
  414. package/src/live-voice/live-voice-photo.ts +88 -7
  415. package/src/live-voice/live-voice-session-manager.ts +15 -0
  416. package/src/live-voice/live-voice-session.ts +1443 -747
  417. package/src/live-voice/live-voice-tts.ts +22 -4
  418. package/src/live-voice/microphone-idle-gate.ts +56 -0
  419. package/src/live-voice/protocol.ts +171 -4
  420. package/src/live-voice/session-controls.ts +11 -3
  421. package/src/live-voice/task-outcomes.ts +131 -0
  422. package/src/live-voice/voice-input-diagnostics.ts +117 -0
  423. package/src/mcp/__tests__/manager-tool-caps.test.ts +156 -0
  424. package/src/mcp/__tests__/startup.test.ts +30 -7
  425. package/src/mcp/__tests__/tool-caps.test.ts +134 -0
  426. package/src/mcp/manager.ts +171 -101
  427. package/src/mcp/startup.ts +33 -13
  428. package/src/mcp/tool-caps.ts +184 -0
  429. package/src/media/__tests__/image-models.test.ts +58 -0
  430. package/src/media/image-credentials.ts +5 -0
  431. package/src/media/image-models.ts +52 -0
  432. package/src/media/image-service.ts +8 -0
  433. package/src/media/openrouter-image-service.test.ts +133 -0
  434. package/src/media/openrouter-image-service.ts +153 -0
  435. package/src/media/types.ts +1 -1
  436. package/src/messaging/provider-message-metadata.ts +3 -3
  437. package/src/messaging/providers/__tests__/transport-dispatch.test.ts +65 -11
  438. package/src/messaging/providers/discord/send.ts +4 -17
  439. package/src/messaging/providers/discord/transport.ts +2 -2
  440. package/src/messaging/providers/outlook/client.ts +16 -22
  441. package/src/messaging/providers/telegram-bot/send.test.ts +236 -1
  442. package/src/messaging/providers/telegram-bot/send.ts +10 -7
  443. package/src/messaging/providers/telegram-bot/transport.ts +3 -10
  444. package/src/messaging/providers/whatsapp/send.test.ts +169 -4
  445. package/src/monitoring/__tests__/control.test.ts +17 -5
  446. package/src/monitoring/control.ts +15 -5
  447. package/src/notifications/AGENTS.md +1 -1
  448. package/src/notifications/README.md +83 -99
  449. package/src/notifications/__tests__/broadcaster.test.ts +1 -1
  450. package/src/notifications/__tests__/channel-allowlist.test.ts +39 -0
  451. package/src/notifications/__tests__/copy-composer.test.ts +71 -1
  452. package/src/notifications/__tests__/decision-engine.test.ts +79 -2
  453. package/src/notifications/__tests__/deterministic-checks.test.ts +10 -10
  454. package/src/notifications/__tests__/home-feed-side-effect.test.ts +93 -4
  455. package/src/notifications/__tests__/notification-utils.test.ts +18 -0
  456. package/src/notifications/adapters/macos.ts +37 -46
  457. package/src/notifications/broadcaster.ts +5 -6
  458. package/src/notifications/channel-allowlist.ts +44 -0
  459. package/src/notifications/copy-composer.ts +11 -26
  460. package/src/notifications/decision-engine.ts +24 -14
  461. package/src/notifications/deterministic-checks.ts +1 -1
  462. package/src/notifications/emit-signal.ts +123 -37
  463. package/src/notifications/home-feed-side-effect.ts +9 -7
  464. package/src/notifications/notification-utils.ts +19 -0
  465. package/src/notifications/schedule-result-producer.ts +4 -6
  466. package/src/notifications/signal.ts +4 -19
  467. package/src/notifications/types.ts +15 -0
  468. package/src/oauth/AGENTS.md +2 -0
  469. package/src/oauth/__tests__/identity-verifier.test.ts +36 -1
  470. package/src/oauth/connection-resolver.ts +1 -2
  471. package/src/oauth/identity-verifier.ts +24 -0
  472. package/src/oauth/oauth-store.ts +12 -4
  473. package/src/oauth/provider-serializer.ts +8 -0
  474. package/src/oauth/seed-providers.ts +42 -0
  475. package/src/onboarding/onboarding-events-store.ts +99 -12
  476. package/src/permissions/confirmation-guardian-request.test.ts +16 -0
  477. package/src/permissions/confirmation-guardian-request.ts +2 -3
  478. package/src/permissions/question-prompter.test.ts +31 -0
  479. package/src/permissions/question-prompter.ts +2 -0
  480. package/src/persistence/__tests__/client-connection-events-store.test.ts +322 -0
  481. package/src/persistence/attachments-store.ts +84 -44
  482. package/src/persistence/client-connection-events-store.ts +377 -0
  483. package/src/persistence/conversation-crud.ts +166 -26
  484. package/src/persistence/conversation-mode-sessions.test.ts +424 -0
  485. package/src/persistence/conversation-mode-sessions.ts +460 -0
  486. package/src/persistence/conversation-plugin-facade.ts +30 -0
  487. package/src/persistence/conversation-queries.ts +6 -19
  488. package/src/persistence/conversation-tool-surface.ts +135 -0
  489. package/src/persistence/conversation-types.test.ts +32 -0
  490. package/src/persistence/conversation-types.ts +26 -4
  491. package/src/persistence/llm-usage-store.ts +3 -9
  492. package/src/persistence/message-metadata.ts +25 -0
  493. package/src/persistence/migrations/378-create-conversation-tool-surfaces.test.ts +78 -0
  494. package/src/persistence/migrations/378-create-conversation-tool-surfaces.ts +29 -0
  495. package/src/persistence/migrations/379-oauth-providers-response-ok-field.test.ts +92 -0
  496. package/src/persistence/migrations/379-oauth-providers-response-ok-field.ts +32 -0
  497. package/src/persistence/migrations/380-conversation-tool-surfaces-delegate-independent-tasks.test.ts +90 -0
  498. package/src/persistence/migrations/380-conversation-tool-surfaces-delegate-independent-tasks.ts +34 -0
  499. package/src/persistence/migrations/381-create-conversation-mode-sessions.ts +41 -0
  500. package/src/persistence/migrations/382-create-client-connection-events.ts +40 -0
  501. package/src/persistence/migrations/__tests__/382-create-client-connection-events.test.ts +56 -0
  502. package/src/persistence/schema/conversation-tool-surfaces.ts +32 -0
  503. package/src/persistence/schema/conversations.ts +67 -0
  504. package/src/persistence/schema/index.ts +1 -0
  505. package/src/persistence/schema/infrastructure.ts +29 -0
  506. package/src/persistence/schema/oauth.ts +1 -0
  507. package/src/persistence/steps.ts +41 -3
  508. package/src/plugin-api/__tests__/plugin-name-env.test.ts +249 -0
  509. package/src/plugin-api/conversation-turn.ts +31 -7
  510. package/src/plugin-api/effective-context-window.ts +33 -0
  511. package/src/plugin-api/index.ts +35 -6
  512. package/src/plugin-api/plugin-channel-turn-trust.test.ts +133 -0
  513. package/src/plugin-api/plugin-channel-turn-trust.ts +71 -0
  514. package/src/plugin-api/plugin-name-env.ts +125 -0
  515. package/src/plugin-api/resolve-credential.ts +79 -20
  516. package/src/plugin-api/vision-support.test.ts +22 -0
  517. package/src/plugin-api/vision-support.ts +17 -5
  518. package/src/plugins/__tests__/active-plugin-env.test.ts +136 -0
  519. package/src/plugins/active-plugin-env.ts +51 -0
  520. package/src/plugins/defaults/index.ts +24 -0
  521. package/src/plugins/defaults/memory/AGENTS.md +39 -5
  522. package/src/plugins/defaults/memory/__tests__/archive-removal-framing.test.ts +64 -0
  523. package/src/plugins/defaults/memory/__tests__/buffer-file.test.ts +365 -0
  524. package/src/plugins/defaults/memory/__tests__/buffer-format.test.ts +43 -0
  525. package/src/plugins/defaults/memory/__tests__/conversation-memory-purge.test.ts +1 -0
  526. package/src/plugins/defaults/memory/__tests__/db-memory-attach.test.ts +2 -0
  527. package/src/plugins/defaults/memory/__tests__/fixtures/buffer-appender.ts +17 -0
  528. package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +74 -0
  529. package/src/plugins/defaults/memory/__tests__/memory-run-evidence.test.ts +161 -0
  530. package/src/plugins/defaults/memory/__tests__/relocated-memory-test-rows.ts +10 -0
  531. package/src/plugins/defaults/memory/__tests__/worker-control.test.ts +19 -6
  532. package/src/plugins/defaults/memory/buffer-file.ts +354 -0
  533. package/src/plugins/defaults/memory/buffer-format.ts +40 -0
  534. package/src/plugins/defaults/memory/context-search/agent-protocol.ts +9 -12
  535. package/src/plugins/defaults/memory/context-search/agent-runner.ts +77 -22
  536. package/src/plugins/defaults/memory/context-search/format.ts +53 -8
  537. package/src/plugins/defaults/memory/context-search/limits.ts +10 -12
  538. package/src/plugins/defaults/memory/context-search/recall-input.ts +38 -0
  539. package/src/plugins/defaults/memory/context-search/search.ts +8 -14
  540. package/src/plugins/defaults/memory/context-search/sources/conversations.ts +1 -0
  541. package/src/plugins/defaults/memory/context-search/sources/memory-v2.ts +2 -1
  542. package/src/plugins/defaults/memory/context-search/sources/workspace.ts +2 -1
  543. package/src/plugins/defaults/memory/context-search/types.ts +1 -11
  544. package/src/plugins/defaults/memory/conversation-memory-purge.ts +4 -0
  545. package/src/plugins/defaults/memory/graph/__tests__/handle-remember-v2.test.ts +11 -4
  546. package/src/plugins/defaults/memory/graph/capability-seed.ts +1 -2
  547. package/src/plugins/defaults/memory/graph/tool-handlers.ts +13 -63
  548. package/src/plugins/defaults/memory/graph/tools.ts +35 -64
  549. package/src/plugins/defaults/memory/host-utils.ts +0 -10
  550. package/src/plugins/defaults/memory/injectors.ts +4 -3
  551. package/src/plugins/defaults/memory/memory-retrospective-job.ts +72 -183
  552. package/src/plugins/defaults/memory/memory-run-evidence.ts +213 -0
  553. package/src/plugins/defaults/memory/src/memory-eval-routes.ts +2 -3
  554. package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +6 -1
  555. package/src/plugins/defaults/memory/src/memory-item-routes.ts +2 -1
  556. package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts +407 -99
  557. package/src/plugins/defaults/memory/substrate/__tests__/consolidation-prompt-flag-gating-guard.test.ts +10 -0
  558. package/src/plugins/defaults/memory/substrate/__tests__/prompts-consolidation.test.ts +107 -7
  559. package/src/plugins/defaults/memory/substrate/consolidation-job.ts +307 -86
  560. package/src/plugins/defaults/memory/substrate/consolidation-tool-surface.ts +34 -0
  561. package/src/plugins/defaults/memory/substrate/page-index.ts +2 -1
  562. package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +91 -51
  563. package/src/plugins/defaults/memory/substrate/sweep-job.ts +1 -1
  564. package/src/plugins/defaults/memory/tools.test.ts +124 -9
  565. package/src/plugins/defaults/memory/tools.ts +38 -11
  566. package/src/plugins/defaults/memory/v1/graph/consolidation.ts +2 -2
  567. package/src/plugins/defaults/memory/v1/graph/extraction.ts +2 -1
  568. package/src/plugins/defaults/memory/v1/graph/retriever.ts +1 -1
  569. package/src/plugins/defaults/memory/v2/__tests__/migration.test.ts +5 -0
  570. package/src/plugins/defaults/memory/v2/__tests__/reranker.test.ts +5 -2
  571. package/src/plugins/defaults/memory/v2/reranker.ts +2 -1
  572. package/src/plugins/defaults/memory/v3/__tests__/injection.test.ts +81 -1
  573. package/src/plugins/defaults/memory/v3/__tests__/orchestrate.test.ts +87 -0
  574. package/src/plugins/defaults/memory/v3/__tests__/plugin-schema.test.ts +12 -0
  575. package/src/plugins/defaults/memory/v3/__tests__/pool-log-store.test.ts +332 -15
  576. package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +626 -27
  577. package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +148 -28
  578. package/src/plugins/defaults/memory/v3/card.ts +2 -1
  579. package/src/plugins/defaults/memory/v3/injector.ts +212 -178
  580. package/src/plugins/defaults/memory/v3/orchestrate.ts +159 -32
  581. package/src/plugins/defaults/memory/v3/plugin-schema.ts +54 -4
  582. package/src/plugins/defaults/memory/v3/pool-log-store.ts +275 -37
  583. package/src/plugins/defaults/memory/v3/pool-select.test.ts +47 -5
  584. package/src/plugins/defaults/memory/v3/pool-select.ts +710 -59
  585. package/src/plugins/defaults/memory/v3/sections.ts +2 -1
  586. package/src/plugins/defaults/memory/v3/shadow-plugin.ts +41 -14
  587. package/src/plugins/defaults/memory/worker-control.ts +15 -5
  588. package/src/plugins/defaults/task-progress/__tests__/registration.test.ts +70 -0
  589. package/src/plugins/defaults/task-progress/__tests__/task-progress-context.test.ts +380 -0
  590. package/src/plugins/defaults/task-progress/hooks/post-compact.ts +19 -0
  591. package/src/plugins/defaults/task-progress/hooks/user-prompt-submit.ts +24 -0
  592. package/src/plugins/defaults/task-progress/package.json +14 -0
  593. package/src/plugins/defaults/task-progress/src/apply-task-progress-context.ts +93 -0
  594. package/src/plugins/defaults/task-progress/src/format-task-progress-context.ts +67 -0
  595. package/src/plugins/defaults/tool-error/hooks/post-tool-use.ts +4 -1
  596. package/src/plugins/defaults/tool-result-truncate/terminal.ts +1 -46
  597. package/src/prompts/__tests__/parallel-tasks-section.test.ts +25 -0
  598. package/src/prompts/delegation-gate.ts +57 -0
  599. package/src/prompts/system-prompt.ts +14 -32
  600. package/src/providers/__tests__/context-overflow-error.test.ts +10 -0
  601. package/src/providers/__tests__/search-provider-catalog.test.ts +18 -0
  602. package/src/providers/anthropic/client.ts +10 -1
  603. package/src/providers/anthropic-gateway-shared.ts +6 -0
  604. package/src/providers/fetch-provider-catalog.ts +11 -0
  605. package/src/providers/gemini/client.ts +25 -8
  606. package/src/providers/inference/adapter-factory.ts +6 -0
  607. package/src/providers/jev/client.test.ts +270 -0
  608. package/src/providers/jev/client.ts +534 -0
  609. package/src/providers/model-catalog.ts +59 -3
  610. package/src/providers/openai/__tests__/chat-completions-extra-body.test.ts +94 -1
  611. package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +106 -1
  612. package/src/providers/openai/__tests__/chat-template-flatten.test.ts +24 -0
  613. package/src/providers/openai/__tests__/google-thought-signature.test.ts +1 -2
  614. package/src/providers/openai/chat-completions-provider.ts +100 -29
  615. package/src/providers/openai/google-thought-signature.ts +0 -6
  616. package/src/providers/openai/responses-provider.ts +23 -1
  617. package/src/providers/ratelimit.ts +1 -3
  618. package/src/providers/search-provider-catalog.ts +24 -0
  619. package/src/providers/server-tool-pairing.ts +16 -7
  620. package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +24 -0
  621. package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +7 -1
  622. package/src/providers/speech-to-text/__tests__/vellum-managed-realtime.test.ts +50 -1
  623. package/src/providers/speech-to-text/deepgram-flux-frames.ts +11 -0
  624. package/src/providers/speech-to-text/vellum-managed-flux-realtime.ts +13 -0
  625. package/src/providers/types.ts +5 -0
  626. package/src/routes/control.ts +7 -5
  627. package/src/runtime/AGENTS.md +6 -4
  628. package/src/runtime/__tests__/agent-wake.test.ts +94 -1
  629. package/src/runtime/actor-trust-resolver.ts +3 -3
  630. package/src/runtime/agent-wake.ts +32 -5
  631. package/src/runtime/approval-message-composer.ts +3 -69
  632. package/src/runtime/assistant-event-hub.ts +73 -75
  633. package/src/runtime/assistant-event-publish-options.ts +7 -0
  634. package/src/runtime/assistant-event-targeting.ts +79 -0
  635. package/src/runtime/assistant-stream-state.ts +13 -100
  636. package/src/runtime/background-job-runner.ts +5 -5
  637. package/src/runtime/channel-retry-sweep.ts +1 -0
  638. package/src/runtime/guardian-action-message-composer.ts +8 -161
  639. package/src/runtime/guardian-action-service.ts +2 -17
  640. package/src/runtime/guardian-reply-router.ts +29 -40
  641. package/src/runtime/http-server.ts +2 -2
  642. package/src/runtime/http-types.ts +6 -0
  643. package/src/runtime/message-composer-types.ts +3 -61
  644. package/src/runtime/migrations/__tests__/vbundle-import-parity.test.ts +67 -1
  645. package/src/runtime/migrations/__tests__/vbundle-import-policy.test.ts +89 -0
  646. package/src/runtime/migrations/vbundle-builder.ts +24 -1
  647. package/src/runtime/migrations/vbundle-import-analyzer.ts +13 -4
  648. package/src/runtime/migrations/vbundle-import-policy.ts +43 -11
  649. package/src/runtime/migrations/vbundle-importer.ts +4 -1
  650. package/src/runtime/migrations/vbundle-streaming-importer.ts +4 -1
  651. package/src/runtime/migrations/vbundle-validator.ts +1 -0
  652. package/src/runtime/pending-interactions-mode-session.test.ts +62 -0
  653. package/src/runtime/pending-interactions.ts +29 -0
  654. package/src/runtime/routes/__tests__/client-history-routes.test.ts +104 -0
  655. package/src/runtime/routes/__tests__/image-generation-routes.test.ts +97 -0
  656. package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +24 -2
  657. package/src/runtime/routes/__tests__/roadmap-routes.test.ts +29 -6
  658. package/src/runtime/routes/browser-context.ts +31 -11
  659. package/src/runtime/routes/browser-routes.ts +43 -4
  660. package/src/runtime/routes/btw-routes.ts +9 -4
  661. package/src/runtime/routes/channel-route-shared.ts +1 -9
  662. package/src/runtime/routes/client-routes.ts +108 -0
  663. package/src/runtime/routes/conversation-management-routes.ts +4 -1
  664. package/src/runtime/routes/conversation-routes.ts +125 -14
  665. package/src/runtime/routes/credential-prompt-routes.ts +4 -3
  666. package/src/runtime/routes/credential-routes.ts +1 -2
  667. package/src/runtime/routes/desktop-setup-routes.test.ts +9 -18
  668. package/src/runtime/routes/desktop-setup-routes.ts +8 -7
  669. package/src/runtime/routes/documents-routes.ts +51 -0
  670. package/src/runtime/routes/email-routes.ts +3 -7
  671. package/src/runtime/routes/events-routes.ts +75 -54
  672. package/src/runtime/routes/guardian-approval-interception.ts +24 -0
  673. package/src/runtime/routes/identity-routes.ts +1 -5
  674. package/src/runtime/routes/image-generation-routes.ts +8 -14
  675. package/src/runtime/routes/inbound-message-handler.ts +37 -41
  676. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.test.ts +168 -0
  677. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.ts +96 -0
  678. package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +39 -10
  679. package/src/runtime/routes/inbound-stages/background-dispatch.ts +9 -9
  680. package/src/runtime/routes/inbound-stages/reaction-intercept.ts +2 -0
  681. package/src/runtime/routes/inference-profile-session-handler.ts +11 -1
  682. package/src/runtime/routes/inference-profiles-routes.ts +9 -0
  683. package/src/runtime/routes/llm-context-normalization.ts +1 -7
  684. package/src/runtime/routes/migration-routes.ts +73 -2
  685. package/src/runtime/routes/notification-routes.ts +10 -0
  686. package/src/runtime/routes/oauth-commands-routes.ts +57 -54
  687. package/src/runtime/routes/oauth-providers.ts +11 -0
  688. package/src/runtime/routes/oauth-request-hints.test.ts +246 -0
  689. package/src/runtime/routes/oauth-request-hints.ts +149 -0
  690. package/src/runtime/routes/plugins-routes.ts +6 -0
  691. package/src/runtime/routes/roadmap-routes.ts +47 -8
  692. package/src/runtime/routes/schedule-routes.ts +2 -0
  693. package/src/runtime/routes/secret-routes.ts +99 -119
  694. package/src/runtime/routes/settings-routes.ts +10 -4
  695. package/src/runtime/routes/watcher-routes.ts +1 -3
  696. package/src/runtime/trust-class.test.ts +7 -6
  697. package/src/runtime/trust-class.ts +9 -17
  698. package/src/schedule/__tests__/worker-control.test.ts +36 -5
  699. package/src/schedule/__tests__/worker-mcp-bootstrap.test.ts +107 -0
  700. package/src/schedule/__tests__/worker-mcp-readiness.test.ts +173 -0
  701. package/src/schedule/__tests__/worker-mcp-tools.test.ts +13 -47
  702. package/src/schedule/run-script.ts +2 -2
  703. package/src/schedule/scheduler.ts +47 -11
  704. package/src/schedule/tool-surface-readiness.ts +65 -0
  705. package/src/schedule/worker-control.ts +15 -5
  706. package/src/schedule/worker-mcp.ts +85 -0
  707. package/src/schedule/worker.ts +9 -31
  708. package/src/security/secure-keys.ts +74 -112
  709. package/src/skills/clawhub.ts +3 -8
  710. package/src/skills/managed-store.ts +1 -5
  711. package/src/stt/types.ts +4 -0
  712. package/src/subagent/notify.ts +54 -34
  713. package/src/subagent/parent-notification.ts +6 -0
  714. package/src/telemetry/__tests__/phone-call-funnel.test.ts +72 -0
  715. package/src/telemetry/phone-call-funnel.ts +144 -0
  716. package/src/tools/__tests__/tool-schema-root-combinator-guard.test.ts +81 -0
  717. package/src/tools/ask-question/ask-question-tool.test.ts +167 -5
  718. package/src/tools/ask-question/ask-question-tool.ts +120 -22
  719. package/src/tools/browser/browser-execution.ts +29 -60
  720. package/src/tools/browser/cdp-client/__tests__/cdp-dom-helpers.test.ts +561 -39
  721. package/src/tools/browser/cdp-client/cdp-dom-helpers.ts +435 -39
  722. package/src/tools/browser/cdp-client/cdp-inspect-client.ts +1 -8
  723. package/src/tools/client-os.ts +15 -0
  724. package/src/tools/computer-use/skill-proxy-bridge.ts +1 -1
  725. package/src/tools/computer-use/target.ts +42 -0
  726. package/src/tools/document/document-tool.ts +13 -18
  727. package/src/tools/execution-target.ts +14 -16
  728. package/src/tools/host-terminal/host-shell.ts +16 -7
  729. package/src/tools/network/__tests__/web-fetch-tinyfish.test.ts +250 -0
  730. package/src/tools/network/__tests__/web-search.test.ts +203 -0
  731. package/src/tools/network/web-fetch.ts +274 -7
  732. package/src/tools/network/web-search.ts +468 -6
  733. package/src/tools/permission-checker.ts +3 -3
  734. package/src/tools/policy-context.ts +3 -1
  735. package/src/tools/shared/filesystem/file-ops-service.ts +1 -31
  736. package/src/tools/shared/shell-output.test.ts +10 -0
  737. package/src/tools/shared/shell-output.ts +14 -2
  738. package/src/tools/skills/load.ts +2 -1
  739. package/src/tools/skills/sandbox-runner.ts +24 -4
  740. package/src/tools/skills/scaffold-managed.ts +2 -1
  741. package/src/tools/skills/skill-script-runner.ts +7 -0
  742. package/src/tools/skills/skill-tool-factory.ts +15 -1
  743. package/src/tools/terminal/__tests__/safe-env.test.ts +44 -4
  744. package/src/tools/terminal/__tests__/sanitized-bash.test.ts +0 -14
  745. package/src/tools/terminal/safe-env.ts +39 -24
  746. package/src/tools/terminal/sanitized-bash.ts +15 -2
  747. package/src/tools/terminal/shell-launch.test.ts +162 -0
  748. package/src/tools/terminal/shell.test.ts +29 -0
  749. package/src/tools/terminal/shell.ts +15 -7
  750. package/src/tools/tool-approval-handler.ts +5 -1
  751. package/src/tools/tool-types.ts +3 -3
  752. package/src/tools/types.ts +8 -2
  753. package/src/util/__tests__/worker-process-command.test.ts +5 -4
  754. package/src/util/abort-reasons.ts +18 -26
  755. package/src/util/browser-human-verification.ts +14 -0
  756. package/src/util/canonicalize-identity.test.ts +21 -0
  757. package/src/util/canonicalize-identity.ts +16 -0
  758. package/src/util/errors.ts +13 -3
  759. package/src/util/host-process.test.ts +17 -1
  760. package/src/util/host-process.ts +24 -0
  761. package/src/util/provider-error-patterns.ts +3 -0
  762. package/src/util/unicode.ts +29 -0
  763. package/src/util/worker-process.ts +77 -35
  764. package/src/workspace/migrations/157-add-desktop-files-launcher.ts +94 -0
  765. package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +4 -10
  766. package/src/workspace/migrations/__tests__/157-add-desktop-files-launcher.test.ts +113 -0
  767. package/src/workspace/migrations/registry.ts +2 -0
  768. package/src/__tests__/computer-use-window-schema.test.ts +0 -24
  769. package/src/api/events/heartbeat-alert.ts +0 -20
  770. package/src/daemon/message-types/web-activity.ts +0 -65
  771. package/src/desktop/desktop-dependencies.test.ts +0 -131
  772. package/src/tools/computer-use/definitions.ts +0 -607
@@ -14,7 +14,7 @@
14
14
  * echo, which is the keep stream's alone.
15
15
  */
16
16
 
17
- import { describe, expect, mock, test } from "bun:test";
17
+ import { describe, expect, mock, spyOn, test } from "bun:test";
18
18
 
19
19
  import {
20
20
  createMockProvider,
@@ -25,13 +25,22 @@ import { waitFor } from "../../__tests__/helpers/wait-for.js";
25
25
 
26
26
  setConfig("memory", { enabled: false });
27
27
 
28
- import type { VoiceTurnOptions } from "../../calls/voice-session-bridge.js";
28
+ import type {
29
+ VoiceTurnHandle,
30
+ VoiceTurnOptions,
31
+ } from "../../calls/voice-session-bridge.js";
29
32
  import { Conversation } from "../../daemon/conversation.js";
30
33
  import {
31
34
  deleteConversation,
35
+ findConversation,
32
36
  setConversation,
33
37
  } from "../../daemon/conversation-registry.js";
34
38
  import {
39
+ evictConversationsForReload,
40
+ getOrCreateConversation,
41
+ } from "../../daemon/conversation-store.js";
42
+ import {
43
+ attachmentExists,
35
44
  getAttachmentsForMessage,
36
45
  uploadAttachment,
37
46
  } from "../../persistence/attachments-store.js";
@@ -40,6 +49,7 @@ import {
40
49
  getMessages,
41
50
  selectSightFrameCaptureTimes,
42
51
  } from "../../persistence/conversation-crud.js";
52
+ import { getConversationModeSession } from "../../persistence/conversation-mode-sessions.js";
43
53
  import { sightFrameAttachmentIdsFromMetadata } from "../../persistence/conversation-types.js";
44
54
  import { initializeDb } from "../../persistence/db-init.js";
45
55
  import type {
@@ -48,6 +58,7 @@ import type {
48
58
  } from "../../stt/types.js";
49
59
  import {
50
60
  LiveVoiceSession,
61
+ type LiveVoiceSessionOptions,
51
62
  type LiveVoiceTurnStarter,
52
63
  } from "../live-voice-session.js";
53
64
  import type { LiveVoiceSessionFactoryContext } from "../live-voice-session-manager.js";
@@ -92,7 +103,16 @@ async function uploadFrame(): Promise<string> {
92
103
  * A session bound to a live conversation, so a kept frame has somewhere real
93
104
  * to land. Returns a `dispose` that unregisters it.
94
105
  */
95
- function createSessionHarness(title: string) {
106
+ function createSessionHarness(
107
+ title: string,
108
+ options?: {
109
+ acquireResidency?: boolean;
110
+ acquireModeSessionResidency?: NonNullable<
111
+ LiveVoiceSessionOptions["acquireModeSessionResidency"]
112
+ >;
113
+ startVoiceTurn?: LiveVoiceTurnStarter;
114
+ },
115
+ ) {
96
116
  const conversation = createConversation(title);
97
117
  const { provider } = createMockProvider([textResponse("")]);
98
118
  const activeConversation = new Conversation(
@@ -125,18 +145,32 @@ function createSessionHarness(title: string) {
125
145
  }),
126
146
  };
127
147
 
128
- const startVoiceTurn: LiveVoiceTurnStarter = mock(
129
- async (_options: VoiceTurnOptions) => ({
130
- turnId: "bridge-turn-1",
131
- abort: mock(),
132
- }),
133
- );
148
+ const startVoiceTurn = mock(
149
+ options?.startVoiceTurn ??
150
+ (async (_options: VoiceTurnOptions) => ({
151
+ turnId: "bridge-turn-1",
152
+ abort: mock(() => {
153
+ const requestId = _options.preacceptedModeSession?.requestId;
154
+ if (requestId) {
155
+ activeConversation.modeSessions.releaseTurn(requestId);
156
+ }
157
+ }),
158
+ })),
159
+ ) satisfies LiveVoiceTurnStarter;
134
160
 
135
161
  const session = new LiveVoiceSession(context, {
136
162
  resolveTranscriber: mock(async () => new MockStreamingTranscriber()),
137
163
  startVoiceTurn,
138
164
  createTurnId: () => "live-turn-1",
139
165
  emitMetrics: false,
166
+ acquireModeSessionResidency:
167
+ options?.acquireModeSessionResidency ??
168
+ (async () => ({
169
+ coordinator: activeConversation.modeSessions,
170
+ release: options?.acquireResidency
171
+ ? activeConversation.acquireLiveVoiceResidency()
172
+ : () => {},
173
+ })),
140
174
  });
141
175
 
142
176
  return {
@@ -153,6 +187,53 @@ function createSessionHarness(title: string) {
153
187
  }
154
188
 
155
189
  describe("live-voice sight_frame frame", () => {
190
+ test("accepts camera lifecycle start, frame, and end", () => {
191
+ expect(
192
+ validateLiveVoiceClientFrame({
193
+ type: "sight_start",
194
+ cameraEpoch: 7,
195
+ source: "ambient",
196
+ }),
197
+ ).toEqual({
198
+ ok: true,
199
+ frame: { type: "sight_start", cameraEpoch: 7, source: "ambient" },
200
+ });
201
+ expect(
202
+ validateLiveVoiceClientFrame({
203
+ type: "sight_frame",
204
+ attachmentId: "att-1",
205
+ cameraEpoch: 7,
206
+ source: "ambient",
207
+ }),
208
+ ).toEqual({
209
+ ok: true,
210
+ frame: {
211
+ type: "sight_frame",
212
+ attachmentId: "att-1",
213
+ cameraEpoch: 7,
214
+ source: "ambient",
215
+ },
216
+ });
217
+ expect(
218
+ validateLiveVoiceClientFrame({ type: "sight_end", cameraEpoch: 7 }),
219
+ ).toEqual({
220
+ ok: true,
221
+ frame: { type: "sight_end", cameraEpoch: 7 },
222
+ });
223
+ });
224
+
225
+ test("rejects invalid epochs, sources, and source without an epoch", () => {
226
+ for (const frame of [
227
+ { type: "sight_start", cameraEpoch: 0 },
228
+ { type: "sight_start", cameraEpoch: 1.5 },
229
+ { type: "sight_start", cameraEpoch: 1, source: "screen" },
230
+ { type: "sight_end", cameraEpoch: Number.MAX_SAFE_INTEGER + 1 },
231
+ { type: "sight_frame", attachmentId: "att-1", source: "live" },
232
+ ]) {
233
+ expect(validateLiveVoiceClientFrame(frame).ok).toBe(false);
234
+ }
235
+ });
236
+
156
237
  test("accepts a well-formed frame", () => {
157
238
  const result = validateLiveVoiceClientFrame({
158
239
  type: "sight_frame",
@@ -296,6 +377,933 @@ describe("live-voice sight_frame frame", () => {
296
377
  });
297
378
 
298
379
  describe("live-voice camera frames kept mid-call", () => {
380
+ test("keeps one canonical conversation across reload and camera restarts", async () => {
381
+ const harness = createSessionHarness("Sight residency across reload", {
382
+ acquireResidency: true,
383
+ });
384
+ try {
385
+ await harness.session.start();
386
+ await harness.session.handleClientFrame({
387
+ type: "sight_start",
388
+ cameraEpoch: 41,
389
+ source: "live",
390
+ });
391
+ const firstAttachmentId = await uploadFrame();
392
+ await harness.session.handleClientFrame({
393
+ type: "sight_frame",
394
+ attachmentId: firstAttachmentId,
395
+ cameraEpoch: 41,
396
+ source: "live",
397
+ });
398
+ await waitFor(() => getMessages(harness.conversationId).length === 1);
399
+ const firstMetadata = JSON.parse(
400
+ getMessages(harness.conversationId)[0]!.metadata ?? "{}",
401
+ ) as { modeSession?: { id: string } };
402
+
403
+ evictConversationsForReload();
404
+ await harness.session.handleClientFrame({
405
+ type: "sight_end",
406
+ cameraEpoch: 41,
407
+ });
408
+ await waitFor(
409
+ () =>
410
+ getConversationModeSession(
411
+ harness.conversationId,
412
+ firstMetadata.modeSession!.id,
413
+ )?.status === "completed",
414
+ );
415
+
416
+ expect(await getOrCreateConversation(harness.conversationId)).toBe(
417
+ harness.activeConversation,
418
+ );
419
+
420
+ await harness.session.handleClientFrame({
421
+ type: "sight_start",
422
+ cameraEpoch: 42,
423
+ source: "ambient",
424
+ });
425
+ const secondAttachmentId = await uploadFrame();
426
+ await harness.session.handleClientFrame({
427
+ type: "sight_frame",
428
+ attachmentId: secondAttachmentId,
429
+ cameraEpoch: 42,
430
+ source: "ambient",
431
+ });
432
+ await waitFor(() => getMessages(harness.conversationId).length === 2);
433
+ const secondMetadata = JSON.parse(
434
+ getMessages(harness.conversationId)[1]!.metadata ?? "{}",
435
+ ) as { modeSession?: { id: string; mode: string } };
436
+ expect(secondMetadata.modeSession).toMatchObject({ mode: "ambient" });
437
+ expect(secondMetadata.modeSession?.id).not.toBe(
438
+ firstMetadata.modeSession?.id,
439
+ );
440
+
441
+ await harness.session.handleClientFrame({
442
+ type: "text",
443
+ text: "Describe the current view.",
444
+ });
445
+ await waitFor(() => harness.startVoiceTurn.mock.calls.length === 1);
446
+ expect(
447
+ harness.startVoiceTurn.mock.calls[0]![0].preacceptedModeSession?.source
448
+ .id,
449
+ ).toBe(secondMetadata.modeSession?.id);
450
+
451
+ await harness.session.handleClientFrame({
452
+ type: "sight_end",
453
+ cameraEpoch: 42,
454
+ });
455
+ await harness.session.close("client_end");
456
+ evictConversationsForReload();
457
+ expect(findConversation(harness.conversationId)).toBeUndefined();
458
+ } finally {
459
+ harness.dispose();
460
+ }
461
+ });
462
+
463
+ test("keeps an accepted voice owner when the camera stops during bridge admission", async () => {
464
+ let resolveTurn!: (handle: VoiceTurnHandle) => void;
465
+ const bridgeTurn = new Promise<VoiceTurnHandle>((resolve) => {
466
+ resolveTurn = resolve;
467
+ });
468
+ const harness = createSessionHarness("Sight stop during admission", {
469
+ startVoiceTurn: async () => bridgeTurn,
470
+ });
471
+ try {
472
+ await harness.session.start();
473
+ await harness.session.handleClientFrame({
474
+ type: "sight_start",
475
+ cameraEpoch: 31,
476
+ source: "live",
477
+ });
478
+ const speaking = harness.session.handleClientFrame({
479
+ type: "text",
480
+ text: "What is in view?",
481
+ });
482
+ await waitFor(() => harness.startVoiceTurn.mock.calls.length > 0);
483
+ const accepted =
484
+ harness.startVoiceTurn.mock.calls[0]![0].preacceptedModeSession!;
485
+
486
+ await harness.session.handleClientFrame({
487
+ type: "sight_end",
488
+ cameraEpoch: 31,
489
+ });
490
+
491
+ expect(
492
+ harness.activeConversation.modeSessions.getTurnOwner(
493
+ accepted.requestId,
494
+ ),
495
+ ).toEqual({ id: accepted.source.id, mode: accepted.source.mode });
496
+ resolveTurn({ turnId: "bridge-turn-1", abort: mock() });
497
+ await speaking;
498
+ harness.activeConversation.modeSessions.releaseTurn(accepted.requestId);
499
+ } finally {
500
+ await harness.session.close("client_end");
501
+ harness.dispose();
502
+ }
503
+ });
504
+
505
+ test("keeps an accepted voice owner when a new camera run replaces it during admission", async () => {
506
+ let resolveTurn!: (handle: VoiceTurnHandle) => void;
507
+ const bridgeTurn = new Promise<VoiceTurnHandle>((resolve) => {
508
+ resolveTurn = resolve;
509
+ });
510
+ const harness = createSessionHarness("Sight replacement during admission", {
511
+ startVoiceTurn: async () => bridgeTurn,
512
+ });
513
+ try {
514
+ await harness.session.start();
515
+ await harness.session.handleClientFrame({
516
+ type: "sight_start",
517
+ cameraEpoch: 32,
518
+ source: "live",
519
+ });
520
+ const speaking = harness.session.handleClientFrame({
521
+ type: "text",
522
+ text: "What is in view?",
523
+ });
524
+ await waitFor(() => harness.startVoiceTurn.mock.calls.length > 0);
525
+ const accepted =
526
+ harness.startVoiceTurn.mock.calls[0]![0].preacceptedModeSession!;
527
+
528
+ await harness.session.handleClientFrame({
529
+ type: "sight_start",
530
+ cameraEpoch: 33,
531
+ source: "ambient",
532
+ });
533
+
534
+ expect(accepted.source.generation).toBe(32);
535
+ expect(
536
+ harness.activeConversation.modeSessions.getTurnOwner(
537
+ accepted.requestId,
538
+ ),
539
+ ).toEqual({ id: accepted.source.id, mode: accepted.source.mode });
540
+ resolveTurn({ turnId: "bridge-turn-1", abort: mock() });
541
+ await speaking;
542
+ harness.activeConversation.modeSessions.releaseTurn(accepted.requestId);
543
+ } finally {
544
+ await harness.session.close("client_end");
545
+ harness.dispose();
546
+ }
547
+ });
548
+
549
+ test("orders sight_end behind deferred camera-session preparation", async () => {
550
+ let resolvePreparation!: (value: Conversation["modeSessions"]) => void;
551
+ const preparation = new Promise<Conversation["modeSessions"]>((resolve) => {
552
+ resolvePreparation = resolve;
553
+ });
554
+ const harness = createSessionHarness("Deferred sight end", {
555
+ acquireModeSessionResidency: async () => ({
556
+ coordinator: await preparation,
557
+ release: () => {},
558
+ }),
559
+ });
560
+ try {
561
+ await harness.session.start();
562
+ const starting = harness.session.handleClientFrame({
563
+ type: "sight_start",
564
+ cameraEpoch: 21,
565
+ source: "live",
566
+ });
567
+ const ending = harness.session.handleClientFrame({
568
+ type: "sight_end",
569
+ cameraEpoch: 21,
570
+ });
571
+
572
+ resolvePreparation(harness.activeConversation.modeSessions);
573
+ await Promise.all([starting, ending]);
574
+ await harness.session.handleClientFrame({
575
+ type: "text",
576
+ text: "Is the camera still active?",
577
+ });
578
+ await waitFor(() => harness.startVoiceTurn.mock.calls.length > 0);
579
+
580
+ expect(
581
+ harness.startVoiceTurn.mock.calls[0]![0].modeSessionSource,
582
+ ).toBeUndefined();
583
+ } finally {
584
+ await harness.session.close("client_end");
585
+ harness.dispose();
586
+ }
587
+ });
588
+
589
+ test("orders a kept frame behind deferred camera-session preparation", async () => {
590
+ let resolvePreparation!: (value: Conversation["modeSessions"]) => void;
591
+ const preparation = new Promise<Conversation["modeSessions"]>((resolve) => {
592
+ resolvePreparation = resolve;
593
+ });
594
+ const harness = createSessionHarness("Deferred sight frame", {
595
+ acquireModeSessionResidency: async () => ({
596
+ coordinator: await preparation,
597
+ release: () => {},
598
+ }),
599
+ });
600
+ try {
601
+ await harness.session.start();
602
+ const attachmentId = await uploadFrame();
603
+ const starting = harness.session.handleClientFrame({
604
+ type: "sight_start",
605
+ cameraEpoch: 22,
606
+ source: "ambient",
607
+ });
608
+ const keeping = harness.session.handleClientFrame({
609
+ type: "sight_frame",
610
+ attachmentId,
611
+ cameraEpoch: 22,
612
+ source: "ambient",
613
+ });
614
+
615
+ resolvePreparation(harness.activeConversation.modeSessions);
616
+ await Promise.all([starting, keeping]);
617
+ await waitFor(() => getMessages(harness.conversationId).length > 0);
618
+
619
+ const row = getMessages(harness.conversationId)[0]!;
620
+ const metadata = JSON.parse(row.metadata ?? "{}") as {
621
+ modeSession?: { id: string; mode: string };
622
+ };
623
+ expect(metadata.modeSession?.mode).toBe("ambient");
624
+ expect(
625
+ harness.frames.filter((frame) => frame.type === "error"),
626
+ ).toHaveLength(0);
627
+ } finally {
628
+ await harness.session.close("client_end");
629
+ harness.dispose();
630
+ }
631
+ });
632
+
633
+ test("does not activate a deferred camera start after session close", async () => {
634
+ let resolvePreparation!: (value: Conversation["modeSessions"]) => void;
635
+ const preparation = new Promise<Conversation["modeSessions"]>((resolve) => {
636
+ resolvePreparation = resolve;
637
+ });
638
+ const release = mock(() => {});
639
+ const acquire = mock(async () => ({
640
+ coordinator: await preparation,
641
+ release,
642
+ }));
643
+ const harness = createSessionHarness("Deferred sight close", {
644
+ acquireModeSessionResidency: acquire,
645
+ });
646
+ const activateSource = spyOn(
647
+ harness.activeConversation.modeSessions,
648
+ "activateSource",
649
+ );
650
+ try {
651
+ await harness.session.start();
652
+ const starting = harness.session.handleClientFrame({
653
+ type: "sight_start",
654
+ cameraEpoch: 23,
655
+ source: "live",
656
+ });
657
+ await waitFor(() => acquire.mock.calls.length === 1);
658
+ const closing = harness.session.close("websocket_close");
659
+
660
+ resolvePreparation(harness.activeConversation.modeSessions);
661
+ await Promise.all([starting, closing]);
662
+
663
+ expect(activateSource).not.toHaveBeenCalled();
664
+ expect(release).toHaveBeenCalledTimes(1);
665
+ await harness.session.close("client_end");
666
+ expect(release).toHaveBeenCalledTimes(1);
667
+ } finally {
668
+ activateSource.mockRestore();
669
+ harness.dispose();
670
+ }
671
+ });
672
+
673
+ test("captures the active camera owner for a typed voice turn", async () => {
674
+ const harness = createSessionHarness("Sight-owned voice turn");
675
+ try {
676
+ await harness.session.start();
677
+ await harness.session.handleClientFrame({
678
+ type: "sight_start",
679
+ cameraEpoch: 11,
680
+ source: "ambient",
681
+ });
682
+ await harness.session.handleClientFrame({
683
+ type: "text",
684
+ text: "What do you see?",
685
+ });
686
+ await waitFor(() => harness.startVoiceTurn.mock.calls.length > 0, {
687
+ message: "Timed out waiting for typed voice turn",
688
+ });
689
+
690
+ const options = harness.startVoiceTurn.mock.calls[0]![0];
691
+ expect(options.preacceptedModeSession?.source).toMatchObject({
692
+ generation: 11,
693
+ mode: "ambient",
694
+ });
695
+ } finally {
696
+ await harness.session.close("client_end");
697
+ harness.dispose();
698
+ }
699
+ });
700
+
701
+ test("dispatches voice ungrouped when the first delivery claim throws", async () => {
702
+ const harness = createSessionHarness("Sight voice first claim failure");
703
+ const coordinator = harness.activeConversation.modeSessions;
704
+ const originalClaimTurn = coordinator.claimTurn.bind(coordinator);
705
+ let attemptedRequestId: string | undefined;
706
+ const claimTurn = spyOn(coordinator, "claimTurn").mockImplementation(
707
+ (turnId, source, at) => {
708
+ const owner = originalClaimTurn(turnId, source, at);
709
+ if (!turnId.startsWith("live-voice-camera:")) {
710
+ attemptedRequestId = turnId;
711
+ throw new Error("session tracking unavailable");
712
+ }
713
+ return owner;
714
+ },
715
+ );
716
+ try {
717
+ await harness.session.start();
718
+ await harness.session.handleClientFrame({
719
+ type: "sight_start",
720
+ cameraEpoch: 54,
721
+ source: "live",
722
+ });
723
+ await harness.session.handleClientFrame({
724
+ type: "text",
725
+ text: "What is here?",
726
+ });
727
+ await waitFor(() => harness.startVoiceTurn.mock.calls.length === 1);
728
+
729
+ expect(
730
+ harness.startVoiceTurn.mock.calls[0]![0].preacceptedModeSession,
731
+ ).toBeUndefined();
732
+ expect(attemptedRequestId).toBeDefined();
733
+ expect(coordinator.getTurnOwner(attemptedRequestId!)).toBeUndefined();
734
+ } finally {
735
+ claimTurn.mockRestore();
736
+ await harness.session.close("client_end");
737
+ harness.dispose();
738
+ }
739
+ });
740
+
741
+ test("releases a mismatched first delivery claim before ungrouped dispatch", async () => {
742
+ const harness = createSessionHarness("Sight voice first claim mismatch");
743
+ const coordinator = harness.activeConversation.modeSessions;
744
+ const originalClaimTurn = coordinator.claimTurn.bind(coordinator);
745
+ let attemptedRequestId: string | undefined;
746
+ const claimTurn = spyOn(coordinator, "claimTurn").mockImplementation(
747
+ (turnId, source, at) => {
748
+ const owner = originalClaimTurn(turnId, source, at);
749
+ if (!turnId.startsWith("live-voice-camera:") && owner) {
750
+ attemptedRequestId = turnId;
751
+ return { id: "different-session", mode: owner.mode };
752
+ }
753
+ return owner;
754
+ },
755
+ );
756
+ try {
757
+ await harness.session.start();
758
+ await harness.session.handleClientFrame({
759
+ type: "sight_start",
760
+ cameraEpoch: 57,
761
+ source: "ambient",
762
+ });
763
+ await harness.session.handleClientFrame({
764
+ type: "text",
765
+ text: "What is here?",
766
+ });
767
+ await waitFor(() => harness.startVoiceTurn.mock.calls.length === 1);
768
+
769
+ expect(
770
+ harness.startVoiceTurn.mock.calls[0]![0].preacceptedModeSession,
771
+ ).toBeUndefined();
772
+ expect(attemptedRequestId).toBeDefined();
773
+ expect(coordinator.getTurnOwner(attemptedRequestId!)).toBeUndefined();
774
+ } finally {
775
+ claimTurn.mockRestore();
776
+ await harness.session.close("client_end");
777
+ harness.dispose();
778
+ }
779
+ });
780
+
781
+ test("releases every delivery claim when a later admission throws", async () => {
782
+ const harness = createSessionHarness("Sight voice later claim failure");
783
+ const coordinator = harness.activeConversation.modeSessions;
784
+ const originalClaimTurn = coordinator.claimTurn.bind(coordinator);
785
+ const attemptedRequestIds: string[] = [];
786
+ const claimTurn = spyOn(coordinator, "claimTurn").mockImplementation(
787
+ (turnId, source, at) => {
788
+ const owner = originalClaimTurn(turnId, source, at);
789
+ if (!turnId.startsWith("live-voice-camera:")) {
790
+ attemptedRequestIds.push(turnId);
791
+ if (attemptedRequestIds.length === 3) {
792
+ throw new Error("session tracking unavailable");
793
+ }
794
+ }
795
+ return owner;
796
+ },
797
+ );
798
+ try {
799
+ await harness.session.start();
800
+ await harness.session.handleClientFrame({
801
+ type: "sight_start",
802
+ cameraEpoch: 55,
803
+ source: "ambient",
804
+ });
805
+ await harness.session.handleClientFrame({
806
+ type: "text",
807
+ text: "What is here?",
808
+ });
809
+ await waitFor(() => harness.startVoiceTurn.mock.calls.length === 1);
810
+
811
+ expect(attemptedRequestIds).toHaveLength(3);
812
+ expect(
813
+ harness.startVoiceTurn.mock.calls[0]![0].preacceptedModeSession,
814
+ ).toBeUndefined();
815
+ expect(
816
+ attemptedRequestIds.map((id) => coordinator.getTurnOwner(id)),
817
+ ).toEqual([undefined, undefined, undefined]);
818
+ } finally {
819
+ claimTurn.mockRestore();
820
+ await harness.session.close("client_end");
821
+ harness.dispose();
822
+ }
823
+ });
824
+
825
+ test("keeps a stopped camera session active until voice delivery settles", async () => {
826
+ const harness = createSessionHarness("Sight voice delivery barrier");
827
+ try {
828
+ await harness.session.start();
829
+ await harness.session.handleClientFrame({
830
+ type: "sight_start",
831
+ cameraEpoch: 12,
832
+ source: "live",
833
+ });
834
+ await harness.session.handleClientFrame({
835
+ type: "text",
836
+ text: "Describe this.",
837
+ });
838
+ await waitFor(() => harness.startVoiceTurn.mock.calls.length > 0);
839
+
840
+ const options = harness.startVoiceTurn.mock.calls[0]![0];
841
+ const sessionId = options.preacceptedModeSession!.source.id;
842
+ await harness.session.handleClientFrame({
843
+ type: "sight_end",
844
+ cameraEpoch: 12,
845
+ });
846
+ expect(
847
+ getConversationModeSession(harness.conversationId, sessionId)?.status,
848
+ ).toBe("active");
849
+
850
+ options.callbacks?.message_complete?.({
851
+ type: "message_complete",
852
+ conversationId: harness.conversationId,
853
+ messageId: "assistant-message-1",
854
+ });
855
+ harness.activeConversation.modeSessions.releaseTurn(
856
+ options.preacceptedModeSession!.requestId,
857
+ );
858
+ await waitFor(
859
+ () =>
860
+ getConversationModeSession(harness.conversationId, sessionId)
861
+ ?.status === "completed",
862
+ { message: "Timed out waiting for voice delivery settlement" },
863
+ );
864
+ expect(harness.frames.some((frame) => frame.type === "tts_done")).toBe(
865
+ true,
866
+ );
867
+ } finally {
868
+ await harness.session.close("client_end");
869
+ harness.dispose();
870
+ }
871
+ });
872
+
873
+ test("releases remaining delivery claims when one release throws", async () => {
874
+ const harness = createSessionHarness("Sight voice release failure");
875
+ const coordinator = harness.activeConversation.modeSessions;
876
+ const originalClaimTurn = coordinator.claimTurn.bind(coordinator);
877
+ const claimedRequestIds: string[] = [];
878
+ const claimTurn = spyOn(coordinator, "claimTurn").mockImplementation(
879
+ (turnId, source, at) => {
880
+ const owner = originalClaimTurn(turnId, source, at);
881
+ if (!turnId.startsWith("live-voice-camera:")) {
882
+ claimedRequestIds.push(turnId);
883
+ }
884
+ return owner;
885
+ },
886
+ );
887
+ const originalReleaseTurn = coordinator.releaseTurn.bind(coordinator);
888
+ const releasedRequestIds: string[] = [];
889
+ let injectedFailure = false;
890
+ const releaseTurn = spyOn(coordinator, "releaseTurn").mockImplementation(
891
+ (turnId) => {
892
+ const released = originalReleaseTurn(turnId);
893
+ if (!turnId.startsWith("live-voice-camera:")) {
894
+ releasedRequestIds.push(turnId);
895
+ if (!injectedFailure) {
896
+ injectedFailure = true;
897
+ throw new Error("session tracking unavailable");
898
+ }
899
+ }
900
+ return released;
901
+ },
902
+ );
903
+ try {
904
+ await harness.session.start();
905
+ await harness.session.handleClientFrame({
906
+ type: "sight_start",
907
+ cameraEpoch: 56,
908
+ source: "live",
909
+ });
910
+ await harness.session.handleClientFrame({
911
+ type: "text",
912
+ text: "Describe this.",
913
+ });
914
+ await waitFor(() => harness.startVoiceTurn.mock.calls.length === 1);
915
+ const options = harness.startVoiceTurn.mock.calls[0]![0];
916
+
917
+ options.callbacks?.message_complete?.({
918
+ type: "message_complete",
919
+ conversationId: harness.conversationId,
920
+ messageId: "assistant-message-release-failure",
921
+ });
922
+ await waitFor(() =>
923
+ harness.frames.some((frame) => frame.type === "tts_done"),
924
+ );
925
+
926
+ expect(claimedRequestIds).toHaveLength(3);
927
+ expect(releasedRequestIds).toContain(claimedRequestIds[1]!);
928
+ expect(releasedRequestIds).toContain(claimedRequestIds[2]!);
929
+ expect(coordinator.getTurnOwner(claimedRequestIds[1]!)).toBeUndefined();
930
+ expect(coordinator.getTurnOwner(claimedRequestIds[2]!)).toBeUndefined();
931
+ } finally {
932
+ releaseTurn.mockRestore();
933
+ claimTurn.mockRestore();
934
+ await harness.session.close("client_end");
935
+ harness.dispose();
936
+ }
937
+ });
938
+
939
+ test("socket loss interrupts a camera run after cancelling its voice hold", async () => {
940
+ const harness = createSessionHarness("Sight socket loss");
941
+ try {
942
+ await harness.session.start();
943
+ await harness.session.handleClientFrame({
944
+ type: "sight_start",
945
+ cameraEpoch: 13,
946
+ source: "ambient",
947
+ });
948
+ await harness.session.handleClientFrame({
949
+ type: "text",
950
+ text: "What is here?",
951
+ });
952
+ await waitFor(() => harness.startVoiceTurn.mock.calls.length > 0);
953
+ const sessionId =
954
+ harness.startVoiceTurn.mock.calls[0]![0].preacceptedModeSession!.source
955
+ .id;
956
+
957
+ await harness.session.close("websocket_close");
958
+
959
+ await waitFor(
960
+ () =>
961
+ getConversationModeSession(harness.conversationId, sessionId)
962
+ ?.status === "interrupted",
963
+ { message: "Timed out waiting for interrupted camera session" },
964
+ );
965
+ expect(
966
+ getConversationModeSession(harness.conversationId, sessionId)
967
+ ?.endReason,
968
+ ).toBe("camera_websocket_close");
969
+ } finally {
970
+ harness.dispose();
971
+ }
972
+ });
973
+
974
+ test("releases a reserved voice owner when close rejects deferred admission", async () => {
975
+ const startVoiceTurn: LiveVoiceTurnStarter = async (options) => {
976
+ await new Promise<void>((_resolve, reject) => {
977
+ const rejectAdmission = () => reject(new Error("admission cancelled"));
978
+ if (options.signal?.aborted) {
979
+ rejectAdmission();
980
+ return;
981
+ }
982
+ options.signal?.addEventListener("abort", rejectAdmission, {
983
+ once: true,
984
+ });
985
+ });
986
+ return { turnId: "unreachable", abort: mock() };
987
+ };
988
+ const harness = createSessionHarness("Sight deferred admission close", {
989
+ startVoiceTurn,
990
+ });
991
+ try {
992
+ await harness.session.start();
993
+ await harness.session.handleClientFrame({
994
+ type: "sight_start",
995
+ cameraEpoch: 14,
996
+ source: "live",
997
+ });
998
+ const speaking = harness.session.handleClientFrame({
999
+ type: "text",
1000
+ text: "What is here?",
1001
+ });
1002
+ await waitFor(() => harness.startVoiceTurn.mock.calls.length === 1);
1003
+ const accepted =
1004
+ harness.startVoiceTurn.mock.calls[0]![0].preacceptedModeSession!;
1005
+
1006
+ await harness.session.close("websocket_close");
1007
+ await speaking;
1008
+
1009
+ await waitFor(
1010
+ () =>
1011
+ getConversationModeSession(harness.conversationId, accepted.source.id)
1012
+ ?.status === "interrupted",
1013
+ );
1014
+ expect(
1015
+ harness.activeConversation.modeSessions.getTurnOwner(
1016
+ accepted.requestId,
1017
+ ),
1018
+ ).toBeUndefined();
1019
+ } finally {
1020
+ harness.dispose();
1021
+ }
1022
+ });
1023
+
1024
+ test("stamps negotiated keeps and finalizes the run on sight_end", async () => {
1025
+ const harness = createSessionHarness("Sight session lifecycle");
1026
+ try {
1027
+ await harness.session.start();
1028
+ await harness.session.handleClientFrame({
1029
+ type: "sight_start",
1030
+ cameraEpoch: 1,
1031
+ source: "live",
1032
+ });
1033
+ const attachmentId = await uploadFrame();
1034
+ await harness.session.handleClientFrame({
1035
+ type: "sight_frame",
1036
+ attachmentId,
1037
+ cameraEpoch: 1,
1038
+ source: "live",
1039
+ });
1040
+ await waitFor(() => getMessages(harness.conversationId).length > 0, {
1041
+ message: "Timed out waiting for grouped camera frame",
1042
+ });
1043
+
1044
+ const row = getMessages(harness.conversationId)[0]!;
1045
+ const metadata = JSON.parse(row.metadata ?? "{}") as {
1046
+ modeSession?: { id: string; mode: string };
1047
+ };
1048
+ expect(metadata.modeSession?.mode).toBe("live_vision");
1049
+ await harness.session.handleClientFrame({
1050
+ type: "sight_end",
1051
+ cameraEpoch: 1,
1052
+ });
1053
+
1054
+ await waitFor(
1055
+ () =>
1056
+ getConversationModeSession(
1057
+ harness.conversationId,
1058
+ metadata.modeSession!.id,
1059
+ )?.status === "completed",
1060
+ { message: "Timed out waiting for camera session finalization" },
1061
+ );
1062
+ } finally {
1063
+ harness.dispose();
1064
+ }
1065
+ });
1066
+
1067
+ test("saves an admitted frame after stop while rejecting a frame delivered after stop", async () => {
1068
+ const harness = createSessionHarness("Sight admitted frame drain", {
1069
+ acquireResidency: true,
1070
+ });
1071
+ const processing = harness.activeConversation.acquireProcessing();
1072
+ expect(processing).not.toBeNull();
1073
+ try {
1074
+ await harness.session.start();
1075
+ await harness.session.handleClientFrame({
1076
+ type: "sight_start",
1077
+ cameraEpoch: 61,
1078
+ source: "live",
1079
+ });
1080
+ const accepted = await uploadFrame();
1081
+ await harness.session.handleClientFrame({
1082
+ type: "sight_frame",
1083
+ attachmentId: accepted,
1084
+ cameraEpoch: 61,
1085
+ source: "live",
1086
+ });
1087
+ await harness.session.handleClientFrame({
1088
+ type: "sight_end",
1089
+ cameraEpoch: 61,
1090
+ });
1091
+ const late = await uploadFrame();
1092
+ await harness.session.handleClientFrame({
1093
+ type: "sight_frame",
1094
+ attachmentId: late,
1095
+ cameraEpoch: 61,
1096
+ source: "live",
1097
+ });
1098
+ expect(attachmentExists(late)).toBe(false);
1099
+ expect(getMessages(harness.conversationId)).toHaveLength(0);
1100
+ expect(harness.activeConversation.modeSessions.hasResidentWork()).toBe(
1101
+ true,
1102
+ );
1103
+ await harness.session.close("client_end");
1104
+ expect(harness.activeConversation.hasInFlightWork()).toBe(true);
1105
+ harness.activeConversation.releaseProcessing(processing!);
1106
+ await waitFor(
1107
+ () =>
1108
+ getMessages(harness.conversationId).length === 1 &&
1109
+ !harness.activeConversation.hasInFlightWork(),
1110
+ );
1111
+ const row = getMessages(harness.conversationId)[0]!;
1112
+ const stamp = JSON.parse(row.metadata ?? "{}").modeSession;
1113
+ expect(
1114
+ getConversationModeSession(harness.conversationId, stamp.id)?.status,
1115
+ ).toBe("completed");
1116
+ expect(getAttachmentsForMessage(row.id)).toHaveLength(1);
1117
+ expect(attachmentExists(accepted)).toBe(true);
1118
+ expect(
1119
+ harness.frames.filter((frame) => frame.type === "error"),
1120
+ ).toMatchObject([{ frameType: "sight_frame", attachmentId: late }]);
1121
+ } finally {
1122
+ harness.activeConversation.releaseProcessing(processing!);
1123
+ await harness.session.close("client_end");
1124
+ harness.dispose();
1125
+ }
1126
+ });
1127
+
1128
+ test.each([false, true])(
1129
+ "persists an accepted camera frame when its claim throws (partial: %s)",
1130
+ async (partial) => {
1131
+ const harness = createSessionHarness("Sight frame ownership failure");
1132
+ const coordinator = harness.activeConversation.modeSessions;
1133
+ const originalClaimTurn = coordinator.claimTurn.bind(coordinator);
1134
+ let failedSessionId: string | undefined;
1135
+ const claimTurn = spyOn(coordinator, "claimTurn").mockImplementation(
1136
+ (turnId, source, at) => {
1137
+ if (!turnId.startsWith("live-voice-camera:")) {
1138
+ if (partial) {
1139
+ originalClaimTurn(turnId, source, at);
1140
+ }
1141
+ throw new Error("session tracking unavailable");
1142
+ }
1143
+ const owner = originalClaimTurn(turnId, source, at);
1144
+ failedSessionId = owner?.id;
1145
+ return owner;
1146
+ },
1147
+ );
1148
+ try {
1149
+ await harness.session.start();
1150
+ await harness.session.handleClientFrame({
1151
+ type: "sight_start",
1152
+ cameraEpoch: 52,
1153
+ source: "ambient",
1154
+ });
1155
+ const attachmentId = await uploadFrame();
1156
+ const keeping = harness.session.handleClientFrame({
1157
+ type: "sight_frame",
1158
+ attachmentId,
1159
+ cameraEpoch: 52,
1160
+ source: "ambient",
1161
+ });
1162
+ const ending = harness.session.handleClientFrame({
1163
+ type: "sight_end",
1164
+ cameraEpoch: 52,
1165
+ });
1166
+ await Promise.all([keeping, ending]);
1167
+
1168
+ await waitFor(
1169
+ () =>
1170
+ failedSessionId !== undefined &&
1171
+ getConversationModeSession(harness.conversationId, failedSessionId)
1172
+ ?.status === "completed" &&
1173
+ getMessages(harness.conversationId).length === 1 &&
1174
+ !coordinator.hasResidentWork(),
1175
+ { message: "Timed out waiting for the accepted frame to settle" },
1176
+ );
1177
+ expect(attachmentExists(attachmentId)).toBe(true);
1178
+ const row = getMessages(harness.conversationId)[0]!;
1179
+ expect(getAttachmentsForMessage(row.id)).toHaveLength(1);
1180
+ expect(
1181
+ harness.frames.filter((frame) => frame.type === "error"),
1182
+ ).toHaveLength(0);
1183
+ } finally {
1184
+ claimTurn.mockRestore();
1185
+ await harness.session.close("client_end");
1186
+ harness.dispose();
1187
+ }
1188
+ },
1189
+ );
1190
+
1191
+ test("releases acquired residency after a refused camera start and accepts ordinary frames", async () => {
1192
+ const harness = createSessionHarness("Sight start tracking failure", {
1193
+ acquireResidency: true,
1194
+ });
1195
+ const claim = spyOn(
1196
+ harness.activeConversation.modeSessions,
1197
+ "claimTurn",
1198
+ ).mockImplementation(() => {
1199
+ throw new Error("tracking unavailable");
1200
+ });
1201
+ try {
1202
+ await harness.session.start();
1203
+ await harness.session.handleClientFrame({
1204
+ type: "sight_start",
1205
+ cameraEpoch: 1,
1206
+ source: "live",
1207
+ });
1208
+ expect(
1209
+ harness.frames.find((frame) => frame.type === "error"),
1210
+ ).toMatchObject({ frameType: "sight_start" });
1211
+ expect(harness.activeConversation.modeSessions.hasResidentWork()).toBe(
1212
+ false,
1213
+ );
1214
+ expect(harness.activeConversation.hasInFlightWork()).toBe(false);
1215
+ const attachmentId = await uploadFrame();
1216
+ await harness.session.handleClientFrame({
1217
+ type: "sight_frame",
1218
+ attachmentId,
1219
+ });
1220
+ await waitFor(() => getMessages(harness.conversationId).length === 1);
1221
+ expect(
1222
+ JSON.parse(getMessages(harness.conversationId)[0]!.metadata ?? "{}")
1223
+ .modeSession,
1224
+ ).toBeUndefined();
1225
+ expect(attachmentExists(attachmentId)).toBe(true);
1226
+ } finally {
1227
+ claim.mockRestore();
1228
+ await harness.session.close("client_end");
1229
+ harness.dispose();
1230
+ }
1231
+ });
1232
+
1233
+ test("keeps a persisted frame successful when bookkeeping and ownership release throw", async () => {
1234
+ const harness = createSessionHarness("Sight frame release failure");
1235
+ const coordinator = harness.activeConversation.modeSessions;
1236
+ const originalReleaseTurn = coordinator.releaseTurn.bind(coordinator);
1237
+ const releaseTurn = spyOn(coordinator, "releaseTurn").mockImplementation(
1238
+ (turnId) => {
1239
+ originalReleaseTurn(turnId);
1240
+ if (!turnId.startsWith("live-voice-camera:")) {
1241
+ throw new Error("session tracking unavailable");
1242
+ }
1243
+ },
1244
+ );
1245
+ const originalTrackRow = coordinator.trackPersistedRow.bind(coordinator);
1246
+ const trackPersistedRow = spyOn(
1247
+ coordinator,
1248
+ "trackPersistedRow",
1249
+ ).mockImplementation((turnId, messageId, at, options) => {
1250
+ originalTrackRow(turnId, messageId, at, options);
1251
+ if (turnId.startsWith("live-voice-camera:")) {
1252
+ throw new Error("tracking unavailable after persistence");
1253
+ }
1254
+ });
1255
+ try {
1256
+ await harness.session.start();
1257
+ await harness.session.handleClientFrame({
1258
+ type: "sight_start",
1259
+ cameraEpoch: 53,
1260
+ source: "live",
1261
+ });
1262
+ const attachmentId = await uploadFrame();
1263
+ const before = harness.frames.length;
1264
+ await harness.session.handleClientFrame({
1265
+ type: "sight_frame",
1266
+ attachmentId,
1267
+ cameraEpoch: 53,
1268
+ source: "live",
1269
+ });
1270
+ await waitFor(() => getMessages(harness.conversationId).length === 1);
1271
+ const row = getMessages(harness.conversationId)[0]!;
1272
+
1273
+ await harness.session.handleClientFrame({
1274
+ type: "sight_end",
1275
+ cameraEpoch: 53,
1276
+ });
1277
+ const metadata = JSON.parse(row.metadata ?? "{}") as {
1278
+ modeSession?: { id: string };
1279
+ };
1280
+ await waitFor(
1281
+ () =>
1282
+ getConversationModeSession(
1283
+ harness.conversationId,
1284
+ metadata.modeSession!.id,
1285
+ )?.status === "completed",
1286
+ );
1287
+
1288
+ expect(attachmentExists(attachmentId)).toBe(true);
1289
+ expect(getAttachmentsForMessage(row.id)).toHaveLength(1);
1290
+ expect(
1291
+ trackPersistedRow.mock.calls.some(
1292
+ ([turnId, messageId]) =>
1293
+ turnId.startsWith("live-voice-camera:") && messageId === row.id,
1294
+ ),
1295
+ ).toBe(true);
1296
+ expect(
1297
+ harness.frames.slice(before).filter((frame) => frame.type === "error"),
1298
+ ).toHaveLength(0);
1299
+ } finally {
1300
+ trackPersistedRow.mockRestore();
1301
+ releaseTurn.mockRestore();
1302
+ await harness.session.close("client_end");
1303
+ harness.dispose();
1304
+ }
1305
+ });
1306
+
299
1307
  test("a kept frame persists as its own tagged message and runs no turn", async () => {
300
1308
  const harness = createSessionHarness("Sight keep stream");
301
1309
  try {