@vellumai/assistant 0.12.3-staging.1 → 0.12.3-staging.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (560) hide show
  1. package/ARCHITECTURE.md +43 -29
  2. package/Dockerfile +63 -0
  3. package/docs/architecture/security.md +10 -5
  4. package/docs/architecture/turn-actor.md +2 -0
  5. package/docs/desktop-browser-cli.md +45 -4
  6. package/docs/live-voice-task-outcomes.md +100 -0
  7. package/docs/mode-sessions.md +183 -0
  8. package/docs/voice-input-diagnostics.md +58 -0
  9. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  10. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  11. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  12. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  13. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  14. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  15. package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +20 -0
  16. package/node_modules/@vellumai/gateway-client/src/index.ts +4 -1
  17. package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +38 -7
  18. package/node_modules/@vellumai/gateway-client/src/trust-verdict-contract.ts +5 -6
  19. package/node_modules/@vellumai/service-contracts/package.json +2 -1
  20. package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  21. package/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  22. package/node_modules/@vellumai/slack-text/src/index.ts +4 -8
  23. package/openapi.yaml +665 -14
  24. package/package.json +1 -1
  25. package/scripts/smoke-desktop-browser-cli.ts +10 -26
  26. package/scripts/smoke-desktop-window-theme.ts +2 -3
  27. package/src/__tests__/agent-loop-output-hooks.test.ts +150 -1
  28. package/src/__tests__/agent-loop-provider-error-recording.test.ts +87 -19
  29. package/src/__tests__/annotate-activity-metadata.test.ts +3 -1
  30. package/src/__tests__/annotate-risk-options.test.ts +2 -0
  31. package/src/__tests__/answered-question-persistence.test.ts +2 -0
  32. package/src/__tests__/anthropic-gateway-shared.test.ts +22 -0
  33. package/src/__tests__/approval-message-composer.test.ts +1 -118
  34. package/src/__tests__/approval-routes-http.test.ts +3 -0
  35. package/src/__tests__/assistant-event-hub-connection-history.test.ts +111 -0
  36. package/src/__tests__/assistant-event-hub-targeted.test.ts +50 -0
  37. package/src/__tests__/assistant-event-targeting.test.ts +186 -0
  38. package/src/__tests__/assistant-stream-state.test.ts +29 -9
  39. package/src/__tests__/browser-fill-credential.test.ts +11 -22
  40. package/src/__tests__/btw-routes.test.ts +4 -1
  41. package/src/__tests__/build-persisted-content.test.ts +1 -1
  42. package/src/__tests__/call-conversation-messages.test.ts +5 -0
  43. package/src/__tests__/call-recovery.test.ts +7 -2
  44. package/src/__tests__/call-routes-http.test.ts +12 -0
  45. package/src/__tests__/call-store.test.ts +7 -2
  46. package/src/__tests__/channel-retry-sweep.test.ts +9 -0
  47. package/src/__tests__/chat-reveal-guard-priming.test.ts +2 -0
  48. package/src/__tests__/checker.test.ts +11 -3
  49. package/src/__tests__/client-os-metadata-persistence.test.ts +35 -1
  50. package/src/__tests__/computer-use-skill-manifest-regression.test.ts +0 -25
  51. package/src/__tests__/computer-use-tools.test.ts +1 -233
  52. package/src/__tests__/config-model-image-provider.test.ts +39 -0
  53. package/src/__tests__/config-schema.test.ts +16 -0
  54. package/src/__tests__/config-watcher.test.ts +24 -0
  55. package/src/__tests__/context-search-agent-runner.test.ts +260 -1
  56. package/src/__tests__/context-search-fanout.test.ts +1 -1
  57. package/src/__tests__/context-search-types.test.ts +7 -7
  58. package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +8 -2
  59. package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +2 -0
  60. package/src/__tests__/conversation-agent-loop.test.ts +546 -16
  61. package/src/__tests__/conversation-app-control-lifecycle.test.ts +53 -1
  62. package/src/__tests__/conversation-evictor.test.ts +62 -37
  63. package/src/__tests__/conversation-history-boundary-repair-failure.test.ts +174 -0
  64. package/src/__tests__/conversation-loop-db-lock-resilience.test.ts +2 -0
  65. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +2 -0
  66. package/src/__tests__/conversation-retry-route.test.ts +22 -0
  67. package/src/__tests__/conversation-routes-disk-view.test.ts +2 -0
  68. package/src/__tests__/conversation-routes-enabled-plugins.test.ts +2 -0
  69. package/src/__tests__/conversation-routes-guardian-reply.test.ts +10 -0
  70. package/src/__tests__/conversation-routes-hidden-queue.test.ts +2 -0
  71. package/src/__tests__/conversation-routes-slash-commands.test.ts +2 -0
  72. package/src/__tests__/conversation-runtime-assembly.test.ts +29 -0
  73. package/src/__tests__/conversation-surface-snapshots.test.ts +543 -0
  74. package/src/__tests__/conversation-surfaces-action-delivery.test.ts +151 -0
  75. package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +63 -1
  76. package/src/__tests__/conversation-tool-setup-attribution.test.ts +23 -0
  77. package/src/__tests__/conversation-undo.test.ts +61 -0
  78. package/src/__tests__/credential-execution-client.test.ts +6 -6
  79. package/src/__tests__/credential-prompt-route.test.ts +35 -3
  80. package/src/__tests__/cu-unified-flow.test.ts +82 -1
  81. package/src/__tests__/db-mode-sessions-migration.test.ts +175 -0
  82. package/src/__tests__/deterministic-verification-control-plane.test.ts +2 -0
  83. package/src/__tests__/dm-backfill.test.ts +91 -1
  84. package/src/__tests__/document-create-route.test.ts +88 -0
  85. package/src/__tests__/document-sync-tags.test.ts +27 -0
  86. package/src/__tests__/emit-signal-routing-intent.test.ts +142 -2
  87. package/src/__tests__/events-tail-route.test.ts +25 -0
  88. package/src/__tests__/evict-conversations-for-reload.test.ts +57 -8
  89. package/src/__tests__/guardian-action-sweep.test.ts +16 -0
  90. package/src/__tests__/guardian-dispatch.test.ts +13 -0
  91. package/src/__tests__/headless-browser-interactions.test.ts +64 -20
  92. package/src/__tests__/heartbeat-service.test.ts +2 -59
  93. package/src/__tests__/helpers/mock-conversation.ts +21 -0
  94. package/src/__tests__/helpers/native-web-search-harness.ts +2 -0
  95. package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +5 -0
  96. package/src/__tests__/host-cu-proxy.test.ts +46 -2
  97. package/src/__tests__/http-user-message-parity.test.ts +2 -0
  98. package/src/__tests__/image-credentials.test.ts +49 -0
  99. package/src/__tests__/image-service-dispatcher.test.ts +45 -0
  100. package/src/__tests__/inference-profile-session-handler.test.ts +1 -1
  101. package/src/__tests__/interrupt-on-send.test.ts +57 -43
  102. package/src/__tests__/interrupt-turn-note.test.ts +5 -6
  103. package/src/__tests__/llm-catalog-parity.test.ts +2 -2
  104. package/src/__tests__/media-generate-image.test.ts +49 -1
  105. package/src/__tests__/message-provenance-contact-id.test.ts +69 -0
  106. package/src/__tests__/migration-export-to-gcs.test.ts +142 -0
  107. package/src/__tests__/mode-session-history.test.ts +245 -0
  108. package/src/__tests__/mode-session-recovery.test.ts +222 -0
  109. package/src/__tests__/mode-session-wire.test.ts +141 -0
  110. package/src/__tests__/notification-guardian-path.test.ts +7 -0
  111. package/src/__tests__/notification-telegram-adapter.test.ts +2 -2
  112. package/src/__tests__/notification-vellum-adapter.test.ts +34 -0
  113. package/src/__tests__/oauth-provider-serializer.test.ts +12 -0
  114. package/src/__tests__/oauth-providers-routes.test.ts +1 -0
  115. package/src/__tests__/outbound-slack-persistence.test.ts +2 -0
  116. package/src/__tests__/persistence-secret-redaction.test.ts +2 -0
  117. package/src/__tests__/plugin-api-provider.test.ts +5 -0
  118. package/src/__tests__/plugin-api-resolve-credential.test.ts +97 -0
  119. package/src/__tests__/plugin-import-boundary-guard.test.ts +9 -0
  120. package/src/__tests__/process-message-display-content.test.ts +56 -0
  121. package/src/__tests__/reaction-persistence.test.ts +5 -0
  122. package/src/__tests__/request-secret-standalone-channel-gate.test.ts +25 -9
  123. package/src/__tests__/runtime-events-sse-reconnect.test.ts +1 -1
  124. package/src/__tests__/scripted-turn-metadata-persistence.test.ts +73 -0
  125. package/src/__tests__/secret-ingress-http.test.ts +2 -0
  126. package/src/__tests__/secure-keys.test.ts +43 -1
  127. package/src/__tests__/send-endpoint-busy.test.ts +163 -1
  128. package/src/__tests__/server-history-render.test.ts +91 -1
  129. package/src/__tests__/settings-routes.test.ts +50 -1
  130. package/src/__tests__/skill-script-runner-sandbox.test.ts +32 -0
  131. package/src/__tests__/subagent-fork-notifications.test.ts +1 -2
  132. package/src/__tests__/subagent-manager-notify.test.ts +1 -2
  133. package/src/__tests__/subagent-notify-parent.test.ts +146 -39
  134. package/src/__tests__/subagent-notify-stale-rebuild.test.ts +2 -4
  135. package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +2 -0
  136. package/src/__tests__/thread-backfill.test.ts +53 -0
  137. package/src/__tests__/tool-preview-lifecycle.test.ts +2 -0
  138. package/src/__tests__/tool-result-metadata-plumbing.test.ts +3 -1
  139. package/src/__tests__/tool-start-timestamp.test.ts +2 -0
  140. package/src/__tests__/twilio-routes.test.ts +1 -0
  141. package/src/__tests__/web-search-backend-failure.test.ts +1 -1
  142. package/src/activation/progress-store.test.ts +12 -7
  143. package/src/agent/loop.ts +99 -41
  144. package/src/api/events/assistant-turn-start.ts +3 -0
  145. package/src/api/events/generation-handoff.ts +2 -0
  146. package/src/api/events/message-complete.ts +2 -0
  147. package/src/api/events/notification-conversation-created.ts +14 -16
  148. package/src/api/events/notification-intent.ts +4 -2
  149. package/src/api/events/tool-result.ts +85 -0
  150. package/src/api/events/tool-use-start.ts +3 -0
  151. package/src/api/events/user-message-echo.ts +3 -0
  152. package/src/api/index.ts +31 -6
  153. package/src/api/mode-session.test.ts +131 -0
  154. package/src/api/mode-session.ts +148 -0
  155. package/src/api/remember-tool.ts +15 -0
  156. package/src/api/responses/conversation-message.ts +6 -0
  157. package/src/approvals/AGENTS.md +4 -2
  158. package/src/browser/__tests__/operations.test.ts +29 -0
  159. package/src/browser/operation-meta.ts +2 -1
  160. package/src/browser/operations.ts +29 -0
  161. package/src/browser/virtual-desktop-target.ts +2 -8
  162. package/src/bundler/bundle-signer.ts +1 -3
  163. package/src/calls/__tests__/barge-in-guard.test.ts +3 -0
  164. package/src/calls/__tests__/call-controller.test.ts +94 -0
  165. package/src/calls/__tests__/call-funnel-telemetry.test.ts +204 -0
  166. package/src/calls/__tests__/voice-control-protocol.test.ts +15 -0
  167. package/src/calls/__tests__/voice-escalation-judge.test.ts +209 -0
  168. package/src/calls/__tests__/voice-leg-coordinator.test.ts +168 -0
  169. package/src/{live-voice/__tests__/live-voice-metrics.test.ts → calls/__tests__/voice-metrics.test.ts} +32 -38
  170. package/src/calls/__tests__/voice-session-bridge.test.ts +576 -122
  171. package/src/calls/barge-in-guard.ts +5 -0
  172. package/src/calls/call-controller.ts +196 -1
  173. package/src/calls/call-domain.ts +4 -0
  174. package/src/calls/call-store.ts +87 -2
  175. package/src/calls/guardian-action-sweep.ts +22 -8
  176. package/src/calls/media-stream-audio-transcode.ts +1 -3
  177. package/src/calls/typesafe-noul.ts +156 -0
  178. package/src/calls/voice-control-protocol.ts +6 -0
  179. package/src/calls/voice-escalation-judge.ts +145 -0
  180. package/src/calls/voice-escalation-target.ts +10 -0
  181. package/src/calls/voice-leg-coordinator.ts +121 -3
  182. package/src/{live-voice/live-voice-metrics.ts → calls/voice-metrics.ts} +94 -87
  183. package/src/calls/voice-session-bridge.ts +256 -112
  184. package/src/calls/voice-triage-escalate.ts +7 -0
  185. package/src/cli/__tests__/roadmap-cli.test.ts +4 -41
  186. package/src/cli/commands/__tests__/notifications.test.ts +35 -10
  187. package/src/cli/commands/browser.help.ts +4 -5
  188. package/src/cli/commands/browser.ts +0 -1
  189. package/src/cli/commands/clients.help.ts +46 -0
  190. package/src/cli/commands/clients.ts +116 -0
  191. package/src/cli/commands/credentials.help.ts +6 -1
  192. package/src/cli/commands/notifications.help.ts +16 -2
  193. package/src/cli/commands/notifications.ts +30 -8
  194. package/src/cli/commands/roadmap.help.ts +18 -21
  195. package/src/cli/commands/roadmap.ts +4 -21
  196. package/src/cli/lib/__tests__/list-installed-plugins.test.ts +5 -2
  197. package/src/cli/lib/bundled-marketplace.json +84 -34
  198. package/src/cli/lib/plugin-marketplace.ts +8 -0
  199. package/src/cli/lib/publish-plugin.ts +1 -2
  200. package/src/config/__tests__/default-provider.test.ts +3 -1
  201. package/src/config/__tests__/memory-retrospective-schema.test.ts +14 -1
  202. package/src/config/bundled-skills/computer-use/SKILL.md +1 -1
  203. package/src/config/bundled-skills/computer-use/TOOLS.json +89 -1
  204. package/src/config/bundled-skills/image-studio/SKILL.md +17 -2
  205. package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +7 -11
  206. package/src/config/bundled-skills/sequences/TOOLS.json +1 -5
  207. package/src/config/call-site-defaults.ts +9 -0
  208. package/src/config/feature-flag-registry.json +16 -0
  209. package/src/config/profile-text-generation.test.ts +2 -2
  210. package/src/config/schemas/__tests__/tools.test.ts +31 -0
  211. package/src/config/schemas/call-site-catalog.ts +14 -0
  212. package/src/config/schemas/channels.ts +0 -18
  213. package/src/config/schemas/llm.ts +3 -1
  214. package/src/config/schemas/mcp.ts +2 -1
  215. package/src/config/schemas/memory-retrospective.ts +11 -2
  216. package/src/config/schemas/services.ts +15 -6
  217. package/src/config/schemas/tools.ts +10 -0
  218. package/src/config/session-groups-gate.ts +21 -0
  219. package/src/config/skills.ts +2 -6
  220. package/src/conversations/__tests__/message-consolidation.test.ts +149 -0
  221. package/src/conversations/message-consolidation.ts +102 -5
  222. package/src/credential-execution/ces-connect.test.ts +69 -0
  223. package/src/credential-execution/ces-connect.ts +115 -0
  224. package/src/credential-execution/ces-runtime.ts +55 -93
  225. package/src/credential-execution/executable-discovery.ts +1 -11
  226. package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +10 -1
  227. package/src/daemon/__tests__/inference-profile-notification.test.ts +2 -0
  228. package/src/daemon/browser-mode-session.test.ts +402 -0
  229. package/src/daemon/browser-mode-session.ts +186 -0
  230. package/src/daemon/computer-use-mode-session.test.ts +210 -0
  231. package/src/daemon/computer-use-mode-session.ts +105 -0
  232. package/src/daemon/config-watcher.ts +10 -0
  233. package/src/daemon/conversation-agent-loop-handlers.ts +58 -10
  234. package/src/daemon/conversation-agent-loop.ts +219 -25
  235. package/src/daemon/conversation-evictor.ts +5 -24
  236. package/src/daemon/conversation-history.ts +10 -0
  237. package/src/daemon/conversation-interrupt-repair.ts +4 -43
  238. package/src/daemon/conversation-interrupt.ts +7 -11
  239. package/src/daemon/conversation-messaging.ts +58 -5
  240. package/src/daemon/conversation-mode-session.test.ts +1667 -0
  241. package/src/daemon/conversation-mode-session.ts +1008 -0
  242. package/src/daemon/conversation-process.ts +5 -0
  243. package/src/daemon/conversation-runtime-assembly.ts +10 -20
  244. package/src/daemon/conversation-skill-tools.ts +1 -0
  245. package/src/daemon/conversation-store.ts +4 -4
  246. package/src/daemon/conversation-surface-snapshots.ts +135 -0
  247. package/src/daemon/conversation-surfaces.ts +113 -5
  248. package/src/daemon/conversation-tool-setup.ts +8 -3
  249. package/src/daemon/conversation.ts +143 -41
  250. package/src/daemon/handlers/config-model.ts +7 -5
  251. package/src/daemon/handlers/shared.ts +55 -16
  252. package/src/daemon/host-cu-proxy.ts +51 -22
  253. package/src/daemon/lifecycle.ts +30 -13
  254. package/src/daemon/message-protocol.ts +0 -1
  255. package/src/daemon/message-provenance.ts +28 -4
  256. package/src/daemon/mode-session-startup-recovery.ts +23 -0
  257. package/src/daemon/mode-session-tracking.ts +54 -0
  258. package/src/daemon/process-message.ts +13 -4
  259. package/src/daemon/virtual-desktop-context.ts +21 -0
  260. package/src/desktop/__tests__/fake-desktop.ts +1 -0
  261. package/src/desktop/desktop-accessibility-script.ts +88 -0
  262. package/src/desktop/desktop-accessibility.test.ts +160 -0
  263. package/src/desktop/desktop-accessibility.ts +169 -0
  264. package/src/desktop/desktop-automation-lease.test.ts +76 -68
  265. package/src/desktop/desktop-automation-lease.ts +41 -50
  266. package/src/desktop/desktop-browser-endpoint.ts +4 -0
  267. package/src/desktop/desktop-computer-use-routing.test.ts +330 -0
  268. package/src/desktop/desktop-computer-use.test.ts +449 -0
  269. package/src/desktop/desktop-computer-use.ts +502 -0
  270. package/src/desktop/desktop-dependencies.ts +47 -293
  271. package/src/desktop/desktop-display.ts +2 -2
  272. package/src/desktop/desktop-panel-config.test.ts +13 -3
  273. package/src/desktop/desktop-panel-config.ts +109 -15
  274. package/src/desktop/desktop-session-manager.test.ts +69 -21
  275. package/src/desktop/desktop-session-manager.ts +76 -22
  276. package/src/desktop/desktop-stream-bridge.test.ts +11 -122
  277. package/src/desktop/desktop-stream-bridge.ts +6 -8
  278. package/src/desktop/desktop-window-theme.ts +5 -25
  279. package/src/desktop/virtual-desktop-feature.ts +52 -2
  280. package/src/desktop/virtual-desktop-platform.test.ts +5 -9
  281. package/src/documents/document-store.ts +23 -0
  282. package/src/heartbeat/__tests__/heartbeat-service.test.ts +22 -26
  283. package/src/heartbeat/heartbeat-service.ts +5 -33
  284. package/src/ipc/__tests__/browser-ipc.test.ts +215 -6
  285. package/src/ipc/assistant-server.ts +2 -0
  286. package/src/ipc/routes/__tests__/channel-reply-ipc-routes.test.ts +111 -0
  287. package/src/ipc/routes/channel-reply-ipc-routes.ts +73 -0
  288. package/src/live-voice/__tests__/continuation-judge.test.ts +151 -0
  289. package/src/live-voice/__tests__/live-activity-reporter.test.ts +154 -15
  290. package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +743 -31
  291. package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +191 -10
  292. package/src/live-voice/__tests__/live-voice-session-groups-gate.test.ts +87 -0
  293. package/src/live-voice/__tests__/live-voice-session-manager.test.ts +5 -1
  294. package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +41 -0
  295. package/src/live-voice/__tests__/live-voice-sight-frame.test.ts +1017 -9
  296. package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +224 -1
  297. package/src/live-voice/__tests__/live-voice-tts-session.test.ts +101 -0
  298. package/src/live-voice/__tests__/live-voice-tts.test.ts +44 -0
  299. package/src/live-voice/__tests__/live-voice-vad.test.ts +939 -75
  300. package/src/live-voice/__tests__/microphone-idle-gate.test.ts +87 -0
  301. package/src/live-voice/__tests__/session-controls.test.ts +23 -0
  302. package/src/live-voice/__tests__/task-outcomes.test.ts +117 -0
  303. package/src/live-voice/__tests__/voice-input-diagnostics.test.ts +117 -0
  304. package/src/live-voice/camera-mode-session.test.ts +265 -0
  305. package/src/live-voice/camera-mode-session.ts +277 -0
  306. package/src/live-voice/continuation-judge.ts +123 -0
  307. package/src/live-voice/live-activity-reporter.ts +115 -6
  308. package/src/live-voice/live-voice-manager.ts +36 -1
  309. package/src/live-voice/live-voice-photo.ts +88 -7
  310. package/src/live-voice/live-voice-session-manager.ts +15 -0
  311. package/src/live-voice/live-voice-session.ts +1443 -747
  312. package/src/live-voice/live-voice-tts.ts +22 -4
  313. package/src/live-voice/microphone-idle-gate.ts +56 -0
  314. package/src/live-voice/protocol.ts +171 -4
  315. package/src/live-voice/session-controls.ts +11 -3
  316. package/src/live-voice/task-outcomes.ts +131 -0
  317. package/src/live-voice/voice-input-diagnostics.ts +117 -0
  318. package/src/mcp/__tests__/manager-tool-caps.test.ts +63 -18
  319. package/src/mcp/__tests__/tool-caps.test.ts +28 -1
  320. package/src/mcp/manager.ts +3 -0
  321. package/src/mcp/tool-caps.ts +12 -4
  322. package/src/media/__tests__/image-models.test.ts +58 -0
  323. package/src/media/image-credentials.ts +5 -0
  324. package/src/media/image-models.ts +52 -0
  325. package/src/media/image-service.ts +8 -0
  326. package/src/media/openrouter-image-service.test.ts +133 -0
  327. package/src/media/openrouter-image-service.ts +153 -0
  328. package/src/media/types.ts +1 -1
  329. package/src/messaging/providers/__tests__/transport-dispatch.test.ts +65 -11
  330. package/src/messaging/providers/discord/send.ts +4 -17
  331. package/src/messaging/providers/discord/transport.ts +2 -2
  332. package/src/messaging/providers/outlook/client.ts +16 -22
  333. package/src/messaging/providers/telegram-bot/send.test.ts +236 -1
  334. package/src/messaging/providers/telegram-bot/send.ts +10 -7
  335. package/src/messaging/providers/telegram-bot/transport.ts +3 -10
  336. package/src/messaging/providers/whatsapp/send.test.ts +169 -4
  337. package/src/monitoring/__tests__/control.test.ts +17 -5
  338. package/src/monitoring/control.ts +15 -5
  339. package/src/notifications/AGENTS.md +1 -1
  340. package/src/notifications/README.md +83 -99
  341. package/src/notifications/__tests__/broadcaster.test.ts +1 -1
  342. package/src/notifications/__tests__/channel-allowlist.test.ts +39 -0
  343. package/src/notifications/__tests__/copy-composer.test.ts +1 -1
  344. package/src/notifications/__tests__/decision-engine.test.ts +79 -2
  345. package/src/notifications/__tests__/deterministic-checks.test.ts +10 -10
  346. package/src/notifications/__tests__/home-feed-side-effect.test.ts +93 -4
  347. package/src/notifications/__tests__/notification-utils.test.ts +18 -0
  348. package/src/notifications/adapters/macos.ts +37 -46
  349. package/src/notifications/broadcaster.ts +5 -6
  350. package/src/notifications/channel-allowlist.ts +44 -0
  351. package/src/notifications/copy-composer.ts +0 -23
  352. package/src/notifications/decision-engine.ts +24 -14
  353. package/src/notifications/deterministic-checks.ts +1 -1
  354. package/src/notifications/emit-signal.ts +123 -37
  355. package/src/notifications/home-feed-side-effect.ts +9 -7
  356. package/src/notifications/notification-utils.ts +19 -0
  357. package/src/notifications/signal.ts +4 -19
  358. package/src/notifications/types.ts +15 -0
  359. package/src/oauth/connection-resolver.ts +1 -2
  360. package/src/oauth/oauth-store.ts +1 -3
  361. package/src/oauth/provider-serializer.ts +7 -0
  362. package/src/oauth/seed-providers.ts +30 -0
  363. package/src/onboarding/onboarding-events-store.ts +99 -12
  364. package/src/persistence/__tests__/client-connection-events-store.test.ts +322 -0
  365. package/src/persistence/client-connection-events-store.ts +377 -0
  366. package/src/persistence/conversation-crud.ts +55 -6
  367. package/src/persistence/conversation-mode-sessions.test.ts +424 -0
  368. package/src/persistence/conversation-mode-sessions.ts +460 -0
  369. package/src/persistence/conversation-queries.ts +6 -19
  370. package/src/persistence/llm-usage-store.ts +3 -9
  371. package/src/persistence/message-metadata.ts +25 -0
  372. package/src/persistence/migrations/381-create-conversation-mode-sessions.ts +41 -0
  373. package/src/persistence/migrations/382-create-client-connection-events.ts +40 -0
  374. package/src/persistence/migrations/__tests__/382-create-client-connection-events.test.ts +56 -0
  375. package/src/persistence/schema/conversations.ts +67 -0
  376. package/src/persistence/schema/infrastructure.ts +29 -0
  377. package/src/persistence/steps.ts +17 -0
  378. package/src/plugin-api/__tests__/plugin-name-env.test.ts +249 -0
  379. package/src/plugin-api/effective-context-window.ts +33 -0
  380. package/src/plugin-api/index.ts +24 -5
  381. package/src/plugin-api/plugin-name-env.ts +125 -0
  382. package/src/plugin-api/resolve-credential.ts +79 -20
  383. package/src/plugin-api/vision-support.test.ts +22 -0
  384. package/src/plugin-api/vision-support.ts +17 -5
  385. package/src/plugins/__tests__/active-plugin-env.test.ts +136 -0
  386. package/src/plugins/active-plugin-env.ts +51 -0
  387. package/src/plugins/defaults/index.ts +24 -0
  388. package/src/plugins/defaults/memory/__tests__/archive-removal-framing.test.ts +64 -0
  389. package/src/plugins/defaults/memory/__tests__/worker-control.test.ts +19 -6
  390. package/src/plugins/defaults/memory/context-search/agent-protocol.ts +9 -12
  391. package/src/plugins/defaults/memory/context-search/agent-runner.ts +78 -22
  392. package/src/plugins/defaults/memory/context-search/format.ts +51 -7
  393. package/src/plugins/defaults/memory/context-search/limits.ts +10 -12
  394. package/src/plugins/defaults/memory/context-search/recall-input.ts +38 -0
  395. package/src/plugins/defaults/memory/context-search/search.ts +8 -14
  396. package/src/plugins/defaults/memory/context-search/sources/conversations.ts +1 -0
  397. package/src/plugins/defaults/memory/context-search/types.ts +1 -11
  398. package/src/plugins/defaults/memory/graph/__tests__/handle-remember-v2.test.ts +11 -4
  399. package/src/plugins/defaults/memory/graph/tool-handlers.ts +12 -21
  400. package/src/plugins/defaults/memory/graph/tools.ts +35 -64
  401. package/src/plugins/defaults/memory/src/memory-eval-routes.ts +2 -3
  402. package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +5 -0
  403. package/src/plugins/defaults/memory/src/memory-item-routes.ts +2 -1
  404. package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +2 -2
  405. package/src/plugins/defaults/memory/tools.test.ts +124 -9
  406. package/src/plugins/defaults/memory/tools.ts +38 -11
  407. package/src/plugins/defaults/memory/v3/__tests__/pool-log-store.test.ts +165 -27
  408. package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +626 -27
  409. package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +46 -27
  410. package/src/plugins/defaults/memory/v3/orchestrate.ts +78 -38
  411. package/src/plugins/defaults/memory/v3/pool-log-store.ts +65 -79
  412. package/src/plugins/defaults/memory/v3/pool-select.test.ts +14 -5
  413. package/src/plugins/defaults/memory/v3/pool-select.ts +683 -44
  414. package/src/plugins/defaults/memory/worker-control.ts +15 -5
  415. package/src/plugins/defaults/task-progress/__tests__/registration.test.ts +70 -0
  416. package/src/plugins/defaults/task-progress/__tests__/task-progress-context.test.ts +380 -0
  417. package/src/plugins/defaults/task-progress/hooks/post-compact.ts +19 -0
  418. package/src/plugins/defaults/task-progress/hooks/user-prompt-submit.ts +24 -0
  419. package/src/plugins/defaults/task-progress/package.json +14 -0
  420. package/src/plugins/defaults/task-progress/src/apply-task-progress-context.ts +93 -0
  421. package/src/plugins/defaults/task-progress/src/format-task-progress-context.ts +67 -0
  422. package/src/providers/__tests__/context-overflow-error.test.ts +10 -0
  423. package/src/providers/__tests__/search-provider-catalog.test.ts +18 -0
  424. package/src/providers/anthropic/client.ts +10 -1
  425. package/src/providers/anthropic-gateway-shared.ts +6 -0
  426. package/src/providers/fetch-provider-catalog.ts +11 -0
  427. package/src/providers/gemini/client.ts +25 -8
  428. package/src/providers/inference/adapter-factory.ts +1 -1
  429. package/src/providers/jev/client.test.ts +10 -0
  430. package/src/providers/jev/client.ts +20 -4
  431. package/src/providers/model-catalog.ts +4 -3
  432. package/src/providers/openai/__tests__/chat-completions-extra-body.test.ts +94 -1
  433. package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +106 -1
  434. package/src/providers/openai/__tests__/chat-template-flatten.test.ts +24 -0
  435. package/src/providers/openai/__tests__/google-thought-signature.test.ts +1 -2
  436. package/src/providers/openai/chat-completions-provider.ts +100 -29
  437. package/src/providers/openai/google-thought-signature.ts +0 -6
  438. package/src/providers/openai/responses-provider.ts +23 -1
  439. package/src/providers/ratelimit.ts +1 -3
  440. package/src/providers/search-provider-catalog.ts +24 -0
  441. package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +24 -0
  442. package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +7 -1
  443. package/src/providers/speech-to-text/__tests__/vellum-managed-realtime.test.ts +50 -1
  444. package/src/providers/speech-to-text/deepgram-flux-frames.ts +11 -0
  445. package/src/providers/speech-to-text/vellum-managed-flux-realtime.ts +13 -0
  446. package/src/providers/types.ts +5 -0
  447. package/src/routes/control.ts +7 -5
  448. package/src/runtime/AGENTS.md +4 -2
  449. package/src/runtime/actor-trust-resolver.ts +3 -3
  450. package/src/runtime/agent-wake.ts +1 -1
  451. package/src/runtime/approval-message-composer.ts +3 -69
  452. package/src/runtime/assistant-event-hub.ts +73 -75
  453. package/src/runtime/assistant-event-publish-options.ts +7 -0
  454. package/src/runtime/assistant-event-targeting.ts +79 -0
  455. package/src/runtime/assistant-stream-state.ts +13 -100
  456. package/src/runtime/background-job-runner.ts +5 -5
  457. package/src/runtime/channel-retry-sweep.ts +1 -0
  458. package/src/runtime/guardian-action-message-composer.ts +8 -161
  459. package/src/runtime/guardian-reply-router.ts +28 -32
  460. package/src/runtime/http-types.ts +6 -0
  461. package/src/runtime/message-composer-types.ts +3 -61
  462. package/src/runtime/migrations/__tests__/vbundle-import-parity.test.ts +67 -1
  463. package/src/runtime/migrations/vbundle-builder.ts +24 -1
  464. package/src/runtime/migrations/vbundle-import-analyzer.ts +13 -4
  465. package/src/runtime/migrations/vbundle-import-policy.ts +10 -0
  466. package/src/runtime/migrations/vbundle-importer.ts +4 -1
  467. package/src/runtime/migrations/vbundle-streaming-importer.ts +4 -1
  468. package/src/runtime/migrations/vbundle-validator.ts +1 -0
  469. package/src/runtime/pending-interactions-mode-session.test.ts +62 -0
  470. package/src/runtime/pending-interactions.ts +29 -0
  471. package/src/runtime/routes/__tests__/client-history-routes.test.ts +104 -0
  472. package/src/runtime/routes/__tests__/image-generation-routes.test.ts +97 -0
  473. package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +1 -1
  474. package/src/runtime/routes/__tests__/roadmap-routes.test.ts +29 -6
  475. package/src/runtime/routes/browser-context.ts +31 -11
  476. package/src/runtime/routes/browser-routes.ts +43 -4
  477. package/src/runtime/routes/btw-routes.ts +9 -4
  478. package/src/runtime/routes/client-routes.ts +108 -0
  479. package/src/runtime/routes/conversation-management-routes.ts +4 -1
  480. package/src/runtime/routes/conversation-routes.ts +113 -27
  481. package/src/runtime/routes/credential-prompt-routes.ts +4 -3
  482. package/src/runtime/routes/credential-routes.ts +1 -2
  483. package/src/runtime/routes/desktop-setup-routes.test.ts +9 -18
  484. package/src/runtime/routes/desktop-setup-routes.ts +4 -6
  485. package/src/runtime/routes/documents-routes.ts +51 -0
  486. package/src/runtime/routes/email-routes.ts +3 -7
  487. package/src/runtime/routes/events-routes.ts +75 -54
  488. package/src/runtime/routes/identity-routes.ts +1 -5
  489. package/src/runtime/routes/image-generation-routes.ts +8 -14
  490. package/src/runtime/routes/inbound-message-handler.ts +35 -38
  491. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.test.ts +168 -0
  492. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.ts +96 -0
  493. package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +38 -9
  494. package/src/runtime/routes/inbound-stages/background-dispatch.ts +2 -5
  495. package/src/runtime/routes/inbound-stages/reaction-intercept.ts +2 -0
  496. package/src/runtime/routes/llm-context-normalization.ts +1 -7
  497. package/src/runtime/routes/migration-routes.ts +73 -2
  498. package/src/runtime/routes/notification-routes.ts +10 -0
  499. package/src/runtime/routes/oauth-providers.ts +7 -0
  500. package/src/runtime/routes/plugins-routes.ts +6 -0
  501. package/src/runtime/routes/roadmap-routes.ts +47 -8
  502. package/src/runtime/routes/secret-routes.ts +1 -1
  503. package/src/runtime/routes/settings-routes.ts +10 -4
  504. package/src/runtime/routes/watcher-routes.ts +1 -3
  505. package/src/runtime/trust-class.test.ts +7 -6
  506. package/src/runtime/trust-class.ts +9 -17
  507. package/src/schedule/__tests__/worker-control.test.ts +36 -5
  508. package/src/schedule/worker-control.ts +15 -5
  509. package/src/security/secure-keys.ts +53 -103
  510. package/src/skills/clawhub.ts +3 -8
  511. package/src/skills/managed-store.ts +1 -5
  512. package/src/stt/types.ts +4 -0
  513. package/src/subagent/notify.ts +54 -34
  514. package/src/subagent/parent-notification.ts +6 -0
  515. package/src/telemetry/__tests__/phone-call-funnel.test.ts +72 -0
  516. package/src/telemetry/phone-call-funnel.ts +144 -0
  517. package/src/tools/__tests__/tool-schema-root-combinator-guard.test.ts +81 -0
  518. package/src/tools/ask-question/ask-question-tool.ts +6 -4
  519. package/src/tools/browser/browser-execution.ts +22 -59
  520. package/src/tools/browser/cdp-client/__tests__/cdp-dom-helpers.test.ts +561 -39
  521. package/src/tools/browser/cdp-client/cdp-dom-helpers.ts +435 -39
  522. package/src/tools/browser/cdp-client/cdp-inspect-client.ts +1 -8
  523. package/src/tools/client-os.ts +15 -0
  524. package/src/tools/computer-use/skill-proxy-bridge.ts +1 -1
  525. package/src/tools/computer-use/target.ts +42 -0
  526. package/src/tools/document/document-tool.ts +13 -18
  527. package/src/tools/execution-target.ts +14 -16
  528. package/src/tools/host-terminal/host-shell.ts +4 -1
  529. package/src/tools/network/__tests__/web-fetch-tinyfish.test.ts +250 -0
  530. package/src/tools/network/__tests__/web-search.test.ts +203 -0
  531. package/src/tools/network/web-fetch.ts +274 -7
  532. package/src/tools/network/web-search.ts +468 -6
  533. package/src/tools/permission-checker.ts +3 -3
  534. package/src/tools/policy-context.ts +3 -1
  535. package/src/tools/skills/load.ts +2 -1
  536. package/src/tools/skills/sandbox-runner.ts +11 -2
  537. package/src/tools/skills/skill-script-runner.ts +7 -0
  538. package/src/tools/skills/skill-tool-factory.ts +15 -1
  539. package/src/tools/terminal/__tests__/safe-env.test.ts +11 -0
  540. package/src/tools/terminal/safe-env.ts +4 -0
  541. package/src/tools/terminal/shell.ts +2 -0
  542. package/src/tools/tool-approval-handler.ts +5 -1
  543. package/src/tools/tool-types.ts +3 -3
  544. package/src/tools/types.ts +8 -2
  545. package/src/util/__tests__/worker-process-command.test.ts +5 -4
  546. package/src/util/abort-reasons.ts +18 -26
  547. package/src/util/canonicalize-identity.test.ts +21 -0
  548. package/src/util/canonicalize-identity.ts +16 -0
  549. package/src/util/errors.ts +13 -3
  550. package/src/util/provider-error-patterns.ts +3 -0
  551. package/src/util/worker-process.ts +77 -35
  552. package/src/workspace/migrations/157-add-desktop-files-launcher.ts +94 -0
  553. package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +4 -10
  554. package/src/workspace/migrations/__tests__/157-add-desktop-files-launcher.test.ts +113 -0
  555. package/src/workspace/migrations/registry.ts +2 -0
  556. package/src/__tests__/computer-use-window-schema.test.ts +0 -24
  557. package/src/api/events/heartbeat-alert.ts +0 -20
  558. package/src/daemon/message-types/web-activity.ts +0 -65
  559. package/src/desktop/desktop-dependencies.test.ts +0 -131
  560. package/src/tools/computer-use/definitions.ts +0 -607
@@ -148,6 +148,23 @@ export async function streamLiveVoiceTtsAudio(
148
148
  // a trailing odd byte is carried into the next chunk to keep frames aligned.
149
149
  const pcmAligner = createPcmChunkAligner(2);
150
150
 
151
+ // Send the first available samples immediately, then frame PCM in 100ms
152
+ // blocks. Provider read sizes must not determine websocket queue pressure.
153
+ const frameBytes = Math.max(2, Math.floor(sampleRate / 10) * 2);
154
+ let pendingPcm: Buffer = Buffer.alloc(0);
155
+ const emitPcm = (audio: Buffer, flush = false): void => {
156
+ pendingPcm =
157
+ pendingPcm.length > 0 ? Buffer.concat([pendingPcm, audio]) : audio;
158
+ while (
159
+ pendingPcm.length >= frameBytes ||
160
+ (pendingPcm.length > 0 && (chunks === 0 || flush))
161
+ ) {
162
+ const length = Math.min(frameBytes, pendingPcm.length);
163
+ emitAudioFrame(chunkContentType, pendingPcm.subarray(0, length));
164
+ pendingPcm = pendingPcm.subarray(length);
165
+ }
166
+ };
167
+
151
168
  try {
152
169
  const result = await synthesizeAndEmit({
153
170
  provider,
@@ -160,16 +177,17 @@ export async function streamLiveVoiceTtsAudio(
160
177
  signal: options.signal,
161
178
  onChunk: (chunk) => {
162
179
  if (canStreamChunks) {
163
- emitAudioFrame(
164
- chunk.contentType || chunkContentType,
165
- pcmAligner.align(chunk.audio),
166
- );
180
+ emitPcm(pcmAligner.align(chunk.audio));
167
181
  } else {
168
182
  bufferedAudio.push(chunk.audio);
169
183
  }
170
184
  },
171
185
  });
172
186
 
187
+ if (canStreamChunks && !result.stopped && !options.signal?.aborted) {
188
+ emitPcm(Buffer.alloc(0), true);
189
+ }
190
+
173
191
  // A dangling final byte is malformed provider output — drop it rather
174
192
  // than emit a torn sample.
175
193
  if (pcmAligner.carryLength() > 0) {
@@ -0,0 +1,56 @@
1
+ import { Buffer } from "node:buffer";
2
+
3
+ const IDLE_TIMEOUT_MS = 1_000;
4
+ const PRE_ROLL_MS = 200;
5
+
6
+ /** Gates PCM16 mono room audio while preserving the provider's audio clock. */
7
+ export class MicrophoneIdleGate {
8
+ private readonly tailBytes: number;
9
+ private readonly preRollBytes: number;
10
+ private quietBytes = 0;
11
+ private pending: Buffer = Buffer.alloc(0);
12
+
13
+ constructor(sampleRate: number) {
14
+ this.tailBytes = Math.round((sampleRate * IDLE_TIMEOUT_MS) / 1_000) * 2;
15
+ this.preRollBytes = Math.round((sampleRate * PRE_ROLL_MS) / 1_000) * 2;
16
+ }
17
+
18
+ get closed(): boolean {
19
+ return this.quietBytes >= this.tailBytes;
20
+ }
21
+
22
+ process(chunk: Buffer, hasSpeech: boolean): Buffer {
23
+ if (hasSpeech) {
24
+ this.quietBytes = 0;
25
+ const pending = this.pending;
26
+ this.pending = Buffer.alloc(0);
27
+ return pending.byteLength > 0 ? Buffer.concat([pending, chunk]) : chunk;
28
+ }
29
+
30
+ const passBytes = Math.min(
31
+ chunk.byteLength,
32
+ this.tailBytes - this.quietBytes,
33
+ );
34
+ this.quietBytes = Math.min(
35
+ this.tailBytes,
36
+ this.quietBytes + chunk.byteLength,
37
+ );
38
+ if (passBytes === chunk.byteLength) {
39
+ return chunk;
40
+ }
41
+
42
+ // Hold only the newest onset audio. Older samples become silence before
43
+ // submission, so reopening never replays an already-sent time interval.
44
+ const buffered = Buffer.concat([this.pending, chunk.subarray(passBytes)]);
45
+ const silenceBytes = Math.max(0, buffered.byteLength - this.preRollBytes);
46
+ this.pending = Buffer.from(buffered.subarray(silenceBytes));
47
+ const output = Buffer.alloc(passBytes + silenceBytes);
48
+ chunk.copy(output, 0, 0, passBytes);
49
+ return output;
50
+ }
51
+
52
+ reset(): void {
53
+ this.quietBytes = 0;
54
+ this.pending = Buffer.alloc(0);
55
+ }
56
+ }
@@ -1,3 +1,4 @@
1
+ import type { VoiceEscalationProfileSource } from "../calls/voice-escalation-target.js";
1
2
  import { type ClientOs, parseClientOs } from "../channels/types.js";
2
3
 
3
4
  const LIVE_VOICE_CLIENT_FRAME_TYPES = [
@@ -9,6 +10,8 @@ const LIVE_VOICE_CLIENT_FRAME_TYPES = [
9
10
  "update_config",
10
11
  "attach_image",
11
12
  "attach_frame",
13
+ "sight_start",
14
+ "sight_end",
12
15
  "sight_frame",
13
16
  "text",
14
17
  ] as const;
@@ -67,6 +70,20 @@ type LiveVoiceParseResult<T> =
67
70
  | { ok: true; frame: T }
68
71
  | { ok: false; error: LiveVoiceProtocolError };
69
72
 
73
+ type LiveVoiceParseFailure = Extract<
74
+ LiveVoiceParseResult<never>,
75
+ { ok: false }
76
+ >;
77
+
78
+ function liveVoiceParseFailure(
79
+ code: LiveVoiceProtocolErrorCode,
80
+ message: string,
81
+ field: string,
82
+ frameType: string,
83
+ ): LiveVoiceParseFailure {
84
+ return { ok: false, error: { code, message, field, frameType } };
85
+ }
86
+
70
87
  export interface LiveVoiceAudioConfig {
71
88
  readonly mimeType: "audio/pcm";
72
89
  readonly sampleRate: number;
@@ -363,6 +380,8 @@ export interface LiveVoiceClientAttachFrameFrame {
363
380
  export interface LiveVoiceClientSightFrameFrame {
364
381
  readonly type: "sight_frame";
365
382
  readonly attachmentId: string;
383
+ readonly cameraEpoch?: number;
384
+ readonly source?: LiveVoiceSightSource;
366
385
  /**
367
386
  * How long the client's half of the frame took, for the daemon's log. The
368
387
  * daemon adds its own half and the distance from the speech onset it
@@ -372,6 +391,19 @@ export interface LiveVoiceClientSightFrameFrame {
372
391
  readonly timing?: LiveVoiceSightFrameTiming;
373
392
  }
374
393
 
394
+ export type LiveVoiceSightSource = "live" | "ambient";
395
+
396
+ export interface LiveVoiceClientSightStartFrame {
397
+ readonly type: "sight_start";
398
+ readonly cameraEpoch: number;
399
+ readonly source?: LiveVoiceSightSource;
400
+ }
401
+
402
+ export interface LiveVoiceClientSightEndFrame {
403
+ readonly type: "sight_end";
404
+ readonly cameraEpoch: number;
405
+ }
406
+
375
407
  /**
376
408
  * The client leg of one kept frame, as durations between its own marks.
377
409
  *
@@ -451,6 +483,8 @@ export type LiveVoiceClientFrame =
451
483
  | LiveVoiceClientUpdateConfigFrame
452
484
  | LiveVoiceClientAttachImageFrame
453
485
  | LiveVoiceClientAttachFrameFrame
486
+ | LiveVoiceClientSightStartFrame
487
+ | LiveVoiceClientSightEndFrame
454
488
  | LiveVoiceClientSightFrameFrame
455
489
  | LiveVoiceClientTextTurnFrame;
456
490
 
@@ -481,6 +515,8 @@ export interface LiveVoiceReadyServerFrame extends LiveVoiceServerFrameBase {
481
515
  * every typed turn with an `unknown_type` error.
482
516
  */
483
517
  readonly textInput?: boolean;
518
+ /** Whether this session accepts camera lifecycle epochs. */
519
+ readonly sightSessions?: boolean;
484
520
  /**
485
521
  * Whether the session's speech-to-text leg is live. Absent means yes, which
486
522
  * is the only thing an older daemon can have meant: it rejects a session it
@@ -575,8 +611,9 @@ export interface LiveVoiceThinkingServerFrame extends LiveVoiceServerFrameBase {
575
611
  * the room) can otherwise only say "Thinking...". The label is composed here
576
612
  * rather than by each surface for the same reason phase wording is composed
577
613
  * once: the Live Activity has two independent drivers (this socket and an APNs
578
- * push the daemon dispatches), they must carry identical content, and the only
579
- * way to guarantee that is for both to be handed the same string.
614
+ * push the daemon dispatches), they must carry compatible content. Structured
615
+ * kinds let in-conversation surfaces use localized copy while system-level
616
+ * surfaces can suppress internal detail.
580
617
  *
581
618
  * An empty `label` means "no current activity", which is what a turn's end
582
619
  * sends. Emitted only on change, never per tool result.
@@ -585,6 +622,12 @@ export interface LiveVoiceActivityServerFrame extends LiveVoiceServerFrameBase {
585
622
  readonly type: "activity";
586
623
  readonly turnId: string;
587
624
  readonly label: string;
625
+ /** Structured reason for the activity, when a client needs custom display. */
626
+ readonly kind?: "escalation";
627
+ /** Selected inference profile for diagnostics, never default UI copy. */
628
+ readonly profile?: string;
629
+ /** Why the selected profile won for this leg. */
630
+ readonly profileSource?: VoiceEscalationProfileSource;
588
631
  /**
589
632
  * The confirmation this turn is blocked on, when the label describes a wait
590
633
  * rather than work in flight. Absent otherwise.
@@ -896,6 +939,10 @@ export function validateLiveVoiceClientFrame(
896
939
  return validateAttachImageFrame(value);
897
940
  case "attach_frame":
898
941
  return validateAttachFrameFrame(value);
942
+ case "sight_start":
943
+ return validateSightStartFrame(value);
944
+ case "sight_end":
945
+ return validateSightEndFrame(value);
899
946
  case "sight_frame":
900
947
  return validateSightFrameFrame(value);
901
948
  case "text":
@@ -1044,10 +1091,19 @@ function validateSightFrameFrame(
1044
1091
  );
1045
1092
  }
1046
1093
 
1094
+ const lifecycle = validateOptionalSightLifecycle(value, "sight_frame");
1095
+ if (!lifecycle.ok) {
1096
+ return lifecycle;
1097
+ }
1098
+
1047
1099
  if (!("timing" in value) || value.timing === undefined) {
1048
1100
  return {
1049
1101
  ok: true,
1050
- frame: { type: "sight_frame", attachmentId: value.attachmentId },
1102
+ frame: {
1103
+ type: "sight_frame",
1104
+ attachmentId: value.attachmentId,
1105
+ ...lifecycle.fields,
1106
+ },
1051
1107
  };
1052
1108
  }
1053
1109
 
@@ -1063,10 +1119,121 @@ function validateSightFrameFrame(
1063
1119
 
1064
1120
  return {
1065
1121
  ok: true,
1066
- frame: { type: "sight_frame", attachmentId: value.attachmentId, timing },
1122
+ frame: {
1123
+ type: "sight_frame",
1124
+ attachmentId: value.attachmentId,
1125
+ ...lifecycle.fields,
1126
+ timing,
1127
+ },
1128
+ };
1129
+ }
1130
+
1131
+ function validateSightStartFrame(
1132
+ value: Record<string, unknown>,
1133
+ ): LiveVoiceParseResult<LiveVoiceClientSightStartFrame> {
1134
+ const lifecycle = validateRequiredSightLifecycle(value, "sight_start");
1135
+ if (!lifecycle.ok) {
1136
+ return lifecycle;
1137
+ }
1138
+ return {
1139
+ ok: true,
1140
+ frame: { type: "sight_start", ...lifecycle.fields },
1141
+ };
1142
+ }
1143
+
1144
+ function validateSightEndFrame(
1145
+ value: Record<string, unknown>,
1146
+ ): LiveVoiceParseResult<LiveVoiceClientSightEndFrame> {
1147
+ if (!isPositiveSafeInteger(value.cameraEpoch)) {
1148
+ return protocolError(
1149
+ "invalid_field",
1150
+ "sight_end frame field cameraEpoch must be a positive safe integer",
1151
+ "cameraEpoch",
1152
+ "sight_end",
1153
+ );
1154
+ }
1155
+ return {
1156
+ ok: true,
1157
+ frame: { type: "sight_end", cameraEpoch: value.cameraEpoch },
1067
1158
  };
1068
1159
  }
1069
1160
 
1161
+ function validateRequiredSightLifecycle(
1162
+ value: Record<string, unknown>,
1163
+ frameType: "sight_start",
1164
+ ):
1165
+ | { ok: true; fields: { cameraEpoch: number; source?: LiveVoiceSightSource } }
1166
+ | LiveVoiceParseFailure {
1167
+ if (!isPositiveSafeInteger(value.cameraEpoch)) {
1168
+ return liveVoiceParseFailure(
1169
+ "invalid_field",
1170
+ `${frameType} frame field cameraEpoch must be a positive safe integer`,
1171
+ "cameraEpoch",
1172
+ frameType,
1173
+ );
1174
+ }
1175
+ if (value.source !== undefined && !isLiveVoiceSightSource(value.source)) {
1176
+ return liveVoiceParseFailure(
1177
+ "invalid_field",
1178
+ `${frameType} frame field source must be live or ambient`,
1179
+ "source",
1180
+ frameType,
1181
+ );
1182
+ }
1183
+ return {
1184
+ ok: true,
1185
+ fields: {
1186
+ cameraEpoch: value.cameraEpoch,
1187
+ ...(value.source ? { source: value.source } : {}),
1188
+ },
1189
+ };
1190
+ }
1191
+
1192
+ function validateOptionalSightLifecycle(
1193
+ value: Record<string, unknown>,
1194
+ frameType: "sight_frame",
1195
+ ):
1196
+ | {
1197
+ ok: true;
1198
+ fields: { cameraEpoch?: number; source?: LiveVoiceSightSource };
1199
+ }
1200
+ | LiveVoiceParseFailure {
1201
+ if (value.cameraEpoch === undefined && value.source === undefined) {
1202
+ return { ok: true, fields: {} };
1203
+ }
1204
+ if (!isPositiveSafeInteger(value.cameraEpoch)) {
1205
+ return liveVoiceParseFailure(
1206
+ "invalid_field",
1207
+ `${frameType} frame field cameraEpoch must be a positive safe integer when lifecycle fields are present`,
1208
+ "cameraEpoch",
1209
+ frameType,
1210
+ );
1211
+ }
1212
+ if (value.source !== undefined && !isLiveVoiceSightSource(value.source)) {
1213
+ return liveVoiceParseFailure(
1214
+ "invalid_field",
1215
+ `${frameType} frame field source must be live or ambient`,
1216
+ "source",
1217
+ frameType,
1218
+ );
1219
+ }
1220
+ return {
1221
+ ok: true,
1222
+ fields: {
1223
+ cameraEpoch: value.cameraEpoch,
1224
+ ...(value.source ? { source: value.source } : {}),
1225
+ },
1226
+ };
1227
+ }
1228
+
1229
+ function isPositiveSafeInteger(value: unknown): value is number {
1230
+ return Number.isSafeInteger(value) && typeof value === "number" && value > 0;
1231
+ }
1232
+
1233
+ function isLiveVoiceSightSource(value: unknown): value is LiveVoiceSightSource {
1234
+ return value === "live" || value === "ambient";
1235
+ }
1236
+
1070
1237
  /**
1071
1238
  * Parse a `sight_frame`'s `timing`. Null for anything off-shape: a timing is
1072
1239
  * for the log, so nothing about it is coerced, but a malformed one still
@@ -44,7 +44,7 @@ const CLIENT_CONTROL_LINES: Record<LiveVoiceSessionControl, string> = {
44
44
  const LOOK_FRAME_CONTROL_LINES: Partial<
45
45
  Record<LiveVoiceSessionControl, string>
46
46
  > = {
47
- look_screen: `- To look at their screen (for example "take a look at my screen", or "can you see it now?" after something on it changed), say in a few words that you are taking a look, then end your reply with ${LOOK_SCREEN_MARKER}. Use it even when their screen is already shared with you, to see it as it is now. You get a fresh view right after you finish speaking and answer from that, so do not describe the screen yet and do not ask them to say anything more.`,
47
+ look_screen: `- To look at their screen (for example "take a look at my screen", or "can you see it now?" after something on it changed), say in a few words that you are taking a look, then end your reply with ${LOOK_SCREEN_MARKER}. Use it even when their screen is already shared with you, to see it as it is now. You get a fresh view right after you finish speaking, so do not describe the screen yet or ask them to say anything more. This only obtains a view; showing them where to click still needs screen annotation after the view arrives.`,
48
48
  look_camera: `- To look through their camera (for example "look at this" or "can you see this?" while they hold something up), say in a few words that you are taking a look, then end your reply with ${LOOK_CAMERA_MARKER}. Use it even when the camera is already on, to see what it shows now. You get a fresh view right after you finish speaking and answer from that, so do not describe it yet and do not ask them to say anything more.`,
49
49
  };
50
50
 
@@ -61,6 +61,11 @@ export type LookSessionControl = Extract<
61
61
  "look_screen" | "look_camera"
62
62
  >;
63
63
 
64
+ export interface LookFollowUp {
65
+ action: LookSessionControl;
66
+ callerUtterance: string;
67
+ }
68
+
64
69
  export function isLookSessionControl(
65
70
  action: string,
66
71
  ): action is LookSessionControl {
@@ -78,9 +83,12 @@ export const LOOK_FOLLOW_UP_CONTENT = "(fresh view taken; answer from it now)";
78
83
  * Appended to the control prompt of the turn that answers a look: the reply
79
84
  * that asked for it only acknowledged, and the frame it asked for has landed.
80
85
  */
81
- export function lookFollowUpNote(action: LookSessionControl): string {
86
+ export function lookFollowUpNote({
87
+ action,
88
+ callerUtterance,
89
+ }: LookFollowUp): string {
82
90
  const what = action === "look_screen" ? "their screen" : "their camera";
83
- return `You just took a fresh look at ${what}, and the newest image in the conversation is what it shows right now. Answer what they wanted you to look at, out loud, in a few spoken sentences. If the image does not show what they meant, say briefly what you do see and ask. You have already said you were taking a look, so do not say it again and do not end with a look marker.`;
91
+ return `You just took a fresh look at ${what}, and the newest image in the conversation is what it shows right now. Resume the caller's request ${JSON.stringify(callerUtterance)} using that view and the latest conversation context. Complete any requested action, not just a spoken description.${action === "look_screen" ? " Showing or pointing out where to click needs the screen-annotation tools." : ""} If the request needs tools and this leg has none, escalate. Otherwise answer in a few spoken sentences. If the image does not show what they meant, say briefly what you do see and ask. You have already said you were taking a look, so do not say it again and do not end with a look marker.`;
84
92
  }
85
93
 
86
94
  // Always taught: narration is the session's own, so no client has to be able
@@ -0,0 +1,131 @@
1
+ import type { SubagentParentNotification } from "../subagent/parent-notification.js";
2
+
3
+ export interface VoiceTaskOutcome extends SubagentParentNotification {
4
+ source: "subagent" | "continuation";
5
+ }
6
+
7
+ interface Delivery {
8
+ outcome: VoiceTaskOutcome | null;
9
+ interrupted: boolean;
10
+ lastPlayedReply: string | null;
11
+ }
12
+
13
+ /** Session-local delivery receipts. Generated text and context exposure are not receipts. */
14
+ export class VoiceTaskOutcomes {
15
+ private readonly deliveries = new Map<string, Delivery>();
16
+ private playback: {
17
+ outcomes: readonly VoiceTaskOutcome[];
18
+ announcement: boolean;
19
+ reply: string;
20
+ untilMs: number;
21
+ } | null = null;
22
+
23
+ get hasPending(): boolean {
24
+ return this.next() !== undefined;
25
+ }
26
+
27
+ enqueue(outcome: VoiceTaskOutcome): void {
28
+ const previous = this.deliveries.get(outcome.taskId);
29
+ this.deliveries.set(outcome.taskId, {
30
+ outcome,
31
+ interrupted: previous?.interrupted ?? false,
32
+ lastPlayedReply: previous?.lastPlayedReply ?? null,
33
+ });
34
+ }
35
+
36
+ next(): VoiceTaskOutcome | undefined {
37
+ for (const delivery of this.deliveries.values()) {
38
+ if (delivery.outcome !== null) {
39
+ return delivery.outcome;
40
+ }
41
+ }
42
+ return undefined;
43
+ }
44
+
45
+ pendingOutcomes(): VoiceTaskOutcome[] {
46
+ return [...this.deliveries.values()].flatMap(({ outcome }) =>
47
+ outcome === null ? [] : [outcome],
48
+ );
49
+ }
50
+
51
+ deliveryContext(outcome: VoiceTaskOutcome): string {
52
+ const delivery = this.deliveries.get(outcome.taskId);
53
+ const status = delivery?.interrupted
54
+ ? "An earlier attempt to deliver this task's findings was interrupted before playback completed. Resume the useful unheard findings briefly."
55
+ : "This update is still pending delivery.";
56
+ const heard = delivery?.lastPlayedReply;
57
+ return `${status} Generated assistant text in conversation history is not evidence that the user heard it. ${heard ? `The latest reply with completed audio playback for this task was: ${JSON.stringify(heard)}. Use that receipt to avoid repeating findings actually heard.` : "There is no completed audio playback receipt for this task."}`;
58
+ }
59
+
60
+ finish(outcome: VoiceTaskOutcome, playbackUntilMs: number, reply = ""): void {
61
+ this.playback = {
62
+ outcomes: [outcome],
63
+ announcement: true,
64
+ reply,
65
+ untilMs: playbackUntilMs,
66
+ };
67
+ }
68
+
69
+ finishContextReply(
70
+ outcomes: readonly VoiceTaskOutcome[],
71
+ playbackUntilMs: number,
72
+ reply: string,
73
+ ): void {
74
+ if (outcomes.length > 0) {
75
+ this.playback = {
76
+ outcomes,
77
+ announcement: false,
78
+ reply,
79
+ untilMs: playbackUntilMs,
80
+ };
81
+ }
82
+ }
83
+
84
+ acknowledgePlayback(now: number): void {
85
+ const playback = this.playback;
86
+ if (playback === null || now < playback.untilMs) {
87
+ return;
88
+ }
89
+ for (const outcome of playback.outcomes) {
90
+ const delivery = this.deliveries.get(outcome.taskId);
91
+ if (delivery === undefined) {
92
+ continue;
93
+ }
94
+ if (playback.reply.length > 0) {
95
+ delivery.lastPlayedReply = playback.reply;
96
+ }
97
+ if (playback.announcement) {
98
+ delivery.interrupted = false;
99
+ }
100
+ if (playback.announcement && delivery.outcome === outcome) {
101
+ delivery.outcome = null;
102
+ }
103
+ }
104
+ this.playback = null;
105
+ }
106
+
107
+ interrupt(outcome: VoiceTaskOutcome): void {
108
+ const delivery = this.deliveries.get(outcome.taskId);
109
+ if (delivery?.outcome != null) {
110
+ delivery.interrupted = true;
111
+ }
112
+ }
113
+
114
+ interruptPlayback(now: number): void {
115
+ this.acknowledgePlayback(now);
116
+ if (this.playback?.announcement) {
117
+ for (const outcome of this.playback.outcomes) {
118
+ this.interrupt(outcome);
119
+ }
120
+ }
121
+ this.playback = null;
122
+ }
123
+
124
+ drain(now: number): VoiceTaskOutcome[] {
125
+ this.acknowledgePlayback(now);
126
+ const outcomes = this.pendingOutcomes();
127
+ this.deliveries.clear();
128
+ this.playback = null;
129
+ return outcomes;
130
+ }
131
+ }
@@ -0,0 +1,117 @@
1
+ export interface VoiceInputSample {
2
+ receivedAtMs: number;
3
+ chunkMs: number;
4
+ meanAmplitude: number;
5
+ baseThreshold: number;
6
+ roomNoiseFloor: number | null;
7
+ echoEnergy: number;
8
+ echoOnsetLapsed: boolean;
9
+ echoWindowMs: number;
10
+ echoGuardCarryover: boolean;
11
+ echoReferenceMs: number;
12
+ echoProbeMs: number;
13
+ echoCorrelation: number | null;
14
+ playbackRemainingMs: number;
15
+ playbackEchoPossible: boolean;
16
+ speechMs: number;
17
+ silenceMs: number;
18
+ echoMs: number;
19
+ }
20
+
21
+ const SUMMARY_INTERVAL_MS = 5_000;
22
+ const TRACE_WINDOW_MS = 2_000;
23
+ const MAX_TRACE_SAMPLES = 40;
24
+
25
+ /** Bounded signal measurements only. Never retains PCM or transcript text. */
26
+ export class VoiceInputDiagnostics {
27
+ private recent: VoiceInputSample[] = [];
28
+ private lastArrivalAtMs: number | null = null;
29
+ private lastSttSubmissionAtMs: number | null = null;
30
+ private window = this.emptyWindow();
31
+
32
+ recordSttSubmission(atMs: number, audioMs: number): void {
33
+ this.window.sttSubmittedChunks += 1;
34
+ this.window.sttSubmittedAudioMs += audioMs;
35
+ if (this.lastSttSubmissionAtMs !== null) {
36
+ this.window.sttMaxSubmissionGapMs = Math.max(
37
+ this.window.sttMaxSubmissionGapMs,
38
+ atMs - this.lastSttSubmissionAtMs,
39
+ );
40
+ }
41
+ this.lastSttSubmissionAtMs = atMs;
42
+ }
43
+
44
+ observe(sample: VoiceInputSample): Record<string, number> | null {
45
+ this.recent.push(sample);
46
+ while (
47
+ this.recent.length > MAX_TRACE_SAMPLES ||
48
+ (this.recent[0] &&
49
+ sample.receivedAtMs - this.recent[0].receivedAtMs > TRACE_WINDOW_MS)
50
+ ) {
51
+ this.recent.shift();
52
+ }
53
+ const window = this.window;
54
+ window.firstAtMs ??= sample.receivedAtMs;
55
+ window.lastAtMs = sample.receivedAtMs;
56
+ window.chunks += 1;
57
+ window.audioMs += sample.chunkMs;
58
+ window.weightedAmplitude += sample.meanAmplitude * sample.chunkMs;
59
+ window.minAmplitude = Math.min(window.minAmplitude, sample.meanAmplitude);
60
+ window.maxAmplitude = Math.max(window.maxAmplitude, sample.meanAmplitude);
61
+ window.zeroAudioMs += sample.meanAmplitude === 0 ? sample.chunkMs : 0;
62
+ window.speechMs += sample.speechMs;
63
+ window.silenceMs += sample.silenceMs;
64
+ window.echoMs += sample.echoMs;
65
+ if (this.lastArrivalAtMs !== null) {
66
+ window.maxArrivalGapMs = Math.max(
67
+ window.maxArrivalGapMs,
68
+ sample.receivedAtMs - this.lastArrivalAtMs,
69
+ );
70
+ }
71
+ this.lastArrivalAtMs = sample.receivedAtMs;
72
+ return sample.receivedAtMs - window.firstAtMs >= SUMMARY_INTERVAL_MS
73
+ ? this.flush()
74
+ : null;
75
+ }
76
+
77
+ snapshot(nowMs: number): VoiceInputSample[] {
78
+ return this.recent
79
+ .filter((entry) => nowMs - entry.receivedAtMs <= TRACE_WINDOW_MS)
80
+ .map((entry) => ({ ...entry }));
81
+ }
82
+
83
+ flush(): Record<string, number> | null {
84
+ const window = this.window;
85
+ if (window.chunks === 0) {
86
+ return null;
87
+ }
88
+ this.window = this.emptyWindow();
89
+ const { weightedAmplitude, firstAtMs, ...summary } = window;
90
+ return {
91
+ ...summary,
92
+ firstAtMs: firstAtMs!,
93
+ meanAmplitude:
94
+ window.audioMs > 0 ? weightedAmplitude / window.audioMs : 0,
95
+ };
96
+ }
97
+
98
+ private emptyWindow() {
99
+ return {
100
+ firstAtMs: null as number | null,
101
+ lastAtMs: 0,
102
+ chunks: 0,
103
+ audioMs: 0,
104
+ weightedAmplitude: 0,
105
+ minAmplitude: Infinity,
106
+ maxAmplitude: 0,
107
+ zeroAudioMs: 0,
108
+ speechMs: 0,
109
+ silenceMs: 0,
110
+ echoMs: 0,
111
+ maxArrivalGapMs: 0,
112
+ sttSubmittedChunks: 0,
113
+ sttSubmittedAudioMs: 0,
114
+ sttMaxSubmissionGapMs: 0,
115
+ };
116
+ }
117
+ }