@vellumai/assistant 0.11.2 → 0.11.3-staging.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (720) hide show
  1. package/ARCHITECTURE.md +58 -3
  2. package/docs/trusted-contact-access.md +9 -5
  3. package/docs/vellum-doctor.md +195 -0
  4. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/channels.ts +39 -0
  5. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
  6. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
  7. package/node_modules/@vellumai/ces-client/src/index.ts +1 -0
  8. package/node_modules/@vellumai/ces-client/src/rpc-client.ts +19 -2
  9. package/node_modules/@vellumai/environments/package.json +2 -1
  10. package/node_modules/@vellumai/environments/src/__tests__/cloud-assistant-hub-url.test.ts +38 -0
  11. package/node_modules/@vellumai/environments/src/__tests__/install-layout.test.ts +98 -0
  12. package/node_modules/@vellumai/environments/src/__tests__/package-boundary.test.ts +5 -5
  13. package/node_modules/@vellumai/environments/src/index.ts +17 -6
  14. package/node_modules/@vellumai/environments/src/install-layout.ts +49 -0
  15. package/node_modules/@vellumai/environments/src/seeds.ts +29 -0
  16. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/channels.ts +39 -0
  17. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
  18. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
  19. package/node_modules/@vellumai/gateway-client/src/index.ts +4 -0
  20. package/node_modules/@vellumai/gateway-client/src/verification-session-contract.ts +91 -6
  21. package/node_modules/@vellumai/ipc-server-utils/src/endpoint.test.ts +36 -0
  22. package/node_modules/@vellumai/ipc-server-utils/src/endpoint.ts +142 -0
  23. package/node_modules/@vellumai/ipc-server-utils/src/index.ts +2 -0
  24. package/node_modules/@vellumai/ipc-server-utils/src/listen-options.ts +3 -0
  25. package/node_modules/@vellumai/ipc-server-utils/src/socket-watchdog.test.ts +15 -1
  26. package/node_modules/@vellumai/ipc-server-utils/src/socket-watchdog.ts +17 -2
  27. package/node_modules/@vellumai/service-contracts/src/channels.ts +39 -0
  28. package/node_modules/@vellumai/service-contracts/src/ingress.ts +10 -0
  29. package/node_modules/@vellumai/service-contracts/src/remote-web-pairing.ts +6 -0
  30. package/node_modules/@vellumai/slack-text/src/index.ts +16 -2
  31. package/openapi.yaml +589 -30
  32. package/package.json +1 -1
  33. package/src/__tests__/agent-image-optimize.test.ts +4 -4
  34. package/src/__tests__/agent-loop-mutable-latest-user-message.test.ts +113 -66
  35. package/src/__tests__/agent-loop-override-profile.test.ts +38 -48
  36. package/src/__tests__/agent-loop-subagent-billing-headers.test.ts +166 -0
  37. package/src/__tests__/agent-wake-disk-pressure-callsite.test.ts +1 -0
  38. package/src/__tests__/agent-wake-override-profile.test.ts +1 -0
  39. package/src/__tests__/answered-question-persistence.test.ts +240 -0
  40. package/src/__tests__/anthropic-provider.test.ts +165 -30
  41. package/src/__tests__/app-control-flow.test.ts +9 -8
  42. package/src/__tests__/assistant-event-hub.test.ts +48 -0
  43. package/src/__tests__/attachments-store-heic-normalize.test.ts +16 -12
  44. package/src/__tests__/attachments-store.test.ts +130 -68
  45. package/src/__tests__/attachments.test.ts +36 -36
  46. package/src/__tests__/background-workers-disk-pressure.test.ts +1 -1
  47. package/src/__tests__/call-start-guardian-guard.test.ts +18 -4
  48. package/src/__tests__/canned-reply-release.test.ts +86 -0
  49. package/src/__tests__/channel-guardian.test.ts +59 -29
  50. package/src/__tests__/channel-readiness-telegram-remote.test.ts +154 -0
  51. package/src/__tests__/channel-setup-panel-ack.test.ts +6 -5
  52. package/src/__tests__/client-os-metadata-persistence.test.ts +103 -4
  53. package/src/__tests__/compactor-image-manifest-trust.test.ts +1 -1
  54. package/src/__tests__/consult-deadline.test.ts +65 -0
  55. package/src/__tests__/conversation-agent-loop.test.ts +438 -2
  56. package/src/__tests__/conversation-disk-view-integration.test.ts +5 -1
  57. package/src/__tests__/conversation-disk-view.test.ts +4 -4
  58. package/src/__tests__/conversation-error.test.ts +22 -3
  59. package/src/__tests__/conversation-fork-crud.test.ts +5 -1
  60. package/src/__tests__/conversation-fork-referential.test.ts +352 -0
  61. package/src/__tests__/conversation-fork-retrospective.test.ts +5 -1
  62. package/src/__tests__/conversation-group-tools.test.ts +32 -2
  63. package/src/__tests__/conversation-inference-profile-route.test.ts +27 -1
  64. package/src/__tests__/conversation-initial-prompt.test.ts +51 -22
  65. package/src/__tests__/conversation-key-store-bootstrap-cleanup.test.ts +119 -0
  66. package/src/__tests__/conversation-key-store-origin.test.ts +105 -0
  67. package/src/__tests__/conversation-list-recency-ordering.test.ts +104 -0
  68. package/src/__tests__/conversation-origin-at-creation.test.ts +153 -0
  69. package/src/__tests__/conversation-origin-channel-filter.test.ts +95 -0
  70. package/src/__tests__/conversation-placement-promotion.test.ts +172 -0
  71. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +123 -0
  72. package/src/__tests__/conversation-queue.test.ts +433 -1
  73. package/src/__tests__/conversation-retry-route.test.ts +3 -1
  74. package/src/__tests__/conversation-routes-hidden-queue.test.ts +64 -1
  75. package/src/__tests__/conversation-store-ephemeral.test.ts +4 -0
  76. package/src/__tests__/conversation-store.test.ts +26 -10
  77. package/src/__tests__/conversation-summarize-route.test.ts +1 -1
  78. package/src/__tests__/conversation-surfaces-action-delivery.test.ts +15 -6
  79. package/src/__tests__/conversation-surfaces-activation-emit.test.ts +6 -5
  80. package/src/__tests__/conversation-surfaces-app-control.test.ts +12 -11
  81. package/src/__tests__/conversation-surfaces-app-open.test.ts +11 -5
  82. package/src/__tests__/conversation-surfaces-data-persist.test.ts +5 -4
  83. package/src/__tests__/conversation-surfaces-history-restored-completion.test.ts +967 -0
  84. package/src/__tests__/conversation-surfaces-persisted-info.test.ts +312 -0
  85. package/src/__tests__/conversation-surfaces-queued-emit.test.ts +6 -5
  86. package/src/__tests__/conversation-surfaces-standalone-payloads.test.ts +12 -8
  87. package/src/__tests__/conversation-surfaces-standalone.test.ts +21 -11
  88. package/src/__tests__/conversation-surfaces-state-update.test.ts +7 -11
  89. package/src/__tests__/conversation-surfaces-table-action.test.ts +7 -6
  90. package/src/__tests__/conversation-surfaces-task-progress.test.ts +11 -6
  91. package/src/__tests__/conversation-tool-setup-app-refresh.test.ts +9 -6
  92. package/src/__tests__/conversation-tool-setup-attribution.test.ts +70 -7
  93. package/src/__tests__/conversation-wait-for-idle.test.ts +6 -11
  94. package/src/__tests__/credential-prompt-route.test.ts +57 -7
  95. package/src/__tests__/credential-security-invariants.test.ts +1 -0
  96. package/src/__tests__/cu-unified-flow.test.ts +89 -28
  97. package/src/__tests__/custom-group-listing-visibility.test.ts +1 -1
  98. package/src/__tests__/db-schedule-syntax-migration.test.ts +4 -1
  99. package/src/__tests__/discord-access-request-privacy.test.ts +366 -0
  100. package/src/__tests__/discord-requester-notice-privacy.test.ts +87 -0
  101. package/src/__tests__/disk-pressure-guard.test.ts +55 -55
  102. package/src/__tests__/disk-pressure-routes.test.ts +10 -10
  103. package/src/__tests__/disk-pressure-tools.test.ts +2 -2
  104. package/src/__tests__/disk-usage.test.ts +35 -20
  105. package/src/__tests__/document-sync-tags.test.ts +379 -0
  106. package/src/__tests__/document-update-default-surface.test.ts +28 -0
  107. package/src/__tests__/dynamic-skill-background-guard.test.ts +2 -2
  108. package/src/__tests__/emit-signal-routing-intent.test.ts +3 -4
  109. package/src/__tests__/ensure-conversation-exists.test.ts +8 -5
  110. package/src/__tests__/events-dev-bypass-actor.test.ts +183 -1
  111. package/src/__tests__/evict-conversations-for-reload.test.ts +90 -0
  112. package/src/__tests__/file-list-tool.test.ts +12 -10
  113. package/src/__tests__/file-ops-service.test.ts +151 -44
  114. package/src/__tests__/file-read-container-boundary.test.ts +137 -0
  115. package/src/__tests__/file-read-tool.test.ts +21 -9
  116. package/src/__tests__/file-url-path-guard.test.ts +83 -0
  117. package/src/__tests__/filesystem-tools.test.ts +82 -66
  118. package/src/__tests__/guardian-outbound-http.test.ts +8 -8
  119. package/src/__tests__/helpers/gateway-verification-sessions-stub.ts +14 -1
  120. package/src/__tests__/helpers/hub-clients.ts +42 -0
  121. package/src/__tests__/helpers/mock-conversation.ts +46 -0
  122. package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +61 -57
  123. package/src/__tests__/history-repair.test.ts +112 -12
  124. package/src/__tests__/host-bash-proxy.test.ts +10 -5
  125. package/src/__tests__/host-bash-routes.test.ts +67 -0
  126. package/src/__tests__/host-browser-proxy.test.ts +2 -2
  127. package/src/__tests__/host-cu-proxy.test.ts +3 -7
  128. package/src/__tests__/host-file-proxy.test.ts +3 -7
  129. package/src/__tests__/host-file-read-tool.test.ts +5 -1
  130. package/src/__tests__/host-transfer-proxy-targeted.test.ts +6 -4
  131. package/src/__tests__/http-conversation-lineage.test.ts +2 -5
  132. package/src/__tests__/http-user-message-parity.test.ts +44 -0
  133. package/src/__tests__/identity-routes.test.ts +13 -13
  134. package/src/__tests__/image-conversion.test.ts +29 -26
  135. package/src/__tests__/image-source-path-reinject.test.ts +2 -2
  136. package/src/__tests__/inference-profile-session-handler.test.ts +95 -0
  137. package/src/__tests__/inference-profile-session-ipc.test.ts +25 -0
  138. package/src/__tests__/introduction-card-resolver.test.ts +6 -1
  139. package/src/__tests__/list-messages-attachments.test.ts +56 -32
  140. package/src/__tests__/list-messages-background-tool-completion.test.ts +4 -4
  141. package/src/__tests__/list-messages-client-message-id.test.ts +2 -2
  142. package/src/__tests__/list-messages-hidden-metadata.test.ts +16 -16
  143. package/src/__tests__/list-messages-page-latest.test.ts +61 -61
  144. package/src/__tests__/list-messages-provider-error.test.ts +6 -6
  145. package/src/__tests__/list-messages-queued.test.ts +49 -6
  146. package/src/__tests__/list-messages-system-card.test.ts +4 -4
  147. package/src/__tests__/list-messages-tool-merge.test.ts +9 -9
  148. package/src/__tests__/llm-resolver-override-or-default.test.ts +141 -0
  149. package/src/__tests__/llm-usage-store.test.ts +55 -0
  150. package/src/__tests__/manual-token-reconciliation.test.ts +45 -0
  151. package/src/__tests__/mcp-tool-annotations-risk.test.ts +133 -0
  152. package/src/__tests__/media-stream-server-integration.test.ts +36 -0
  153. package/src/__tests__/migration-export-http.test.ts +1 -1
  154. package/src/__tests__/migration-import-commit-http.test.ts +1 -1
  155. package/src/__tests__/migration-import-preflight-http.test.ts +1 -1
  156. package/src/__tests__/migration-validate-http.test.ts +1 -1
  157. package/src/__tests__/mtime-cache.test.ts +65 -3
  158. package/src/__tests__/notification-platform-adapter.test.ts +104 -0
  159. package/src/__tests__/notification-schedule-notify-dedup.test.ts +1 -1
  160. package/src/__tests__/oauth-provider-profiles.test.ts +4 -2
  161. package/src/__tests__/oauth-provider-seed-logos.test.ts +65 -0
  162. package/src/__tests__/oauth-provider-serializer.test.ts +17 -0
  163. package/src/__tests__/oauth-providers-routes.test.ts +1 -0
  164. package/src/__tests__/openai-responses-prompt-cache.test.ts +5 -2
  165. package/src/__tests__/path-policy.test.ts +164 -1
  166. package/src/__tests__/persist-media-references.test.ts +2 -2
  167. package/src/__tests__/persist-unsendable-image-downscale.test.ts +5 -5
  168. package/src/__tests__/persist-unsendable-image.test.ts +23 -23
  169. package/src/__tests__/persona-resolver.test.ts +154 -0
  170. package/src/__tests__/plugin-api-webhook-url.test.ts +170 -0
  171. package/src/__tests__/plugin-import-boundary-guard.test.ts +6 -0
  172. package/src/__tests__/processing-flag-persist-failure.test.ts +15 -10
  173. package/src/__tests__/profile-availability-incomplete.test.ts +113 -0
  174. package/src/__tests__/profiler-routes.test.ts +13 -13
  175. package/src/__tests__/prompt-cache-cross-turn-stability.test.ts +593 -0
  176. package/src/__tests__/proxy-approval-callback.test.ts +3 -3
  177. package/src/__tests__/queued-message-cancel-and-steer.test.ts +189 -0
  178. package/src/__tests__/queued-message-steer-actor-scoping.test.ts +164 -0
  179. package/src/__tests__/resolve-trust-class.test.ts +20 -0
  180. package/src/__tests__/run-conversation-turn-persistence.test.ts +1 -1
  181. package/src/__tests__/run-due-schedules.test.ts +174 -0
  182. package/src/__tests__/runtime-attachment-metadata.test.ts +15 -7
  183. package/src/__tests__/scaffold-managed-skill-tool.test.ts +166 -0
  184. package/src/__tests__/schedule-routes.test.ts +730 -8
  185. package/src/__tests__/schedule-store.test.ts +540 -0
  186. package/src/__tests__/schedule-tools.test.ts +103 -0
  187. package/src/__tests__/scheduler-wake.test.ts +5 -0
  188. package/src/__tests__/secret-routes-platform-proxy.test.ts +29 -1
  189. package/src/__tests__/secure-keys.test.ts +91 -0
  190. package/src/__tests__/server-history-render.test.ts +97 -7
  191. package/src/__tests__/size-guard.test.ts +8 -8
  192. package/src/__tests__/skill-docs-sync-guard.test.ts +3 -2
  193. package/src/__tests__/skill-tool-factory.test.ts +247 -0
  194. package/src/__tests__/skills-install-staging.test.ts +15 -6
  195. package/src/__tests__/slack-edit-ordering-characterization.test.ts +103 -0
  196. package/src/__tests__/slack-inbound-verification.test.ts +4 -1
  197. package/src/__tests__/slack-mention-binding-invariance.test.ts +103 -0
  198. package/src/__tests__/slack-mention-provider-leak-audit.test.ts +135 -0
  199. package/src/__tests__/starter-task-flow.test.ts +5 -4
  200. package/src/__tests__/steer-on-enqueue-question.test.ts +3 -2
  201. package/src/__tests__/stt-language-catalog-parity.test.ts +38 -0
  202. package/src/__tests__/subagent-allowlist-validation.test.ts +26 -4
  203. package/src/__tests__/subagent-cron-run-attribution.test.ts +301 -0
  204. package/src/__tests__/subagent-disposal.test.ts +10 -0
  205. package/src/__tests__/subagent-fork-notifications.test.ts +10 -0
  206. package/src/__tests__/subagent-fork-prompt-role.test.ts +85 -11
  207. package/src/__tests__/subagent-fork-spawn.test.ts +15 -7
  208. package/src/__tests__/subagent-manager-notify.test.ts +148 -0
  209. package/src/__tests__/subagent-notify-parent.test.ts +1 -1
  210. package/src/__tests__/subagent-persistence.test.ts +141 -0
  211. package/src/__tests__/subagent-role-registry.test.ts +172 -63
  212. package/src/__tests__/subagent-role-resolution.test.ts +88 -0
  213. package/src/__tests__/subagent-spawn-and-await.test.ts +48 -0
  214. package/src/__tests__/subagent-spawn-tool-fork.test.ts +111 -10
  215. package/src/__tests__/subagent-terminal-message.test.ts +88 -1
  216. package/src/__tests__/subagent-tool-gate-mode.test.ts +225 -10
  217. package/src/__tests__/subagent-tools.test.ts +1901 -108
  218. package/src/__tests__/surface-completion-compaction-boundary.test.ts +124 -0
  219. package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +298 -0
  220. package/src/__tests__/system-prompt.test.ts +82 -7
  221. package/src/__tests__/tool-grant-request-escalation.test.ts +40 -1
  222. package/src/__tests__/tool-result-spool.test.ts +74 -18
  223. package/src/__tests__/tool-side-effects-documents.test.ts +174 -0
  224. package/src/__tests__/tool-side-effects-hook-failures.test.ts +129 -0
  225. package/src/__tests__/tools-audio-read.test.ts +10 -10
  226. package/src/__tests__/tools-get-route.test.ts +85 -12
  227. package/src/__tests__/tools-list-cli.test.ts +171 -0
  228. package/src/__tests__/top-level-renderer.test.ts +54 -1
  229. package/src/__tests__/trusted-contact-inline-approval-integration.test.ts +68 -1
  230. package/src/__tests__/trusted-contact-multichannel.test.ts +4 -1
  231. package/src/__tests__/turn-tail-chain.test.ts +135 -0
  232. package/src/__tests__/ui-choice-copy-surfaces.test.ts +6 -4
  233. package/src/__tests__/ui-visual-surface.test.ts +5 -4
  234. package/src/__tests__/ui-work-result-surface.test.ts +5 -4
  235. package/src/__tests__/unified-turn-context-visible-app.test.ts +7 -8
  236. package/src/__tests__/usage-routes.test.ts +62 -0
  237. package/src/__tests__/verification-outbound-delivery.test.ts +176 -0
  238. package/src/__tests__/visible-app-context.test.ts +20 -4
  239. package/src/__tests__/web-search-history.test.ts +96 -0
  240. package/src/__tests__/workspace-migration-139-clear-renamed-cost-profile-label.test.ts +6 -6
  241. package/src/__tests__/workspace-migration-140-repair-seed-pinned-memory-v3-live.test.ts +330 -0
  242. package/src/acp/failure-error.ts +3 -5
  243. package/src/agent/attachments.ts +26 -24
  244. package/src/agent/history-repair/history-repair.ts +27 -26
  245. package/src/agent/image-optimize.ts +6 -6
  246. package/src/agent/loop.ts +40 -22
  247. package/src/agent/message-types.ts +3 -3
  248. package/src/api/constants/document-tools.ts +55 -0
  249. package/src/api/events/question-answered.ts +53 -0
  250. package/src/api/events/tool-result.ts +7 -0
  251. package/src/api/index.ts +13 -0
  252. package/src/api/responses/conversation-message.ts +11 -0
  253. package/src/api/surfaces-oauth-connect.test.ts +96 -0
  254. package/src/api/surfaces.ts +36 -8
  255. package/src/approvals/guardian-channel-delivery.ts +83 -0
  256. package/src/approvals/guardian-expiry-notifier.ts +13 -11
  257. package/src/approvals/guardian-request-resolvers.ts +222 -135
  258. package/src/calls/__tests__/voice-session-bridge.test.ts +50 -0
  259. package/src/calls/media-stream-server.ts +9 -1
  260. package/src/calls/voice-session-bridge.ts +38 -0
  261. package/src/channels/__tests__/gateway-verification-sessions.test.ts +3 -3
  262. package/src/channels/config.ts +7 -2
  263. package/src/channels/gateway-verification-sessions.ts +7 -2
  264. package/src/channels/types.ts +7 -1
  265. package/src/cli/commands/__tests__/cache.test.ts +7 -4
  266. package/src/cli/commands/__tests__/cli-test-harness.ts +82 -0
  267. package/src/cli/commands/__tests__/conversations-wake.test.ts +51 -6
  268. package/src/cli/commands/__tests__/inference-models.test.ts +14 -0
  269. package/src/cli/commands/__tests__/inference-profiles.test.ts +44 -1
  270. package/src/cli/commands/__tests__/inference-providers.test.ts +7 -0
  271. package/src/cli/commands/__tests__/plugins.test.ts +692 -0
  272. package/src/cli/commands/__tests__/schedules.test.ts +136 -37
  273. package/src/cli/commands/conversations.ts +7 -3
  274. package/src/cli/commands/credentials.help.ts +7 -0
  275. package/src/cli/commands/inference-profiles.ts +9 -3
  276. package/src/cli/commands/inference.help.ts +16 -2
  277. package/src/cli/commands/mcp.help.ts +4 -2
  278. package/src/cli/commands/memory/memory-retrospective.ts +18 -0
  279. package/src/cli/commands/notifications.ts +3 -6
  280. package/src/cli/commands/oauth/connect-surface-guidance.test.ts +16 -0
  281. package/src/cli/commands/oauth/connect-surface-guidance.ts +25 -7
  282. package/src/cli/commands/oauth/connect.test.ts +26 -1
  283. package/src/cli/commands/oauth/connect.ts +4 -1
  284. package/src/cli/commands/oauth/index.help.ts +3 -1
  285. package/src/cli/commands/oauth/providers.ts +3 -0
  286. package/src/cli/commands/plugins.help.ts +25 -2
  287. package/src/cli/commands/plugins.ts +289 -10
  288. package/src/cli/commands/schedules.help.ts +14 -12
  289. package/src/cli/commands/schedules.ts +19 -5
  290. package/src/cli/commands/tools.help.ts +4 -2
  291. package/src/cli/commands/tools.ts +38 -0
  292. package/src/cli/lib/__tests__/inspect-plugin.test.ts +1 -0
  293. package/src/cli/lib/__tests__/install-from-github.test.ts +79 -0
  294. package/src/cli/lib/__tests__/install-from-platform.test.ts +47 -1
  295. package/src/cli/lib/__tests__/plugin-surfaces.test.ts +86 -7
  296. package/src/cli/lib/__tests__/upgrade-plugin.test.ts +122 -0
  297. package/src/cli/lib/install-from-github.ts +61 -0
  298. package/src/cli/lib/install-from-platform.ts +13 -2
  299. package/src/cli/lib/plugin-surfaces.ts +124 -4
  300. package/src/cli/lib/upgrade-plugin.ts +18 -0
  301. package/src/cli.ts +1 -1
  302. package/src/config/__tests__/deployment-context-defaults.test.ts +73 -0
  303. package/src/config/__tests__/memory-retrospective-schema.test.ts +40 -0
  304. package/src/config/__tests__/plugin-updates-schema.test.ts +43 -0
  305. package/src/config/__tests__/profile-tool-support.test.ts +131 -0
  306. package/src/config/bundled-skills/AGENTS.md +1 -1
  307. package/src/config/bundled-skills/app-builder/SKILL.md +1 -1
  308. package/src/config/bundled-skills/app-builder/references/RESPONSIVE.md +3 -3
  309. package/src/config/bundled-skills/schedule/SKILL.md +5 -1
  310. package/src/config/bundled-skills/schedule/TOOLS.json +2 -2
  311. package/src/config/bundled-skills/settings/TOOLS.json +2 -2
  312. package/src/config/bundled-skills/settings/tools/voice-config-update.ts +1 -0
  313. package/src/config/bundled-skills/subagent/SKILL.md +53 -22
  314. package/src/config/bundled-skills/subagent/TOOLS.json +14 -13
  315. package/src/config/bundled-skills/visualize/SKILL.md +1 -1
  316. package/src/config/default-profile-catalog.ts +6 -2
  317. package/src/config/feature-flag-registry.json +25 -17
  318. package/src/config/llm-resolver.ts +45 -15
  319. package/src/config/loader.ts +70 -1
  320. package/src/config/profile-tool-support.ts +110 -0
  321. package/src/config/sanitize-for-transfer.ts +11 -0
  322. package/src/config/schema.ts +4 -0
  323. package/src/config/schemas/__tests__/stt.test.ts +20 -5
  324. package/src/config/schemas/channels.ts +13 -0
  325. package/src/config/schemas/memory-retrospective.ts +25 -0
  326. package/src/config/schemas/plugin-updates.ts +55 -0
  327. package/src/config/schemas/services.ts +8 -4
  328. package/src/config/schemas/stt.ts +17 -5
  329. package/src/config/schemas/timeouts.ts +1 -1
  330. package/src/contacts/__tests__/guardian-delivery-reader.test.ts +16 -0
  331. package/src/contacts/guardian-delivery-reader.ts +29 -2
  332. package/src/context/compactor.ts +4 -4
  333. package/src/context/post-turn-tool-result-truncation.ts +32 -16
  334. package/src/context/strip-injections.ts +42 -0
  335. package/src/context/tool-result-spool.ts +32 -18
  336. package/src/credential-execution/ces-runtime.ts +3 -0
  337. package/src/credential-execution/client.ts +22 -1
  338. package/src/credential-execution/process-manager.test.ts +99 -0
  339. package/src/credential-execution/process-manager.ts +62 -16
  340. package/src/daemon/__tests__/abort-null-controller.test.ts +141 -5
  341. package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +8 -12
  342. package/src/daemon/__tests__/install-assistant-command.test.ts +234 -0
  343. package/src/daemon/__tests__/turn-tail-assistant-reply-notify.test.ts +2 -4
  344. package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +20 -29
  345. package/src/daemon/channel-ui-capability.ts +4 -4
  346. package/src/daemon/conversation-agent-loop-handlers.ts +117 -61
  347. package/src/daemon/conversation-agent-loop.ts +318 -148
  348. package/src/daemon/conversation-attachments.ts +1 -1
  349. package/src/daemon/conversation-error.ts +8 -1
  350. package/src/daemon/conversation-initial-prompt.ts +16 -38
  351. package/src/daemon/conversation-lifecycle.ts +154 -32
  352. package/src/daemon/conversation-messaging.ts +106 -43
  353. package/src/daemon/conversation-process.ts +131 -33
  354. package/src/daemon/conversation-queue-manager.ts +10 -0
  355. package/src/daemon/conversation-runtime-assembly.ts +9 -4
  356. package/src/daemon/conversation-store.ts +52 -3
  357. package/src/daemon/conversation-surface-state.ts +9 -0
  358. package/src/daemon/conversation-surfaces.ts +403 -210
  359. package/src/daemon/conversation-tool-setup.ts +56 -7
  360. package/src/daemon/conversation-turn-finalize.ts +134 -89
  361. package/src/daemon/conversation.ts +111 -27
  362. package/src/daemon/disk-pressure-guard-lifecycle.ts +2 -2
  363. package/src/daemon/disk-pressure-guard.ts +3 -3
  364. package/src/daemon/doordash-steps.ts +5 -5
  365. package/src/daemon/external-plugins-bootstrap.ts +7 -1
  366. package/src/daemon/handlers/config-channels.ts +48 -5
  367. package/src/daemon/handlers/conversation-history.ts +1 -1
  368. package/src/daemon/handlers/conversations.ts +62 -17
  369. package/src/daemon/handlers/shared.ts +12 -0
  370. package/src/daemon/handlers/skills.ts +11 -1
  371. package/src/daemon/identity-helpers.ts +38 -10
  372. package/src/daemon/install-assistant-command.ts +322 -0
  373. package/src/daemon/lifecycle.ts +14 -3
  374. package/src/daemon/message-types/conversations.ts +6 -7
  375. package/src/daemon/message-types/sync.ts +1 -0
  376. package/src/daemon/persist-media-references.ts +23 -14
  377. package/src/daemon/process-message.ts +6 -6
  378. package/src/daemon/tool-setup-types.ts +19 -122
  379. package/src/daemon/tool-side-effects.ts +17 -94
  380. package/src/daemon/trust-context-types.ts +39 -0
  381. package/src/daemon/trust-context.ts +14 -6
  382. package/src/daemon/turn-tail-chain.ts +73 -0
  383. package/src/daemon/web-search-history.ts +47 -33
  384. package/src/heartbeat/heartbeat-service.ts +6 -6
  385. package/src/ipc/__tests__/socket-path.test.ts +21 -0
  386. package/src/ipc/assistant-server.ts +17 -10
  387. package/src/ipc/cli-client.ts +10 -1
  388. package/src/ipc/routes/__tests__/documents-sync-ipc-routes.test.ts +56 -0
  389. package/src/ipc/routes/documents-sync-ipc-routes.ts +42 -0
  390. package/src/ipc/socket-cleanup.ts +9 -7
  391. package/src/ipc/socket-path.ts +4 -96
  392. package/src/live-voice/__tests__/live-voice-archive.test.ts +16 -16
  393. package/src/live-voice/__tests__/live-voice-attach-image.test.ts +209 -0
  394. package/src/live-voice/__tests__/live-voice-connection.test.ts +20 -0
  395. package/src/live-voice/__tests__/live-voice-integration.test.ts +7 -5
  396. package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +265 -0
  397. package/src/live-voice/__tests__/live-voice-stt.test.ts +6 -1
  398. package/src/live-voice/__tests__/live-voice-vad.test.ts +35 -15
  399. package/src/live-voice/__tests__/protocol.test.ts +52 -0
  400. package/src/live-voice/__tests__/runtime-websocket-shell.test.ts +3 -0
  401. package/src/live-voice/live-voice-archive.ts +13 -13
  402. package/src/live-voice/live-voice-connection.ts +7 -1
  403. package/src/live-voice/live-voice-photo.ts +169 -0
  404. package/src/live-voice/live-voice-session.ts +108 -4
  405. package/src/live-voice/protocol.ts +89 -1
  406. package/src/mcp/client.ts +10 -0
  407. package/src/messaging/providers/__tests__/callback-routing.test.ts +5 -1
  408. package/src/messaging/providers/__tests__/transport-dispatch.test.ts +66 -4
  409. package/src/messaging/providers/callback-routing.ts +1 -0
  410. package/src/messaging/providers/discord/api.ts +312 -0
  411. package/src/messaging/providers/discord/dm-delivery.test.ts +204 -0
  412. package/src/messaging/providers/discord/render.test.ts +204 -0
  413. package/src/messaging/providers/discord/render.ts +195 -0
  414. package/src/messaging/providers/discord/send.test.ts +179 -0
  415. package/src/messaging/providers/discord/send.ts +212 -0
  416. package/src/messaging/providers/discord/transport.ts +81 -0
  417. package/src/messaging/providers/index.ts +3 -1
  418. package/src/messaging/providers/retry-policy.test.ts +247 -0
  419. package/src/messaging/providers/retry-policy.ts +206 -0
  420. package/src/messaging/providers/slack/mention-source.test.ts +285 -0
  421. package/src/messaging/providers/slack/mention-source.ts +287 -0
  422. package/src/messaging/providers/telegram-bot/api.ts +47 -130
  423. package/src/messaging/providers/whatsapp/api.ts +47 -136
  424. package/src/monitoring/__tests__/db-integrity-sample.test.ts +4 -1
  425. package/src/monitoring/__tests__/plugin-auto-update.test.ts +332 -0
  426. package/src/monitoring/db-integrity-sample.ts +6 -1
  427. package/src/monitoring/plugin-auto-update.ts +359 -0
  428. package/src/monitoring/resource-sampler.ts +21 -5
  429. package/src/monitoring/worker.ts +13 -0
  430. package/src/notifications/__tests__/assistant-reply-producer.test.ts +375 -2
  431. package/src/notifications/__tests__/broadcaster.test.ts +37 -1
  432. package/src/notifications/__tests__/copy-composer.test.ts +165 -0
  433. package/src/notifications/__tests__/emit-signal-pipeline-failure.test.ts +205 -0
  434. package/src/notifications/__tests__/notification-utils.test.ts +249 -0
  435. package/src/notifications/adapters/platform.ts +31 -2
  436. package/src/notifications/assistant-reply-producer.ts +79 -32
  437. package/src/notifications/broadcaster.ts +30 -10
  438. package/src/notifications/copy-composer.ts +69 -0
  439. package/src/notifications/emit-signal.ts +82 -15
  440. package/src/notifications/events-store.ts +10 -2
  441. package/src/notifications/home-feed-side-effect.ts +6 -3
  442. package/src/notifications/notification-utils.ts +173 -5
  443. package/src/notifications/signal.ts +12 -0
  444. package/src/oauth/AGENTS.md +4 -2
  445. package/src/oauth/byo-connection.ts +6 -1
  446. package/src/oauth/credential-token-resolver.ts +3 -10
  447. package/src/oauth/manual-token-connection.ts +29 -64
  448. package/src/oauth/manual-token-providers.ts +62 -0
  449. package/src/oauth/provider-serializer.ts +13 -0
  450. package/src/oauth/seed-providers.ts +62 -11
  451. package/src/onboarding/onboarding-events-store.ts +91 -1
  452. package/src/permissions/question-prompter.test.ts +4 -0
  453. package/src/permissions/question-prompter.ts +21 -4
  454. package/src/persistence/__tests__/conversation-lineage.test.ts +248 -0
  455. package/src/persistence/attachments-store.ts +15 -15
  456. package/src/persistence/conversation-attention-store.ts +30 -10
  457. package/src/persistence/conversation-bootstrap.ts +17 -0
  458. package/src/persistence/conversation-crud.ts +363 -97
  459. package/src/persistence/conversation-group-migration.ts +1 -1
  460. package/src/persistence/conversation-key-store.ts +38 -10
  461. package/src/persistence/conversation-lineage.ts +226 -0
  462. package/src/persistence/conversation-plugin-facade.ts +28 -3
  463. package/src/persistence/conversation-queries.ts +207 -33
  464. package/src/persistence/conversation-types.ts +107 -1
  465. package/src/persistence/jobs-store.ts +32 -8
  466. package/src/persistence/llm-usage-store.ts +22 -0
  467. package/src/persistence/migrations/361-normalize-managed-connection-rows.test.ts +168 -0
  468. package/src/persistence/migrations/361-normalize-managed-connection-rows.ts +76 -0
  469. package/src/persistence/migrations/362-add-conversation-subagent-kind.test.ts +187 -0
  470. package/src/persistence/migrations/362-add-conversation-subagent-kind.ts +93 -0
  471. package/src/persistence/migrations/363-backfill-schedule-inference-profile.test.ts +229 -0
  472. package/src/persistence/migrations/363-backfill-schedule-inference-profile.ts +127 -0
  473. package/src/persistence/migrations/364-add-schedule-source-key.test.ts +120 -0
  474. package/src/persistence/migrations/364-add-schedule-source-key.ts +43 -0
  475. package/src/persistence/migrations/365-add-conversation-fork-strategy.ts +27 -0
  476. package/src/persistence/schema/conversations.ts +32 -0
  477. package/src/persistence/schema/infrastructure.ts +3 -0
  478. package/src/persistence/steps.ts +30 -0
  479. package/src/persistence/subagent-store.ts +148 -6
  480. package/src/platform/managed-speech.test.ts +20 -0
  481. package/src/platform/managed-speech.ts +9 -0
  482. package/src/plugin-api/index.ts +8 -0
  483. package/src/plugin-api/webhook-url.ts +151 -0
  484. package/src/plugins/__tests__/installed-plugin-dirs.test.ts +82 -0
  485. package/src/plugins/__tests__/source-fingerprint.test.ts +5 -1
  486. package/src/plugins/collect-source-versions.ts +4 -27
  487. package/src/plugins/defaults/exploration-drift/hooks/post-tool-use.ts +5 -5
  488. package/src/plugins/defaults/exploration-drift/package.json +1 -1
  489. package/src/plugins/defaults/image-recovery/hooks/post-model-call.ts +4 -2
  490. package/src/plugins/defaults/image-recovery/recover.ts +67 -57
  491. package/src/plugins/defaults/index.ts +1 -1
  492. package/src/plugins/defaults/memory/__tests__/conversation-queries.test.ts +64 -18
  493. package/src/plugins/defaults/memory/__tests__/jobs-worker-outcome-contract.test.ts +242 -0
  494. package/src/plugins/defaults/memory/__tests__/jobs-worker-retrospective-sweep.test.ts +24 -1
  495. package/src/plugins/defaults/memory/__tests__/memory-retrospective-enqueue.test.ts +48 -3
  496. package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +511 -19
  497. package/src/plugins/defaults/memory/__tests__/memory-retrospective-prompt.test.ts +39 -3
  498. package/src/plugins/defaults/memory/__tests__/memory-retrospective-provider-path.test.ts +493 -0
  499. package/src/plugins/defaults/memory/__tests__/memory-retrospective-wake-chain.test.ts +5 -4
  500. package/src/plugins/defaults/memory/__tests__/memory-tier-boundary-guard.test.ts +5 -1
  501. package/src/plugins/defaults/memory/job-handlers.ts +87 -5
  502. package/src/plugins/defaults/memory/jobs-worker.ts +60 -6
  503. package/src/plugins/defaults/memory/memory-retrospective-constants.ts +11 -0
  504. package/src/plugins/defaults/memory/memory-retrospective-enqueue.ts +29 -0
  505. package/src/plugins/defaults/memory/memory-retrospective-job.ts +342 -60
  506. package/src/plugins/defaults/memory/memory-retrospective-prompt.ts +23 -3
  507. package/src/plugins/defaults/memory/substrate/consolidation-lock.ts +9 -4
  508. package/src/plugins/defaults/memory/substrate/sweep-job.ts +7 -0
  509. package/src/plugins/defaults/memory/v1/graph/injection.ts +1 -1
  510. package/src/plugins/defaults/memory/v2/router.ts +8 -0
  511. package/src/plugins/defaults/turn-context/unified-turn-context.ts +1 -2
  512. package/src/plugins/external-plugin-loader.ts +17 -8
  513. package/src/plugins/installed-plugin-dirs.ts +101 -0
  514. package/src/plugins/mtime-cache.ts +34 -22
  515. package/src/plugins/types.ts +69 -8
  516. package/src/prompts/persona-resolver.ts +6 -2
  517. package/src/prompts/system-prompt.ts +72 -17
  518. package/src/providers/__tests__/inference.test.ts +40 -0
  519. package/src/providers/__tests__/preflight-resolved-config.test.ts +58 -1
  520. package/src/providers/__tests__/retry-callsite.test.ts +67 -0
  521. package/src/providers/__tests__/vellum-connection-routing.test.ts +42 -11
  522. package/src/providers/anthropic/client.ts +112 -95
  523. package/src/providers/connection-resolution.ts +35 -1
  524. package/src/providers/gemini/client.ts +5 -5
  525. package/src/providers/inference/adapter-factory.ts +16 -10
  526. package/src/providers/inference/auth.ts +16 -0
  527. package/src/providers/inference/connection-availability.ts +164 -12
  528. package/src/providers/inference/connections.ts +25 -1
  529. package/src/providers/media-resolve.ts +21 -11
  530. package/src/providers/model-catalog.ts +15 -0
  531. package/src/providers/openai/chat-completions-provider.ts +4 -4
  532. package/src/providers/openai/responses-provider.ts +12 -25
  533. package/src/providers/registry.ts +9 -4
  534. package/src/providers/retry.ts +25 -0
  535. package/src/providers/server-tool-pairing.ts +134 -0
  536. package/src/providers/speech-to-text/__tests__/resolve.test.ts +187 -6
  537. package/src/providers/speech-to-text/deepgram.ts +1 -0
  538. package/src/providers/speech-to-text/resolve.ts +80 -10
  539. package/src/providers/speech-to-text/vellum-managed.ts +7 -1
  540. package/src/providers/speech-to-text/vellum-speech-relay-connection.ts +3 -1
  541. package/src/providers/types.ts +26 -6
  542. package/src/providers/vellum-model-routing.ts +6 -7
  543. package/src/runtime/AGENTS.md +2 -0
  544. package/src/runtime/__tests__/agent-wake.test.ts +16 -16
  545. package/src/runtime/__tests__/background-job-runner.test.ts +0 -144
  546. package/src/runtime/__tests__/desktop-presence.test.ts +162 -0
  547. package/src/runtime/agent-wake.ts +63 -24
  548. package/src/runtime/assistant-event-hub.ts +87 -12
  549. package/src/runtime/auth/__tests__/guard-tests.test.ts +2 -0
  550. package/src/runtime/auth/__tests__/scopes.test.ts +20 -0
  551. package/src/runtime/auth/same-actor.ts +56 -8
  552. package/src/runtime/auth/scopes.ts +12 -0
  553. package/src/runtime/auth/types.ts +2 -0
  554. package/src/runtime/background-job-runner.ts +5 -27
  555. package/src/runtime/channel-readiness-service.ts +59 -3
  556. package/src/runtime/channel-readiness-types.ts +18 -0
  557. package/src/runtime/desktop-presence.ts +69 -0
  558. package/src/runtime/http-server.ts +12 -0
  559. package/src/runtime/local-actor-identity.ts +11 -9
  560. package/src/runtime/migrations/__tests__/vbundle-symlink-streaming-importer.test.ts +0 -0
  561. package/src/runtime/routes/__tests__/channel-verification-routes.test.ts +1 -1
  562. package/src/runtime/routes/__tests__/client-routes.test.ts +186 -14
  563. package/src/runtime/routes/__tests__/conversation-list-routes.test.ts +547 -1
  564. package/src/runtime/routes/__tests__/conversation-management-routes.test.ts +162 -7
  565. package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +14 -10
  566. package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +340 -3
  567. package/src/runtime/routes/__tests__/llm-call-sites-routes.test.ts +28 -1
  568. package/src/runtime/routes/__tests__/plugins-routes.test.ts +173 -1
  569. package/src/runtime/routes/__tests__/question-routes.test.ts +64 -0
  570. package/src/runtime/routes/__tests__/retrospective-routes.test.ts +20 -0
  571. package/src/runtime/routes/__tests__/skill-history-route.test.ts +116 -0
  572. package/src/runtime/routes/__tests__/wake-conversation-routes.test.ts +60 -2
  573. package/src/runtime/routes/archive-utils.ts +16 -12
  574. package/src/runtime/routes/attachment-routes.ts +13 -6
  575. package/src/runtime/routes/canned-reply-release.ts +54 -0
  576. package/src/runtime/routes/channel-verification-routes.ts +2 -27
  577. package/src/runtime/routes/chatgpt-subscription-auth-routes.ts +5 -0
  578. package/src/runtime/routes/client-routes.ts +114 -2
  579. package/src/runtime/routes/conversation-cli-routes.ts +10 -12
  580. package/src/runtime/routes/conversation-list-routes.ts +59 -10
  581. package/src/runtime/routes/conversation-management-routes.ts +47 -18
  582. package/src/runtime/routes/conversation-query-routes.ts +32 -53
  583. package/src/runtime/routes/conversation-routes.ts +360 -293
  584. package/src/runtime/routes/credential-prompt-routes.ts +23 -2
  585. package/src/runtime/routes/debug-routes.ts +4 -1
  586. package/src/runtime/routes/default-provider-routes.ts +7 -11
  587. package/src/runtime/routes/documents-routes.ts +44 -1
  588. package/src/runtime/routes/events-routes.ts +28 -1
  589. package/src/runtime/routes/host-app-control-routes.ts +2 -0
  590. package/src/runtime/routes/host-bash-routes.ts +2 -0
  591. package/src/runtime/routes/host-browser-routes.ts +2 -0
  592. package/src/runtime/routes/host-cu-routes.ts +2 -0
  593. package/src/runtime/routes/host-file-routes.ts +2 -0
  594. package/src/runtime/routes/host-transfer-routes.ts +4 -0
  595. package/src/runtime/routes/identity-routes.ts +6 -6
  596. package/src/runtime/routes/inbound-message-handler.ts +5 -5
  597. package/src/runtime/routes/inbound-stages/acl-enforcement.ts +7 -3
  598. package/src/runtime/routes/inbound-stages/background-dispatch.ts +14 -0
  599. package/src/runtime/routes/inbound-stages/bootstrap-intercept.ts +2 -2
  600. package/src/runtime/routes/inbound-stages/guardian-activation-intercept.test.ts +14 -11
  601. package/src/runtime/routes/inbound-stages/guardian-activation-intercept.ts +14 -17
  602. package/src/runtime/routes/inference-profile-availability-guard.ts +171 -0
  603. package/src/runtime/routes/inference-profile-session-handler.ts +26 -2
  604. package/src/runtime/routes/inference-profile-session-routes.ts +2 -1
  605. package/src/runtime/routes/inference-profiles-routes.ts +152 -73
  606. package/src/runtime/routes/inference-provider-connection-routes.ts +0 -16
  607. package/src/runtime/routes/llm-call-sites-routes.ts +13 -0
  608. package/src/runtime/routes/log-export-routes.ts +1 -1
  609. package/src/runtime/routes/notification-routes.ts +0 -14
  610. package/src/runtime/routes/oauth-providers.ts +1 -0
  611. package/src/runtime/routes/plugins-routes.ts +106 -1
  612. package/src/runtime/routes/profiler-routes.ts +4 -2
  613. package/src/runtime/routes/question-routes.ts +28 -10
  614. package/src/runtime/routes/retrospective-routes.ts +9 -3
  615. package/src/runtime/routes/schedule-routes.ts +210 -12
  616. package/src/runtime/routes/secret-routes.ts +40 -23
  617. package/src/runtime/routes/settings-routes.ts +41 -13
  618. package/src/runtime/routes/skills-routes.ts +79 -0
  619. package/src/runtime/routes/ui-snapshot-routes.ts +1 -0
  620. package/src/runtime/routes/wake-conversation-routes.ts +23 -0
  621. package/src/runtime/services/__tests__/conversation-serializer.test.ts +1 -0
  622. package/src/runtime/services/conversation-serializer.ts +24 -9
  623. package/src/runtime/sync/documents-sidecar-publish.test.ts +101 -0
  624. package/src/runtime/sync/resource-sync-events.ts +79 -1
  625. package/src/runtime/sync/worker-daemon-notify.test.ts +34 -0
  626. package/src/runtime/sync/worker-daemon-notify.ts +44 -6
  627. package/src/runtime/verification-outbound-actions.ts +250 -222
  628. package/src/runtime/verification-templates.ts +41 -30
  629. package/src/schedule/__tests__/plugin-schedule-declarations.test.ts +614 -0
  630. package/src/schedule/__tests__/plugin-schedule-reconciler.test.ts +1066 -0
  631. package/src/schedule/inference-profile.ts +64 -3
  632. package/src/schedule/plugin-schedule-declarations.ts +518 -0
  633. package/src/schedule/plugin-schedule-reconciler.ts +605 -0
  634. package/src/schedule/plugin-schedules-gate.ts +23 -0
  635. package/src/schedule/schedule-store.ts +520 -26
  636. package/src/schedule/scheduler.ts +38 -0
  637. package/src/security/credential-backend.ts +51 -4
  638. package/src/security/encrypted-store.ts +64 -42
  639. package/src/security/secure-keys.ts +55 -7
  640. package/src/skills/catalog-install.ts +20 -7
  641. package/src/skills/inline-command-runner.ts +2 -7
  642. package/src/skills/skill-history.test.ts +269 -0
  643. package/src/skills/skill-history.ts +250 -0
  644. package/src/skills/skillssh-registry.ts +1 -1
  645. package/src/stt/daemon-batch-transcriber.ts +8 -6
  646. package/src/subagent/__tests__/consult-prompt.test.ts +25 -0
  647. package/src/subagent/consult-prompt.ts +3 -1
  648. package/src/subagent/consult-transcript.ts +1 -1
  649. package/src/subagent/manager.ts +304 -44
  650. package/src/subagent/progress-events.ts +35 -0
  651. package/src/subagent/role-resolution.ts +88 -0
  652. package/src/subagent/types.ts +302 -67
  653. package/src/subagent/validate-allowlists.ts +3 -4
  654. package/src/telegram/__tests__/webhook-health.test.ts +73 -1
  655. package/src/telegram/webhook-health.ts +101 -3
  656. package/src/telemetry/__tests__/telemetry-event-fixtures.ts +2 -0
  657. package/src/telemetry/activation-funnel.ts +6 -1
  658. package/src/telemetry/live-voice-funnel.ts +75 -0
  659. package/src/telemetry/telemetry-event-sources.ts +6 -0
  660. package/src/telemetry/telemetry-wire-source.json +1 -1
  661. package/src/telemetry/telemetry-wire.generated.ts +2 -0
  662. package/src/telemetry/types.ts +23 -4
  663. package/src/telemetry/usage-telemetry-reporter.test.ts +4 -0
  664. package/src/tools/__tests__/subagent-workflow-tool-input-schemas.test.ts +1 -1
  665. package/src/tools/ask-question/ask-question-tool.test.ts +118 -3
  666. package/src/tools/ask-question/ask-question-tool.ts +56 -2
  667. package/src/tools/browser/browser-execution.ts +1 -1
  668. package/src/tools/calls/call-start.ts +8 -5
  669. package/src/tools/conversation-groups/move_to_group.ts +10 -12
  670. package/src/tools/document/document-tool.ts +11 -4
  671. package/src/tools/execution-timeout.ts +9 -2
  672. package/src/tools/executor.ts +7 -6
  673. package/src/tools/filesystem/edit.ts +1 -1
  674. package/src/tools/filesystem/list.ts +1 -1
  675. package/src/tools/filesystem/read.ts +7 -13
  676. package/src/tools/filesystem/search.ts +23 -21
  677. package/src/tools/filesystem/write.ts +4 -1
  678. package/src/tools/host-filesystem/edit.ts +7 -2
  679. package/src/tools/host-filesystem/read.ts +21 -8
  680. package/src/tools/host-filesystem/write.ts +10 -2
  681. package/src/tools/host-terminal/host-shell.ts +2 -2
  682. package/src/tools/mcp/mcp-tool-factory.ts +25 -1
  683. package/src/tools/permission-checker.ts +8 -3
  684. package/src/tools/schedule/delete.ts +13 -1
  685. package/src/tools/schedule/list.ts +14 -3
  686. package/src/tools/schedule/update.ts +56 -33
  687. package/src/tools/shared/filesystem/audio-read.ts +6 -4
  688. package/src/tools/shared/filesystem/file-ops-service.ts +176 -116
  689. package/src/tools/shared/filesystem/image-read.ts +12 -12
  690. package/src/tools/shared/filesystem/path-policy.ts +70 -5
  691. package/src/tools/shared/filesystem/size-guard.ts +6 -6
  692. package/src/tools/shared/input-misuse.ts +72 -0
  693. package/src/tools/skills/scaffold-managed.ts +82 -0
  694. package/src/tools/skills/skill-tool-factory.ts +14 -1
  695. package/src/tools/subagent/consult-deadline.ts +12 -7
  696. package/src/tools/subagent/message.ts +3 -1
  697. package/src/tools/subagent/read.ts +66 -4
  698. package/src/tools/subagent/spawn.ts +500 -78
  699. package/src/tools/tool-approval-handler.ts +45 -4
  700. package/src/tools/types.ts +22 -3
  701. package/src/tools/ui-surface/surface-shape-docs.ts +1 -1
  702. package/src/tools/ui-surface/visual-validation.ts +2 -0
  703. package/src/usage/__tests__/subagent-attribution.test.ts +109 -0
  704. package/src/usage/subagent-attribution.ts +108 -0
  705. package/src/util/__tests__/sqlite-retry.test.ts +112 -0
  706. package/src/util/abort-reasons.ts +16 -0
  707. package/src/util/ansi.ts +44 -0
  708. package/src/util/disk-usage.ts +17 -13
  709. package/src/util/image-conversion.ts +20 -13
  710. package/src/util/schedule-source-key.ts +23 -0
  711. package/src/util/worker-process.ts +7 -1
  712. package/src/workspace/git-service.ts +33 -9
  713. package/src/workspace/migrations/140-repair-seed-pinned-memory-v3-live.ts +260 -0
  714. package/src/workspace/migrations/141-stt-english-default-to-multilingual.ts +111 -0
  715. package/src/workspace/migrations/__tests__/141-stt-english-default-to-multilingual.test.ts +126 -0
  716. package/src/workspace/migrations/registry.ts +4 -0
  717. package/src/workspace/top-level-renderer.ts +15 -2
  718. package/src/__tests__/tool-side-effects-slack-dm.test.ts +0 -357
  719. package/src/daemon/install-symlink.ts +0 -207
  720. package/src/notifications/deferred-emit.ts +0 -147
@@ -0,0 +1,169 @@
1
+ /**
2
+ * Persisting a photo taken during a live-voice call.
3
+ *
4
+ * A photo lands in the conversation as its own user message, the moment the
5
+ * shutter fires, and **runs no turn**. That single choice is what makes the
6
+ * order of shutter and speech irrelevant: whatever the user says next, before
7
+ * or after the snap, is answered by a model whose history already contains the
8
+ * image.
9
+ *
10
+ * The alternatives were both worse and both were tried. Attaching the photo to
11
+ * the *next* spoken turn strands it whenever the user speaks first: the
12
+ * question "what's this?" dispatches without an image and the photo waits for
13
+ * a sentence that has already been said. Dispatching a turn *for* the photo
14
+ * races the sentence the user is in the middle of saying, so the assistant
15
+ * answers the picture and the words separately.
16
+ *
17
+ * The cost, accepted: media survives a context-overflow retry only on the most
18
+ * recent user message (`conversation-media-retry.ts`), so on a long call a
19
+ * photo can be stripped where one riding the spoken turn would have survived.
20
+ * Stranding is the worse failure, and it happens every time rather than only
21
+ * under overflow.
22
+ */
23
+
24
+ import { v7 as uuidv7 } from "uuid";
25
+
26
+ import { persistQueuedMessageBody } from "../daemon/conversation-messaging.js";
27
+ import { getOrCreateConversation } from "../daemon/conversation-store.js";
28
+ import {
29
+ getAttachmentsByIds,
30
+ getSourcePathsForAttachments,
31
+ } from "../persistence/attachments-store.js";
32
+ import { recordConversationPersistedSeq } from "../persistence/conversation-crud.js";
33
+ import { broadcastMessage } from "../runtime/assistant-event-hub.js";
34
+ import { getCurrentSeq } from "../runtime/assistant-stream-state.js";
35
+ import { publishConversationMessagesChanged } from "../runtime/sync/resource-sync-events.js";
36
+ import { getLogger } from "../util/logger.js";
37
+
38
+ const log = getLogger("live-voice-photo");
39
+
40
+ /**
41
+ * How long to wait for an in-flight turn before giving up on a photo.
42
+ *
43
+ * Persisting takes the conversation's processing lock, which a running turn
44
+ * holds for as long as it runs, tools included. The wait is generous because
45
+ * the alternative is dropping a photo the user watched themselves take, and
46
+ * the photo is not urgent: nothing is blocked on it except the next thing they
47
+ * say.
48
+ */
49
+ const PROCESSING_WAIT_MS = 30_000;
50
+ const PROCESSING_POLL_MS = 100;
51
+
52
+ /** What the user message says. The image blocks carry the actual content. */
53
+ const PHOTO_MESSAGE_CONTENT = "[photo]";
54
+
55
+ export interface LiveVoicePhotoResult {
56
+ readonly ok: boolean;
57
+ readonly messageId?: string;
58
+ }
59
+
60
+ /**
61
+ * Hydrate attachment ids into the shape `persistQueuedMessageBody` stores.
62
+ * Mirrors the HTTP send path's own `resolveAttachments`, so a photo taken
63
+ * mid-call is stored exactly like one attached to a typed message.
64
+ */
65
+ function resolvePhotoAttachments(attachmentIds: string[]) {
66
+ const resolved = getAttachmentsByIds(attachmentIds, {
67
+ hydrateFileData: true,
68
+ });
69
+ const sourcePaths = getSourcePathsForAttachments(attachmentIds);
70
+ return resolved.map((a) => ({
71
+ id: a.id,
72
+ filename: a.originalFilename,
73
+ mimeType: a.mimeType,
74
+ data: a.dataBase64,
75
+ ...(sourcePaths.has(a.id) ? { filePath: sourcePaths.get(a.id) } : {}),
76
+ }));
77
+ }
78
+
79
+ /** Resolve once the conversation is not mid-turn, or false on timeout. */
80
+ async function waitForIdle(conversation: {
81
+ isProcessing: () => boolean;
82
+ }): Promise<boolean> {
83
+ const deadline = Date.now() + PROCESSING_WAIT_MS;
84
+ while (conversation.isProcessing()) {
85
+ if (Date.now() >= deadline) {
86
+ return false;
87
+ }
88
+ await new Promise((resolve) => setTimeout(resolve, PROCESSING_POLL_MS));
89
+ }
90
+ return true;
91
+ }
92
+
93
+ /**
94
+ * Persist a photo into the conversation as a user message, running no turn.
95
+ *
96
+ * Takes and releases the processing lock the way the built-in slash commands
97
+ * do: they are the existing precedent for "persist a user message and answer
98
+ * without the agent loop", and the lock is what keeps this write from
99
+ * interleaving with a turn's own persist.
100
+ *
101
+ * Never throws. A photo that cannot be stored must not take the call down with
102
+ * it, and the caller reports the failure to the user instead.
103
+ */
104
+ export async function persistLiveVoicePhoto(
105
+ conversationId: string,
106
+ attachmentId: string,
107
+ ): Promise<LiveVoicePhotoResult> {
108
+ try {
109
+ const attachments = resolvePhotoAttachments([attachmentId]);
110
+ if (attachments.length === 0) {
111
+ log.warn({ attachmentId }, "Live-voice photo attachment did not resolve");
112
+ return { ok: false };
113
+ }
114
+
115
+ const conversation = await getOrCreateConversation(conversationId);
116
+
117
+ // A turn holds the lock for its whole run. Waiting rather than queueing:
118
+ // the conversation's queue drains into a turn, which is the one thing this
119
+ // must not cause.
120
+ if (!(await waitForIdle(conversation))) {
121
+ log.warn(
122
+ { conversationId, attachmentId },
123
+ "Live-voice photo timed out waiting for the conversation to go idle",
124
+ );
125
+ return { ok: false };
126
+ }
127
+
128
+ conversation.setProcessing(true);
129
+ try {
130
+ const persisted = await persistQueuedMessageBody(conversation, {
131
+ content: PHOTO_MESSAGE_CONTENT,
132
+ attachments,
133
+ requestId: uuidv7(),
134
+ metadata: {
135
+ // Marks the row as something the user did on a call rather than
136
+ // typed, the same way a voice turn's own user message is marked.
137
+ voiceSessionTurn: true,
138
+ livePhoto: true,
139
+ },
140
+ });
141
+
142
+ // The row exists but no turn will announce it, so the clients that
143
+ // render this conversation have to be told directly, or the photo does
144
+ // not appear until something else forces a refetch.
145
+ broadcastMessage({
146
+ type: "user_message_echo",
147
+ text: PHOTO_MESSAGE_CONTENT,
148
+ conversationId,
149
+ messageId: persisted.id,
150
+ });
151
+ recordConversationPersistedSeq(conversationId, getCurrentSeq());
152
+ publishConversationMessagesChanged(conversationId);
153
+
154
+ return { ok: true, messageId: persisted.id };
155
+ } finally {
156
+ conversation.setProcessing(false);
157
+ // Anything queued behind the lock we just held still has to run. Without
158
+ // this a message queued during the photo's write sits until the next
159
+ // turn ends.
160
+ void conversation.kickDrainQueue("loop_complete", "live_voice_photo");
161
+ }
162
+ } catch (err) {
163
+ log.warn(
164
+ { err, conversationId, attachmentId },
165
+ "Failed to persist a live-voice photo",
166
+ );
167
+ return { ok: false };
168
+ }
169
+ }
@@ -38,6 +38,10 @@ import {
38
38
  import { ABORT_WATCHDOG_MS } from "../daemon/abort-watchdog.js";
39
39
  import { isRefusedInReadOnlyPass } from "../daemon/conversation-tool-setup.js";
40
40
  import type { TrustContext } from "../daemon/trust-context-types.js";
41
+ import {
42
+ recordLiveVoiceSessionEnded,
43
+ recordLiveVoiceSessionStarted,
44
+ } from "../onboarding/onboarding-events-store.js";
41
45
  import { isInstalledStaticSkillLoad } from "../permissions/checker.js";
42
46
  import { ensureConversationExists } from "../persistence/conversation-crud.js";
43
47
  import {
@@ -54,6 +58,7 @@ import type {
54
58
  SttStreamServerEvent,
55
59
  } from "../stt/types.js";
56
60
  import { getSubagentManager } from "../subagent/index.js";
61
+ import { liveVoiceEndScreen } from "../telemetry/live-voice-funnel.js";
57
62
  import { getToolOwner } from "../tools/registry.js";
58
63
  import { extractSpeakableSegments } from "../tts/speakable-segments.js";
59
64
  import { createAbortReason } from "../util/abort-reasons.js";
@@ -84,6 +89,7 @@ import {
84
89
  type LiveVoiceTurnSeedMarks,
85
90
  type VoiceEndpointAction,
86
91
  } from "./live-voice-metrics.js";
92
+ import { persistLiveVoicePhoto } from "./live-voice-photo.js";
87
93
  import {
88
94
  type LiveVoiceSession as LiveVoiceSessionContract,
89
95
  type LiveVoiceSessionCloseReason,
@@ -97,6 +103,7 @@ import type {
97
103
  } from "./live-voice-tts.js";
98
104
  import { pickProgressPhrase } from "./progress-phrases.js";
99
105
  import {
106
+ type LiveVoiceClientAttachImageFrame,
100
107
  type LiveVoiceClientFrame,
101
108
  type LiveVoiceClientUpdateConfigFrame,
102
109
  LiveVoiceProtocolErrorCode,
@@ -972,6 +979,16 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
972
979
  private outboundFrames: Promise<void> = Promise.resolve();
973
980
  private activeAssistantTurn: ActiveAssistantTurn | null = null;
974
981
  private sessionEndMetricsEmitted = false;
982
+ /**
983
+ * Protocol error code of the failure that killed the session, latched by
984
+ * {@link sendFrame} when an error frame goes out on an already-`failed`
985
+ * session. Both fatal paths (`failStartup` before `ready`, a failed
986
+ * utterance arm after it) set `state = "failed"` before sending, and no
987
+ * other error frame does, so this catches exactly the session-ending
988
+ * failures and ignores the recoverable mid-session ones. `null` on a
989
+ * session that never failed.
990
+ */
991
+ private failureCode: LiveVoiceProtocolErrorCode | null = null;
975
992
  // Non-null iff the start frame requested turnDetection "server_vad".
976
993
  private readonly turnDetector: MediaTurnDetector | null;
977
994
  // Energy gate for server-VAD speech classification; undefined defers to
@@ -1184,6 +1201,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1184
1201
  return;
1185
1202
  }
1186
1203
 
1204
+ // Before the preflight, not after: a session rejected for missing
1205
+ // credentials is precisely the one the failure rate needs to count, and
1206
+ // recording the start only once `ready` goes out would hide every such
1207
+ // session from both the numerator and the denominator.
1208
+ recordLiveVoiceSessionStarted(this.context.sessionId);
1209
+
1187
1210
  if (this.resolveCredentialReadiness) {
1188
1211
  const readiness = await this.resolveCredentialReadiness();
1189
1212
  if (readiness.status === "not-ready") {
@@ -1237,9 +1260,44 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1237
1260
  case "update_config":
1238
1261
  this.applyConfigUpdate(frame);
1239
1262
  return;
1263
+ case "attach_image":
1264
+ this.persistPhoto(frame);
1265
+ return;
1240
1266
  }
1241
1267
  }
1242
1268
 
1269
+ /**
1270
+ * Persist a photo taken mid-call into the conversation, running no turn.
1271
+ *
1272
+ * Fire-and-forget on purpose: the persist waits out any in-flight turn, and
1273
+ * the socket must keep pumping audio meanwhile. The client already showed a
1274
+ * thumbnail from the local frame, so nothing on screen is waiting on this.
1275
+ *
1276
+ * The photo becomes its own user message rather than riding the next spoken
1277
+ * turn, which is what makes shutter-then-speak and speak-then-shutter
1278
+ * behave the same: either way the model's history has the image by the time
1279
+ * it answers. See `live-voice-photo.ts` for the full reasoning.
1280
+ */
1281
+ private persistPhoto(frame: LiveVoiceClientAttachImageFrame): void {
1282
+ void persistLiveVoicePhoto(this.conversationId, frame.attachmentId).then(
1283
+ (result) => {
1284
+ if (!result.ok && !this.isClosed) {
1285
+ void this.sendFrame({
1286
+ type: "error",
1287
+ code: LiveVoiceProtocolErrorCode.InvalidFrame,
1288
+ message: "Could not attach that photo to the conversation.",
1289
+ // Names the photo as the casualty so the client can retract the
1290
+ // thumbnail it already showed, rather than filing this with the
1291
+ // transient transcriber and TTS blips that share `recoverable`.
1292
+ frameType: "attach_image",
1293
+ // The session is fine; only this photo failed.
1294
+ recoverable: true,
1295
+ });
1296
+ }
1297
+ },
1298
+ );
1299
+ }
1300
+
1243
1301
  /**
1244
1302
  * Apply a mid-session `update_config` frame: retune the live turn detector's
1245
1303
  * pause ("pause before reply") and/or the barge-in guard ("interrupt
@@ -1261,11 +1319,24 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
1261
1319
  await this.handleAudio(Buffer.from(chunk));
1262
1320
  }
1263
1321
 
1264
- async close(_reason: LiveVoiceSessionCloseReason): Promise<void> {
1322
+ async close(reason: LiveVoiceSessionCloseReason): Promise<void> {
1265
1323
  if (this.isClosed) {
1266
1324
  return;
1267
1325
  }
1268
1326
 
1327
+ // Recorded first, and independently of `shouldEmitSessionEndMetrics`
1328
+ // below: that flag governs the client-facing `metrics` frame, and a
1329
+ // failed session (which suppresses the frame) is the one whose end
1330
+ // telemetry matters most. Session duration downstream is this row's
1331
+ // `recorded_at` minus the started row's, so it must be written before the
1332
+ // teardown below, which awaits a pending continuation and can run long.
1333
+ const failed = this.state === "failed";
1334
+ recordLiveVoiceSessionEnded({
1335
+ sessionId: this.context.sessionId,
1336
+ screen: liveVoiceEndScreen(reason, failed ? this.failureCode : null),
1337
+ outcome: failed ? "failed" : "completed",
1338
+ });
1339
+
1269
1340
  const shouldEmitSessionEndMetrics = this.state !== "failed";
1270
1341
  this.state = "closed";
1271
1342
  // Retire the island before the teardown below starts awaiting things. A
@@ -2864,8 +2935,9 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
2864
2935
  ): void {
2865
2936
  turn.speculativePending = false;
2866
2937
  // The dispatch is being unwound, so the barge-in merge note, the finished
2867
- // continuation's answer, and its queued announcement all go back to the
2868
- // session the turn that would have delivered them is gone.
2938
+ // continuation's answer, its queued announcement, and any photos it claimed
2939
+ // all go back to the session. The turn that would have delivered them is
2940
+ // gone, and the utterance they belong to is about to be sent by another.
2869
2941
  this.restorePendingTurnContext(turn);
2870
2942
  // Latched before the handle check: when the discard beats the bridge
2871
2943
  // handle's resolution (startVoiceTurn still persisting), the handle's
@@ -3807,8 +3879,20 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
3807
3879
  voiceSessionId: this.context.sessionId,
3808
3880
  userMessageChannel: "vellum",
3809
3881
  assistantMessageChannel: "vellum",
3882
+ // Fixed, and NOT the originating client: this pair resolves the turn's
3883
+ // channel capabilities, where `macos` is what grants a live-voice turn
3884
+ // desktop UI and dynamic surfaces. Reporting the true client here would
3885
+ // strip `supportsDynamicUi` from every iOS session: a behavior change
3886
+ // wearing an attribution fix's clothes. The originating client travels
3887
+ // as telemetry instead, on `voiceTelemetry` below.
3810
3888
  userMessageInterface: "macos",
3811
3889
  assistantMessageInterface: "macos",
3890
+ voiceTelemetry: {
3891
+ sessionId: this.context.sessionId,
3892
+ ...(this.context.startFrame.client
3893
+ ? { client: this.context.startFrame.client }
3894
+ : {}),
3895
+ },
3812
3896
  voiceControlPrompt: buildVoiceControlPrompt(activeTurn, {
3813
3897
  ...(leg.frontDoor !== undefined ? { frontDoor: leg.frontDoor } : {}),
3814
3898
  }),
@@ -5506,6 +5590,13 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
5506
5590
  // method, and a per-call-site hook would drift from it on the first frame
5507
5591
  // anyone added. Fire-and-forget by contract — see the reporter.
5508
5592
  this.liveActivityReporter.report(frame);
5593
+ // Latched here for the same reason: both fatal paths mark the session
5594
+ // `failed` and then send their error frame through this method, so the
5595
+ // code that ended the session is readable at close without either path
5596
+ // having to remember to stash it.
5597
+ if (frame.type === "error" && this.state === "failed") {
5598
+ this.failureCode ??= frame.code;
5599
+ }
5509
5600
  let sent = false;
5510
5601
  this.outboundFrames = this.outboundFrames
5511
5602
  .catch(() => {})
@@ -5675,6 +5766,12 @@ export async function defaultSpawnBackgroundContinuation(args: {
5675
5766
  objective: args.objective,
5676
5767
  fork: true,
5677
5768
  sendResultToUser: false,
5769
+ // Distinct spawn mode so this unattended continuation is separable in
5770
+ // cost telemetry from a tool-initiated fork. Mechanically both are
5771
+ // forks, but they come from different call sites with different cost
5772
+ // profiles, and lumping them together is what made delegated spend
5773
+ // opaque in the first place.
5774
+ spawnMode: "voice_continuation",
5678
5775
  // Full subagent abilities: the continuation runs like any other
5679
5776
  // background subagent, so it can genuinely finish build-shaped work
5680
5777
  // (JARVIS-1354). Side effects are governed by the standard
@@ -5735,7 +5832,14 @@ async function defaultStartVoiceTurn(
5735
5832
  // inside `startVoiceTurn` trips `FOREIGN KEY constraint failed`. Ensure it
5736
5833
  // exists (idempotent) before persisting. Lives in the production wiring, not
5737
5834
  // the session state machine, so session unit tests stay DB-free.
5738
- const createdConversation = ensureConversationExists(options.conversationId);
5835
+ // Native: this is the local live-voice session, which adopts a conversation
5836
+ // id the app supplied. Its trust context resolves through
5837
+ // `resolveLocalLiveVoiceTrustContext`, and a phone call reaches the assistant
5838
+ // through the telephony path rather than here.
5839
+ const createdConversation = ensureConversationExists(
5840
+ options.conversationId,
5841
+ "vellum",
5842
+ );
5739
5843
  if (createdConversation) {
5740
5844
  // The row was created outside the normal send-message route, which is where
5741
5845
  // sibling clients/sidebars learn about a new conversation. Emit the same
@@ -1,3 +1,5 @@
1
+ import { type ClientOs, parseClientOs } from "../channels/types.js";
2
+
1
3
  const LIVE_VOICE_CLIENT_FRAME_TYPES = [
2
4
  "start",
3
5
  "audio",
@@ -5,6 +7,7 @@ const LIVE_VOICE_CLIENT_FRAME_TYPES = [
5
7
  "interrupt",
6
8
  "end",
7
9
  "update_config",
10
+ "attach_image",
8
11
  ] as const;
9
12
 
10
13
  type LiveVoiceClientFrameType = (typeof LIVE_VOICE_CLIENT_FRAME_TYPES)[number];
@@ -98,6 +101,21 @@ export interface LiveVoiceClientStartFrame {
98
101
  * [{@link MIN_BARGE_IN_MIN_SPEECH_MS}, {@link MAX_BARGE_IN_MIN_SPEECH_MS}].
99
102
  */
100
103
  readonly bargeInMinSpeechMs?: number;
104
+ /**
105
+ * Which client opened the session. Absent from clients that predate the
106
+ * field, in which case the originating client is simply unknown.
107
+ *
108
+ * A {@link ClientOs}, not an `InterfaceId`, and deliberately so: the iOS and
109
+ * macOS apps run the same web bundle over the same transport, so the thing
110
+ * that actually differs between them is the OS surface. `ClientOs` is also
111
+ * the vocabulary that is barred by contract from answering transport
112
+ * questions (see `channels/types.ts`), which is the invariant wanted here:
113
+ * this value is **analytics only**. It rides the voice turn's telemetry
114
+ * `client` bag (see `voice-session-bridge.ts`) and never reaches
115
+ * `userMessageInterface`, which feeds `resolveChannelCapabilities` and is
116
+ * load-bearing for what a voice turn may do.
117
+ */
118
+ readonly client?: ClientOs;
101
119
  }
102
120
 
103
121
  /**
@@ -141,13 +159,35 @@ export interface LiveVoiceClientUpdateConfigFrame {
141
159
  readonly bargeInMinSpeechMs?: number;
142
160
  }
143
161
 
162
+ /**
163
+ * A photo the user took mid-call, already uploaded over the normal attachment
164
+ * route (`POST /v1/assistants/{id}/attachments`). Only the resulting id
165
+ * travels here.
166
+ *
167
+ * The bytes deliberately do not: the upload endpoint already normalizes HEIF,
168
+ * caps size, and stores the blob, and routing an image through this socket
169
+ * would duplicate all of it on a transport tuned for 50 ms audio frames.
170
+ *
171
+ * The id is *parked*, not dispatched. It rides the next turn's own user
172
+ * message (see {@link LiveVoiceSession}'s pending-attachment handling), so the
173
+ * photo and the words spoken about it are one message rather than two, which
174
+ * is what lets "what's this?" resolve, and what keeps the image attached to the
175
+ * newest user message, the only one a context-overflow retry preserves media on
176
+ * (`conversation-media-retry.ts`).
177
+ */
178
+ export interface LiveVoiceClientAttachImageFrame {
179
+ readonly type: "attach_image";
180
+ readonly attachmentId: string;
181
+ }
182
+
144
183
  export type LiveVoiceClientFrame =
145
184
  | LiveVoiceClientStartFrame
146
185
  | LiveVoiceClientAudioFrame
147
186
  | LiveVoiceClientPttReleaseFrame
148
187
  | LiveVoiceClientInterruptFrame
149
188
  | LiveVoiceClientEndFrame
150
- | LiveVoiceClientUpdateConfigFrame;
189
+ | LiveVoiceClientUpdateConfigFrame
190
+ | LiveVoiceClientAttachImageFrame;
151
191
 
152
192
  interface LiveVoiceBinaryAudioFrame {
153
193
  readonly type: "binary_audio";
@@ -337,6 +377,19 @@ export interface LiveVoiceErrorServerFrame extends LiveVoiceServerFrameBase {
337
377
  readonly type: "error";
338
378
  readonly code: LiveVoiceProtocolErrorCode;
339
379
  readonly message: string;
380
+ /**
381
+ * The client frame this error is about, when the failure was a parse or
382
+ * validation failure of a specific frame. Absent otherwise, and absent from
383
+ * daemons predating the field.
384
+ *
385
+ * It exists so an `unknown_type` is attributable. A client that sends more
386
+ * than one optional frame (today: `update_config` and `attach_image`) gets
387
+ * the same code for either, and without this has to assume which one was
388
+ * refused. The wrong assumption is silent in both directions: settings stop
389
+ * applying for a session, or a photo the user watched themselves take is
390
+ * dropped with nothing said.
391
+ */
392
+ readonly frameType?: string;
340
393
  /**
341
394
  * True when the session continues past the error (e.g. a transient
342
395
  * transcriber blip or one failed TTS segment). Absent (including on frames
@@ -469,9 +522,38 @@ export function validateLiveVoiceClientFrame(
469
522
  return { ok: true, frame: { type: "end" } };
470
523
  case "update_config":
471
524
  return validateUpdateConfigFrame(value);
525
+ case "attach_image":
526
+ return validateAttachImageFrame(value);
472
527
  }
473
528
  }
474
529
 
530
+ function validateAttachImageFrame(
531
+ value: Record<string, unknown>,
532
+ ): LiveVoiceParseResult<LiveVoiceClientAttachImageFrame> {
533
+ if (!("attachmentId" in value)) {
534
+ return protocolError(
535
+ "missing_required_field",
536
+ "attach_image frame is missing required field attachmentId",
537
+ "attachmentId",
538
+ "attach_image",
539
+ );
540
+ }
541
+
542
+ if (!isNonEmptyString(value.attachmentId)) {
543
+ return protocolError(
544
+ "invalid_field",
545
+ "attach_image frame field attachmentId must be a non-empty string",
546
+ "attachmentId",
547
+ "attach_image",
548
+ );
549
+ }
550
+
551
+ return {
552
+ ok: true,
553
+ frame: { type: "attach_image", attachmentId: value.attachmentId },
554
+ };
555
+ }
556
+
475
557
  function validateUpdateConfigFrame(
476
558
  value: Record<string, unknown>,
477
559
  ): LiveVoiceParseResult<LiveVoiceClientUpdateConfigFrame> {
@@ -642,6 +724,11 @@ function validateStartFrame(
642
724
  );
643
725
  }
644
726
 
727
+ // An unrecognized client is dropped rather than rejected: the field is an
728
+ // analytics dimension, and failing a session's startup over it would trade a
729
+ // gap in a chart for a user who cannot talk to their assistant.
730
+ const client = parseClientOs(value.client);
731
+
645
732
  return {
646
733
  ok: true,
647
734
  frame: {
@@ -649,6 +736,7 @@ function validateStartFrame(
649
736
  ...(typeof value.conversationId === "string"
650
737
  ? { conversationId: value.conversationId }
651
738
  : {}),
739
+ ...(client ? { client } : {}),
652
740
  audio: audioConfig.frame,
653
741
  ...(isLiveVoiceTurnDetectionMode(value.turnDetection)
654
742
  ? { turnDetection: value.turnDetection }
package/src/mcp/client.ts CHANGED
@@ -3,6 +3,7 @@ import { Client } from "@modelcontextprotocol/sdk/client/index.js";
3
3
  import { SSEClientTransport } from "@modelcontextprotocol/sdk/client/sse.js";
4
4
  import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
5
5
  import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
6
+ import type { ToolAnnotations } from "@modelcontextprotocol/sdk/types.js";
6
7
 
7
8
  import { getIsPlatform } from "../config/env-registry.js";
8
9
  import type { McpTransport } from "../config/schemas/mcp.js";
@@ -33,10 +34,18 @@ function normalizeInputSchema(
33
34
  return { type: "object", properties: {} };
34
35
  }
35
36
 
37
+ /**
38
+ * Behavioral hints a server may attach to a tool (MCP `ToolAnnotations`).
39
+ * Self-reported by the server, so consumers treat them as hints rather than
40
+ * guarantees.
41
+ */
42
+ export type McpToolAnnotations = ToolAnnotations;
43
+
36
44
  export interface McpToolInfo {
37
45
  name: string;
38
46
  description: string;
39
47
  inputSchema: Record<string, unknown>;
48
+ annotations?: McpToolAnnotations;
40
49
  }
41
50
 
42
51
  export interface McpCallResult {
@@ -197,6 +206,7 @@ export class McpClient {
197
206
  name: tool.name,
198
207
  description: tool.description ?? "",
199
208
  inputSchema: normalizeInputSchema(tool.inputSchema, tool.name),
209
+ annotations: tool.annotations,
200
210
  }));
201
211
  }
202
212
 
@@ -10,11 +10,14 @@ describe("channelForCallback", () => {
10
10
  expect(channelForCallback("http://gw/deliver/telegram")).toBe("telegram");
11
11
  expect(channelForCallback("http://gw/deliver/whatsapp")).toBe("whatsapp");
12
12
  expect(channelForCallback("http://gw/deliver/a2a?taskId=t1")).toBe("a2a");
13
+ expect(channelForCallback("http://gw/deliver/discord?threadId=T1")).toBe(
14
+ "discord",
15
+ );
13
16
  });
14
17
 
15
18
  test("returns undefined for channels not delivered directly", () => {
16
- expect(channelForCallback("http://gw/deliver/discord")).toBeUndefined();
17
19
  expect(channelForCallback("http://gw/deliver/phone")).toBeUndefined();
20
+ expect(channelForCallback("http://gw/deliver/email")).toBeUndefined();
18
21
  });
19
22
 
20
23
  test("returns undefined for non-delivery paths", () => {
@@ -41,5 +44,6 @@ describe("channelForCallback", () => {
41
44
  expect(channelForCallback("/deliver/slack")).toBe("slack");
42
45
  expect(channelForCallback("/deliver/telegram")).toBe("telegram");
43
46
  expect(channelForCallback("/deliver/whatsapp")).toBe("whatsapp");
47
+ expect(channelForCallback("/deliver/discord")).toBe("discord");
44
48
  });
45
49
  });