@vellumai/assistant 0.12.3-staging.1 → 0.12.3-staging.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (560) hide show
  1. package/ARCHITECTURE.md +43 -29
  2. package/Dockerfile +63 -0
  3. package/docs/architecture/security.md +10 -5
  4. package/docs/architecture/turn-actor.md +2 -0
  5. package/docs/desktop-browser-cli.md +45 -4
  6. package/docs/live-voice-task-outcomes.md +100 -0
  7. package/docs/mode-sessions.md +183 -0
  8. package/docs/voice-input-diagnostics.md +58 -0
  9. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  10. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  11. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  12. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  13. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  14. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  15. package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +20 -0
  16. package/node_modules/@vellumai/gateway-client/src/index.ts +4 -1
  17. package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +38 -7
  18. package/node_modules/@vellumai/gateway-client/src/trust-verdict-contract.ts +5 -6
  19. package/node_modules/@vellumai/service-contracts/package.json +2 -1
  20. package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  21. package/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  22. package/node_modules/@vellumai/slack-text/src/index.ts +4 -8
  23. package/openapi.yaml +665 -14
  24. package/package.json +1 -1
  25. package/scripts/smoke-desktop-browser-cli.ts +10 -26
  26. package/scripts/smoke-desktop-window-theme.ts +2 -3
  27. package/src/__tests__/agent-loop-output-hooks.test.ts +150 -1
  28. package/src/__tests__/agent-loop-provider-error-recording.test.ts +87 -19
  29. package/src/__tests__/annotate-activity-metadata.test.ts +3 -1
  30. package/src/__tests__/annotate-risk-options.test.ts +2 -0
  31. package/src/__tests__/answered-question-persistence.test.ts +2 -0
  32. package/src/__tests__/anthropic-gateway-shared.test.ts +22 -0
  33. package/src/__tests__/approval-message-composer.test.ts +1 -118
  34. package/src/__tests__/approval-routes-http.test.ts +3 -0
  35. package/src/__tests__/assistant-event-hub-connection-history.test.ts +111 -0
  36. package/src/__tests__/assistant-event-hub-targeted.test.ts +50 -0
  37. package/src/__tests__/assistant-event-targeting.test.ts +186 -0
  38. package/src/__tests__/assistant-stream-state.test.ts +29 -9
  39. package/src/__tests__/browser-fill-credential.test.ts +11 -22
  40. package/src/__tests__/btw-routes.test.ts +4 -1
  41. package/src/__tests__/build-persisted-content.test.ts +1 -1
  42. package/src/__tests__/call-conversation-messages.test.ts +5 -0
  43. package/src/__tests__/call-recovery.test.ts +7 -2
  44. package/src/__tests__/call-routes-http.test.ts +12 -0
  45. package/src/__tests__/call-store.test.ts +7 -2
  46. package/src/__tests__/channel-retry-sweep.test.ts +9 -0
  47. package/src/__tests__/chat-reveal-guard-priming.test.ts +2 -0
  48. package/src/__tests__/checker.test.ts +11 -3
  49. package/src/__tests__/client-os-metadata-persistence.test.ts +35 -1
  50. package/src/__tests__/computer-use-skill-manifest-regression.test.ts +0 -25
  51. package/src/__tests__/computer-use-tools.test.ts +1 -233
  52. package/src/__tests__/config-model-image-provider.test.ts +39 -0
  53. package/src/__tests__/config-schema.test.ts +16 -0
  54. package/src/__tests__/config-watcher.test.ts +24 -0
  55. package/src/__tests__/context-search-agent-runner.test.ts +260 -1
  56. package/src/__tests__/context-search-fanout.test.ts +1 -1
  57. package/src/__tests__/context-search-types.test.ts +7 -7
  58. package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +8 -2
  59. package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +2 -0
  60. package/src/__tests__/conversation-agent-loop.test.ts +546 -16
  61. package/src/__tests__/conversation-app-control-lifecycle.test.ts +53 -1
  62. package/src/__tests__/conversation-evictor.test.ts +62 -37
  63. package/src/__tests__/conversation-history-boundary-repair-failure.test.ts +174 -0
  64. package/src/__tests__/conversation-loop-db-lock-resilience.test.ts +2 -0
  65. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +2 -0
  66. package/src/__tests__/conversation-retry-route.test.ts +22 -0
  67. package/src/__tests__/conversation-routes-disk-view.test.ts +2 -0
  68. package/src/__tests__/conversation-routes-enabled-plugins.test.ts +2 -0
  69. package/src/__tests__/conversation-routes-guardian-reply.test.ts +10 -0
  70. package/src/__tests__/conversation-routes-hidden-queue.test.ts +2 -0
  71. package/src/__tests__/conversation-routes-slash-commands.test.ts +2 -0
  72. package/src/__tests__/conversation-runtime-assembly.test.ts +29 -0
  73. package/src/__tests__/conversation-surface-snapshots.test.ts +543 -0
  74. package/src/__tests__/conversation-surfaces-action-delivery.test.ts +151 -0
  75. package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +63 -1
  76. package/src/__tests__/conversation-tool-setup-attribution.test.ts +23 -0
  77. package/src/__tests__/conversation-undo.test.ts +61 -0
  78. package/src/__tests__/credential-execution-client.test.ts +6 -6
  79. package/src/__tests__/credential-prompt-route.test.ts +35 -3
  80. package/src/__tests__/cu-unified-flow.test.ts +82 -1
  81. package/src/__tests__/db-mode-sessions-migration.test.ts +175 -0
  82. package/src/__tests__/deterministic-verification-control-plane.test.ts +2 -0
  83. package/src/__tests__/dm-backfill.test.ts +91 -1
  84. package/src/__tests__/document-create-route.test.ts +88 -0
  85. package/src/__tests__/document-sync-tags.test.ts +27 -0
  86. package/src/__tests__/emit-signal-routing-intent.test.ts +142 -2
  87. package/src/__tests__/events-tail-route.test.ts +25 -0
  88. package/src/__tests__/evict-conversations-for-reload.test.ts +57 -8
  89. package/src/__tests__/guardian-action-sweep.test.ts +16 -0
  90. package/src/__tests__/guardian-dispatch.test.ts +13 -0
  91. package/src/__tests__/headless-browser-interactions.test.ts +64 -20
  92. package/src/__tests__/heartbeat-service.test.ts +2 -59
  93. package/src/__tests__/helpers/mock-conversation.ts +21 -0
  94. package/src/__tests__/helpers/native-web-search-harness.ts +2 -0
  95. package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +5 -0
  96. package/src/__tests__/host-cu-proxy.test.ts +46 -2
  97. package/src/__tests__/http-user-message-parity.test.ts +2 -0
  98. package/src/__tests__/image-credentials.test.ts +49 -0
  99. package/src/__tests__/image-service-dispatcher.test.ts +45 -0
  100. package/src/__tests__/inference-profile-session-handler.test.ts +1 -1
  101. package/src/__tests__/interrupt-on-send.test.ts +57 -43
  102. package/src/__tests__/interrupt-turn-note.test.ts +5 -6
  103. package/src/__tests__/llm-catalog-parity.test.ts +2 -2
  104. package/src/__tests__/media-generate-image.test.ts +49 -1
  105. package/src/__tests__/message-provenance-contact-id.test.ts +69 -0
  106. package/src/__tests__/migration-export-to-gcs.test.ts +142 -0
  107. package/src/__tests__/mode-session-history.test.ts +245 -0
  108. package/src/__tests__/mode-session-recovery.test.ts +222 -0
  109. package/src/__tests__/mode-session-wire.test.ts +141 -0
  110. package/src/__tests__/notification-guardian-path.test.ts +7 -0
  111. package/src/__tests__/notification-telegram-adapter.test.ts +2 -2
  112. package/src/__tests__/notification-vellum-adapter.test.ts +34 -0
  113. package/src/__tests__/oauth-provider-serializer.test.ts +12 -0
  114. package/src/__tests__/oauth-providers-routes.test.ts +1 -0
  115. package/src/__tests__/outbound-slack-persistence.test.ts +2 -0
  116. package/src/__tests__/persistence-secret-redaction.test.ts +2 -0
  117. package/src/__tests__/plugin-api-provider.test.ts +5 -0
  118. package/src/__tests__/plugin-api-resolve-credential.test.ts +97 -0
  119. package/src/__tests__/plugin-import-boundary-guard.test.ts +9 -0
  120. package/src/__tests__/process-message-display-content.test.ts +56 -0
  121. package/src/__tests__/reaction-persistence.test.ts +5 -0
  122. package/src/__tests__/request-secret-standalone-channel-gate.test.ts +25 -9
  123. package/src/__tests__/runtime-events-sse-reconnect.test.ts +1 -1
  124. package/src/__tests__/scripted-turn-metadata-persistence.test.ts +73 -0
  125. package/src/__tests__/secret-ingress-http.test.ts +2 -0
  126. package/src/__tests__/secure-keys.test.ts +43 -1
  127. package/src/__tests__/send-endpoint-busy.test.ts +163 -1
  128. package/src/__tests__/server-history-render.test.ts +91 -1
  129. package/src/__tests__/settings-routes.test.ts +50 -1
  130. package/src/__tests__/skill-script-runner-sandbox.test.ts +32 -0
  131. package/src/__tests__/subagent-fork-notifications.test.ts +1 -2
  132. package/src/__tests__/subagent-manager-notify.test.ts +1 -2
  133. package/src/__tests__/subagent-notify-parent.test.ts +146 -39
  134. package/src/__tests__/subagent-notify-stale-rebuild.test.ts +2 -4
  135. package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +2 -0
  136. package/src/__tests__/thread-backfill.test.ts +53 -0
  137. package/src/__tests__/tool-preview-lifecycle.test.ts +2 -0
  138. package/src/__tests__/tool-result-metadata-plumbing.test.ts +3 -1
  139. package/src/__tests__/tool-start-timestamp.test.ts +2 -0
  140. package/src/__tests__/twilio-routes.test.ts +1 -0
  141. package/src/__tests__/web-search-backend-failure.test.ts +1 -1
  142. package/src/activation/progress-store.test.ts +12 -7
  143. package/src/agent/loop.ts +99 -41
  144. package/src/api/events/assistant-turn-start.ts +3 -0
  145. package/src/api/events/generation-handoff.ts +2 -0
  146. package/src/api/events/message-complete.ts +2 -0
  147. package/src/api/events/notification-conversation-created.ts +14 -16
  148. package/src/api/events/notification-intent.ts +4 -2
  149. package/src/api/events/tool-result.ts +85 -0
  150. package/src/api/events/tool-use-start.ts +3 -0
  151. package/src/api/events/user-message-echo.ts +3 -0
  152. package/src/api/index.ts +31 -6
  153. package/src/api/mode-session.test.ts +131 -0
  154. package/src/api/mode-session.ts +148 -0
  155. package/src/api/remember-tool.ts +15 -0
  156. package/src/api/responses/conversation-message.ts +6 -0
  157. package/src/approvals/AGENTS.md +4 -2
  158. package/src/browser/__tests__/operations.test.ts +29 -0
  159. package/src/browser/operation-meta.ts +2 -1
  160. package/src/browser/operations.ts +29 -0
  161. package/src/browser/virtual-desktop-target.ts +2 -8
  162. package/src/bundler/bundle-signer.ts +1 -3
  163. package/src/calls/__tests__/barge-in-guard.test.ts +3 -0
  164. package/src/calls/__tests__/call-controller.test.ts +94 -0
  165. package/src/calls/__tests__/call-funnel-telemetry.test.ts +204 -0
  166. package/src/calls/__tests__/voice-control-protocol.test.ts +15 -0
  167. package/src/calls/__tests__/voice-escalation-judge.test.ts +209 -0
  168. package/src/calls/__tests__/voice-leg-coordinator.test.ts +168 -0
  169. package/src/{live-voice/__tests__/live-voice-metrics.test.ts → calls/__tests__/voice-metrics.test.ts} +32 -38
  170. package/src/calls/__tests__/voice-session-bridge.test.ts +576 -122
  171. package/src/calls/barge-in-guard.ts +5 -0
  172. package/src/calls/call-controller.ts +196 -1
  173. package/src/calls/call-domain.ts +4 -0
  174. package/src/calls/call-store.ts +87 -2
  175. package/src/calls/guardian-action-sweep.ts +22 -8
  176. package/src/calls/media-stream-audio-transcode.ts +1 -3
  177. package/src/calls/typesafe-noul.ts +156 -0
  178. package/src/calls/voice-control-protocol.ts +6 -0
  179. package/src/calls/voice-escalation-judge.ts +145 -0
  180. package/src/calls/voice-escalation-target.ts +10 -0
  181. package/src/calls/voice-leg-coordinator.ts +121 -3
  182. package/src/{live-voice/live-voice-metrics.ts → calls/voice-metrics.ts} +94 -87
  183. package/src/calls/voice-session-bridge.ts +256 -112
  184. package/src/calls/voice-triage-escalate.ts +7 -0
  185. package/src/cli/__tests__/roadmap-cli.test.ts +4 -41
  186. package/src/cli/commands/__tests__/notifications.test.ts +35 -10
  187. package/src/cli/commands/browser.help.ts +4 -5
  188. package/src/cli/commands/browser.ts +0 -1
  189. package/src/cli/commands/clients.help.ts +46 -0
  190. package/src/cli/commands/clients.ts +116 -0
  191. package/src/cli/commands/credentials.help.ts +6 -1
  192. package/src/cli/commands/notifications.help.ts +16 -2
  193. package/src/cli/commands/notifications.ts +30 -8
  194. package/src/cli/commands/roadmap.help.ts +18 -21
  195. package/src/cli/commands/roadmap.ts +4 -21
  196. package/src/cli/lib/__tests__/list-installed-plugins.test.ts +5 -2
  197. package/src/cli/lib/bundled-marketplace.json +84 -34
  198. package/src/cli/lib/plugin-marketplace.ts +8 -0
  199. package/src/cli/lib/publish-plugin.ts +1 -2
  200. package/src/config/__tests__/default-provider.test.ts +3 -1
  201. package/src/config/__tests__/memory-retrospective-schema.test.ts +14 -1
  202. package/src/config/bundled-skills/computer-use/SKILL.md +1 -1
  203. package/src/config/bundled-skills/computer-use/TOOLS.json +89 -1
  204. package/src/config/bundled-skills/image-studio/SKILL.md +17 -2
  205. package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +7 -11
  206. package/src/config/bundled-skills/sequences/TOOLS.json +1 -5
  207. package/src/config/call-site-defaults.ts +9 -0
  208. package/src/config/feature-flag-registry.json +16 -0
  209. package/src/config/profile-text-generation.test.ts +2 -2
  210. package/src/config/schemas/__tests__/tools.test.ts +31 -0
  211. package/src/config/schemas/call-site-catalog.ts +14 -0
  212. package/src/config/schemas/channels.ts +0 -18
  213. package/src/config/schemas/llm.ts +3 -1
  214. package/src/config/schemas/mcp.ts +2 -1
  215. package/src/config/schemas/memory-retrospective.ts +11 -2
  216. package/src/config/schemas/services.ts +15 -6
  217. package/src/config/schemas/tools.ts +10 -0
  218. package/src/config/session-groups-gate.ts +21 -0
  219. package/src/config/skills.ts +2 -6
  220. package/src/conversations/__tests__/message-consolidation.test.ts +149 -0
  221. package/src/conversations/message-consolidation.ts +102 -5
  222. package/src/credential-execution/ces-connect.test.ts +69 -0
  223. package/src/credential-execution/ces-connect.ts +115 -0
  224. package/src/credential-execution/ces-runtime.ts +55 -93
  225. package/src/credential-execution/executable-discovery.ts +1 -11
  226. package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +10 -1
  227. package/src/daemon/__tests__/inference-profile-notification.test.ts +2 -0
  228. package/src/daemon/browser-mode-session.test.ts +402 -0
  229. package/src/daemon/browser-mode-session.ts +186 -0
  230. package/src/daemon/computer-use-mode-session.test.ts +210 -0
  231. package/src/daemon/computer-use-mode-session.ts +105 -0
  232. package/src/daemon/config-watcher.ts +10 -0
  233. package/src/daemon/conversation-agent-loop-handlers.ts +58 -10
  234. package/src/daemon/conversation-agent-loop.ts +219 -25
  235. package/src/daemon/conversation-evictor.ts +5 -24
  236. package/src/daemon/conversation-history.ts +10 -0
  237. package/src/daemon/conversation-interrupt-repair.ts +4 -43
  238. package/src/daemon/conversation-interrupt.ts +7 -11
  239. package/src/daemon/conversation-messaging.ts +58 -5
  240. package/src/daemon/conversation-mode-session.test.ts +1667 -0
  241. package/src/daemon/conversation-mode-session.ts +1008 -0
  242. package/src/daemon/conversation-process.ts +5 -0
  243. package/src/daemon/conversation-runtime-assembly.ts +10 -20
  244. package/src/daemon/conversation-skill-tools.ts +1 -0
  245. package/src/daemon/conversation-store.ts +4 -4
  246. package/src/daemon/conversation-surface-snapshots.ts +135 -0
  247. package/src/daemon/conversation-surfaces.ts +113 -5
  248. package/src/daemon/conversation-tool-setup.ts +8 -3
  249. package/src/daemon/conversation.ts +143 -41
  250. package/src/daemon/handlers/config-model.ts +7 -5
  251. package/src/daemon/handlers/shared.ts +55 -16
  252. package/src/daemon/host-cu-proxy.ts +51 -22
  253. package/src/daemon/lifecycle.ts +30 -13
  254. package/src/daemon/message-protocol.ts +0 -1
  255. package/src/daemon/message-provenance.ts +28 -4
  256. package/src/daemon/mode-session-startup-recovery.ts +23 -0
  257. package/src/daemon/mode-session-tracking.ts +54 -0
  258. package/src/daemon/process-message.ts +13 -4
  259. package/src/daemon/virtual-desktop-context.ts +21 -0
  260. package/src/desktop/__tests__/fake-desktop.ts +1 -0
  261. package/src/desktop/desktop-accessibility-script.ts +88 -0
  262. package/src/desktop/desktop-accessibility.test.ts +160 -0
  263. package/src/desktop/desktop-accessibility.ts +169 -0
  264. package/src/desktop/desktop-automation-lease.test.ts +76 -68
  265. package/src/desktop/desktop-automation-lease.ts +41 -50
  266. package/src/desktop/desktop-browser-endpoint.ts +4 -0
  267. package/src/desktop/desktop-computer-use-routing.test.ts +330 -0
  268. package/src/desktop/desktop-computer-use.test.ts +449 -0
  269. package/src/desktop/desktop-computer-use.ts +502 -0
  270. package/src/desktop/desktop-dependencies.ts +47 -293
  271. package/src/desktop/desktop-display.ts +2 -2
  272. package/src/desktop/desktop-panel-config.test.ts +13 -3
  273. package/src/desktop/desktop-panel-config.ts +109 -15
  274. package/src/desktop/desktop-session-manager.test.ts +69 -21
  275. package/src/desktop/desktop-session-manager.ts +76 -22
  276. package/src/desktop/desktop-stream-bridge.test.ts +11 -122
  277. package/src/desktop/desktop-stream-bridge.ts +6 -8
  278. package/src/desktop/desktop-window-theme.ts +5 -25
  279. package/src/desktop/virtual-desktop-feature.ts +52 -2
  280. package/src/desktop/virtual-desktop-platform.test.ts +5 -9
  281. package/src/documents/document-store.ts +23 -0
  282. package/src/heartbeat/__tests__/heartbeat-service.test.ts +22 -26
  283. package/src/heartbeat/heartbeat-service.ts +5 -33
  284. package/src/ipc/__tests__/browser-ipc.test.ts +215 -6
  285. package/src/ipc/assistant-server.ts +2 -0
  286. package/src/ipc/routes/__tests__/channel-reply-ipc-routes.test.ts +111 -0
  287. package/src/ipc/routes/channel-reply-ipc-routes.ts +73 -0
  288. package/src/live-voice/__tests__/continuation-judge.test.ts +151 -0
  289. package/src/live-voice/__tests__/live-activity-reporter.test.ts +154 -15
  290. package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +743 -31
  291. package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +191 -10
  292. package/src/live-voice/__tests__/live-voice-session-groups-gate.test.ts +87 -0
  293. package/src/live-voice/__tests__/live-voice-session-manager.test.ts +5 -1
  294. package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +41 -0
  295. package/src/live-voice/__tests__/live-voice-sight-frame.test.ts +1017 -9
  296. package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +224 -1
  297. package/src/live-voice/__tests__/live-voice-tts-session.test.ts +101 -0
  298. package/src/live-voice/__tests__/live-voice-tts.test.ts +44 -0
  299. package/src/live-voice/__tests__/live-voice-vad.test.ts +939 -75
  300. package/src/live-voice/__tests__/microphone-idle-gate.test.ts +87 -0
  301. package/src/live-voice/__tests__/session-controls.test.ts +23 -0
  302. package/src/live-voice/__tests__/task-outcomes.test.ts +117 -0
  303. package/src/live-voice/__tests__/voice-input-diagnostics.test.ts +117 -0
  304. package/src/live-voice/camera-mode-session.test.ts +265 -0
  305. package/src/live-voice/camera-mode-session.ts +277 -0
  306. package/src/live-voice/continuation-judge.ts +123 -0
  307. package/src/live-voice/live-activity-reporter.ts +115 -6
  308. package/src/live-voice/live-voice-manager.ts +36 -1
  309. package/src/live-voice/live-voice-photo.ts +88 -7
  310. package/src/live-voice/live-voice-session-manager.ts +15 -0
  311. package/src/live-voice/live-voice-session.ts +1443 -747
  312. package/src/live-voice/live-voice-tts.ts +22 -4
  313. package/src/live-voice/microphone-idle-gate.ts +56 -0
  314. package/src/live-voice/protocol.ts +171 -4
  315. package/src/live-voice/session-controls.ts +11 -3
  316. package/src/live-voice/task-outcomes.ts +131 -0
  317. package/src/live-voice/voice-input-diagnostics.ts +117 -0
  318. package/src/mcp/__tests__/manager-tool-caps.test.ts +63 -18
  319. package/src/mcp/__tests__/tool-caps.test.ts +28 -1
  320. package/src/mcp/manager.ts +3 -0
  321. package/src/mcp/tool-caps.ts +12 -4
  322. package/src/media/__tests__/image-models.test.ts +58 -0
  323. package/src/media/image-credentials.ts +5 -0
  324. package/src/media/image-models.ts +52 -0
  325. package/src/media/image-service.ts +8 -0
  326. package/src/media/openrouter-image-service.test.ts +133 -0
  327. package/src/media/openrouter-image-service.ts +153 -0
  328. package/src/media/types.ts +1 -1
  329. package/src/messaging/providers/__tests__/transport-dispatch.test.ts +65 -11
  330. package/src/messaging/providers/discord/send.ts +4 -17
  331. package/src/messaging/providers/discord/transport.ts +2 -2
  332. package/src/messaging/providers/outlook/client.ts +16 -22
  333. package/src/messaging/providers/telegram-bot/send.test.ts +236 -1
  334. package/src/messaging/providers/telegram-bot/send.ts +10 -7
  335. package/src/messaging/providers/telegram-bot/transport.ts +3 -10
  336. package/src/messaging/providers/whatsapp/send.test.ts +169 -4
  337. package/src/monitoring/__tests__/control.test.ts +17 -5
  338. package/src/monitoring/control.ts +15 -5
  339. package/src/notifications/AGENTS.md +1 -1
  340. package/src/notifications/README.md +83 -99
  341. package/src/notifications/__tests__/broadcaster.test.ts +1 -1
  342. package/src/notifications/__tests__/channel-allowlist.test.ts +39 -0
  343. package/src/notifications/__tests__/copy-composer.test.ts +1 -1
  344. package/src/notifications/__tests__/decision-engine.test.ts +79 -2
  345. package/src/notifications/__tests__/deterministic-checks.test.ts +10 -10
  346. package/src/notifications/__tests__/home-feed-side-effect.test.ts +93 -4
  347. package/src/notifications/__tests__/notification-utils.test.ts +18 -0
  348. package/src/notifications/adapters/macos.ts +37 -46
  349. package/src/notifications/broadcaster.ts +5 -6
  350. package/src/notifications/channel-allowlist.ts +44 -0
  351. package/src/notifications/copy-composer.ts +0 -23
  352. package/src/notifications/decision-engine.ts +24 -14
  353. package/src/notifications/deterministic-checks.ts +1 -1
  354. package/src/notifications/emit-signal.ts +123 -37
  355. package/src/notifications/home-feed-side-effect.ts +9 -7
  356. package/src/notifications/notification-utils.ts +19 -0
  357. package/src/notifications/signal.ts +4 -19
  358. package/src/notifications/types.ts +15 -0
  359. package/src/oauth/connection-resolver.ts +1 -2
  360. package/src/oauth/oauth-store.ts +1 -3
  361. package/src/oauth/provider-serializer.ts +7 -0
  362. package/src/oauth/seed-providers.ts +30 -0
  363. package/src/onboarding/onboarding-events-store.ts +99 -12
  364. package/src/persistence/__tests__/client-connection-events-store.test.ts +322 -0
  365. package/src/persistence/client-connection-events-store.ts +377 -0
  366. package/src/persistence/conversation-crud.ts +55 -6
  367. package/src/persistence/conversation-mode-sessions.test.ts +424 -0
  368. package/src/persistence/conversation-mode-sessions.ts +460 -0
  369. package/src/persistence/conversation-queries.ts +6 -19
  370. package/src/persistence/llm-usage-store.ts +3 -9
  371. package/src/persistence/message-metadata.ts +25 -0
  372. package/src/persistence/migrations/381-create-conversation-mode-sessions.ts +41 -0
  373. package/src/persistence/migrations/382-create-client-connection-events.ts +40 -0
  374. package/src/persistence/migrations/__tests__/382-create-client-connection-events.test.ts +56 -0
  375. package/src/persistence/schema/conversations.ts +67 -0
  376. package/src/persistence/schema/infrastructure.ts +29 -0
  377. package/src/persistence/steps.ts +17 -0
  378. package/src/plugin-api/__tests__/plugin-name-env.test.ts +249 -0
  379. package/src/plugin-api/effective-context-window.ts +33 -0
  380. package/src/plugin-api/index.ts +24 -5
  381. package/src/plugin-api/plugin-name-env.ts +125 -0
  382. package/src/plugin-api/resolve-credential.ts +79 -20
  383. package/src/plugin-api/vision-support.test.ts +22 -0
  384. package/src/plugin-api/vision-support.ts +17 -5
  385. package/src/plugins/__tests__/active-plugin-env.test.ts +136 -0
  386. package/src/plugins/active-plugin-env.ts +51 -0
  387. package/src/plugins/defaults/index.ts +24 -0
  388. package/src/plugins/defaults/memory/__tests__/archive-removal-framing.test.ts +64 -0
  389. package/src/plugins/defaults/memory/__tests__/worker-control.test.ts +19 -6
  390. package/src/plugins/defaults/memory/context-search/agent-protocol.ts +9 -12
  391. package/src/plugins/defaults/memory/context-search/agent-runner.ts +78 -22
  392. package/src/plugins/defaults/memory/context-search/format.ts +51 -7
  393. package/src/plugins/defaults/memory/context-search/limits.ts +10 -12
  394. package/src/plugins/defaults/memory/context-search/recall-input.ts +38 -0
  395. package/src/plugins/defaults/memory/context-search/search.ts +8 -14
  396. package/src/plugins/defaults/memory/context-search/sources/conversations.ts +1 -0
  397. package/src/plugins/defaults/memory/context-search/types.ts +1 -11
  398. package/src/plugins/defaults/memory/graph/__tests__/handle-remember-v2.test.ts +11 -4
  399. package/src/plugins/defaults/memory/graph/tool-handlers.ts +12 -21
  400. package/src/plugins/defaults/memory/graph/tools.ts +35 -64
  401. package/src/plugins/defaults/memory/src/memory-eval-routes.ts +2 -3
  402. package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +5 -0
  403. package/src/plugins/defaults/memory/src/memory-item-routes.ts +2 -1
  404. package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +2 -2
  405. package/src/plugins/defaults/memory/tools.test.ts +124 -9
  406. package/src/plugins/defaults/memory/tools.ts +38 -11
  407. package/src/plugins/defaults/memory/v3/__tests__/pool-log-store.test.ts +165 -27
  408. package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +626 -27
  409. package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +46 -27
  410. package/src/plugins/defaults/memory/v3/orchestrate.ts +78 -38
  411. package/src/plugins/defaults/memory/v3/pool-log-store.ts +65 -79
  412. package/src/plugins/defaults/memory/v3/pool-select.test.ts +14 -5
  413. package/src/plugins/defaults/memory/v3/pool-select.ts +683 -44
  414. package/src/plugins/defaults/memory/worker-control.ts +15 -5
  415. package/src/plugins/defaults/task-progress/__tests__/registration.test.ts +70 -0
  416. package/src/plugins/defaults/task-progress/__tests__/task-progress-context.test.ts +380 -0
  417. package/src/plugins/defaults/task-progress/hooks/post-compact.ts +19 -0
  418. package/src/plugins/defaults/task-progress/hooks/user-prompt-submit.ts +24 -0
  419. package/src/plugins/defaults/task-progress/package.json +14 -0
  420. package/src/plugins/defaults/task-progress/src/apply-task-progress-context.ts +93 -0
  421. package/src/plugins/defaults/task-progress/src/format-task-progress-context.ts +67 -0
  422. package/src/providers/__tests__/context-overflow-error.test.ts +10 -0
  423. package/src/providers/__tests__/search-provider-catalog.test.ts +18 -0
  424. package/src/providers/anthropic/client.ts +10 -1
  425. package/src/providers/anthropic-gateway-shared.ts +6 -0
  426. package/src/providers/fetch-provider-catalog.ts +11 -0
  427. package/src/providers/gemini/client.ts +25 -8
  428. package/src/providers/inference/adapter-factory.ts +1 -1
  429. package/src/providers/jev/client.test.ts +10 -0
  430. package/src/providers/jev/client.ts +20 -4
  431. package/src/providers/model-catalog.ts +4 -3
  432. package/src/providers/openai/__tests__/chat-completions-extra-body.test.ts +94 -1
  433. package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +106 -1
  434. package/src/providers/openai/__tests__/chat-template-flatten.test.ts +24 -0
  435. package/src/providers/openai/__tests__/google-thought-signature.test.ts +1 -2
  436. package/src/providers/openai/chat-completions-provider.ts +100 -29
  437. package/src/providers/openai/google-thought-signature.ts +0 -6
  438. package/src/providers/openai/responses-provider.ts +23 -1
  439. package/src/providers/ratelimit.ts +1 -3
  440. package/src/providers/search-provider-catalog.ts +24 -0
  441. package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +24 -0
  442. package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +7 -1
  443. package/src/providers/speech-to-text/__tests__/vellum-managed-realtime.test.ts +50 -1
  444. package/src/providers/speech-to-text/deepgram-flux-frames.ts +11 -0
  445. package/src/providers/speech-to-text/vellum-managed-flux-realtime.ts +13 -0
  446. package/src/providers/types.ts +5 -0
  447. package/src/routes/control.ts +7 -5
  448. package/src/runtime/AGENTS.md +4 -2
  449. package/src/runtime/actor-trust-resolver.ts +3 -3
  450. package/src/runtime/agent-wake.ts +1 -1
  451. package/src/runtime/approval-message-composer.ts +3 -69
  452. package/src/runtime/assistant-event-hub.ts +73 -75
  453. package/src/runtime/assistant-event-publish-options.ts +7 -0
  454. package/src/runtime/assistant-event-targeting.ts +79 -0
  455. package/src/runtime/assistant-stream-state.ts +13 -100
  456. package/src/runtime/background-job-runner.ts +5 -5
  457. package/src/runtime/channel-retry-sweep.ts +1 -0
  458. package/src/runtime/guardian-action-message-composer.ts +8 -161
  459. package/src/runtime/guardian-reply-router.ts +28 -32
  460. package/src/runtime/http-types.ts +6 -0
  461. package/src/runtime/message-composer-types.ts +3 -61
  462. package/src/runtime/migrations/__tests__/vbundle-import-parity.test.ts +67 -1
  463. package/src/runtime/migrations/vbundle-builder.ts +24 -1
  464. package/src/runtime/migrations/vbundle-import-analyzer.ts +13 -4
  465. package/src/runtime/migrations/vbundle-import-policy.ts +10 -0
  466. package/src/runtime/migrations/vbundle-importer.ts +4 -1
  467. package/src/runtime/migrations/vbundle-streaming-importer.ts +4 -1
  468. package/src/runtime/migrations/vbundle-validator.ts +1 -0
  469. package/src/runtime/pending-interactions-mode-session.test.ts +62 -0
  470. package/src/runtime/pending-interactions.ts +29 -0
  471. package/src/runtime/routes/__tests__/client-history-routes.test.ts +104 -0
  472. package/src/runtime/routes/__tests__/image-generation-routes.test.ts +97 -0
  473. package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +1 -1
  474. package/src/runtime/routes/__tests__/roadmap-routes.test.ts +29 -6
  475. package/src/runtime/routes/browser-context.ts +31 -11
  476. package/src/runtime/routes/browser-routes.ts +43 -4
  477. package/src/runtime/routes/btw-routes.ts +9 -4
  478. package/src/runtime/routes/client-routes.ts +108 -0
  479. package/src/runtime/routes/conversation-management-routes.ts +4 -1
  480. package/src/runtime/routes/conversation-routes.ts +113 -27
  481. package/src/runtime/routes/credential-prompt-routes.ts +4 -3
  482. package/src/runtime/routes/credential-routes.ts +1 -2
  483. package/src/runtime/routes/desktop-setup-routes.test.ts +9 -18
  484. package/src/runtime/routes/desktop-setup-routes.ts +4 -6
  485. package/src/runtime/routes/documents-routes.ts +51 -0
  486. package/src/runtime/routes/email-routes.ts +3 -7
  487. package/src/runtime/routes/events-routes.ts +75 -54
  488. package/src/runtime/routes/identity-routes.ts +1 -5
  489. package/src/runtime/routes/image-generation-routes.ts +8 -14
  490. package/src/runtime/routes/inbound-message-handler.ts +35 -38
  491. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.test.ts +168 -0
  492. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.ts +96 -0
  493. package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +38 -9
  494. package/src/runtime/routes/inbound-stages/background-dispatch.ts +2 -5
  495. package/src/runtime/routes/inbound-stages/reaction-intercept.ts +2 -0
  496. package/src/runtime/routes/llm-context-normalization.ts +1 -7
  497. package/src/runtime/routes/migration-routes.ts +73 -2
  498. package/src/runtime/routes/notification-routes.ts +10 -0
  499. package/src/runtime/routes/oauth-providers.ts +7 -0
  500. package/src/runtime/routes/plugins-routes.ts +6 -0
  501. package/src/runtime/routes/roadmap-routes.ts +47 -8
  502. package/src/runtime/routes/secret-routes.ts +1 -1
  503. package/src/runtime/routes/settings-routes.ts +10 -4
  504. package/src/runtime/routes/watcher-routes.ts +1 -3
  505. package/src/runtime/trust-class.test.ts +7 -6
  506. package/src/runtime/trust-class.ts +9 -17
  507. package/src/schedule/__tests__/worker-control.test.ts +36 -5
  508. package/src/schedule/worker-control.ts +15 -5
  509. package/src/security/secure-keys.ts +53 -103
  510. package/src/skills/clawhub.ts +3 -8
  511. package/src/skills/managed-store.ts +1 -5
  512. package/src/stt/types.ts +4 -0
  513. package/src/subagent/notify.ts +54 -34
  514. package/src/subagent/parent-notification.ts +6 -0
  515. package/src/telemetry/__tests__/phone-call-funnel.test.ts +72 -0
  516. package/src/telemetry/phone-call-funnel.ts +144 -0
  517. package/src/tools/__tests__/tool-schema-root-combinator-guard.test.ts +81 -0
  518. package/src/tools/ask-question/ask-question-tool.ts +6 -4
  519. package/src/tools/browser/browser-execution.ts +22 -59
  520. package/src/tools/browser/cdp-client/__tests__/cdp-dom-helpers.test.ts +561 -39
  521. package/src/tools/browser/cdp-client/cdp-dom-helpers.ts +435 -39
  522. package/src/tools/browser/cdp-client/cdp-inspect-client.ts +1 -8
  523. package/src/tools/client-os.ts +15 -0
  524. package/src/tools/computer-use/skill-proxy-bridge.ts +1 -1
  525. package/src/tools/computer-use/target.ts +42 -0
  526. package/src/tools/document/document-tool.ts +13 -18
  527. package/src/tools/execution-target.ts +14 -16
  528. package/src/tools/host-terminal/host-shell.ts +4 -1
  529. package/src/tools/network/__tests__/web-fetch-tinyfish.test.ts +250 -0
  530. package/src/tools/network/__tests__/web-search.test.ts +203 -0
  531. package/src/tools/network/web-fetch.ts +274 -7
  532. package/src/tools/network/web-search.ts +468 -6
  533. package/src/tools/permission-checker.ts +3 -3
  534. package/src/tools/policy-context.ts +3 -1
  535. package/src/tools/skills/load.ts +2 -1
  536. package/src/tools/skills/sandbox-runner.ts +11 -2
  537. package/src/tools/skills/skill-script-runner.ts +7 -0
  538. package/src/tools/skills/skill-tool-factory.ts +15 -1
  539. package/src/tools/terminal/__tests__/safe-env.test.ts +11 -0
  540. package/src/tools/terminal/safe-env.ts +4 -0
  541. package/src/tools/terminal/shell.ts +2 -0
  542. package/src/tools/tool-approval-handler.ts +5 -1
  543. package/src/tools/tool-types.ts +3 -3
  544. package/src/tools/types.ts +8 -2
  545. package/src/util/__tests__/worker-process-command.test.ts +5 -4
  546. package/src/util/abort-reasons.ts +18 -26
  547. package/src/util/canonicalize-identity.test.ts +21 -0
  548. package/src/util/canonicalize-identity.ts +16 -0
  549. package/src/util/errors.ts +13 -3
  550. package/src/util/provider-error-patterns.ts +3 -0
  551. package/src/util/worker-process.ts +77 -35
  552. package/src/workspace/migrations/157-add-desktop-files-launcher.ts +94 -0
  553. package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +4 -10
  554. package/src/workspace/migrations/__tests__/157-add-desktop-files-launcher.test.ts +113 -0
  555. package/src/workspace/migrations/registry.ts +2 -0
  556. package/src/__tests__/computer-use-window-schema.test.ts +0 -24
  557. package/src/api/events/heartbeat-alert.ts +0 -20
  558. package/src/daemon/message-types/web-activity.ts +0 -65
  559. package/src/desktop/desktop-dependencies.test.ts +0 -131
  560. package/src/tools/computer-use/definitions.ts +0 -607
@@ -21,11 +21,16 @@
21
21
  },
22
22
  "full_tree": {
23
23
  "type": "boolean",
24
- "description": "Walk the accessibility tree to full depth. Observations list the tree to a limited depth and say when it was cut off; pass this when the element you need is not in the tree."
24
+ "description": "On connected personal computers, walk the accessibility tree to full depth. The virtual desktop uses a bounded tree; use screenshot coordinates when an element is not listed."
25
25
  },
26
26
  "target_client_id": {
27
27
  "type": "string",
28
28
  "description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
29
+ },
30
+ "target": {
31
+ "type": "string",
32
+ "enum": ["connected-computer", "assistant-desktop"],
33
+ "description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
29
34
  }
30
35
  },
31
36
  "required": []
@@ -65,6 +70,15 @@
65
70
  "target_client_id": {
66
71
  "type": "string",
67
72
  "description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
73
+ },
74
+ "target": {
75
+ "type": "string",
76
+ "enum": ["connected-computer", "assistant-desktop"],
77
+ "description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
78
+ },
79
+ "observation_id": {
80
+ "type": "string",
81
+ "description": "Required on assistant-desktop: the ID from its latest screenshot. Observe again after browser actions, user handoff, or interruption. Omit for connected-computer. For a sequence, applies to the whole batch."
68
82
  }
69
83
  },
70
84
  "required": ["reasoning"]
@@ -91,6 +105,15 @@
91
105
  "target_client_id": {
92
106
  "type": "string",
93
107
  "description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
108
+ },
109
+ "target": {
110
+ "type": "string",
111
+ "enum": ["connected-computer", "assistant-desktop"],
112
+ "description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
113
+ },
114
+ "observation_id": {
115
+ "type": "string",
116
+ "description": "Required on assistant-desktop: the ID from its latest screenshot. Observe again after browser actions, user handoff, or interruption. Omit for connected-computer. For a sequence, applies to the whole batch."
94
117
  }
95
118
  },
96
119
  "required": ["text", "reasoning"]
@@ -117,6 +140,15 @@
117
140
  "target_client_id": {
118
141
  "type": "string",
119
142
  "description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
143
+ },
144
+ "target": {
145
+ "type": "string",
146
+ "enum": ["connected-computer", "assistant-desktop"],
147
+ "description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
148
+ },
149
+ "observation_id": {
150
+ "type": "string",
151
+ "description": "Required on assistant-desktop: the ID from its latest screenshot. Observe again after browser actions, user handoff, or interruption. Omit for connected-computer. For a sequence, applies to the whole batch."
120
152
  }
121
153
  },
122
154
  "required": ["key", "reasoning"]
@@ -160,6 +192,15 @@
160
192
  "target_client_id": {
161
193
  "type": "string",
162
194
  "description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
195
+ },
196
+ "target": {
197
+ "type": "string",
198
+ "enum": ["connected-computer", "assistant-desktop"],
199
+ "description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
200
+ },
201
+ "observation_id": {
202
+ "type": "string",
203
+ "description": "Required on assistant-desktop: the ID from its latest screenshot. Observe again after browser actions, user handoff, or interruption. Omit for connected-computer. For a sequence, applies to the whole batch."
163
204
  }
164
205
  },
165
206
  "required": ["direction", "amount", "reasoning"]
@@ -206,6 +247,15 @@
206
247
  "target_client_id": {
207
248
  "type": "string",
208
249
  "description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
250
+ },
251
+ "target": {
252
+ "type": "string",
253
+ "enum": ["connected-computer", "assistant-desktop"],
254
+ "description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
255
+ },
256
+ "observation_id": {
257
+ "type": "string",
258
+ "description": "Required on assistant-desktop: the ID from its latest screenshot. Observe again after browser actions, user handoff, or interruption. Omit for connected-computer. For a sequence, applies to the whole batch."
209
259
  }
210
260
  },
211
261
  "required": ["reasoning"]
@@ -233,6 +283,15 @@
233
283
  "target_client_id": {
234
284
  "type": "string",
235
285
  "description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
286
+ },
287
+ "target": {
288
+ "type": "string",
289
+ "enum": ["connected-computer", "assistant-desktop"],
290
+ "description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
291
+ },
292
+ "observation_id": {
293
+ "type": "string",
294
+ "description": "Required on assistant-desktop: the ID from its latest screenshot. Observe again after browser actions, user handoff, or interruption. Omit for connected-computer. For a sequence, applies to the whole batch."
236
295
  }
237
296
  },
238
297
  "required": ["duration_ms", "reasoning"]
@@ -259,6 +318,11 @@
259
318
  "target_client_id": {
260
319
  "type": "string",
261
320
  "description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
321
+ },
322
+ "target": {
323
+ "type": "string",
324
+ "enum": ["connected-computer", "assistant-desktop"],
325
+ "description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
262
326
  }
263
327
  },
264
328
  "required": ["app_name", "reasoning"]
@@ -286,6 +350,11 @@
286
350
  "target_client_id": {
287
351
  "type": "string",
288
352
  "description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
353
+ },
354
+ "target": {
355
+ "type": "string",
356
+ "enum": ["connected-computer", "assistant-desktop"],
357
+ "description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
289
358
  }
290
359
  },
291
360
  "required": ["script", "reasoning"]
@@ -377,6 +446,15 @@
377
446
  "target_client_id": {
378
447
  "type": "string",
379
448
  "description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
449
+ },
450
+ "target": {
451
+ "type": "string",
452
+ "enum": ["connected-computer", "assistant-desktop"],
453
+ "description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
454
+ },
455
+ "observation_id": {
456
+ "type": "string",
457
+ "description": "Required on assistant-desktop: the ID from its latest screenshot. Observe again after browser actions, user handoff, or interruption. Omit for connected-computer. For a sequence, applies to the whole batch."
380
458
  }
381
459
  },
382
460
  "required": ["actions"]
@@ -396,6 +474,11 @@
396
474
  "summary": {
397
475
  "type": "string",
398
476
  "description": "Human-readable summary of what was accomplished"
477
+ },
478
+ "target": {
479
+ "type": "string",
480
+ "enum": ["connected-computer", "assistant-desktop"],
481
+ "description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
399
482
  }
400
483
  },
401
484
  "required": ["summary"]
@@ -418,6 +501,11 @@
418
501
  "reasoning": {
419
502
  "type": "string",
420
503
  "description": "Explanation of how you determined the answer"
504
+ },
505
+ "target": {
506
+ "type": "string",
507
+ "enum": ["connected-computer", "assistant-desktop"],
508
+ "description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
421
509
  }
422
510
  },
423
511
  "required": ["answer", "reasoning"]
@@ -37,13 +37,27 @@ Pass a concrete model ID only if the user names one explicitly. If the tool reje
37
37
  Generate (no model parameter, default is correct):
38
38
 
39
39
  ```json
40
- { "tool": "media_generate_image", "input": { "prompt": "A sunset over the ocean, golden hour, soft haze, 35mm photo style", "variants": 2 } }
40
+ {
41
+ "tool": "media_generate_image",
42
+ "input": {
43
+ "prompt": "A sunset over the ocean, golden hour, soft haze, 35mm photo style",
44
+ "variants": 2
45
+ }
46
+ }
41
47
  ```
42
48
 
43
49
  Edit:
44
50
 
45
51
  ```json
46
- { "tool": "media_generate_image", "input": { "prompt": "Remove the watermark text from the background. Keep the subject, framing, lighting, and colors exactly identical. Change nothing else.", "mode": "edit", "source_paths": ["conversations/<conv-id>/attachments/photo.jpeg"], "model": "openai" } }
52
+ {
53
+ "tool": "media_generate_image",
54
+ "input": {
55
+ "prompt": "Remove the watermark text from the background. Keep the subject, framing, lighting, and colors exactly identical. Change nothing else.",
56
+ "mode": "edit",
57
+ "source_paths": ["conversations/<conv-id>/attachments/photo.jpeg"],
58
+ "model": "openai"
59
+ }
60
+ }
47
61
  ```
48
62
 
49
63
  `source_paths` is a flat array of file path strings. Do NOT pass objects:
@@ -72,6 +86,7 @@ Edits on large photos are slow (1 to 2 minutes). If the tool reports a timeout (
72
86
  Each generated image is saved into the workspace under `media/generated/` and the tool result lists the saved paths. The images also come back as inline content blocks so you can judge the result before presenting it.
73
87
 
74
88
  - Present an image to the user by embedding its saved path in your reply: `![short description](vellum://workspace/media/generated/<file>.png)`. The app renders it inline where your text refers to it, and chat channels (Slack, Telegram, WhatsApp) deliver it as a native image upload.
89
+ - Label a plain image link "View image", for example `[View image](vellum://workspace/media/generated/<file>.png)`. The link opens a preview; do not call it "Download" or promise a download in the surrounding text. To save the image, the user can choose Download in the preview.
75
90
  - If you do not embed it, the image is still auto-attached to your reply as a file, so it is never lost. Prefer embedding: an attachment chip at the end of the message is a worse presentation than the image inline.
76
91
  - To iterate on a result, pass its saved path via `source_paths` with `mode: "edit"`.
77
92
 
@@ -5,10 +5,7 @@ import {
5
5
  resolveImageGenCredentials,
6
6
  resolveImageGenRouting,
7
7
  } from "../../../../media/image-credentials.js";
8
- import {
9
- describeImageModels,
10
- resolveImageModel,
11
- } from "../../../../media/image-models.js";
8
+ import { resolveRequestedImageModel } from "../../../../media/image-models.js";
12
9
  import {
13
10
  generateImage,
14
11
  mapImageGenError,
@@ -115,18 +112,17 @@ export async function run(
115
112
  const config = getConfig();
116
113
  const svc = config.services["image-generation"];
117
114
  let modelOverride = input.model;
118
- // Resolve tier aliases (fast, quality, openai) to concrete model IDs via
119
- // the registry. Unknown values get an error listing the current catalog so
120
- // callers can self-correct without a stale schema enum.
115
+ // Resolve aliases and OpenRouter slugs against the configured provider.
116
+ // Built-in providers still list the current catalog on unknown values.
121
117
  if (typeof modelOverride === "string" && modelOverride) {
122
- const entry = resolveImageModel(modelOverride);
123
- if (!entry) {
118
+ const resolved = resolveRequestedImageModel(modelOverride, svc.provider);
119
+ if (resolved.error) {
124
120
  return {
125
- content: `Unknown model "${modelOverride}". Available models and aliases:\n${describeImageModels()}\n\nRetry with one of the aliases above, or omit the model parameter to use the configured default.`,
121
+ content: `${resolved.error}\n\nRetry with one of the aliases above, or omit the model parameter to use the configured default.`,
126
122
  isError: true,
127
123
  };
128
124
  }
129
- modelOverride = entry.id;
125
+ modelOverride = resolved.model;
130
126
  }
131
127
  // Backend and managed-ness resolve together: an explicit model re-routes
132
128
  // to the model's backend (e.g. `gpt-image-2` under a gemini config routes
@@ -165,11 +165,7 @@
165
165
  },
166
166
  "description": "Replacement steps (replaces all existing steps)"
167
167
  }
168
- },
169
- "oneOf": [
170
- { "required": ["id"] },
171
- { "required": ["enrollment_id", "enrollment_action"] }
172
- ]
168
+ }
173
169
  },
174
170
  "executor": "tools/sequence-update.ts",
175
171
  "execution_target": "host"
@@ -153,6 +153,15 @@ export const CALL_SITE_DEFAULTS: Record<LLMCallSite, CallSiteDefaultConfig> = {
153
153
  effort: "low",
154
154
  thinking: { enabled: false },
155
155
  },
156
+ // The escalation judge only runs when this site resolves to the TypeSafe
157
+ // provider (see calls/voice-escalation-judge.ts), so the shipped default is
158
+ // any ordinary profile: the judge stays off until a user pins a TypeSafe
159
+ // profile here.
160
+ voiceEscalationJudge: { profile: "cost-optimized" },
161
+ // Same arrangement for the barge-in continuation judge
162
+ // (live-voice/continuation-judge.ts): off until a TypeSafe profile is
163
+ // pinned here.
164
+ voiceContinuationJudge: { profile: "cost-optimized" },
156
165
  // Names the background continuation a barge-in spawns, from the interrupted
157
166
  // transcript. The label is fixed at spawn, so the call runs under a short
158
167
  // timeout and the deterministic label stands in when it misses; that makes
@@ -536,6 +536,14 @@
536
536
  "description": "Route all user-facing assistant text through the send_user_message tool instead of streamed assistant text",
537
537
  "defaultEnabled": false
538
538
  },
539
+ {
540
+ "id": "session-groups",
541
+ "scope": "both",
542
+ "key": "session-groups",
543
+ "label": "Session Groups",
544
+ "description": "Records eligible computer use, browser, live vision, and ambient runs and presents them as collapsible transcript groups. Off preserves ordinary actions and the flat transcript without new session tracking.",
545
+ "defaultEnabled": false
546
+ },
539
547
  {
540
548
  "id": "assistant-desktop",
541
549
  "scope": "assistant",
@@ -543,6 +551,14 @@
543
551
  "label": "Virtual desktop",
544
552
  "description": "Streams an interactive virtual desktop from a platform-hosted, containerized assistant over /v1/desktop/stream. Requires platform hosting; self-hosted assistants are excluded. Off: the virtual desktop is hidden. On: the chat header offers its viewer and controls.",
545
553
  "defaultEnabled": false
554
+ },
555
+ {
556
+ "id": "assistant-inbox",
557
+ "scope": "client",
558
+ "key": "assistant-inbox",
559
+ "label": "Assistant Inbox",
560
+ "description": "Shows an Assistant Inbox entry above Preferences in the web sidebar and the /assistant/inbox page it opens: a mailbox for the assistant's managed vellum.me address (received and sent, with a reading pane), the inline email setup card for entitled orgs with no address yet, and an upgrade card for orgs without the managed_email entitlement. Off hides the entry and redirects the route to chat.",
561
+ "defaultEnabled": false
546
562
  }
547
563
  ]
548
564
  }
@@ -9,7 +9,7 @@ describe("profileSupportsTextGeneration", () => {
9
9
  test("false for a structured-decision catalog model", () => {
10
10
  expect(
11
11
  profileSupportsTextGeneration(
12
- { provider: "jev", model: "jev-latest" },
12
+ { provider: "typesafe", model: "jev-latest" },
13
13
  {},
14
14
  ),
15
15
  ).toBe(false);
@@ -35,7 +35,7 @@ describe("profileSupportsTextGeneration", () => {
35
35
  profileSupportsTextGeneration(
36
36
  { mix: [{ profile: "jev" }, { profile: "balanced" }] },
37
37
  {
38
- jev: { provider: "jev", model: "jev-latest" },
38
+ jev: { provider: "typesafe", model: "jev-latest" },
39
39
  balanced: { provider: "anthropic", model: "claude-opus-4-8" },
40
40
  },
41
41
  ),
@@ -0,0 +1,31 @@
1
+ import { describe, expect, test } from "bun:test";
2
+
3
+ import { MCP_GLOBAL_MAX_TOOLS } from "../mcp.js";
4
+ import { ToolsConfigSchema } from "../tools.js";
5
+
6
+ describe("ToolsConfigSchema.mcpGlobalMaxTools", () => {
7
+ test("omits the override so the shipped MCP cap remains in force", () => {
8
+ const parsed = ToolsConfigSchema.parse({});
9
+ expect(parsed.mcpGlobalMaxTools).toBeUndefined();
10
+ expect(parsed.exclude).toEqual([]);
11
+ });
12
+
13
+ test("accepts a workspace override", () => {
14
+ expect(
15
+ ToolsConfigSchema.parse({ mcpGlobalMaxTools: 80 }).mcpGlobalMaxTools,
16
+ ).toBe(80);
17
+ expect(
18
+ ToolsConfigSchema.parse({
19
+ mcpGlobalMaxTools: MCP_GLOBAL_MAX_TOOLS,
20
+ }).mcpGlobalMaxTools,
21
+ ).toBe(MCP_GLOBAL_MAX_TOOLS);
22
+ expect(
23
+ ToolsConfigSchema.parse({ mcpGlobalMaxTools: 1000 }).mcpGlobalMaxTools,
24
+ ).toBe(1000);
25
+ });
26
+
27
+ test("rejects zero and fractions", () => {
28
+ expect(() => ToolsConfigSchema.parse({ mcpGlobalMaxTools: 0 })).toThrow();
29
+ expect(() => ToolsConfigSchema.parse({ mcpGlobalMaxTools: 1.5 })).toThrow();
30
+ });
31
+ });
@@ -303,6 +303,20 @@ const CATALOG_RECORD: CatalogRecord = {
303
303
  "Fast front-door leg fronting live-voice turns under triage-and-escalate: leading-token verdict, holding phrase, or the direct answer.",
304
304
  domain: "agentLoop",
305
305
  },
306
+ voiceEscalationJudge: {
307
+ id: "voiceEscalationJudge",
308
+ displayName: "Voice Escalation Judge",
309
+ description:
310
+ "Second opinion on the voice front door's escalate decision. Runs only when pinned to a TypeSafe profile; a confident 'needs tools' verdict hands the turn to the escalated leg.",
311
+ domain: "agentLoop",
312
+ },
313
+ voiceContinuationJudge: {
314
+ id: "voiceContinuationJudge",
315
+ displayName: "Voice Continuation Judge",
316
+ description:
317
+ "Decides from the caller's interruption whether a barged-in request should keep running in the background. Runs only when pinned to a TypeSafe profile.",
318
+ domain: "agentLoop",
319
+ },
306
320
  voiceContinuationLabel: {
307
321
  id: "voiceContinuationLabel",
308
322
  displayName: "Voice Continuation Label",
@@ -23,12 +23,6 @@ export const WhatsAppConfigSchema = z
23
23
  .string({ error: "whatsapp.phoneNumber must be a string" })
24
24
  .default("")
25
25
  .describe("WhatsApp Business phone number"),
26
- deliverAuthBypass: z
27
- .boolean({ error: "whatsapp.deliverAuthBypass must be a boolean" })
28
- .default(false)
29
- .describe(
30
- "Whether to bypass authentication when delivering WhatsApp messages",
31
- ),
32
26
  timeoutMs: z
33
27
  .number({ error: "whatsapp.timeoutMs must be a number" })
34
28
  .int("whatsapp.timeoutMs must be an integer")
@@ -81,12 +75,6 @@ export const TelegramConfigSchema = z
81
75
  .describe(
82
76
  "Webhook URL this deployment last registered with Telegram (managed by the gateway)",
83
77
  ),
84
- deliverAuthBypass: z
85
- .boolean({ error: "telegram.deliverAuthBypass must be a boolean" })
86
- .default(false)
87
- .describe(
88
- "Whether to bypass authentication when delivering Telegram messages",
89
- ),
90
78
  timeoutMs: z
91
79
  .number({ error: "telegram.timeoutMs must be a number" })
92
80
  .int("telegram.timeoutMs must be an integer")
@@ -155,12 +143,6 @@ export const DiscordConfigSchema = z
155
143
 
156
144
  export const SlackConfigSchema = z
157
145
  .object({
158
- deliverAuthBypass: z
159
- .boolean({ error: "slack.deliverAuthBypass must be a boolean" })
160
- .default(false)
161
- .describe(
162
- "Whether to bypass authentication when delivering Slack messages",
163
- ),
164
146
  teamId: z
165
147
  .string({ error: "slack.teamId must be a string" })
166
148
  .default("")
@@ -62,7 +62,7 @@ export const KNOWN_LLM_PROVIDERS = [
62
62
  "opencode",
63
63
  "baseten",
64
64
  "poolside",
65
- "jev",
65
+ "typesafe",
66
66
  // Routing identities: "vellum" = the platform-managed route (upstream
67
67
  // derived from the model at dispatch) and the catalog owner of
68
68
  // Vellum-hosted GPU models; "chatgpt" = the subscription route to OpenAI.
@@ -229,6 +229,8 @@ export const LLMCallSiteEnum = z.enum([
229
229
  "vision",
230
230
  "voiceProgressNarration",
231
231
  "voiceFrontDoor",
232
+ "voiceEscalationJudge",
233
+ "voiceContinuationJudge",
232
234
  "voiceContinuationLabel",
233
235
  "trustRuleSuggestion",
234
236
  "homeGreeting",
@@ -18,7 +18,8 @@ export const PLUGIN_MCP_RISK_LEVEL = "low" as const;
18
18
  export const MCP_MAX_TOOLS_PER_SERVER = 20;
19
19
 
20
20
  /**
21
- * Cap on tools registered across every MCP server. Selection is a
21
+ * Cap on tools registered across every MCP server when the workspace
22
+ * does not set `tools.mcpGlobalMaxTools` in config.json. Selection is a
22
23
  * deterministic round-robin by server id (see `mcp/tool-caps.ts`), so a
23
24
  * later server is not emptied just because earlier ones filled the budget.
24
25
  */
@@ -11,14 +11,23 @@ export const MemoryRetrospectiveConfigSchema = z
11
11
 
12
12
  skillImprovement: z
13
13
  .boolean({
14
- error:
15
- "memory.retrospective.skillImprovement must be a boolean",
14
+ error: "memory.retrospective.skillImprovement must be a boolean",
16
15
  })
17
16
  .default(true)
18
17
  .describe(
19
18
  "Whether retrospectives may discover, refine, and create managed skills from observed procedures. When false, retrospectives still capture ordinary memories through `remember`, but cannot load skill management, search for similar skills, or scaffold managed skills.",
20
19
  ),
21
20
 
21
+ skillImprovementMonitoring: z
22
+ .boolean({
23
+ error:
24
+ "memory.retrospective.skillImprovementMonitoring must be a boolean",
25
+ })
26
+ .default(false)
27
+ .describe(
28
+ "Reserved opt-in for monitoring retrospective skill-improvement decisions. This setting currently has no effect.",
29
+ ),
30
+
22
31
  timeThresholdMs: z
23
32
  .number({
24
33
  error: "memory.retrospective.timeThresholdMs must be a number",
@@ -22,7 +22,12 @@ export const VALID_INFERENCE_PROVIDERS = [
22
22
 
23
23
  // `vellum` generates through the platform runtime proxy; the backend
24
24
  // (gemini/openai) derives from the selected model's prefix at request time.
25
- const VALID_IMAGE_GEN_PROVIDERS = ["vellum", "gemini", "openai"] as const;
25
+ const VALID_IMAGE_GEN_PROVIDERS = [
26
+ "vellum",
27
+ "gemini",
28
+ "openai",
29
+ "openrouter",
30
+ ] as const;
26
31
 
27
32
  /**
28
33
  * Derived from `SEARCH_PROVIDER_CATALOG`. Adding a new web-search provider
@@ -56,8 +61,9 @@ const InferenceServiceSchema = z.object({});
56
61
  /**
57
62
  * Image generation carries no `mode`: `provider` is the only axis. `"vellum"`
58
63
  * generates through the platform runtime proxy for the model-appropriate
59
- * backend, billed to Vellum credits; `"gemini"`/`"openai"` use the user's
60
- * key. A `mode` key sent by an older client is stripped at parse.
64
+ * backend, billed to Vellum credits; `"gemini"`/`"openai"`/`"openrouter"`
65
+ * use the user's key. A `mode` key sent by an older client is stripped at
66
+ * parse.
61
67
  */
62
68
  const ImageGenerationServiceSchema = z.object({
63
69
  provider: z.enum(VALID_IMAGE_GEN_PROVIDERS).default("gemini"),
@@ -76,8 +82,10 @@ const WebSearchServiceSchema = z.object({
76
82
  provider: z
77
83
  .enum(VALID_WEB_SEARCH_PROVIDERS)
78
84
  .default("inference-provider-native"),
79
- // Origin for providers that support a custom API base (e.g. fastCRW).
80
- // Empty / omitted uses the provider's cloud default.
85
+ // Origin for providers that support a custom API base (e.g. fastCRW,
86
+ // SearXNG, TinyFish). Empty / omitted uses the provider's cloud default
87
+ // when one exists. SearXNG has no cloud default and requires a user-supplied
88
+ // URL.
81
89
  apiBase: z.string().optional(),
82
90
  });
83
91
 
@@ -91,7 +99,8 @@ const WebFetchServiceSchema = z.object({
91
99
  // `firecrawl`) scrape via their hosted API and reuse the same stored key as
92
100
  // their web-search counterpart.
93
101
  provider: z.enum(VALID_WEB_FETCH_PROVIDERS).default("default"),
94
- // Origin for providers that support a custom API base (e.g. fastCRW).
102
+ // Origin for providers that support a custom API base (e.g. fastCRW,
103
+ // TinyFish).
95
104
  // Empty / omitted uses the provider's cloud default.
96
105
  apiBase: z.string().optional(),
97
106
  });
@@ -1,5 +1,7 @@
1
1
  import { z } from "zod";
2
2
 
3
+ import { MCP_GLOBAL_MAX_TOOLS } from "./mcp.js";
4
+
3
5
  export const ToolsConfigSchema = z
4
6
  .object({
5
7
  exclude: z
@@ -8,6 +10,14 @@ export const ToolsConfigSchema = z
8
10
  .describe(
9
11
  "Tool names to suppress. Excluded tools are not sent to the LLM. Names match `ToolDefinition.name` exactly (e.g. `bash`, `mcp__server__tool`).",
10
12
  ),
13
+ mcpGlobalMaxTools: z
14
+ .number({ error: "tools.mcpGlobalMaxTools must be a number" })
15
+ .int("tools.mcpGlobalMaxTools must be an integer")
16
+ .min(1, "tools.mcpGlobalMaxTools must be >= 1")
17
+ .optional()
18
+ .describe(
19
+ `Override the code-owned cap on MCP tools registered across every server. Omit to use the shipped default of ${MCP_GLOBAL_MAX_TOOLS}. Selection is a fair round-robin by server id.`,
20
+ ),
11
21
  })
12
22
  .describe("Tool visibility configuration");
13
23
 
@@ -0,0 +1,21 @@
1
+ import { isAssistantFeatureFlagEnabled } from "./assistant-feature-flags.js";
2
+
3
+ export const SESSION_GROUPS_FLAG_KEY = "session-groups";
4
+
5
+ let recoveryHealthy = true;
6
+
7
+ /** Recovery is a process prerequisite, independent of rollout targeting. */
8
+ export function setModeSessionRecoveryHealthy(healthy: boolean): void {
9
+ recoveryHealthy = healthy;
10
+ }
11
+
12
+ export function isModeSessionRecoveryHealthy(): boolean {
13
+ return recoveryHealthy;
14
+ }
15
+
16
+ /** Whether this assistant may admit new transcript mode-session tracking. */
17
+ export function isSessionGroupsEnabled(): boolean {
18
+ return (
19
+ recoveryHealthy && isAssistantFeatureFlagEnabled(SESSION_GROUPS_FLAG_KEY)
20
+ );
21
+ }
@@ -225,10 +225,6 @@ export interface SkillToolManifestMeta {
225
225
 
226
226
  // ─── Helpers ─────────────────────────────────────────────────────────────────
227
227
 
228
- function getSkillsDir(): string {
229
- return getWorkspaceSkillsDir();
230
- }
231
-
232
228
  export function getBundledSkillsDir(): string {
233
229
  const dir = import.meta.dir;
234
230
 
@@ -1114,7 +1110,7 @@ export function loadSkillCatalog(
1114
1110
  }
1115
1111
 
1116
1112
  // Load managed (user) skills, which take precedence over bundled skills with the same ID
1117
- const skillsDir = getSkillsDir();
1113
+ const skillsDir = getWorkspaceSkillsDir();
1118
1114
  const directories = discoverSkillDirectories(skillsDir);
1119
1115
 
1120
1116
  for (const directory of directories) {
@@ -1354,7 +1350,7 @@ function loadSkillDefinition(skill: SkillSummary): SkillLookupResult {
1354
1350
  } else {
1355
1351
  loaded = readSkillFromDirectory(
1356
1352
  skill.directoryPath,
1357
- getSkillsDir(),
1353
+ getWorkspaceSkillsDir(),
1358
1354
  skill.source,
1359
1355
  );
1360
1356
  }