@vellumai/assistant 0.12.2 → 0.12.3-staging.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (772) hide show
  1. package/.env.example +2 -0
  2. package/AGENTS.md +1 -1
  3. package/ARCHITECTURE.md +43 -29
  4. package/Dockerfile +63 -0
  5. package/docs/architecture/integrations.md +7 -0
  6. package/docs/architecture/memory.md +17 -2
  7. package/docs/architecture/security.md +10 -5
  8. package/docs/architecture/turn-actor.md +2 -0
  9. package/docs/credential-execution-service.md +1 -1
  10. package/docs/desktop-browser-cli.md +50 -5
  11. package/docs/live-voice-task-outcomes.md +100 -0
  12. package/docs/mode-sessions.md +183 -0
  13. package/docs/voice-input-diagnostics.md +58 -0
  14. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  15. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
  16. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  17. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  18. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
  19. package/node_modules/@vellumai/environments/src/shell.test.ts +21 -0
  20. package/node_modules/@vellumai/environments/src/shell.ts +24 -0
  21. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  22. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
  23. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  24. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  25. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
  26. package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +20 -0
  27. package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +8 -2
  28. package/node_modules/@vellumai/gateway-client/src/index.ts +4 -1
  29. package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +38 -7
  30. package/node_modules/@vellumai/gateway-client/src/trust-verdict-contract.ts +5 -6
  31. package/node_modules/@vellumai/service-contracts/package.json +2 -1
  32. package/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
  33. package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  34. package/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  35. package/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
  36. package/node_modules/@vellumai/slack-text/src/index.ts +4 -8
  37. package/openapi.yaml +716 -16
  38. package/package.json +1 -1
  39. package/scripts/smoke-desktop-browser-cli.ts +11 -26
  40. package/scripts/smoke-desktop-window-theme.ts +2 -3
  41. package/scripts/sync-llm-catalog.ts +3 -0
  42. package/scripts/voice-ttft-spike.ts +2 -2
  43. package/src/__tests__/agent-loop-output-hooks.test.ts +150 -1
  44. package/src/__tests__/agent-loop-provider-error-recording.test.ts +87 -19
  45. package/src/__tests__/agent-loop.test.ts +259 -0
  46. package/src/__tests__/agent-wake-delegation-prompt.test.ts +64 -2
  47. package/src/__tests__/annotate-activity-metadata.test.ts +3 -1
  48. package/src/__tests__/annotate-risk-options.test.ts +2 -0
  49. package/src/__tests__/answered-question-persistence.test.ts +2 -0
  50. package/src/__tests__/anthropic-gateway-shared.test.ts +22 -0
  51. package/src/__tests__/anthropic-provider.test.ts +55 -0
  52. package/src/__tests__/approval-interception-trust-gates.test.ts +40 -0
  53. package/src/__tests__/approval-message-composer.test.ts +1 -118
  54. package/src/__tests__/approval-routes-http.test.ts +3 -0
  55. package/src/__tests__/assistant-event-hub-connection-history.test.ts +111 -0
  56. package/src/__tests__/assistant-event-hub-targeted.test.ts +50 -0
  57. package/src/__tests__/assistant-event-targeting.test.ts +186 -0
  58. package/src/__tests__/assistant-stream-state.test.ts +29 -9
  59. package/src/__tests__/attachments-store.test.ts +22 -3
  60. package/src/__tests__/browser-fill-credential.test.ts +11 -22
  61. package/src/__tests__/btw-routes.test.ts +4 -1
  62. package/src/__tests__/build-persisted-content.test.ts +1 -1
  63. package/src/__tests__/call-conversation-messages.test.ts +5 -0
  64. package/src/__tests__/call-recovery.test.ts +7 -2
  65. package/src/__tests__/call-routes-http.test.ts +12 -0
  66. package/src/__tests__/call-store.test.ts +7 -2
  67. package/src/__tests__/channel-approval.test.ts +9 -14
  68. package/src/__tests__/channel-reply-delivery.test.ts +56 -0
  69. package/src/__tests__/channel-retry-sweep.test.ts +9 -0
  70. package/src/__tests__/chat-credential-redaction.test.ts +23 -0
  71. package/src/__tests__/chat-reveal-guard-priming.test.ts +2 -0
  72. package/src/__tests__/checker.test.ts +11 -3
  73. package/src/__tests__/client-os-metadata-persistence.test.ts +35 -1
  74. package/src/__tests__/computer-use-screenshot-attachments.test.ts +411 -0
  75. package/src/__tests__/computer-use-screenshot-selection.test.ts +50 -0
  76. package/src/__tests__/computer-use-skill-manifest-regression.test.ts +0 -25
  77. package/src/__tests__/computer-use-tools.test.ts +1 -233
  78. package/src/__tests__/config-model-image-provider.test.ts +39 -0
  79. package/src/__tests__/config-schema.test.ts +16 -0
  80. package/src/__tests__/config-watcher.test.ts +24 -0
  81. package/src/__tests__/context-search-agent-runner.test.ts +260 -1
  82. package/src/__tests__/context-search-fanout.test.ts +1 -1
  83. package/src/__tests__/context-search-types.test.ts +7 -7
  84. package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +8 -2
  85. package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +2 -0
  86. package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +2 -0
  87. package/src/__tests__/conversation-agent-loop-overflow.test.ts +2 -0
  88. package/src/__tests__/conversation-agent-loop.test.ts +797 -16
  89. package/src/__tests__/conversation-app-control-lifecycle.test.ts +53 -1
  90. package/src/__tests__/conversation-attachments.test.ts +106 -0
  91. package/src/__tests__/conversation-evictor.test.ts +62 -37
  92. package/src/__tests__/conversation-fork-crud.test.ts +191 -0
  93. package/src/__tests__/conversation-history-boundary-repair-failure.test.ts +174 -0
  94. package/src/__tests__/conversation-loop-db-lock-resilience.test.ts +2 -0
  95. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +2 -0
  96. package/src/__tests__/conversation-rendered-delegation-state.test.ts +276 -0
  97. package/src/__tests__/conversation-retry-route.test.ts +22 -0
  98. package/src/__tests__/conversation-routes-disk-view.test.ts +2 -0
  99. package/src/__tests__/conversation-routes-enabled-plugins.test.ts +2 -0
  100. package/src/__tests__/conversation-routes-guardian-reply.test.ts +10 -0
  101. package/src/__tests__/conversation-routes-hidden-queue.test.ts +2 -0
  102. package/src/__tests__/conversation-routes-slash-commands.test.ts +2 -0
  103. package/src/__tests__/conversation-runtime-assembly.test.ts +29 -0
  104. package/src/__tests__/conversation-surface-snapshots.test.ts +543 -0
  105. package/src/__tests__/conversation-surfaces-action-delivery.test.ts +151 -0
  106. package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +63 -1
  107. package/src/__tests__/conversation-tool-setup-attribution.test.ts +23 -0
  108. package/src/__tests__/conversation-undo.test.ts +61 -0
  109. package/src/__tests__/credential-execution-client.test.ts +72 -55
  110. package/src/__tests__/credential-prompt-route.test.ts +35 -3
  111. package/src/__tests__/cu-unified-flow.test.ts +82 -1
  112. package/src/__tests__/db-conversation-tool-surface.test.ts +221 -0
  113. package/src/__tests__/db-mode-sessions-migration.test.ts +175 -0
  114. package/src/__tests__/deterministic-verification-control-plane.test.ts +2 -0
  115. package/src/__tests__/dm-backfill.test.ts +91 -1
  116. package/src/__tests__/document-create-route.test.ts +88 -0
  117. package/src/__tests__/document-sync-tags.test.ts +27 -0
  118. package/src/__tests__/emit-signal-routing-intent.test.ts +142 -2
  119. package/src/__tests__/events-tail-route.test.ts +58 -0
  120. package/src/__tests__/evict-conversations-for-reload.test.ts +57 -8
  121. package/src/__tests__/guardian-action-sweep.test.ts +16 -0
  122. package/src/__tests__/guardian-dispatch.test.ts +13 -0
  123. package/src/__tests__/headless-browser-interactions.test.ts +64 -20
  124. package/src/__tests__/heartbeat-service.test.ts +2 -59
  125. package/src/__tests__/helpers/mock-conversation.ts +21 -0
  126. package/src/__tests__/helpers/native-web-search-harness.ts +2 -0
  127. package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +5 -0
  128. package/src/__tests__/history-repair.test.ts +53 -0
  129. package/src/__tests__/host-cu-proxy.test.ts +46 -2
  130. package/src/__tests__/http-user-message-parity.test.ts +2 -0
  131. package/src/__tests__/image-credentials.test.ts +49 -0
  132. package/src/__tests__/image-service-dispatcher.test.ts +45 -0
  133. package/src/__tests__/inference-profile-session-handler.test.ts +26 -0
  134. package/src/__tests__/interrupt-on-send.test.ts +57 -43
  135. package/src/__tests__/interrupt-turn-note.test.ts +5 -6
  136. package/src/__tests__/list-messages-tool-merge.test.ts +85 -1
  137. package/src/__tests__/llm-catalog-parity.test.ts +27 -4
  138. package/src/__tests__/media-generate-image.test.ts +49 -1
  139. package/src/__tests__/message-provenance-contact-id.test.ts +69 -0
  140. package/src/__tests__/migration-export-to-gcs.test.ts +142 -0
  141. package/src/__tests__/mode-session-history.test.ts +245 -0
  142. package/src/__tests__/mode-session-recovery.test.ts +222 -0
  143. package/src/__tests__/mode-session-wire.test.ts +141 -0
  144. package/src/__tests__/notification-guardian-path.test.ts +7 -0
  145. package/src/__tests__/notification-telegram-adapter.test.ts +2 -2
  146. package/src/__tests__/notification-vellum-adapter.test.ts +34 -0
  147. package/src/__tests__/oauth-apps-routes.test.ts +1 -0
  148. package/src/__tests__/oauth-commands-routes.test.ts +114 -101
  149. package/src/__tests__/oauth-connect-orchestrator.test.ts +2 -0
  150. package/src/__tests__/oauth-provider-serializer.test.ts +13 -0
  151. package/src/__tests__/oauth-providers-routes.test.ts +3 -0
  152. package/src/__tests__/outbound-slack-persistence.test.ts +2 -0
  153. package/src/__tests__/persist-media-references.test.ts +50 -0
  154. package/src/__tests__/persistence-secret-redaction.test.ts +2 -0
  155. package/src/__tests__/plugin-api-provider.test.ts +5 -0
  156. package/src/__tests__/plugin-api-resolve-credential.test.ts +97 -0
  157. package/src/__tests__/plugin-import-boundary-guard.test.ts +9 -1
  158. package/src/__tests__/process-message-display-content.test.ts +56 -0
  159. package/src/__tests__/reaction-persistence.test.ts +5 -0
  160. package/src/__tests__/request-secret-standalone-channel-gate.test.ts +25 -9
  161. package/src/__tests__/run-conversation-turn-persistence.test.ts +138 -1
  162. package/src/__tests__/runtime-events-sse-reconnect.test.ts +1 -1
  163. package/src/__tests__/schedule-routes.test.ts +20 -0
  164. package/src/__tests__/scheduler-result-notification.test.ts +23 -4
  165. package/src/__tests__/script-proxy-certs.test.ts +1 -1
  166. package/src/__tests__/scripted-turn-metadata-persistence.test.ts +73 -0
  167. package/src/__tests__/secret-ingress-http.test.ts +2 -0
  168. package/src/__tests__/secret-routes-platform-proxy.test.ts +86 -31
  169. package/src/__tests__/secret-routes-scrub.test.ts +22 -27
  170. package/src/__tests__/secret-scanner.test.ts +20 -0
  171. package/src/__tests__/secure-keys-managed-failover.test.ts +8 -0
  172. package/src/__tests__/secure-keys.test.ts +50 -4
  173. package/src/__tests__/send-endpoint-busy.test.ts +163 -1
  174. package/src/__tests__/server-history-render.test.ts +91 -1
  175. package/src/__tests__/server-tool-pairing.test.ts +107 -0
  176. package/src/__tests__/settings-routes.test.ts +50 -1
  177. package/src/__tests__/skill-script-runner-sandbox.test.ts +32 -0
  178. package/src/__tests__/skills.test.ts +5 -4
  179. package/src/__tests__/subagent-fork-notifications.test.ts +1 -2
  180. package/src/__tests__/subagent-manager-notify.test.ts +1 -2
  181. package/src/__tests__/subagent-notify-parent.test.ts +146 -39
  182. package/src/__tests__/subagent-notify-stale-rebuild.test.ts +2 -4
  183. package/src/__tests__/subagent-tool-gate-mode.test.ts +226 -0
  184. package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +2 -0
  185. package/src/__tests__/terminal-tools.test.ts +8 -0
  186. package/src/__tests__/thread-backfill.test.ts +53 -0
  187. package/src/__tests__/tool-preview-lifecycle.test.ts +2 -0
  188. package/src/__tests__/tool-result-follow-up.test.ts +99 -0
  189. package/src/__tests__/tool-result-metadata-plumbing.test.ts +66 -1
  190. package/src/__tests__/tool-start-timestamp.test.ts +2 -0
  191. package/src/__tests__/twilio-routes.test.ts +1 -0
  192. package/src/__tests__/unicode.test.ts +36 -0
  193. package/src/__tests__/web-search-backend-failure.test.ts +1 -1
  194. package/src/activation/progress-store.test.ts +12 -7
  195. package/src/agent/loop.ts +136 -53
  196. package/src/agent/tool-result-follow-up.ts +70 -0
  197. package/src/api/attachment-provenance.test.ts +68 -0
  198. package/src/api/computer-use-tool.test.ts +55 -0
  199. package/src/api/computer-use-tool.ts +27 -0
  200. package/src/api/events/assistant-outbound-attachment.ts +3 -0
  201. package/src/api/events/assistant-turn-start.ts +3 -0
  202. package/src/api/events/desktop-activity-changed.ts +10 -0
  203. package/src/api/events/generation-handoff.ts +2 -0
  204. package/src/api/events/message-complete.ts +2 -0
  205. package/src/api/events/notification-conversation-created.ts +14 -16
  206. package/src/api/events/notification-intent.ts +4 -2
  207. package/src/api/events/question-request.ts +1 -0
  208. package/src/api/events/tool-result.ts +85 -0
  209. package/src/api/events/tool-use-start.ts +3 -0
  210. package/src/api/events/user-message-echo.ts +3 -0
  211. package/src/api/index.ts +42 -6
  212. package/src/api/mode-session.test.ts +131 -0
  213. package/src/api/mode-session.ts +148 -0
  214. package/src/api/remember-tool.ts +15 -0
  215. package/src/api/responses/conversation-message.ts +9 -0
  216. package/src/approvals/AGENTS.md +4 -2
  217. package/src/approvals/approval-primitive.ts +5 -2
  218. package/src/approvals/scoped-approval-grants.ts +6 -2
  219. package/src/browser/__tests__/operations.test.ts +29 -0
  220. package/src/browser/operation-meta.ts +2 -1
  221. package/src/browser/operations.ts +29 -0
  222. package/src/browser/virtual-desktop-target.ts +2 -9
  223. package/src/bundler/bundle-signer.ts +1 -3
  224. package/src/calls/__tests__/barge-in-guard.test.ts +3 -0
  225. package/src/calls/__tests__/call-controller.test.ts +94 -0
  226. package/src/calls/__tests__/call-funnel-telemetry.test.ts +204 -0
  227. package/src/calls/__tests__/voice-control-protocol.test.ts +15 -0
  228. package/src/calls/__tests__/voice-escalation-judge.test.ts +209 -0
  229. package/src/calls/__tests__/voice-leg-coordinator.test.ts +168 -0
  230. package/src/{live-voice/__tests__/live-voice-metrics.test.ts → calls/__tests__/voice-metrics.test.ts} +32 -38
  231. package/src/calls/__tests__/voice-session-bridge.test.ts +576 -122
  232. package/src/calls/barge-in-guard.ts +5 -0
  233. package/src/calls/call-controller.ts +196 -1
  234. package/src/calls/call-domain.ts +4 -0
  235. package/src/calls/call-store.ts +87 -2
  236. package/src/calls/guardian-action-sweep.ts +22 -8
  237. package/src/calls/media-stream-audio-transcode.ts +1 -3
  238. package/src/calls/typesafe-noul.ts +156 -0
  239. package/src/calls/voice-control-protocol.ts +6 -0
  240. package/src/calls/voice-escalation-judge.ts +145 -0
  241. package/src/calls/voice-escalation-target.ts +10 -0
  242. package/src/calls/voice-leg-coordinator.ts +121 -3
  243. package/src/{live-voice/live-voice-metrics.ts → calls/voice-metrics.ts} +94 -87
  244. package/src/calls/voice-session-bridge.ts +256 -112
  245. package/src/calls/voice-triage-escalate.ts +7 -0
  246. package/src/cli/__tests__/roadmap-cli.test.ts +4 -41
  247. package/src/cli/commands/__tests__/cli-test-harness.ts +21 -3
  248. package/src/cli/commands/__tests__/notifications.test.ts +35 -10
  249. package/src/cli/commands/__tests__/plugins.test.ts +20 -1
  250. package/src/cli/commands/__tests__/schedules.test.ts +14 -0
  251. package/src/cli/commands/bash.help.ts +4 -3
  252. package/src/cli/commands/browser.help.ts +9 -6
  253. package/src/cli/commands/browser.ts +0 -1
  254. package/src/cli/commands/clients.help.ts +46 -0
  255. package/src/cli/commands/clients.ts +116 -0
  256. package/src/cli/commands/credentials.help.ts +9 -4
  257. package/src/cli/commands/notifications.help.ts +16 -2
  258. package/src/cli/commands/notifications.ts +30 -8
  259. package/src/cli/commands/oauth/index.help.ts +10 -0
  260. package/src/cli/commands/oauth/providers.ts +12 -0
  261. package/src/cli/commands/oauth/request.test.ts +179 -2
  262. package/src/cli/commands/oauth/request.ts +41 -10
  263. package/src/cli/commands/plugins.ts +12 -5
  264. package/src/cli/commands/roadmap.help.ts +18 -21
  265. package/src/cli/commands/roadmap.ts +4 -21
  266. package/src/cli/commands/schedules.ts +2 -0
  267. package/src/cli/lib/__tests__/inspect-plugin.test.ts +54 -0
  268. package/src/cli/lib/__tests__/install-from-github.test.ts +41 -0
  269. package/src/cli/lib/__tests__/list-installed-plugins.test.ts +5 -2
  270. package/src/cli/lib/__tests__/local-plugin-upgrade.test.ts +133 -51
  271. package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +8 -2
  272. package/src/cli/lib/bundled-marketplace.json +906 -1
  273. package/src/cli/lib/bundled-plugin-packages.json +783 -1
  274. package/src/cli/lib/inspect-plugin.ts +11 -4
  275. package/src/cli/lib/plugin-marketplace.ts +8 -0
  276. package/src/cli/lib/publish-plugin.ts +1 -2
  277. package/src/cli/lib/upgrade-plugin.ts +19 -11
  278. package/src/cli/output.ts +18 -4
  279. package/src/config/__tests__/default-provider.test.ts +6 -0
  280. package/src/config/__tests__/memory-retrospective-schema.test.ts +14 -1
  281. package/src/config/bundled-skills/computer-use/SKILL.md +1 -1
  282. package/src/config/bundled-skills/computer-use/TOOLS.json +89 -1
  283. package/src/config/bundled-skills/image-studio/SKILL.md +17 -2
  284. package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +7 -11
  285. package/src/config/bundled-skills/schedule/SKILL.md +6 -6
  286. package/src/config/bundled-skills/sequences/TOOLS.json +1 -5
  287. package/src/config/call-site-defaults.ts +9 -0
  288. package/src/config/feature-flag-registry.json +16 -0
  289. package/src/config/loader.ts +9 -17
  290. package/src/config/platform-identity.ts +8 -8
  291. package/src/config/profile-text-generation.test.ts +51 -0
  292. package/src/config/profile-text-generation.ts +51 -0
  293. package/src/config/schemas/__tests__/memory-v3.test.ts +12 -0
  294. package/src/config/schemas/__tests__/tools.test.ts +31 -0
  295. package/src/config/schemas/call-site-catalog.ts +14 -0
  296. package/src/config/schemas/channels.ts +0 -18
  297. package/src/config/schemas/llm.ts +9 -2
  298. package/src/config/schemas/mcp.ts +6 -1
  299. package/src/config/schemas/memory-retrospective.ts +11 -2
  300. package/src/config/schemas/memory-v3.ts +12 -0
  301. package/src/config/schemas/services.ts +15 -6
  302. package/src/config/schemas/tools.ts +10 -0
  303. package/src/config/session-groups-gate.ts +21 -0
  304. package/src/config/skills.ts +2 -6
  305. package/src/conversations/__tests__/message-consolidation.test.ts +149 -0
  306. package/src/conversations/message-consolidation.ts +102 -5
  307. package/src/credential-execution/ces-connect.test.ts +69 -0
  308. package/src/credential-execution/ces-connect.ts +115 -0
  309. package/src/credential-execution/ces-runtime.ts +55 -93
  310. package/src/credential-execution/executable-discovery.ts +34 -142
  311. package/src/credential-execution/process-manager.test.ts +20 -24
  312. package/src/credential-execution/process-manager.ts +24 -19
  313. package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +10 -1
  314. package/src/daemon/__tests__/conversation-tool-setup.test.ts +43 -0
  315. package/src/daemon/__tests__/inference-profile-notification.test.ts +2 -0
  316. package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +36 -0
  317. package/src/daemon/assistant-attachments.ts +20 -12
  318. package/src/daemon/browser-mode-session.test.ts +402 -0
  319. package/src/daemon/browser-mode-session.ts +186 -0
  320. package/src/daemon/chat-credential-redaction.ts +16 -1
  321. package/src/daemon/computer-use-mode-session.test.ts +210 -0
  322. package/src/daemon/computer-use-mode-session.ts +105 -0
  323. package/src/daemon/config-watcher.ts +10 -0
  324. package/src/daemon/conversation-agent-loop-handlers.ts +132 -13
  325. package/src/daemon/conversation-agent-loop.ts +231 -25
  326. package/src/daemon/conversation-attachments.ts +93 -12
  327. package/src/daemon/conversation-evictor.ts +5 -24
  328. package/src/daemon/conversation-history.ts +10 -0
  329. package/src/daemon/conversation-interrupt-repair.ts +4 -43
  330. package/src/daemon/conversation-interrupt.ts +7 -11
  331. package/src/daemon/conversation-messaging.ts +58 -5
  332. package/src/daemon/conversation-mode-session.test.ts +1667 -0
  333. package/src/daemon/conversation-mode-session.ts +1008 -0
  334. package/src/daemon/conversation-process.ts +5 -0
  335. package/src/daemon/conversation-runtime-assembly.ts +10 -20
  336. package/src/daemon/conversation-skill-tools.ts +1 -0
  337. package/src/daemon/conversation-store.ts +4 -4
  338. package/src/daemon/conversation-surface-snapshots.ts +135 -0
  339. package/src/daemon/conversation-surfaces.ts +113 -5
  340. package/src/daemon/conversation-tool-setup.ts +79 -6
  341. package/src/daemon/conversation-turn-finalize.ts +26 -13
  342. package/src/daemon/conversation.ts +215 -58
  343. package/src/daemon/daemon-control.ts +3 -7
  344. package/src/daemon/handlers/config-model.ts +7 -5
  345. package/src/daemon/handlers/shared.ts +55 -16
  346. package/src/daemon/host-cu-proxy.ts +51 -22
  347. package/src/daemon/lifecycle.ts +30 -13
  348. package/src/daemon/mcp-reload-service.ts +1 -1
  349. package/src/daemon/message-protocol.ts +0 -1
  350. package/src/daemon/message-provenance.ts +28 -4
  351. package/src/daemon/message-types/shared.ts +1 -0
  352. package/src/daemon/mode-session-startup-recovery.ts +23 -0
  353. package/src/daemon/mode-session-tracking.ts +54 -0
  354. package/src/daemon/orphan-reaper.ts +4 -3
  355. package/src/daemon/persist-media-references.ts +20 -3
  356. package/src/daemon/process-message.ts +13 -4
  357. package/src/daemon/tool-setup-types.ts +6 -0
  358. package/src/daemon/virtual-desktop-context.ts +21 -0
  359. package/src/daemon/wake-conversation-ops.ts +50 -15
  360. package/src/desktop/__tests__/fake-desktop.ts +1 -0
  361. package/src/desktop/desktop-accessibility-script.ts +88 -0
  362. package/src/desktop/desktop-accessibility.test.ts +160 -0
  363. package/src/desktop/desktop-accessibility.ts +169 -0
  364. package/src/desktop/desktop-automation-lease.test.ts +302 -64
  365. package/src/desktop/desktop-automation-lease.ts +129 -54
  366. package/src/desktop/desktop-browser-endpoint.ts +4 -0
  367. package/src/desktop/desktop-computer-use-routing.test.ts +330 -0
  368. package/src/desktop/desktop-computer-use.test.ts +449 -0
  369. package/src/desktop/desktop-computer-use.ts +502 -0
  370. package/src/desktop/desktop-dependencies.ts +47 -293
  371. package/src/desktop/desktop-display.ts +2 -2
  372. package/src/desktop/desktop-help.ts +15 -0
  373. package/src/desktop/desktop-panel-config.test.ts +13 -3
  374. package/src/desktop/desktop-panel-config.ts +109 -15
  375. package/src/desktop/desktop-session-manager.test.ts +69 -21
  376. package/src/desktop/desktop-session-manager.ts +76 -22
  377. package/src/desktop/desktop-stream-bridge.test.ts +76 -132
  378. package/src/desktop/desktop-stream-bridge.ts +9 -11
  379. package/src/desktop/desktop-wallpaper-renderer.ts +132 -0
  380. package/src/desktop/desktop-wallpaper-worker.ts +17 -0
  381. package/src/desktop/desktop-wallpaper.test.ts +48 -7
  382. package/src/desktop/desktop-wallpaper.ts +34 -102
  383. package/src/desktop/desktop-window-theme.ts +5 -25
  384. package/src/desktop/virtual-desktop-feature.ts +53 -3
  385. package/src/desktop/virtual-desktop-platform.test.ts +5 -9
  386. package/src/documents/document-store.ts +23 -0
  387. package/src/heartbeat/__tests__/heartbeat-service.test.ts +22 -26
  388. package/src/heartbeat/heartbeat-service.ts +5 -33
  389. package/src/ipc/__tests__/browser-ipc.test.ts +215 -6
  390. package/src/ipc/assistant-server.ts +2 -0
  391. package/src/ipc/routes/__tests__/channel-reply-ipc-routes.test.ts +111 -0
  392. package/src/ipc/routes/channel-reply-ipc-routes.ts +73 -0
  393. package/src/live-voice/__tests__/continuation-judge.test.ts +151 -0
  394. package/src/live-voice/__tests__/live-activity-reporter.test.ts +154 -15
  395. package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +743 -31
  396. package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +191 -10
  397. package/src/live-voice/__tests__/live-voice-session-groups-gate.test.ts +87 -0
  398. package/src/live-voice/__tests__/live-voice-session-manager.test.ts +5 -1
  399. package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +41 -0
  400. package/src/live-voice/__tests__/live-voice-sight-frame.test.ts +1017 -9
  401. package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +224 -1
  402. package/src/live-voice/__tests__/live-voice-tts-session.test.ts +101 -0
  403. package/src/live-voice/__tests__/live-voice-tts.test.ts +44 -0
  404. package/src/live-voice/__tests__/live-voice-vad.test.ts +939 -75
  405. package/src/live-voice/__tests__/microphone-idle-gate.test.ts +87 -0
  406. package/src/live-voice/__tests__/session-controls.test.ts +23 -0
  407. package/src/live-voice/__tests__/task-outcomes.test.ts +117 -0
  408. package/src/live-voice/__tests__/voice-input-diagnostics.test.ts +117 -0
  409. package/src/live-voice/camera-mode-session.test.ts +265 -0
  410. package/src/live-voice/camera-mode-session.ts +277 -0
  411. package/src/live-voice/continuation-judge.ts +123 -0
  412. package/src/live-voice/live-activity-reporter.ts +115 -6
  413. package/src/live-voice/live-voice-manager.ts +36 -1
  414. package/src/live-voice/live-voice-photo.ts +88 -7
  415. package/src/live-voice/live-voice-session-manager.ts +15 -0
  416. package/src/live-voice/live-voice-session.ts +1443 -747
  417. package/src/live-voice/live-voice-tts.ts +22 -4
  418. package/src/live-voice/microphone-idle-gate.ts +56 -0
  419. package/src/live-voice/protocol.ts +171 -4
  420. package/src/live-voice/session-controls.ts +11 -3
  421. package/src/live-voice/task-outcomes.ts +131 -0
  422. package/src/live-voice/voice-input-diagnostics.ts +117 -0
  423. package/src/mcp/__tests__/manager-tool-caps.test.ts +156 -0
  424. package/src/mcp/__tests__/startup.test.ts +30 -7
  425. package/src/mcp/__tests__/tool-caps.test.ts +134 -0
  426. package/src/mcp/manager.ts +171 -101
  427. package/src/mcp/startup.ts +33 -13
  428. package/src/mcp/tool-caps.ts +184 -0
  429. package/src/media/__tests__/image-models.test.ts +58 -0
  430. package/src/media/image-credentials.ts +5 -0
  431. package/src/media/image-models.ts +52 -0
  432. package/src/media/image-service.ts +8 -0
  433. package/src/media/openrouter-image-service.test.ts +133 -0
  434. package/src/media/openrouter-image-service.ts +153 -0
  435. package/src/media/types.ts +1 -1
  436. package/src/messaging/provider-message-metadata.ts +3 -3
  437. package/src/messaging/providers/__tests__/transport-dispatch.test.ts +65 -11
  438. package/src/messaging/providers/discord/send.ts +4 -17
  439. package/src/messaging/providers/discord/transport.ts +2 -2
  440. package/src/messaging/providers/outlook/client.ts +16 -22
  441. package/src/messaging/providers/telegram-bot/send.test.ts +236 -1
  442. package/src/messaging/providers/telegram-bot/send.ts +10 -7
  443. package/src/messaging/providers/telegram-bot/transport.ts +3 -10
  444. package/src/messaging/providers/whatsapp/send.test.ts +169 -4
  445. package/src/monitoring/__tests__/control.test.ts +17 -5
  446. package/src/monitoring/control.ts +15 -5
  447. package/src/notifications/AGENTS.md +1 -1
  448. package/src/notifications/README.md +83 -99
  449. package/src/notifications/__tests__/broadcaster.test.ts +1 -1
  450. package/src/notifications/__tests__/channel-allowlist.test.ts +39 -0
  451. package/src/notifications/__tests__/copy-composer.test.ts +71 -1
  452. package/src/notifications/__tests__/decision-engine.test.ts +79 -2
  453. package/src/notifications/__tests__/deterministic-checks.test.ts +10 -10
  454. package/src/notifications/__tests__/home-feed-side-effect.test.ts +93 -4
  455. package/src/notifications/__tests__/notification-utils.test.ts +18 -0
  456. package/src/notifications/adapters/macos.ts +37 -46
  457. package/src/notifications/broadcaster.ts +5 -6
  458. package/src/notifications/channel-allowlist.ts +44 -0
  459. package/src/notifications/copy-composer.ts +11 -26
  460. package/src/notifications/decision-engine.ts +24 -14
  461. package/src/notifications/deterministic-checks.ts +1 -1
  462. package/src/notifications/emit-signal.ts +123 -37
  463. package/src/notifications/home-feed-side-effect.ts +9 -7
  464. package/src/notifications/notification-utils.ts +19 -0
  465. package/src/notifications/schedule-result-producer.ts +4 -6
  466. package/src/notifications/signal.ts +4 -19
  467. package/src/notifications/types.ts +15 -0
  468. package/src/oauth/AGENTS.md +2 -0
  469. package/src/oauth/__tests__/identity-verifier.test.ts +36 -1
  470. package/src/oauth/connection-resolver.ts +1 -2
  471. package/src/oauth/identity-verifier.ts +24 -0
  472. package/src/oauth/oauth-store.ts +12 -4
  473. package/src/oauth/provider-serializer.ts +8 -0
  474. package/src/oauth/seed-providers.ts +42 -0
  475. package/src/onboarding/onboarding-events-store.ts +99 -12
  476. package/src/permissions/confirmation-guardian-request.test.ts +16 -0
  477. package/src/permissions/confirmation-guardian-request.ts +2 -3
  478. package/src/permissions/question-prompter.test.ts +31 -0
  479. package/src/permissions/question-prompter.ts +2 -0
  480. package/src/persistence/__tests__/client-connection-events-store.test.ts +322 -0
  481. package/src/persistence/attachments-store.ts +84 -44
  482. package/src/persistence/client-connection-events-store.ts +377 -0
  483. package/src/persistence/conversation-crud.ts +166 -26
  484. package/src/persistence/conversation-mode-sessions.test.ts +424 -0
  485. package/src/persistence/conversation-mode-sessions.ts +460 -0
  486. package/src/persistence/conversation-plugin-facade.ts +30 -0
  487. package/src/persistence/conversation-queries.ts +6 -19
  488. package/src/persistence/conversation-tool-surface.ts +135 -0
  489. package/src/persistence/conversation-types.test.ts +32 -0
  490. package/src/persistence/conversation-types.ts +26 -4
  491. package/src/persistence/llm-usage-store.ts +3 -9
  492. package/src/persistence/message-metadata.ts +25 -0
  493. package/src/persistence/migrations/378-create-conversation-tool-surfaces.test.ts +78 -0
  494. package/src/persistence/migrations/378-create-conversation-tool-surfaces.ts +29 -0
  495. package/src/persistence/migrations/379-oauth-providers-response-ok-field.test.ts +92 -0
  496. package/src/persistence/migrations/379-oauth-providers-response-ok-field.ts +32 -0
  497. package/src/persistence/migrations/380-conversation-tool-surfaces-delegate-independent-tasks.test.ts +90 -0
  498. package/src/persistence/migrations/380-conversation-tool-surfaces-delegate-independent-tasks.ts +34 -0
  499. package/src/persistence/migrations/381-create-conversation-mode-sessions.ts +41 -0
  500. package/src/persistence/migrations/382-create-client-connection-events.ts +40 -0
  501. package/src/persistence/migrations/__tests__/382-create-client-connection-events.test.ts +56 -0
  502. package/src/persistence/schema/conversation-tool-surfaces.ts +32 -0
  503. package/src/persistence/schema/conversations.ts +67 -0
  504. package/src/persistence/schema/index.ts +1 -0
  505. package/src/persistence/schema/infrastructure.ts +29 -0
  506. package/src/persistence/schema/oauth.ts +1 -0
  507. package/src/persistence/steps.ts +41 -3
  508. package/src/plugin-api/__tests__/plugin-name-env.test.ts +249 -0
  509. package/src/plugin-api/conversation-turn.ts +31 -7
  510. package/src/plugin-api/effective-context-window.ts +33 -0
  511. package/src/plugin-api/index.ts +35 -6
  512. package/src/plugin-api/plugin-channel-turn-trust.test.ts +133 -0
  513. package/src/plugin-api/plugin-channel-turn-trust.ts +71 -0
  514. package/src/plugin-api/plugin-name-env.ts +125 -0
  515. package/src/plugin-api/resolve-credential.ts +79 -20
  516. package/src/plugin-api/vision-support.test.ts +22 -0
  517. package/src/plugin-api/vision-support.ts +17 -5
  518. package/src/plugins/__tests__/active-plugin-env.test.ts +136 -0
  519. package/src/plugins/active-plugin-env.ts +51 -0
  520. package/src/plugins/defaults/index.ts +24 -0
  521. package/src/plugins/defaults/memory/AGENTS.md +39 -5
  522. package/src/plugins/defaults/memory/__tests__/archive-removal-framing.test.ts +64 -0
  523. package/src/plugins/defaults/memory/__tests__/buffer-file.test.ts +365 -0
  524. package/src/plugins/defaults/memory/__tests__/buffer-format.test.ts +43 -0
  525. package/src/plugins/defaults/memory/__tests__/conversation-memory-purge.test.ts +1 -0
  526. package/src/plugins/defaults/memory/__tests__/db-memory-attach.test.ts +2 -0
  527. package/src/plugins/defaults/memory/__tests__/fixtures/buffer-appender.ts +17 -0
  528. package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +74 -0
  529. package/src/plugins/defaults/memory/__tests__/memory-run-evidence.test.ts +161 -0
  530. package/src/plugins/defaults/memory/__tests__/relocated-memory-test-rows.ts +10 -0
  531. package/src/plugins/defaults/memory/__tests__/worker-control.test.ts +19 -6
  532. package/src/plugins/defaults/memory/buffer-file.ts +354 -0
  533. package/src/plugins/defaults/memory/buffer-format.ts +40 -0
  534. package/src/plugins/defaults/memory/context-search/agent-protocol.ts +9 -12
  535. package/src/plugins/defaults/memory/context-search/agent-runner.ts +77 -22
  536. package/src/plugins/defaults/memory/context-search/format.ts +53 -8
  537. package/src/plugins/defaults/memory/context-search/limits.ts +10 -12
  538. package/src/plugins/defaults/memory/context-search/recall-input.ts +38 -0
  539. package/src/plugins/defaults/memory/context-search/search.ts +8 -14
  540. package/src/plugins/defaults/memory/context-search/sources/conversations.ts +1 -0
  541. package/src/plugins/defaults/memory/context-search/sources/memory-v2.ts +2 -1
  542. package/src/plugins/defaults/memory/context-search/sources/workspace.ts +2 -1
  543. package/src/plugins/defaults/memory/context-search/types.ts +1 -11
  544. package/src/plugins/defaults/memory/conversation-memory-purge.ts +4 -0
  545. package/src/plugins/defaults/memory/graph/__tests__/handle-remember-v2.test.ts +11 -4
  546. package/src/plugins/defaults/memory/graph/capability-seed.ts +1 -2
  547. package/src/plugins/defaults/memory/graph/tool-handlers.ts +13 -63
  548. package/src/plugins/defaults/memory/graph/tools.ts +35 -64
  549. package/src/plugins/defaults/memory/host-utils.ts +0 -10
  550. package/src/plugins/defaults/memory/injectors.ts +4 -3
  551. package/src/plugins/defaults/memory/memory-retrospective-job.ts +72 -183
  552. package/src/plugins/defaults/memory/memory-run-evidence.ts +213 -0
  553. package/src/plugins/defaults/memory/src/memory-eval-routes.ts +2 -3
  554. package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +6 -1
  555. package/src/plugins/defaults/memory/src/memory-item-routes.ts +2 -1
  556. package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts +407 -99
  557. package/src/plugins/defaults/memory/substrate/__tests__/consolidation-prompt-flag-gating-guard.test.ts +10 -0
  558. package/src/plugins/defaults/memory/substrate/__tests__/prompts-consolidation.test.ts +107 -7
  559. package/src/plugins/defaults/memory/substrate/consolidation-job.ts +307 -86
  560. package/src/plugins/defaults/memory/substrate/consolidation-tool-surface.ts +34 -0
  561. package/src/plugins/defaults/memory/substrate/page-index.ts +2 -1
  562. package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +91 -51
  563. package/src/plugins/defaults/memory/substrate/sweep-job.ts +1 -1
  564. package/src/plugins/defaults/memory/tools.test.ts +124 -9
  565. package/src/plugins/defaults/memory/tools.ts +38 -11
  566. package/src/plugins/defaults/memory/v1/graph/consolidation.ts +2 -2
  567. package/src/plugins/defaults/memory/v1/graph/extraction.ts +2 -1
  568. package/src/plugins/defaults/memory/v1/graph/retriever.ts +1 -1
  569. package/src/plugins/defaults/memory/v2/__tests__/migration.test.ts +5 -0
  570. package/src/plugins/defaults/memory/v2/__tests__/reranker.test.ts +5 -2
  571. package/src/plugins/defaults/memory/v2/reranker.ts +2 -1
  572. package/src/plugins/defaults/memory/v3/__tests__/injection.test.ts +81 -1
  573. package/src/plugins/defaults/memory/v3/__tests__/orchestrate.test.ts +87 -0
  574. package/src/plugins/defaults/memory/v3/__tests__/plugin-schema.test.ts +12 -0
  575. package/src/plugins/defaults/memory/v3/__tests__/pool-log-store.test.ts +332 -15
  576. package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +626 -27
  577. package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +148 -28
  578. package/src/plugins/defaults/memory/v3/card.ts +2 -1
  579. package/src/plugins/defaults/memory/v3/injector.ts +212 -178
  580. package/src/plugins/defaults/memory/v3/orchestrate.ts +159 -32
  581. package/src/plugins/defaults/memory/v3/plugin-schema.ts +54 -4
  582. package/src/plugins/defaults/memory/v3/pool-log-store.ts +275 -37
  583. package/src/plugins/defaults/memory/v3/pool-select.test.ts +47 -5
  584. package/src/plugins/defaults/memory/v3/pool-select.ts +710 -59
  585. package/src/plugins/defaults/memory/v3/sections.ts +2 -1
  586. package/src/plugins/defaults/memory/v3/shadow-plugin.ts +41 -14
  587. package/src/plugins/defaults/memory/worker-control.ts +15 -5
  588. package/src/plugins/defaults/task-progress/__tests__/registration.test.ts +70 -0
  589. package/src/plugins/defaults/task-progress/__tests__/task-progress-context.test.ts +380 -0
  590. package/src/plugins/defaults/task-progress/hooks/post-compact.ts +19 -0
  591. package/src/plugins/defaults/task-progress/hooks/user-prompt-submit.ts +24 -0
  592. package/src/plugins/defaults/task-progress/package.json +14 -0
  593. package/src/plugins/defaults/task-progress/src/apply-task-progress-context.ts +93 -0
  594. package/src/plugins/defaults/task-progress/src/format-task-progress-context.ts +67 -0
  595. package/src/plugins/defaults/tool-error/hooks/post-tool-use.ts +4 -1
  596. package/src/plugins/defaults/tool-result-truncate/terminal.ts +1 -46
  597. package/src/prompts/__tests__/parallel-tasks-section.test.ts +25 -0
  598. package/src/prompts/delegation-gate.ts +57 -0
  599. package/src/prompts/system-prompt.ts +14 -32
  600. package/src/providers/__tests__/context-overflow-error.test.ts +10 -0
  601. package/src/providers/__tests__/search-provider-catalog.test.ts +18 -0
  602. package/src/providers/anthropic/client.ts +10 -1
  603. package/src/providers/anthropic-gateway-shared.ts +6 -0
  604. package/src/providers/fetch-provider-catalog.ts +11 -0
  605. package/src/providers/gemini/client.ts +25 -8
  606. package/src/providers/inference/adapter-factory.ts +6 -0
  607. package/src/providers/jev/client.test.ts +270 -0
  608. package/src/providers/jev/client.ts +534 -0
  609. package/src/providers/model-catalog.ts +59 -3
  610. package/src/providers/openai/__tests__/chat-completions-extra-body.test.ts +94 -1
  611. package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +106 -1
  612. package/src/providers/openai/__tests__/chat-template-flatten.test.ts +24 -0
  613. package/src/providers/openai/__tests__/google-thought-signature.test.ts +1 -2
  614. package/src/providers/openai/chat-completions-provider.ts +100 -29
  615. package/src/providers/openai/google-thought-signature.ts +0 -6
  616. package/src/providers/openai/responses-provider.ts +23 -1
  617. package/src/providers/ratelimit.ts +1 -3
  618. package/src/providers/search-provider-catalog.ts +24 -0
  619. package/src/providers/server-tool-pairing.ts +16 -7
  620. package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +24 -0
  621. package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +7 -1
  622. package/src/providers/speech-to-text/__tests__/vellum-managed-realtime.test.ts +50 -1
  623. package/src/providers/speech-to-text/deepgram-flux-frames.ts +11 -0
  624. package/src/providers/speech-to-text/vellum-managed-flux-realtime.ts +13 -0
  625. package/src/providers/types.ts +5 -0
  626. package/src/routes/control.ts +7 -5
  627. package/src/runtime/AGENTS.md +6 -4
  628. package/src/runtime/__tests__/agent-wake.test.ts +94 -1
  629. package/src/runtime/actor-trust-resolver.ts +3 -3
  630. package/src/runtime/agent-wake.ts +32 -5
  631. package/src/runtime/approval-message-composer.ts +3 -69
  632. package/src/runtime/assistant-event-hub.ts +73 -75
  633. package/src/runtime/assistant-event-publish-options.ts +7 -0
  634. package/src/runtime/assistant-event-targeting.ts +79 -0
  635. package/src/runtime/assistant-stream-state.ts +13 -100
  636. package/src/runtime/background-job-runner.ts +5 -5
  637. package/src/runtime/channel-retry-sweep.ts +1 -0
  638. package/src/runtime/guardian-action-message-composer.ts +8 -161
  639. package/src/runtime/guardian-action-service.ts +2 -17
  640. package/src/runtime/guardian-reply-router.ts +29 -40
  641. package/src/runtime/http-server.ts +2 -2
  642. package/src/runtime/http-types.ts +6 -0
  643. package/src/runtime/message-composer-types.ts +3 -61
  644. package/src/runtime/migrations/__tests__/vbundle-import-parity.test.ts +67 -1
  645. package/src/runtime/migrations/__tests__/vbundle-import-policy.test.ts +89 -0
  646. package/src/runtime/migrations/vbundle-builder.ts +24 -1
  647. package/src/runtime/migrations/vbundle-import-analyzer.ts +13 -4
  648. package/src/runtime/migrations/vbundle-import-policy.ts +43 -11
  649. package/src/runtime/migrations/vbundle-importer.ts +4 -1
  650. package/src/runtime/migrations/vbundle-streaming-importer.ts +4 -1
  651. package/src/runtime/migrations/vbundle-validator.ts +1 -0
  652. package/src/runtime/pending-interactions-mode-session.test.ts +62 -0
  653. package/src/runtime/pending-interactions.ts +29 -0
  654. package/src/runtime/routes/__tests__/client-history-routes.test.ts +104 -0
  655. package/src/runtime/routes/__tests__/image-generation-routes.test.ts +97 -0
  656. package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +24 -2
  657. package/src/runtime/routes/__tests__/roadmap-routes.test.ts +29 -6
  658. package/src/runtime/routes/browser-context.ts +31 -11
  659. package/src/runtime/routes/browser-routes.ts +43 -4
  660. package/src/runtime/routes/btw-routes.ts +9 -4
  661. package/src/runtime/routes/channel-route-shared.ts +1 -9
  662. package/src/runtime/routes/client-routes.ts +108 -0
  663. package/src/runtime/routes/conversation-management-routes.ts +4 -1
  664. package/src/runtime/routes/conversation-routes.ts +125 -14
  665. package/src/runtime/routes/credential-prompt-routes.ts +4 -3
  666. package/src/runtime/routes/credential-routes.ts +1 -2
  667. package/src/runtime/routes/desktop-setup-routes.test.ts +9 -18
  668. package/src/runtime/routes/desktop-setup-routes.ts +8 -7
  669. package/src/runtime/routes/documents-routes.ts +51 -0
  670. package/src/runtime/routes/email-routes.ts +3 -7
  671. package/src/runtime/routes/events-routes.ts +75 -54
  672. package/src/runtime/routes/guardian-approval-interception.ts +24 -0
  673. package/src/runtime/routes/identity-routes.ts +1 -5
  674. package/src/runtime/routes/image-generation-routes.ts +8 -14
  675. package/src/runtime/routes/inbound-message-handler.ts +37 -41
  676. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.test.ts +168 -0
  677. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.ts +96 -0
  678. package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +39 -10
  679. package/src/runtime/routes/inbound-stages/background-dispatch.ts +9 -9
  680. package/src/runtime/routes/inbound-stages/reaction-intercept.ts +2 -0
  681. package/src/runtime/routes/inference-profile-session-handler.ts +11 -1
  682. package/src/runtime/routes/inference-profiles-routes.ts +9 -0
  683. package/src/runtime/routes/llm-context-normalization.ts +1 -7
  684. package/src/runtime/routes/migration-routes.ts +73 -2
  685. package/src/runtime/routes/notification-routes.ts +10 -0
  686. package/src/runtime/routes/oauth-commands-routes.ts +57 -54
  687. package/src/runtime/routes/oauth-providers.ts +11 -0
  688. package/src/runtime/routes/oauth-request-hints.test.ts +246 -0
  689. package/src/runtime/routes/oauth-request-hints.ts +149 -0
  690. package/src/runtime/routes/plugins-routes.ts +6 -0
  691. package/src/runtime/routes/roadmap-routes.ts +47 -8
  692. package/src/runtime/routes/schedule-routes.ts +2 -0
  693. package/src/runtime/routes/secret-routes.ts +99 -119
  694. package/src/runtime/routes/settings-routes.ts +10 -4
  695. package/src/runtime/routes/watcher-routes.ts +1 -3
  696. package/src/runtime/trust-class.test.ts +7 -6
  697. package/src/runtime/trust-class.ts +9 -17
  698. package/src/schedule/__tests__/worker-control.test.ts +36 -5
  699. package/src/schedule/__tests__/worker-mcp-bootstrap.test.ts +107 -0
  700. package/src/schedule/__tests__/worker-mcp-readiness.test.ts +173 -0
  701. package/src/schedule/__tests__/worker-mcp-tools.test.ts +13 -47
  702. package/src/schedule/run-script.ts +2 -2
  703. package/src/schedule/scheduler.ts +47 -11
  704. package/src/schedule/tool-surface-readiness.ts +65 -0
  705. package/src/schedule/worker-control.ts +15 -5
  706. package/src/schedule/worker-mcp.ts +85 -0
  707. package/src/schedule/worker.ts +9 -31
  708. package/src/security/secure-keys.ts +74 -112
  709. package/src/skills/clawhub.ts +3 -8
  710. package/src/skills/managed-store.ts +1 -5
  711. package/src/stt/types.ts +4 -0
  712. package/src/subagent/notify.ts +54 -34
  713. package/src/subagent/parent-notification.ts +6 -0
  714. package/src/telemetry/__tests__/phone-call-funnel.test.ts +72 -0
  715. package/src/telemetry/phone-call-funnel.ts +144 -0
  716. package/src/tools/__tests__/tool-schema-root-combinator-guard.test.ts +81 -0
  717. package/src/tools/ask-question/ask-question-tool.test.ts +167 -5
  718. package/src/tools/ask-question/ask-question-tool.ts +120 -22
  719. package/src/tools/browser/browser-execution.ts +29 -60
  720. package/src/tools/browser/cdp-client/__tests__/cdp-dom-helpers.test.ts +561 -39
  721. package/src/tools/browser/cdp-client/cdp-dom-helpers.ts +435 -39
  722. package/src/tools/browser/cdp-client/cdp-inspect-client.ts +1 -8
  723. package/src/tools/client-os.ts +15 -0
  724. package/src/tools/computer-use/skill-proxy-bridge.ts +1 -1
  725. package/src/tools/computer-use/target.ts +42 -0
  726. package/src/tools/document/document-tool.ts +13 -18
  727. package/src/tools/execution-target.ts +14 -16
  728. package/src/tools/host-terminal/host-shell.ts +16 -7
  729. package/src/tools/network/__tests__/web-fetch-tinyfish.test.ts +250 -0
  730. package/src/tools/network/__tests__/web-search.test.ts +203 -0
  731. package/src/tools/network/web-fetch.ts +274 -7
  732. package/src/tools/network/web-search.ts +468 -6
  733. package/src/tools/permission-checker.ts +3 -3
  734. package/src/tools/policy-context.ts +3 -1
  735. package/src/tools/shared/filesystem/file-ops-service.ts +1 -31
  736. package/src/tools/shared/shell-output.test.ts +10 -0
  737. package/src/tools/shared/shell-output.ts +14 -2
  738. package/src/tools/skills/load.ts +2 -1
  739. package/src/tools/skills/sandbox-runner.ts +24 -4
  740. package/src/tools/skills/scaffold-managed.ts +2 -1
  741. package/src/tools/skills/skill-script-runner.ts +7 -0
  742. package/src/tools/skills/skill-tool-factory.ts +15 -1
  743. package/src/tools/terminal/__tests__/safe-env.test.ts +44 -4
  744. package/src/tools/terminal/__tests__/sanitized-bash.test.ts +0 -14
  745. package/src/tools/terminal/safe-env.ts +39 -24
  746. package/src/tools/terminal/sanitized-bash.ts +15 -2
  747. package/src/tools/terminal/shell-launch.test.ts +162 -0
  748. package/src/tools/terminal/shell.test.ts +29 -0
  749. package/src/tools/terminal/shell.ts +15 -7
  750. package/src/tools/tool-approval-handler.ts +5 -1
  751. package/src/tools/tool-types.ts +3 -3
  752. package/src/tools/types.ts +8 -2
  753. package/src/util/__tests__/worker-process-command.test.ts +5 -4
  754. package/src/util/abort-reasons.ts +18 -26
  755. package/src/util/browser-human-verification.ts +14 -0
  756. package/src/util/canonicalize-identity.test.ts +21 -0
  757. package/src/util/canonicalize-identity.ts +16 -0
  758. package/src/util/errors.ts +13 -3
  759. package/src/util/host-process.test.ts +17 -1
  760. package/src/util/host-process.ts +24 -0
  761. package/src/util/provider-error-patterns.ts +3 -0
  762. package/src/util/unicode.ts +29 -0
  763. package/src/util/worker-process.ts +77 -35
  764. package/src/workspace/migrations/157-add-desktop-files-launcher.ts +94 -0
  765. package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +4 -10
  766. package/src/workspace/migrations/__tests__/157-add-desktop-files-launcher.test.ts +113 -0
  767. package/src/workspace/migrations/registry.ts +2 -0
  768. package/src/__tests__/computer-use-window-schema.test.ts +0 -24
  769. package/src/api/events/heartbeat-alert.ts +0 -20
  770. package/src/daemon/message-types/web-activity.ts +0 -65
  771. package/src/desktop/desktop-dependencies.test.ts +0 -131
  772. package/src/tools/computer-use/definitions.ts +0 -607
@@ -27,7 +27,10 @@ import {
27
27
  type InstalledPluginInfo,
28
28
  readInstalledPlugin,
29
29
  } from "./list-installed-plugins.js";
30
- import { readBundledLocalPluginCatalog } from "./plugin-catalog-local.js";
30
+ import {
31
+ arePlatformFeaturesEnabled,
32
+ readBundledLocalPluginCatalog,
33
+ } from "./plugin-catalog-local.js";
31
34
  import { DEFAULT_PLUGIN_REF } from "./plugin-constants.js";
32
35
  import {
33
36
  compareFingerprint,
@@ -338,17 +341,21 @@ export async function inspectPlugin(
338
341
  let remote: PluginRemoteInfo | null = null;
339
342
  let remoteError: string | null = null;
340
343
  const installedLocalSource = local?.source?.kind === "local";
344
+ const useBundledLocalCatalog =
345
+ installedLocalSource || (!installed && !arePlatformFeaturesEnabled());
341
346
  try {
342
- if (installedLocalSource) {
347
+ if (useBundledLocalCatalog) {
343
348
  const catalog = deps.localCatalog ?? readBundledLocalPluginCatalog();
344
349
  const match = catalog.matches.find(
345
350
  (candidate) =>
346
351
  candidate.name === name &&
347
352
  candidate.source.kind === "local" &&
348
- candidate.source.path === local.source?.path,
353
+ (!installedLocalSource ||
354
+ candidate.source.path === local.source?.path),
349
355
  );
350
356
  remote = match ? readLocalPackageRemote(match, catalog.ref) : null;
351
- } else {
357
+ }
358
+ if (!remote && !installedLocalSource) {
352
359
  const entries = await fetchMarketplaceEntries(
353
360
  { fetch: deps.fetch },
354
361
  { ref: marketplaceRef },
@@ -127,6 +127,14 @@ export const mcpIntegrationSchema = z.object({
127
127
  }),
128
128
  /** Filename under the web client's bundled integration-image directory. */
129
129
  logo: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9_.-]*$/),
130
+ /**
131
+ * Where the integrations catalog files the entry. Any string at this
132
+ * boundary: a catalog newer than this build can name a category it does not
133
+ * know, and one unfamiliar slug must not reject the whole catalog. Clients
134
+ * normalize against the shared vocabulary; the bundled inventory test holds
135
+ * every entry in this repo to it.
136
+ */
137
+ category: z.string().optional(),
130
138
  /** Existing main OAuth provider used to group a related MCP connection. */
131
139
  oauthProvider: z.string().regex(PLUGIN_NAME_RE).optional(),
132
140
  });
@@ -337,8 +337,7 @@ export async function postPublishRequest(
337
337
  },
338
338
  );
339
339
 
340
- const data = (await resp.json()) as PublishResult;
341
- return data;
340
+ return (await resp.json()) as PublishResult;
342
341
  }
343
342
 
344
343
  /**
@@ -412,6 +412,24 @@ export async function upgradePlugin(
412
412
  };
413
413
  }
414
414
 
415
+ const usesMergeStrategy =
416
+ strategy === "ours" || strategy === "theirs" || strategy === "assistant";
417
+ const canOverwriteCleanBundledInstall =
418
+ remote.kind === "local" &&
419
+ strategy === "theirs" &&
420
+ local.localChanges?.clean === true;
421
+
422
+ if (
423
+ remote.kind === "local" &&
424
+ usesMergeStrategy &&
425
+ !canOverwriteCleanBundledInstall
426
+ ) {
427
+ throw new PluginMergeBaselineError(
428
+ name,
429
+ "bundled packages do not retain the previous package version needed for a three-way merge",
430
+ );
431
+ }
432
+
415
433
  if (dryRun) {
416
434
  return {
417
435
  name,
@@ -433,17 +451,7 @@ export async function upgradePlugin(
433
451
  // `ours`/`theirs`/`assistant` carry local edits forward via a three-way
434
452
  // merge; the default `overwrite` discards them and re-installs the pin
435
453
  // wholesale.
436
- if (
437
- strategy === "ours" ||
438
- strategy === "theirs" ||
439
- strategy === "assistant"
440
- ) {
441
- if (remote.kind === "local") {
442
- throw new PluginMergeBaselineError(
443
- name,
444
- "bundled packages do not retain the previous package version needed for a three-way merge",
445
- );
446
- }
454
+ if (usesMergeStrategy && remote.kind !== "local") {
447
455
  const [remoteOwner, remoteRepo] = remote.repo.split("/");
448
456
  return mergeUpgrade(
449
457
  {
package/src/cli/output.ts CHANGED
@@ -11,20 +11,34 @@ export function shouldOutputJson(cmd: Command): boolean {
11
11
  return false;
12
12
  }
13
13
 
14
- export function writeOutput(cmd: Command, payload: unknown): void {
14
+ /**
15
+ * Write a JSON payload to stdout. `onFlushed` runs once the bytes have left
16
+ * the process, for a command that must exit itself and would otherwise cut
17
+ * a piped payload short.
18
+ */
19
+ export function writeOutput(
20
+ cmd: Command,
21
+ payload: unknown,
22
+ onFlushed?: () => void,
23
+ ): void {
15
24
  const compact = shouldOutputJson(cmd);
16
25
  process.stdout.write(
17
26
  compact
18
27
  ? JSON.stringify(payload) + "\n"
19
28
  : JSON.stringify(payload, null, 2) + "\n",
29
+ onFlushed,
20
30
  );
21
31
  }
22
32
 
23
33
  /** Format-aware error output: JSON envelope with --json, stderr otherwise. */
24
- export function writeError(cmd: Command, message: string): void {
34
+ export function writeError(
35
+ cmd: Command,
36
+ message: string,
37
+ onFlushed?: () => void,
38
+ ): void {
25
39
  if (shouldOutputJson(cmd)) {
26
- writeOutput(cmd, { ok: false, error: message });
40
+ writeOutput(cmd, { ok: false, error: message }, onFlushed);
27
41
  } else {
28
- process.stderr.write(`Error: ${message}\n`);
42
+ process.stderr.write(`Error: ${message}\n`, onFlushed);
29
43
  }
30
44
  }
@@ -72,6 +72,12 @@ describe("LLMSchema.defaultProvider", () => {
72
72
  ).toThrow();
73
73
  });
74
74
 
75
+ test("rejects a structured-decision catalog provider", () => {
76
+ expect(() =>
77
+ DefaultProviderSchema.parse({ provider: "typesafe" }),
78
+ ).toThrow();
79
+ });
80
+
75
81
  test("rejects an empty connectionName", () => {
76
82
  expect(() =>
77
83
  DefaultProviderSchema.parse({
@@ -12,10 +12,11 @@ import { describe, expect, test } from "bun:test";
12
12
  import { MemoryRetrospectiveConfigSchema } from "../schemas/memory-retrospective.js";
13
13
 
14
14
  describe("memory.retrospective config schema", () => {
15
- test("an empty block leaves retrospectives and skill improvement on", () => {
15
+ test("an empty block leaves skill improvement on and monitoring off", () => {
16
16
  const parsed = MemoryRetrospectiveConfigSchema.parse({});
17
17
  expect(parsed.enabled).toBe(true);
18
18
  expect(parsed.skillImprovement).toBe(true);
19
+ expect(parsed.skillImprovementMonitoring).toBe(false);
19
20
  expect(parsed).not.toHaveProperty("forkStrategy");
20
21
  });
21
22
 
@@ -41,6 +42,18 @@ describe("memory.retrospective config schema", () => {
41
42
  ).toBe(false);
42
43
  });
43
44
 
45
+ test("skillImprovementMonitoring is a boolean-only opt-in", () => {
46
+ const parsed = MemoryRetrospectiveConfigSchema.parse({
47
+ skillImprovementMonitoring: true,
48
+ });
49
+ expect(parsed.skillImprovementMonitoring).toBe(true);
50
+ expect(
51
+ MemoryRetrospectiveConfigSchema.safeParse({
52
+ skillImprovementMonitoring: "true",
53
+ }).success,
54
+ ).toBe(false);
55
+ });
56
+
44
57
  test("a leftover forkStrategy key is ignored", () => {
45
58
  const parsed = MemoryRetrospectiveConfigSchema.parse({
46
59
  forkStrategy: "cloning",
@@ -21,7 +21,7 @@ connected desktop. CU tools run through the main agent loop via HostCuProxy.
21
21
  The skill is internally preactivated for conversations with a connected desktop client.
22
22
 
23
23
  Tools in this skill are proxy tools. Execution is forwarded to a connected
24
- desktop client and is never handled locally by the assistant.
24
+ desktop client or handled by the assistant's virtual desktop.
25
25
 
26
26
  ## Observations
27
27
 
@@ -21,11 +21,16 @@
21
21
  },
22
22
  "full_tree": {
23
23
  "type": "boolean",
24
- "description": "Walk the accessibility tree to full depth. Observations list the tree to a limited depth and say when it was cut off; pass this when the element you need is not in the tree."
24
+ "description": "On connected personal computers, walk the accessibility tree to full depth. The virtual desktop uses a bounded tree; use screenshot coordinates when an element is not listed."
25
25
  },
26
26
  "target_client_id": {
27
27
  "type": "string",
28
28
  "description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
29
+ },
30
+ "target": {
31
+ "type": "string",
32
+ "enum": ["connected-computer", "assistant-desktop"],
33
+ "description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
29
34
  }
30
35
  },
31
36
  "required": []
@@ -65,6 +70,15 @@
65
70
  "target_client_id": {
66
71
  "type": "string",
67
72
  "description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
73
+ },
74
+ "target": {
75
+ "type": "string",
76
+ "enum": ["connected-computer", "assistant-desktop"],
77
+ "description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
78
+ },
79
+ "observation_id": {
80
+ "type": "string",
81
+ "description": "Required on assistant-desktop: the ID from its latest screenshot. Observe again after browser actions, user handoff, or interruption. Omit for connected-computer. For a sequence, applies to the whole batch."
68
82
  }
69
83
  },
70
84
  "required": ["reasoning"]
@@ -91,6 +105,15 @@
91
105
  "target_client_id": {
92
106
  "type": "string",
93
107
  "description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
108
+ },
109
+ "target": {
110
+ "type": "string",
111
+ "enum": ["connected-computer", "assistant-desktop"],
112
+ "description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
113
+ },
114
+ "observation_id": {
115
+ "type": "string",
116
+ "description": "Required on assistant-desktop: the ID from its latest screenshot. Observe again after browser actions, user handoff, or interruption. Omit for connected-computer. For a sequence, applies to the whole batch."
94
117
  }
95
118
  },
96
119
  "required": ["text", "reasoning"]
@@ -117,6 +140,15 @@
117
140
  "target_client_id": {
118
141
  "type": "string",
119
142
  "description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
143
+ },
144
+ "target": {
145
+ "type": "string",
146
+ "enum": ["connected-computer", "assistant-desktop"],
147
+ "description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
148
+ },
149
+ "observation_id": {
150
+ "type": "string",
151
+ "description": "Required on assistant-desktop: the ID from its latest screenshot. Observe again after browser actions, user handoff, or interruption. Omit for connected-computer. For a sequence, applies to the whole batch."
120
152
  }
121
153
  },
122
154
  "required": ["key", "reasoning"]
@@ -160,6 +192,15 @@
160
192
  "target_client_id": {
161
193
  "type": "string",
162
194
  "description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
195
+ },
196
+ "target": {
197
+ "type": "string",
198
+ "enum": ["connected-computer", "assistant-desktop"],
199
+ "description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
200
+ },
201
+ "observation_id": {
202
+ "type": "string",
203
+ "description": "Required on assistant-desktop: the ID from its latest screenshot. Observe again after browser actions, user handoff, or interruption. Omit for connected-computer. For a sequence, applies to the whole batch."
163
204
  }
164
205
  },
165
206
  "required": ["direction", "amount", "reasoning"]
@@ -206,6 +247,15 @@
206
247
  "target_client_id": {
207
248
  "type": "string",
208
249
  "description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
250
+ },
251
+ "target": {
252
+ "type": "string",
253
+ "enum": ["connected-computer", "assistant-desktop"],
254
+ "description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
255
+ },
256
+ "observation_id": {
257
+ "type": "string",
258
+ "description": "Required on assistant-desktop: the ID from its latest screenshot. Observe again after browser actions, user handoff, or interruption. Omit for connected-computer. For a sequence, applies to the whole batch."
209
259
  }
210
260
  },
211
261
  "required": ["reasoning"]
@@ -233,6 +283,15 @@
233
283
  "target_client_id": {
234
284
  "type": "string",
235
285
  "description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
286
+ },
287
+ "target": {
288
+ "type": "string",
289
+ "enum": ["connected-computer", "assistant-desktop"],
290
+ "description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
291
+ },
292
+ "observation_id": {
293
+ "type": "string",
294
+ "description": "Required on assistant-desktop: the ID from its latest screenshot. Observe again after browser actions, user handoff, or interruption. Omit for connected-computer. For a sequence, applies to the whole batch."
236
295
  }
237
296
  },
238
297
  "required": ["duration_ms", "reasoning"]
@@ -259,6 +318,11 @@
259
318
  "target_client_id": {
260
319
  "type": "string",
261
320
  "description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
321
+ },
322
+ "target": {
323
+ "type": "string",
324
+ "enum": ["connected-computer", "assistant-desktop"],
325
+ "description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
262
326
  }
263
327
  },
264
328
  "required": ["app_name", "reasoning"]
@@ -286,6 +350,11 @@
286
350
  "target_client_id": {
287
351
  "type": "string",
288
352
  "description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
353
+ },
354
+ "target": {
355
+ "type": "string",
356
+ "enum": ["connected-computer", "assistant-desktop"],
357
+ "description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
289
358
  }
290
359
  },
291
360
  "required": ["script", "reasoning"]
@@ -377,6 +446,15 @@
377
446
  "target_client_id": {
378
447
  "type": "string",
379
448
  "description": "ID of the specific client to target. Required when multiple clients support host_cu; omit when only one is connected. Obtain IDs from `assistant clients list --capability host_cu`."
449
+ },
450
+ "target": {
451
+ "type": "string",
452
+ "enum": ["connected-computer", "assistant-desktop"],
453
+ "description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
454
+ },
455
+ "observation_id": {
456
+ "type": "string",
457
+ "description": "Required on assistant-desktop: the ID from its latest screenshot. Observe again after browser actions, user handoff, or interruption. Omit for connected-computer. For a sequence, applies to the whole batch."
380
458
  }
381
459
  },
382
460
  "required": ["actions"]
@@ -396,6 +474,11 @@
396
474
  "summary": {
397
475
  "type": "string",
398
476
  "description": "Human-readable summary of what was accomplished"
477
+ },
478
+ "target": {
479
+ "type": "string",
480
+ "enum": ["connected-computer", "assistant-desktop"],
481
+ "description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
399
482
  }
400
483
  },
401
484
  "required": ["summary"]
@@ -418,6 +501,11 @@
418
501
  "reasoning": {
419
502
  "type": "string",
420
503
  "description": "Explanation of how you determined the answer"
504
+ },
505
+ "target": {
506
+ "type": "string",
507
+ "enum": ["connected-computer", "assistant-desktop"],
508
+ "description": "Desktop to control. Platform-hosted web conversations default to assistant-desktop; native desktop apps and non-platform assistants default to connected-computer. Explicit target or target_client_id overrides the default. Use assistant-desktop from a native app only when explicitly requested; do not combine it with target_client_id."
421
509
  }
422
510
  },
423
511
  "required": ["answer", "reasoning"]
@@ -37,13 +37,27 @@ Pass a concrete model ID only if the user names one explicitly. If the tool reje
37
37
  Generate (no model parameter, default is correct):
38
38
 
39
39
  ```json
40
- { "tool": "media_generate_image", "input": { "prompt": "A sunset over the ocean, golden hour, soft haze, 35mm photo style", "variants": 2 } }
40
+ {
41
+ "tool": "media_generate_image",
42
+ "input": {
43
+ "prompt": "A sunset over the ocean, golden hour, soft haze, 35mm photo style",
44
+ "variants": 2
45
+ }
46
+ }
41
47
  ```
42
48
 
43
49
  Edit:
44
50
 
45
51
  ```json
46
- { "tool": "media_generate_image", "input": { "prompt": "Remove the watermark text from the background. Keep the subject, framing, lighting, and colors exactly identical. Change nothing else.", "mode": "edit", "source_paths": ["conversations/<conv-id>/attachments/photo.jpeg"], "model": "openai" } }
52
+ {
53
+ "tool": "media_generate_image",
54
+ "input": {
55
+ "prompt": "Remove the watermark text from the background. Keep the subject, framing, lighting, and colors exactly identical. Change nothing else.",
56
+ "mode": "edit",
57
+ "source_paths": ["conversations/<conv-id>/attachments/photo.jpeg"],
58
+ "model": "openai"
59
+ }
60
+ }
47
61
  ```
48
62
 
49
63
  `source_paths` is a flat array of file path strings. Do NOT pass objects:
@@ -72,6 +86,7 @@ Edits on large photos are slow (1 to 2 minutes). If the tool reports a timeout (
72
86
  Each generated image is saved into the workspace under `media/generated/` and the tool result lists the saved paths. The images also come back as inline content blocks so you can judge the result before presenting it.
73
87
 
74
88
  - Present an image to the user by embedding its saved path in your reply: `![short description](vellum://workspace/media/generated/<file>.png)`. The app renders it inline where your text refers to it, and chat channels (Slack, Telegram, WhatsApp) deliver it as a native image upload.
89
+ - Label a plain image link "View image", for example `[View image](vellum://workspace/media/generated/<file>.png)`. The link opens a preview; do not call it "Download" or promise a download in the surrounding text. To save the image, the user can choose Download in the preview.
75
90
  - If you do not embed it, the image is still auto-attached to your reply as a file, so it is never lost. Prefer embedding: an attachment chip at the end of the message is a worse presentation than the image inline.
76
91
  - To iterate on a result, pass its saved path via `source_paths` with `mode: "edit"`.
77
92
 
@@ -5,10 +5,7 @@ import {
5
5
  resolveImageGenCredentials,
6
6
  resolveImageGenRouting,
7
7
  } from "../../../../media/image-credentials.js";
8
- import {
9
- describeImageModels,
10
- resolveImageModel,
11
- } from "../../../../media/image-models.js";
8
+ import { resolveRequestedImageModel } from "../../../../media/image-models.js";
12
9
  import {
13
10
  generateImage,
14
11
  mapImageGenError,
@@ -115,18 +112,17 @@ export async function run(
115
112
  const config = getConfig();
116
113
  const svc = config.services["image-generation"];
117
114
  let modelOverride = input.model;
118
- // Resolve tier aliases (fast, quality, openai) to concrete model IDs via
119
- // the registry. Unknown values get an error listing the current catalog so
120
- // callers can self-correct without a stale schema enum.
115
+ // Resolve aliases and OpenRouter slugs against the configured provider.
116
+ // Built-in providers still list the current catalog on unknown values.
121
117
  if (typeof modelOverride === "string" && modelOverride) {
122
- const entry = resolveImageModel(modelOverride);
123
- if (!entry) {
118
+ const resolved = resolveRequestedImageModel(modelOverride, svc.provider);
119
+ if (resolved.error) {
124
120
  return {
125
- content: `Unknown model "${modelOverride}". Available models and aliases:\n${describeImageModels()}\n\nRetry with one of the aliases above, or omit the model parameter to use the configured default.`,
121
+ content: `${resolved.error}\n\nRetry with one of the aliases above, or omit the model parameter to use the configured default.`,
126
122
  isError: true,
127
123
  };
128
124
  }
129
- modelOverride = entry.id;
125
+ modelOverride = resolved.model;
130
126
  }
131
127
  // Backend and managed-ness resolve together: an explicit model re-routes
132
128
  // to the model's backend (e.g. `gpt-image-2` under a gemini config routes
@@ -249,18 +249,18 @@ If any required capability is missing:
249
249
 
250
250
  Scheduled messages run without user interaction, in a conversation nobody has open. If the task produces output the user should see (a digest, summary, report, or a check whose answer is "nothing changed"), the scheduled message **must** end with an explicit instruction to deliver it. Without one, the output lives in a conversation log the user never opens.
251
251
 
252
- Write the delivery step into the `message` when you create the schedule — not as a vague "let me know", but as the actual call, with a real title:
252
+ Write the delivery step into the `message` when you create the schedule, not as a vague "let me know", but as the actual call, with a real title:
253
253
 
254
- > "…then send the summary with `assistant notifications send --title \"Inbox digest\" --message \"<the summary>\"`."
254
+ > "…then send the summary with `assistant notifications send --source-channel scheduler --title \"Inbox digest\" --message \"<the summary>\"`."
255
255
 
256
256
  A schedule whose message has no delivery step is not finished. Before calling `schedule_create` in `execute` mode, read your own message back and check that it says where the output goes.
257
257
 
258
- There is a safety net, and it is not a substitute for the above. When an execute-mode run finishes with user-facing output and nothing delivered it, the assistant sends a notification carrying the run's final reply, so a schedule can no longer run and leave no trace. `messaging_send` counts as a delivery only when the call reported success, and `assistant notifications send` once the notification pipeline has taken it: a send the channel refused, or one that never reported at all, leaves the run eligible for the net, so the failure reaches you instead of vanishing into a conversation nobody has open. A Slack `chat.postMessage` never counts, because Slack refuses a call with a successful HTTP status and `ok: false` in the body, so the command succeeding proves nothing; a run that posts Block Kit that way also gets the net's notification. It fires on the raw reply, whatever the run happened to end on, at whatever length. An authored delivery step gets a title and body you chose, sent at the moment you chose. Rely on the net and you get the machine's guess instead.
258
+ There is a safety net, and it is not a substitute for the above. When an execute-mode run finishes with user-facing output and nothing delivered it, the assistant sends a notification carrying the run's final reply, so a schedule can no longer run and leave no trace. `messaging_send` counts as a delivery only when the call reported success, and `assistant notifications send` once the notification pipeline has taken it: a send the channel refused, or one that never reported at all, leaves the run eligible for the net, so the failure reaches you instead of vanishing into a conversation nobody has open. A Slack `chat.postMessage` never counts, because the net recognizes only the recorded routes; a run that posts Block Kit that way also gets the net's notification. It fires on the raw reply, whatever the run happened to end on, at whatever length. An authored delivery step gets a title and body you chose, sent at the moment you chose. Rely on the net and you get the machine's guess instead.
259
259
 
260
- Choose the right delivery tool based on the content:
260
+ Choose the right delivery tool based on the destination:
261
261
 
262
- - **Rich content** (digests, summaries, reports): Use `messaging_send` with the target platform and conversation ID. It reaches Gmail or Outlook as a draft, and posts to a Slack, Telegram, or Discord chat through that channel's own transport, where the post is recorded and can be found again afterwards. Reach for `chat.postMessage` through `assistant oauth request` only for a Block Kit shape `messaging_send` cannot carry, with the provider the **slack** skill says to pass for posting on this workspace (`slack_channel` when the bot is set up; where only the `slack` integration exists, the skill says to tell the user before posting as them), never through `curl` or a revealed token. Either way the full content posts directly.
263
- - **Short alerts** (status updates, completion notices): Use `assistant notifications send` via `bash` to let the notification router pick the best channel. Note: the router's decision engine rewrites content into short alerts, so it is not suitable for rich content.
262
+ - **A specific chat or mailbox** (a named Slack, Telegram, or Discord conversation, or an email draft): Use `messaging_send` with the target platform and conversation ID. It reaches Gmail or Outlook as a draft, and posts to a Slack, Telegram, or Discord chat through that channel's own transport, where the post is recorded and can be found again afterwards. Reach for `chat.postMessage` through `assistant oauth request` only for a Block Kit shape `messaging_send` cannot carry, with the provider the **slack** skill says to pass for posting on this workspace (`slack_channel` when the bot is set up; where only the `slack` integration exists, the skill says to tell the user before posting as them), never through `curl` or a revealed token. Either way the full content posts directly.
263
+ - **The user's notification surfaces** (Vellum inbox, Telegram, platform push): Use `assistant notifications send` via `bash`. Pass `--source-channel scheduler` and the complete authored body as `--message`. The pipeline keeps that body. Do not summarize the report before sending.
264
264
 
265
265
  Example schedule message for a Slack digest:
266
266
 
@@ -165,11 +165,7 @@
165
165
  },
166
166
  "description": "Replacement steps (replaces all existing steps)"
167
167
  }
168
- },
169
- "oneOf": [
170
- { "required": ["id"] },
171
- { "required": ["enrollment_id", "enrollment_action"] }
172
- ]
168
+ }
173
169
  },
174
170
  "executor": "tools/sequence-update.ts",
175
171
  "execution_target": "host"
@@ -153,6 +153,15 @@ export const CALL_SITE_DEFAULTS: Record<LLMCallSite, CallSiteDefaultConfig> = {
153
153
  effort: "low",
154
154
  thinking: { enabled: false },
155
155
  },
156
+ // The escalation judge only runs when this site resolves to the TypeSafe
157
+ // provider (see calls/voice-escalation-judge.ts), so the shipped default is
158
+ // any ordinary profile: the judge stays off until a user pins a TypeSafe
159
+ // profile here.
160
+ voiceEscalationJudge: { profile: "cost-optimized" },
161
+ // Same arrangement for the barge-in continuation judge
162
+ // (live-voice/continuation-judge.ts): off until a TypeSafe profile is
163
+ // pinned here.
164
+ voiceContinuationJudge: { profile: "cost-optimized" },
156
165
  // Names the background continuation a barge-in spawns, from the interrupted
157
166
  // transcript. The label is fixed at spawn, so the call runs under a short
158
167
  // timeout and the deterministic label stands in when it misses; that makes
@@ -536,6 +536,14 @@
536
536
  "description": "Route all user-facing assistant text through the send_user_message tool instead of streamed assistant text",
537
537
  "defaultEnabled": false
538
538
  },
539
+ {
540
+ "id": "session-groups",
541
+ "scope": "both",
542
+ "key": "session-groups",
543
+ "label": "Session Groups",
544
+ "description": "Records eligible computer use, browser, live vision, and ambient runs and presents them as collapsible transcript groups. Off preserves ordinary actions and the flat transcript without new session tracking.",
545
+ "defaultEnabled": false
546
+ },
539
547
  {
540
548
  "id": "assistant-desktop",
541
549
  "scope": "assistant",
@@ -543,6 +551,14 @@
543
551
  "label": "Virtual desktop",
544
552
  "description": "Streams an interactive virtual desktop from a platform-hosted, containerized assistant over /v1/desktop/stream. Requires platform hosting; self-hosted assistants are excluded. Off: the virtual desktop is hidden. On: the chat header offers its viewer and controls.",
545
553
  "defaultEnabled": false
554
+ },
555
+ {
556
+ "id": "assistant-inbox",
557
+ "scope": "client",
558
+ "key": "assistant-inbox",
559
+ "label": "Assistant Inbox",
560
+ "description": "Shows an Assistant Inbox entry above Preferences in the web sidebar and the /assistant/inbox page it opens: a mailbox for the assistant's managed vellum.me address (received and sent, with a reading pane), the inline email setup card for entitled orgs with no address yet, and an upgrade card for orgs without the managed_email entitlement. Off hides the entry and redirects the route to chat.",
561
+ "defaultEnabled": false
546
562
  }
547
563
  ]
548
564
  }
@@ -58,14 +58,6 @@ type ConfigFileSignature =
58
58
  exists: false;
59
59
  };
60
60
 
61
- function getConfigPath(): string {
62
- return getWorkspaceConfigPath();
63
- }
64
-
65
- function ensureMigratedDataDir(): void {
66
- ensureDataDir();
67
- }
68
-
69
61
  function readConfigFileSignature(configPath: string): ConfigFileSignature {
70
62
  const stats = safeStatSync(configPath);
71
63
  if (!stats) {
@@ -104,7 +96,7 @@ function getCachedConfigIfFresh(): AssistantConfig | null {
104
96
  return null;
105
97
  }
106
98
 
107
- const currentSignature = readConfigFileSignature(getConfigPath());
99
+ const currentSignature = readConfigFileSignature(getWorkspaceConfigPath());
108
100
  if (configFileSignaturesEqual(cachedFileSignature, currentSignature)) {
109
101
  return cached;
110
102
  }
@@ -1002,7 +994,7 @@ export function mergeDefaultWorkspaceConfig(): DefaultWorkspaceConfigMergeResult
1002
994
  Object.prototype.hasOwnProperty.call(llmDefaults, "activeProfile"),
1003
995
  };
1004
996
 
1005
- const configPath = getConfigPath();
997
+ const configPath = getWorkspaceConfigPath();
1006
998
  let existing: Record<string, unknown> = {};
1007
999
  if (existsSync(configPath)) {
1008
1000
  try {
@@ -1070,8 +1062,8 @@ export function loadConfig(): AssistantConfig {
1070
1062
  loading = true;
1071
1063
 
1072
1064
  try {
1073
- ensureMigratedDataDir();
1074
- const configPath = getConfigPath();
1065
+ ensureDataDir();
1066
+ const configPath = getWorkspaceConfigPath();
1075
1067
 
1076
1068
  let fileConfig: Record<string, unknown> = {};
1077
1069
  let configFileExisted = true;
@@ -1259,7 +1251,7 @@ export function getConfigReadOnly(): AssistantConfig {
1259
1251
  return freshCached;
1260
1252
  }
1261
1253
 
1262
- const configPath = getConfigPath();
1254
+ const configPath = getWorkspaceConfigPath();
1263
1255
  let fileConfig: Record<string, unknown> = {};
1264
1256
  if (existsSync(configPath)) {
1265
1257
  try {
@@ -1312,8 +1304,8 @@ export function withSuppressedConfigDiskWritesSync<T>(fn: () => T): T {
1312
1304
  * type without runtime shape-checking — the boundary check happens here.
1313
1305
  */
1314
1306
  export function loadRawConfig(): Record<string, unknown> {
1315
- ensureMigratedDataDir();
1316
- const configPath = getConfigPath();
1307
+ ensureDataDir();
1308
+ const configPath = getWorkspaceConfigPath();
1317
1309
  if (!existsSync(configPath)) {
1318
1310
  return {};
1319
1311
  }
@@ -1372,8 +1364,8 @@ function describeJsonShape(value: unknown): string {
1372
1364
  }
1373
1365
 
1374
1366
  export function saveRawConfig(config: Record<string, unknown>): void {
1375
- ensureMigratedDataDir();
1376
- const configPath = getConfigPath();
1367
+ ensureDataDir();
1368
+ const configPath = getWorkspaceConfigPath();
1377
1369
 
1378
1370
  // Strip legacy apiKeys — provider keys belong in secure storage, not plaintext config
1379
1371
  delete config.apiKeys;