@vellumai/assistant 0.12.2 → 0.12.3-staging.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (772) hide show
  1. package/.env.example +2 -0
  2. package/AGENTS.md +1 -1
  3. package/ARCHITECTURE.md +43 -29
  4. package/Dockerfile +63 -0
  5. package/docs/architecture/integrations.md +7 -0
  6. package/docs/architecture/memory.md +17 -2
  7. package/docs/architecture/security.md +10 -5
  8. package/docs/architecture/turn-actor.md +2 -0
  9. package/docs/credential-execution-service.md +1 -1
  10. package/docs/desktop-browser-cli.md +50 -5
  11. package/docs/live-voice-task-outcomes.md +100 -0
  12. package/docs/mode-sessions.md +183 -0
  13. package/docs/voice-input-diagnostics.md +58 -0
  14. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  15. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
  16. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  17. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  18. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
  19. package/node_modules/@vellumai/environments/src/shell.test.ts +21 -0
  20. package/node_modules/@vellumai/environments/src/shell.ts +24 -0
  21. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  22. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
  23. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  24. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  25. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
  26. package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +20 -0
  27. package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +8 -2
  28. package/node_modules/@vellumai/gateway-client/src/index.ts +4 -1
  29. package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +38 -7
  30. package/node_modules/@vellumai/gateway-client/src/trust-verdict-contract.ts +5 -6
  31. package/node_modules/@vellumai/service-contracts/package.json +2 -1
  32. package/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
  33. package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  34. package/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  35. package/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
  36. package/node_modules/@vellumai/slack-text/src/index.ts +4 -8
  37. package/openapi.yaml +716 -16
  38. package/package.json +1 -1
  39. package/scripts/smoke-desktop-browser-cli.ts +11 -26
  40. package/scripts/smoke-desktop-window-theme.ts +2 -3
  41. package/scripts/sync-llm-catalog.ts +3 -0
  42. package/scripts/voice-ttft-spike.ts +2 -2
  43. package/src/__tests__/agent-loop-output-hooks.test.ts +150 -1
  44. package/src/__tests__/agent-loop-provider-error-recording.test.ts +87 -19
  45. package/src/__tests__/agent-loop.test.ts +259 -0
  46. package/src/__tests__/agent-wake-delegation-prompt.test.ts +64 -2
  47. package/src/__tests__/annotate-activity-metadata.test.ts +3 -1
  48. package/src/__tests__/annotate-risk-options.test.ts +2 -0
  49. package/src/__tests__/answered-question-persistence.test.ts +2 -0
  50. package/src/__tests__/anthropic-gateway-shared.test.ts +22 -0
  51. package/src/__tests__/anthropic-provider.test.ts +55 -0
  52. package/src/__tests__/approval-interception-trust-gates.test.ts +40 -0
  53. package/src/__tests__/approval-message-composer.test.ts +1 -118
  54. package/src/__tests__/approval-routes-http.test.ts +3 -0
  55. package/src/__tests__/assistant-event-hub-connection-history.test.ts +111 -0
  56. package/src/__tests__/assistant-event-hub-targeted.test.ts +50 -0
  57. package/src/__tests__/assistant-event-targeting.test.ts +186 -0
  58. package/src/__tests__/assistant-stream-state.test.ts +29 -9
  59. package/src/__tests__/attachments-store.test.ts +22 -3
  60. package/src/__tests__/browser-fill-credential.test.ts +11 -22
  61. package/src/__tests__/btw-routes.test.ts +4 -1
  62. package/src/__tests__/build-persisted-content.test.ts +1 -1
  63. package/src/__tests__/call-conversation-messages.test.ts +5 -0
  64. package/src/__tests__/call-recovery.test.ts +7 -2
  65. package/src/__tests__/call-routes-http.test.ts +12 -0
  66. package/src/__tests__/call-store.test.ts +7 -2
  67. package/src/__tests__/channel-approval.test.ts +9 -14
  68. package/src/__tests__/channel-reply-delivery.test.ts +56 -0
  69. package/src/__tests__/channel-retry-sweep.test.ts +9 -0
  70. package/src/__tests__/chat-credential-redaction.test.ts +23 -0
  71. package/src/__tests__/chat-reveal-guard-priming.test.ts +2 -0
  72. package/src/__tests__/checker.test.ts +11 -3
  73. package/src/__tests__/client-os-metadata-persistence.test.ts +35 -1
  74. package/src/__tests__/computer-use-screenshot-attachments.test.ts +411 -0
  75. package/src/__tests__/computer-use-screenshot-selection.test.ts +50 -0
  76. package/src/__tests__/computer-use-skill-manifest-regression.test.ts +0 -25
  77. package/src/__tests__/computer-use-tools.test.ts +1 -233
  78. package/src/__tests__/config-model-image-provider.test.ts +39 -0
  79. package/src/__tests__/config-schema.test.ts +16 -0
  80. package/src/__tests__/config-watcher.test.ts +24 -0
  81. package/src/__tests__/context-search-agent-runner.test.ts +260 -1
  82. package/src/__tests__/context-search-fanout.test.ts +1 -1
  83. package/src/__tests__/context-search-types.test.ts +7 -7
  84. package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +8 -2
  85. package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +2 -0
  86. package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +2 -0
  87. package/src/__tests__/conversation-agent-loop-overflow.test.ts +2 -0
  88. package/src/__tests__/conversation-agent-loop.test.ts +797 -16
  89. package/src/__tests__/conversation-app-control-lifecycle.test.ts +53 -1
  90. package/src/__tests__/conversation-attachments.test.ts +106 -0
  91. package/src/__tests__/conversation-evictor.test.ts +62 -37
  92. package/src/__tests__/conversation-fork-crud.test.ts +191 -0
  93. package/src/__tests__/conversation-history-boundary-repair-failure.test.ts +174 -0
  94. package/src/__tests__/conversation-loop-db-lock-resilience.test.ts +2 -0
  95. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +2 -0
  96. package/src/__tests__/conversation-rendered-delegation-state.test.ts +276 -0
  97. package/src/__tests__/conversation-retry-route.test.ts +22 -0
  98. package/src/__tests__/conversation-routes-disk-view.test.ts +2 -0
  99. package/src/__tests__/conversation-routes-enabled-plugins.test.ts +2 -0
  100. package/src/__tests__/conversation-routes-guardian-reply.test.ts +10 -0
  101. package/src/__tests__/conversation-routes-hidden-queue.test.ts +2 -0
  102. package/src/__tests__/conversation-routes-slash-commands.test.ts +2 -0
  103. package/src/__tests__/conversation-runtime-assembly.test.ts +29 -0
  104. package/src/__tests__/conversation-surface-snapshots.test.ts +543 -0
  105. package/src/__tests__/conversation-surfaces-action-delivery.test.ts +151 -0
  106. package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +63 -1
  107. package/src/__tests__/conversation-tool-setup-attribution.test.ts +23 -0
  108. package/src/__tests__/conversation-undo.test.ts +61 -0
  109. package/src/__tests__/credential-execution-client.test.ts +72 -55
  110. package/src/__tests__/credential-prompt-route.test.ts +35 -3
  111. package/src/__tests__/cu-unified-flow.test.ts +82 -1
  112. package/src/__tests__/db-conversation-tool-surface.test.ts +221 -0
  113. package/src/__tests__/db-mode-sessions-migration.test.ts +175 -0
  114. package/src/__tests__/deterministic-verification-control-plane.test.ts +2 -0
  115. package/src/__tests__/dm-backfill.test.ts +91 -1
  116. package/src/__tests__/document-create-route.test.ts +88 -0
  117. package/src/__tests__/document-sync-tags.test.ts +27 -0
  118. package/src/__tests__/emit-signal-routing-intent.test.ts +142 -2
  119. package/src/__tests__/events-tail-route.test.ts +58 -0
  120. package/src/__tests__/evict-conversations-for-reload.test.ts +57 -8
  121. package/src/__tests__/guardian-action-sweep.test.ts +16 -0
  122. package/src/__tests__/guardian-dispatch.test.ts +13 -0
  123. package/src/__tests__/headless-browser-interactions.test.ts +64 -20
  124. package/src/__tests__/heartbeat-service.test.ts +2 -59
  125. package/src/__tests__/helpers/mock-conversation.ts +21 -0
  126. package/src/__tests__/helpers/native-web-search-harness.ts +2 -0
  127. package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +5 -0
  128. package/src/__tests__/history-repair.test.ts +53 -0
  129. package/src/__tests__/host-cu-proxy.test.ts +46 -2
  130. package/src/__tests__/http-user-message-parity.test.ts +2 -0
  131. package/src/__tests__/image-credentials.test.ts +49 -0
  132. package/src/__tests__/image-service-dispatcher.test.ts +45 -0
  133. package/src/__tests__/inference-profile-session-handler.test.ts +26 -0
  134. package/src/__tests__/interrupt-on-send.test.ts +57 -43
  135. package/src/__tests__/interrupt-turn-note.test.ts +5 -6
  136. package/src/__tests__/list-messages-tool-merge.test.ts +85 -1
  137. package/src/__tests__/llm-catalog-parity.test.ts +27 -4
  138. package/src/__tests__/media-generate-image.test.ts +49 -1
  139. package/src/__tests__/message-provenance-contact-id.test.ts +69 -0
  140. package/src/__tests__/migration-export-to-gcs.test.ts +142 -0
  141. package/src/__tests__/mode-session-history.test.ts +245 -0
  142. package/src/__tests__/mode-session-recovery.test.ts +222 -0
  143. package/src/__tests__/mode-session-wire.test.ts +141 -0
  144. package/src/__tests__/notification-guardian-path.test.ts +7 -0
  145. package/src/__tests__/notification-telegram-adapter.test.ts +2 -2
  146. package/src/__tests__/notification-vellum-adapter.test.ts +34 -0
  147. package/src/__tests__/oauth-apps-routes.test.ts +1 -0
  148. package/src/__tests__/oauth-commands-routes.test.ts +114 -101
  149. package/src/__tests__/oauth-connect-orchestrator.test.ts +2 -0
  150. package/src/__tests__/oauth-provider-serializer.test.ts +13 -0
  151. package/src/__tests__/oauth-providers-routes.test.ts +3 -0
  152. package/src/__tests__/outbound-slack-persistence.test.ts +2 -0
  153. package/src/__tests__/persist-media-references.test.ts +50 -0
  154. package/src/__tests__/persistence-secret-redaction.test.ts +2 -0
  155. package/src/__tests__/plugin-api-provider.test.ts +5 -0
  156. package/src/__tests__/plugin-api-resolve-credential.test.ts +97 -0
  157. package/src/__tests__/plugin-import-boundary-guard.test.ts +9 -1
  158. package/src/__tests__/process-message-display-content.test.ts +56 -0
  159. package/src/__tests__/reaction-persistence.test.ts +5 -0
  160. package/src/__tests__/request-secret-standalone-channel-gate.test.ts +25 -9
  161. package/src/__tests__/run-conversation-turn-persistence.test.ts +138 -1
  162. package/src/__tests__/runtime-events-sse-reconnect.test.ts +1 -1
  163. package/src/__tests__/schedule-routes.test.ts +20 -0
  164. package/src/__tests__/scheduler-result-notification.test.ts +23 -4
  165. package/src/__tests__/script-proxy-certs.test.ts +1 -1
  166. package/src/__tests__/scripted-turn-metadata-persistence.test.ts +73 -0
  167. package/src/__tests__/secret-ingress-http.test.ts +2 -0
  168. package/src/__tests__/secret-routes-platform-proxy.test.ts +86 -31
  169. package/src/__tests__/secret-routes-scrub.test.ts +22 -27
  170. package/src/__tests__/secret-scanner.test.ts +20 -0
  171. package/src/__tests__/secure-keys-managed-failover.test.ts +8 -0
  172. package/src/__tests__/secure-keys.test.ts +50 -4
  173. package/src/__tests__/send-endpoint-busy.test.ts +163 -1
  174. package/src/__tests__/server-history-render.test.ts +91 -1
  175. package/src/__tests__/server-tool-pairing.test.ts +107 -0
  176. package/src/__tests__/settings-routes.test.ts +50 -1
  177. package/src/__tests__/skill-script-runner-sandbox.test.ts +32 -0
  178. package/src/__tests__/skills.test.ts +5 -4
  179. package/src/__tests__/subagent-fork-notifications.test.ts +1 -2
  180. package/src/__tests__/subagent-manager-notify.test.ts +1 -2
  181. package/src/__tests__/subagent-notify-parent.test.ts +146 -39
  182. package/src/__tests__/subagent-notify-stale-rebuild.test.ts +2 -4
  183. package/src/__tests__/subagent-tool-gate-mode.test.ts +226 -0
  184. package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +2 -0
  185. package/src/__tests__/terminal-tools.test.ts +8 -0
  186. package/src/__tests__/thread-backfill.test.ts +53 -0
  187. package/src/__tests__/tool-preview-lifecycle.test.ts +2 -0
  188. package/src/__tests__/tool-result-follow-up.test.ts +99 -0
  189. package/src/__tests__/tool-result-metadata-plumbing.test.ts +66 -1
  190. package/src/__tests__/tool-start-timestamp.test.ts +2 -0
  191. package/src/__tests__/twilio-routes.test.ts +1 -0
  192. package/src/__tests__/unicode.test.ts +36 -0
  193. package/src/__tests__/web-search-backend-failure.test.ts +1 -1
  194. package/src/activation/progress-store.test.ts +12 -7
  195. package/src/agent/loop.ts +136 -53
  196. package/src/agent/tool-result-follow-up.ts +70 -0
  197. package/src/api/attachment-provenance.test.ts +68 -0
  198. package/src/api/computer-use-tool.test.ts +55 -0
  199. package/src/api/computer-use-tool.ts +27 -0
  200. package/src/api/events/assistant-outbound-attachment.ts +3 -0
  201. package/src/api/events/assistant-turn-start.ts +3 -0
  202. package/src/api/events/desktop-activity-changed.ts +10 -0
  203. package/src/api/events/generation-handoff.ts +2 -0
  204. package/src/api/events/message-complete.ts +2 -0
  205. package/src/api/events/notification-conversation-created.ts +14 -16
  206. package/src/api/events/notification-intent.ts +4 -2
  207. package/src/api/events/question-request.ts +1 -0
  208. package/src/api/events/tool-result.ts +85 -0
  209. package/src/api/events/tool-use-start.ts +3 -0
  210. package/src/api/events/user-message-echo.ts +3 -0
  211. package/src/api/index.ts +42 -6
  212. package/src/api/mode-session.test.ts +131 -0
  213. package/src/api/mode-session.ts +148 -0
  214. package/src/api/remember-tool.ts +15 -0
  215. package/src/api/responses/conversation-message.ts +9 -0
  216. package/src/approvals/AGENTS.md +4 -2
  217. package/src/approvals/approval-primitive.ts +5 -2
  218. package/src/approvals/scoped-approval-grants.ts +6 -2
  219. package/src/browser/__tests__/operations.test.ts +29 -0
  220. package/src/browser/operation-meta.ts +2 -1
  221. package/src/browser/operations.ts +29 -0
  222. package/src/browser/virtual-desktop-target.ts +2 -9
  223. package/src/bundler/bundle-signer.ts +1 -3
  224. package/src/calls/__tests__/barge-in-guard.test.ts +3 -0
  225. package/src/calls/__tests__/call-controller.test.ts +94 -0
  226. package/src/calls/__tests__/call-funnel-telemetry.test.ts +204 -0
  227. package/src/calls/__tests__/voice-control-protocol.test.ts +15 -0
  228. package/src/calls/__tests__/voice-escalation-judge.test.ts +209 -0
  229. package/src/calls/__tests__/voice-leg-coordinator.test.ts +168 -0
  230. package/src/{live-voice/__tests__/live-voice-metrics.test.ts → calls/__tests__/voice-metrics.test.ts} +32 -38
  231. package/src/calls/__tests__/voice-session-bridge.test.ts +576 -122
  232. package/src/calls/barge-in-guard.ts +5 -0
  233. package/src/calls/call-controller.ts +196 -1
  234. package/src/calls/call-domain.ts +4 -0
  235. package/src/calls/call-store.ts +87 -2
  236. package/src/calls/guardian-action-sweep.ts +22 -8
  237. package/src/calls/media-stream-audio-transcode.ts +1 -3
  238. package/src/calls/typesafe-noul.ts +156 -0
  239. package/src/calls/voice-control-protocol.ts +6 -0
  240. package/src/calls/voice-escalation-judge.ts +145 -0
  241. package/src/calls/voice-escalation-target.ts +10 -0
  242. package/src/calls/voice-leg-coordinator.ts +121 -3
  243. package/src/{live-voice/live-voice-metrics.ts → calls/voice-metrics.ts} +94 -87
  244. package/src/calls/voice-session-bridge.ts +256 -112
  245. package/src/calls/voice-triage-escalate.ts +7 -0
  246. package/src/cli/__tests__/roadmap-cli.test.ts +4 -41
  247. package/src/cli/commands/__tests__/cli-test-harness.ts +21 -3
  248. package/src/cli/commands/__tests__/notifications.test.ts +35 -10
  249. package/src/cli/commands/__tests__/plugins.test.ts +20 -1
  250. package/src/cli/commands/__tests__/schedules.test.ts +14 -0
  251. package/src/cli/commands/bash.help.ts +4 -3
  252. package/src/cli/commands/browser.help.ts +9 -6
  253. package/src/cli/commands/browser.ts +0 -1
  254. package/src/cli/commands/clients.help.ts +46 -0
  255. package/src/cli/commands/clients.ts +116 -0
  256. package/src/cli/commands/credentials.help.ts +9 -4
  257. package/src/cli/commands/notifications.help.ts +16 -2
  258. package/src/cli/commands/notifications.ts +30 -8
  259. package/src/cli/commands/oauth/index.help.ts +10 -0
  260. package/src/cli/commands/oauth/providers.ts +12 -0
  261. package/src/cli/commands/oauth/request.test.ts +179 -2
  262. package/src/cli/commands/oauth/request.ts +41 -10
  263. package/src/cli/commands/plugins.ts +12 -5
  264. package/src/cli/commands/roadmap.help.ts +18 -21
  265. package/src/cli/commands/roadmap.ts +4 -21
  266. package/src/cli/commands/schedules.ts +2 -0
  267. package/src/cli/lib/__tests__/inspect-plugin.test.ts +54 -0
  268. package/src/cli/lib/__tests__/install-from-github.test.ts +41 -0
  269. package/src/cli/lib/__tests__/list-installed-plugins.test.ts +5 -2
  270. package/src/cli/lib/__tests__/local-plugin-upgrade.test.ts +133 -51
  271. package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +8 -2
  272. package/src/cli/lib/bundled-marketplace.json +906 -1
  273. package/src/cli/lib/bundled-plugin-packages.json +783 -1
  274. package/src/cli/lib/inspect-plugin.ts +11 -4
  275. package/src/cli/lib/plugin-marketplace.ts +8 -0
  276. package/src/cli/lib/publish-plugin.ts +1 -2
  277. package/src/cli/lib/upgrade-plugin.ts +19 -11
  278. package/src/cli/output.ts +18 -4
  279. package/src/config/__tests__/default-provider.test.ts +6 -0
  280. package/src/config/__tests__/memory-retrospective-schema.test.ts +14 -1
  281. package/src/config/bundled-skills/computer-use/SKILL.md +1 -1
  282. package/src/config/bundled-skills/computer-use/TOOLS.json +89 -1
  283. package/src/config/bundled-skills/image-studio/SKILL.md +17 -2
  284. package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +7 -11
  285. package/src/config/bundled-skills/schedule/SKILL.md +6 -6
  286. package/src/config/bundled-skills/sequences/TOOLS.json +1 -5
  287. package/src/config/call-site-defaults.ts +9 -0
  288. package/src/config/feature-flag-registry.json +16 -0
  289. package/src/config/loader.ts +9 -17
  290. package/src/config/platform-identity.ts +8 -8
  291. package/src/config/profile-text-generation.test.ts +51 -0
  292. package/src/config/profile-text-generation.ts +51 -0
  293. package/src/config/schemas/__tests__/memory-v3.test.ts +12 -0
  294. package/src/config/schemas/__tests__/tools.test.ts +31 -0
  295. package/src/config/schemas/call-site-catalog.ts +14 -0
  296. package/src/config/schemas/channels.ts +0 -18
  297. package/src/config/schemas/llm.ts +9 -2
  298. package/src/config/schemas/mcp.ts +6 -1
  299. package/src/config/schemas/memory-retrospective.ts +11 -2
  300. package/src/config/schemas/memory-v3.ts +12 -0
  301. package/src/config/schemas/services.ts +15 -6
  302. package/src/config/schemas/tools.ts +10 -0
  303. package/src/config/session-groups-gate.ts +21 -0
  304. package/src/config/skills.ts +2 -6
  305. package/src/conversations/__tests__/message-consolidation.test.ts +149 -0
  306. package/src/conversations/message-consolidation.ts +102 -5
  307. package/src/credential-execution/ces-connect.test.ts +69 -0
  308. package/src/credential-execution/ces-connect.ts +115 -0
  309. package/src/credential-execution/ces-runtime.ts +55 -93
  310. package/src/credential-execution/executable-discovery.ts +34 -142
  311. package/src/credential-execution/process-manager.test.ts +20 -24
  312. package/src/credential-execution/process-manager.ts +24 -19
  313. package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +10 -1
  314. package/src/daemon/__tests__/conversation-tool-setup.test.ts +43 -0
  315. package/src/daemon/__tests__/inference-profile-notification.test.ts +2 -0
  316. package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +36 -0
  317. package/src/daemon/assistant-attachments.ts +20 -12
  318. package/src/daemon/browser-mode-session.test.ts +402 -0
  319. package/src/daemon/browser-mode-session.ts +186 -0
  320. package/src/daemon/chat-credential-redaction.ts +16 -1
  321. package/src/daemon/computer-use-mode-session.test.ts +210 -0
  322. package/src/daemon/computer-use-mode-session.ts +105 -0
  323. package/src/daemon/config-watcher.ts +10 -0
  324. package/src/daemon/conversation-agent-loop-handlers.ts +132 -13
  325. package/src/daemon/conversation-agent-loop.ts +231 -25
  326. package/src/daemon/conversation-attachments.ts +93 -12
  327. package/src/daemon/conversation-evictor.ts +5 -24
  328. package/src/daemon/conversation-history.ts +10 -0
  329. package/src/daemon/conversation-interrupt-repair.ts +4 -43
  330. package/src/daemon/conversation-interrupt.ts +7 -11
  331. package/src/daemon/conversation-messaging.ts +58 -5
  332. package/src/daemon/conversation-mode-session.test.ts +1667 -0
  333. package/src/daemon/conversation-mode-session.ts +1008 -0
  334. package/src/daemon/conversation-process.ts +5 -0
  335. package/src/daemon/conversation-runtime-assembly.ts +10 -20
  336. package/src/daemon/conversation-skill-tools.ts +1 -0
  337. package/src/daemon/conversation-store.ts +4 -4
  338. package/src/daemon/conversation-surface-snapshots.ts +135 -0
  339. package/src/daemon/conversation-surfaces.ts +113 -5
  340. package/src/daemon/conversation-tool-setup.ts +79 -6
  341. package/src/daemon/conversation-turn-finalize.ts +26 -13
  342. package/src/daemon/conversation.ts +215 -58
  343. package/src/daemon/daemon-control.ts +3 -7
  344. package/src/daemon/handlers/config-model.ts +7 -5
  345. package/src/daemon/handlers/shared.ts +55 -16
  346. package/src/daemon/host-cu-proxy.ts +51 -22
  347. package/src/daemon/lifecycle.ts +30 -13
  348. package/src/daemon/mcp-reload-service.ts +1 -1
  349. package/src/daemon/message-protocol.ts +0 -1
  350. package/src/daemon/message-provenance.ts +28 -4
  351. package/src/daemon/message-types/shared.ts +1 -0
  352. package/src/daemon/mode-session-startup-recovery.ts +23 -0
  353. package/src/daemon/mode-session-tracking.ts +54 -0
  354. package/src/daemon/orphan-reaper.ts +4 -3
  355. package/src/daemon/persist-media-references.ts +20 -3
  356. package/src/daemon/process-message.ts +13 -4
  357. package/src/daemon/tool-setup-types.ts +6 -0
  358. package/src/daemon/virtual-desktop-context.ts +21 -0
  359. package/src/daemon/wake-conversation-ops.ts +50 -15
  360. package/src/desktop/__tests__/fake-desktop.ts +1 -0
  361. package/src/desktop/desktop-accessibility-script.ts +88 -0
  362. package/src/desktop/desktop-accessibility.test.ts +160 -0
  363. package/src/desktop/desktop-accessibility.ts +169 -0
  364. package/src/desktop/desktop-automation-lease.test.ts +302 -64
  365. package/src/desktop/desktop-automation-lease.ts +129 -54
  366. package/src/desktop/desktop-browser-endpoint.ts +4 -0
  367. package/src/desktop/desktop-computer-use-routing.test.ts +330 -0
  368. package/src/desktop/desktop-computer-use.test.ts +449 -0
  369. package/src/desktop/desktop-computer-use.ts +502 -0
  370. package/src/desktop/desktop-dependencies.ts +47 -293
  371. package/src/desktop/desktop-display.ts +2 -2
  372. package/src/desktop/desktop-help.ts +15 -0
  373. package/src/desktop/desktop-panel-config.test.ts +13 -3
  374. package/src/desktop/desktop-panel-config.ts +109 -15
  375. package/src/desktop/desktop-session-manager.test.ts +69 -21
  376. package/src/desktop/desktop-session-manager.ts +76 -22
  377. package/src/desktop/desktop-stream-bridge.test.ts +76 -132
  378. package/src/desktop/desktop-stream-bridge.ts +9 -11
  379. package/src/desktop/desktop-wallpaper-renderer.ts +132 -0
  380. package/src/desktop/desktop-wallpaper-worker.ts +17 -0
  381. package/src/desktop/desktop-wallpaper.test.ts +48 -7
  382. package/src/desktop/desktop-wallpaper.ts +34 -102
  383. package/src/desktop/desktop-window-theme.ts +5 -25
  384. package/src/desktop/virtual-desktop-feature.ts +53 -3
  385. package/src/desktop/virtual-desktop-platform.test.ts +5 -9
  386. package/src/documents/document-store.ts +23 -0
  387. package/src/heartbeat/__tests__/heartbeat-service.test.ts +22 -26
  388. package/src/heartbeat/heartbeat-service.ts +5 -33
  389. package/src/ipc/__tests__/browser-ipc.test.ts +215 -6
  390. package/src/ipc/assistant-server.ts +2 -0
  391. package/src/ipc/routes/__tests__/channel-reply-ipc-routes.test.ts +111 -0
  392. package/src/ipc/routes/channel-reply-ipc-routes.ts +73 -0
  393. package/src/live-voice/__tests__/continuation-judge.test.ts +151 -0
  394. package/src/live-voice/__tests__/live-activity-reporter.test.ts +154 -15
  395. package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +743 -31
  396. package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +191 -10
  397. package/src/live-voice/__tests__/live-voice-session-groups-gate.test.ts +87 -0
  398. package/src/live-voice/__tests__/live-voice-session-manager.test.ts +5 -1
  399. package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +41 -0
  400. package/src/live-voice/__tests__/live-voice-sight-frame.test.ts +1017 -9
  401. package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +224 -1
  402. package/src/live-voice/__tests__/live-voice-tts-session.test.ts +101 -0
  403. package/src/live-voice/__tests__/live-voice-tts.test.ts +44 -0
  404. package/src/live-voice/__tests__/live-voice-vad.test.ts +939 -75
  405. package/src/live-voice/__tests__/microphone-idle-gate.test.ts +87 -0
  406. package/src/live-voice/__tests__/session-controls.test.ts +23 -0
  407. package/src/live-voice/__tests__/task-outcomes.test.ts +117 -0
  408. package/src/live-voice/__tests__/voice-input-diagnostics.test.ts +117 -0
  409. package/src/live-voice/camera-mode-session.test.ts +265 -0
  410. package/src/live-voice/camera-mode-session.ts +277 -0
  411. package/src/live-voice/continuation-judge.ts +123 -0
  412. package/src/live-voice/live-activity-reporter.ts +115 -6
  413. package/src/live-voice/live-voice-manager.ts +36 -1
  414. package/src/live-voice/live-voice-photo.ts +88 -7
  415. package/src/live-voice/live-voice-session-manager.ts +15 -0
  416. package/src/live-voice/live-voice-session.ts +1443 -747
  417. package/src/live-voice/live-voice-tts.ts +22 -4
  418. package/src/live-voice/microphone-idle-gate.ts +56 -0
  419. package/src/live-voice/protocol.ts +171 -4
  420. package/src/live-voice/session-controls.ts +11 -3
  421. package/src/live-voice/task-outcomes.ts +131 -0
  422. package/src/live-voice/voice-input-diagnostics.ts +117 -0
  423. package/src/mcp/__tests__/manager-tool-caps.test.ts +156 -0
  424. package/src/mcp/__tests__/startup.test.ts +30 -7
  425. package/src/mcp/__tests__/tool-caps.test.ts +134 -0
  426. package/src/mcp/manager.ts +171 -101
  427. package/src/mcp/startup.ts +33 -13
  428. package/src/mcp/tool-caps.ts +184 -0
  429. package/src/media/__tests__/image-models.test.ts +58 -0
  430. package/src/media/image-credentials.ts +5 -0
  431. package/src/media/image-models.ts +52 -0
  432. package/src/media/image-service.ts +8 -0
  433. package/src/media/openrouter-image-service.test.ts +133 -0
  434. package/src/media/openrouter-image-service.ts +153 -0
  435. package/src/media/types.ts +1 -1
  436. package/src/messaging/provider-message-metadata.ts +3 -3
  437. package/src/messaging/providers/__tests__/transport-dispatch.test.ts +65 -11
  438. package/src/messaging/providers/discord/send.ts +4 -17
  439. package/src/messaging/providers/discord/transport.ts +2 -2
  440. package/src/messaging/providers/outlook/client.ts +16 -22
  441. package/src/messaging/providers/telegram-bot/send.test.ts +236 -1
  442. package/src/messaging/providers/telegram-bot/send.ts +10 -7
  443. package/src/messaging/providers/telegram-bot/transport.ts +3 -10
  444. package/src/messaging/providers/whatsapp/send.test.ts +169 -4
  445. package/src/monitoring/__tests__/control.test.ts +17 -5
  446. package/src/monitoring/control.ts +15 -5
  447. package/src/notifications/AGENTS.md +1 -1
  448. package/src/notifications/README.md +83 -99
  449. package/src/notifications/__tests__/broadcaster.test.ts +1 -1
  450. package/src/notifications/__tests__/channel-allowlist.test.ts +39 -0
  451. package/src/notifications/__tests__/copy-composer.test.ts +71 -1
  452. package/src/notifications/__tests__/decision-engine.test.ts +79 -2
  453. package/src/notifications/__tests__/deterministic-checks.test.ts +10 -10
  454. package/src/notifications/__tests__/home-feed-side-effect.test.ts +93 -4
  455. package/src/notifications/__tests__/notification-utils.test.ts +18 -0
  456. package/src/notifications/adapters/macos.ts +37 -46
  457. package/src/notifications/broadcaster.ts +5 -6
  458. package/src/notifications/channel-allowlist.ts +44 -0
  459. package/src/notifications/copy-composer.ts +11 -26
  460. package/src/notifications/decision-engine.ts +24 -14
  461. package/src/notifications/deterministic-checks.ts +1 -1
  462. package/src/notifications/emit-signal.ts +123 -37
  463. package/src/notifications/home-feed-side-effect.ts +9 -7
  464. package/src/notifications/notification-utils.ts +19 -0
  465. package/src/notifications/schedule-result-producer.ts +4 -6
  466. package/src/notifications/signal.ts +4 -19
  467. package/src/notifications/types.ts +15 -0
  468. package/src/oauth/AGENTS.md +2 -0
  469. package/src/oauth/__tests__/identity-verifier.test.ts +36 -1
  470. package/src/oauth/connection-resolver.ts +1 -2
  471. package/src/oauth/identity-verifier.ts +24 -0
  472. package/src/oauth/oauth-store.ts +12 -4
  473. package/src/oauth/provider-serializer.ts +8 -0
  474. package/src/oauth/seed-providers.ts +42 -0
  475. package/src/onboarding/onboarding-events-store.ts +99 -12
  476. package/src/permissions/confirmation-guardian-request.test.ts +16 -0
  477. package/src/permissions/confirmation-guardian-request.ts +2 -3
  478. package/src/permissions/question-prompter.test.ts +31 -0
  479. package/src/permissions/question-prompter.ts +2 -0
  480. package/src/persistence/__tests__/client-connection-events-store.test.ts +322 -0
  481. package/src/persistence/attachments-store.ts +84 -44
  482. package/src/persistence/client-connection-events-store.ts +377 -0
  483. package/src/persistence/conversation-crud.ts +166 -26
  484. package/src/persistence/conversation-mode-sessions.test.ts +424 -0
  485. package/src/persistence/conversation-mode-sessions.ts +460 -0
  486. package/src/persistence/conversation-plugin-facade.ts +30 -0
  487. package/src/persistence/conversation-queries.ts +6 -19
  488. package/src/persistence/conversation-tool-surface.ts +135 -0
  489. package/src/persistence/conversation-types.test.ts +32 -0
  490. package/src/persistence/conversation-types.ts +26 -4
  491. package/src/persistence/llm-usage-store.ts +3 -9
  492. package/src/persistence/message-metadata.ts +25 -0
  493. package/src/persistence/migrations/378-create-conversation-tool-surfaces.test.ts +78 -0
  494. package/src/persistence/migrations/378-create-conversation-tool-surfaces.ts +29 -0
  495. package/src/persistence/migrations/379-oauth-providers-response-ok-field.test.ts +92 -0
  496. package/src/persistence/migrations/379-oauth-providers-response-ok-field.ts +32 -0
  497. package/src/persistence/migrations/380-conversation-tool-surfaces-delegate-independent-tasks.test.ts +90 -0
  498. package/src/persistence/migrations/380-conversation-tool-surfaces-delegate-independent-tasks.ts +34 -0
  499. package/src/persistence/migrations/381-create-conversation-mode-sessions.ts +41 -0
  500. package/src/persistence/migrations/382-create-client-connection-events.ts +40 -0
  501. package/src/persistence/migrations/__tests__/382-create-client-connection-events.test.ts +56 -0
  502. package/src/persistence/schema/conversation-tool-surfaces.ts +32 -0
  503. package/src/persistence/schema/conversations.ts +67 -0
  504. package/src/persistence/schema/index.ts +1 -0
  505. package/src/persistence/schema/infrastructure.ts +29 -0
  506. package/src/persistence/schema/oauth.ts +1 -0
  507. package/src/persistence/steps.ts +41 -3
  508. package/src/plugin-api/__tests__/plugin-name-env.test.ts +249 -0
  509. package/src/plugin-api/conversation-turn.ts +31 -7
  510. package/src/plugin-api/effective-context-window.ts +33 -0
  511. package/src/plugin-api/index.ts +35 -6
  512. package/src/plugin-api/plugin-channel-turn-trust.test.ts +133 -0
  513. package/src/plugin-api/plugin-channel-turn-trust.ts +71 -0
  514. package/src/plugin-api/plugin-name-env.ts +125 -0
  515. package/src/plugin-api/resolve-credential.ts +79 -20
  516. package/src/plugin-api/vision-support.test.ts +22 -0
  517. package/src/plugin-api/vision-support.ts +17 -5
  518. package/src/plugins/__tests__/active-plugin-env.test.ts +136 -0
  519. package/src/plugins/active-plugin-env.ts +51 -0
  520. package/src/plugins/defaults/index.ts +24 -0
  521. package/src/plugins/defaults/memory/AGENTS.md +39 -5
  522. package/src/plugins/defaults/memory/__tests__/archive-removal-framing.test.ts +64 -0
  523. package/src/plugins/defaults/memory/__tests__/buffer-file.test.ts +365 -0
  524. package/src/plugins/defaults/memory/__tests__/buffer-format.test.ts +43 -0
  525. package/src/plugins/defaults/memory/__tests__/conversation-memory-purge.test.ts +1 -0
  526. package/src/plugins/defaults/memory/__tests__/db-memory-attach.test.ts +2 -0
  527. package/src/plugins/defaults/memory/__tests__/fixtures/buffer-appender.ts +17 -0
  528. package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +74 -0
  529. package/src/plugins/defaults/memory/__tests__/memory-run-evidence.test.ts +161 -0
  530. package/src/plugins/defaults/memory/__tests__/relocated-memory-test-rows.ts +10 -0
  531. package/src/plugins/defaults/memory/__tests__/worker-control.test.ts +19 -6
  532. package/src/plugins/defaults/memory/buffer-file.ts +354 -0
  533. package/src/plugins/defaults/memory/buffer-format.ts +40 -0
  534. package/src/plugins/defaults/memory/context-search/agent-protocol.ts +9 -12
  535. package/src/plugins/defaults/memory/context-search/agent-runner.ts +77 -22
  536. package/src/plugins/defaults/memory/context-search/format.ts +53 -8
  537. package/src/plugins/defaults/memory/context-search/limits.ts +10 -12
  538. package/src/plugins/defaults/memory/context-search/recall-input.ts +38 -0
  539. package/src/plugins/defaults/memory/context-search/search.ts +8 -14
  540. package/src/plugins/defaults/memory/context-search/sources/conversations.ts +1 -0
  541. package/src/plugins/defaults/memory/context-search/sources/memory-v2.ts +2 -1
  542. package/src/plugins/defaults/memory/context-search/sources/workspace.ts +2 -1
  543. package/src/plugins/defaults/memory/context-search/types.ts +1 -11
  544. package/src/plugins/defaults/memory/conversation-memory-purge.ts +4 -0
  545. package/src/plugins/defaults/memory/graph/__tests__/handle-remember-v2.test.ts +11 -4
  546. package/src/plugins/defaults/memory/graph/capability-seed.ts +1 -2
  547. package/src/plugins/defaults/memory/graph/tool-handlers.ts +13 -63
  548. package/src/plugins/defaults/memory/graph/tools.ts +35 -64
  549. package/src/plugins/defaults/memory/host-utils.ts +0 -10
  550. package/src/plugins/defaults/memory/injectors.ts +4 -3
  551. package/src/plugins/defaults/memory/memory-retrospective-job.ts +72 -183
  552. package/src/plugins/defaults/memory/memory-run-evidence.ts +213 -0
  553. package/src/plugins/defaults/memory/src/memory-eval-routes.ts +2 -3
  554. package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +6 -1
  555. package/src/plugins/defaults/memory/src/memory-item-routes.ts +2 -1
  556. package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts +407 -99
  557. package/src/plugins/defaults/memory/substrate/__tests__/consolidation-prompt-flag-gating-guard.test.ts +10 -0
  558. package/src/plugins/defaults/memory/substrate/__tests__/prompts-consolidation.test.ts +107 -7
  559. package/src/plugins/defaults/memory/substrate/consolidation-job.ts +307 -86
  560. package/src/plugins/defaults/memory/substrate/consolidation-tool-surface.ts +34 -0
  561. package/src/plugins/defaults/memory/substrate/page-index.ts +2 -1
  562. package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +91 -51
  563. package/src/plugins/defaults/memory/substrate/sweep-job.ts +1 -1
  564. package/src/plugins/defaults/memory/tools.test.ts +124 -9
  565. package/src/plugins/defaults/memory/tools.ts +38 -11
  566. package/src/plugins/defaults/memory/v1/graph/consolidation.ts +2 -2
  567. package/src/plugins/defaults/memory/v1/graph/extraction.ts +2 -1
  568. package/src/plugins/defaults/memory/v1/graph/retriever.ts +1 -1
  569. package/src/plugins/defaults/memory/v2/__tests__/migration.test.ts +5 -0
  570. package/src/plugins/defaults/memory/v2/__tests__/reranker.test.ts +5 -2
  571. package/src/plugins/defaults/memory/v2/reranker.ts +2 -1
  572. package/src/plugins/defaults/memory/v3/__tests__/injection.test.ts +81 -1
  573. package/src/plugins/defaults/memory/v3/__tests__/orchestrate.test.ts +87 -0
  574. package/src/plugins/defaults/memory/v3/__tests__/plugin-schema.test.ts +12 -0
  575. package/src/plugins/defaults/memory/v3/__tests__/pool-log-store.test.ts +332 -15
  576. package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +626 -27
  577. package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +148 -28
  578. package/src/plugins/defaults/memory/v3/card.ts +2 -1
  579. package/src/plugins/defaults/memory/v3/injector.ts +212 -178
  580. package/src/plugins/defaults/memory/v3/orchestrate.ts +159 -32
  581. package/src/plugins/defaults/memory/v3/plugin-schema.ts +54 -4
  582. package/src/plugins/defaults/memory/v3/pool-log-store.ts +275 -37
  583. package/src/plugins/defaults/memory/v3/pool-select.test.ts +47 -5
  584. package/src/plugins/defaults/memory/v3/pool-select.ts +710 -59
  585. package/src/plugins/defaults/memory/v3/sections.ts +2 -1
  586. package/src/plugins/defaults/memory/v3/shadow-plugin.ts +41 -14
  587. package/src/plugins/defaults/memory/worker-control.ts +15 -5
  588. package/src/plugins/defaults/task-progress/__tests__/registration.test.ts +70 -0
  589. package/src/plugins/defaults/task-progress/__tests__/task-progress-context.test.ts +380 -0
  590. package/src/plugins/defaults/task-progress/hooks/post-compact.ts +19 -0
  591. package/src/plugins/defaults/task-progress/hooks/user-prompt-submit.ts +24 -0
  592. package/src/plugins/defaults/task-progress/package.json +14 -0
  593. package/src/plugins/defaults/task-progress/src/apply-task-progress-context.ts +93 -0
  594. package/src/plugins/defaults/task-progress/src/format-task-progress-context.ts +67 -0
  595. package/src/plugins/defaults/tool-error/hooks/post-tool-use.ts +4 -1
  596. package/src/plugins/defaults/tool-result-truncate/terminal.ts +1 -46
  597. package/src/prompts/__tests__/parallel-tasks-section.test.ts +25 -0
  598. package/src/prompts/delegation-gate.ts +57 -0
  599. package/src/prompts/system-prompt.ts +14 -32
  600. package/src/providers/__tests__/context-overflow-error.test.ts +10 -0
  601. package/src/providers/__tests__/search-provider-catalog.test.ts +18 -0
  602. package/src/providers/anthropic/client.ts +10 -1
  603. package/src/providers/anthropic-gateway-shared.ts +6 -0
  604. package/src/providers/fetch-provider-catalog.ts +11 -0
  605. package/src/providers/gemini/client.ts +25 -8
  606. package/src/providers/inference/adapter-factory.ts +6 -0
  607. package/src/providers/jev/client.test.ts +270 -0
  608. package/src/providers/jev/client.ts +534 -0
  609. package/src/providers/model-catalog.ts +59 -3
  610. package/src/providers/openai/__tests__/chat-completions-extra-body.test.ts +94 -1
  611. package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +106 -1
  612. package/src/providers/openai/__tests__/chat-template-flatten.test.ts +24 -0
  613. package/src/providers/openai/__tests__/google-thought-signature.test.ts +1 -2
  614. package/src/providers/openai/chat-completions-provider.ts +100 -29
  615. package/src/providers/openai/google-thought-signature.ts +0 -6
  616. package/src/providers/openai/responses-provider.ts +23 -1
  617. package/src/providers/ratelimit.ts +1 -3
  618. package/src/providers/search-provider-catalog.ts +24 -0
  619. package/src/providers/server-tool-pairing.ts +16 -7
  620. package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +24 -0
  621. package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +7 -1
  622. package/src/providers/speech-to-text/__tests__/vellum-managed-realtime.test.ts +50 -1
  623. package/src/providers/speech-to-text/deepgram-flux-frames.ts +11 -0
  624. package/src/providers/speech-to-text/vellum-managed-flux-realtime.ts +13 -0
  625. package/src/providers/types.ts +5 -0
  626. package/src/routes/control.ts +7 -5
  627. package/src/runtime/AGENTS.md +6 -4
  628. package/src/runtime/__tests__/agent-wake.test.ts +94 -1
  629. package/src/runtime/actor-trust-resolver.ts +3 -3
  630. package/src/runtime/agent-wake.ts +32 -5
  631. package/src/runtime/approval-message-composer.ts +3 -69
  632. package/src/runtime/assistant-event-hub.ts +73 -75
  633. package/src/runtime/assistant-event-publish-options.ts +7 -0
  634. package/src/runtime/assistant-event-targeting.ts +79 -0
  635. package/src/runtime/assistant-stream-state.ts +13 -100
  636. package/src/runtime/background-job-runner.ts +5 -5
  637. package/src/runtime/channel-retry-sweep.ts +1 -0
  638. package/src/runtime/guardian-action-message-composer.ts +8 -161
  639. package/src/runtime/guardian-action-service.ts +2 -17
  640. package/src/runtime/guardian-reply-router.ts +29 -40
  641. package/src/runtime/http-server.ts +2 -2
  642. package/src/runtime/http-types.ts +6 -0
  643. package/src/runtime/message-composer-types.ts +3 -61
  644. package/src/runtime/migrations/__tests__/vbundle-import-parity.test.ts +67 -1
  645. package/src/runtime/migrations/__tests__/vbundle-import-policy.test.ts +89 -0
  646. package/src/runtime/migrations/vbundle-builder.ts +24 -1
  647. package/src/runtime/migrations/vbundle-import-analyzer.ts +13 -4
  648. package/src/runtime/migrations/vbundle-import-policy.ts +43 -11
  649. package/src/runtime/migrations/vbundle-importer.ts +4 -1
  650. package/src/runtime/migrations/vbundle-streaming-importer.ts +4 -1
  651. package/src/runtime/migrations/vbundle-validator.ts +1 -0
  652. package/src/runtime/pending-interactions-mode-session.test.ts +62 -0
  653. package/src/runtime/pending-interactions.ts +29 -0
  654. package/src/runtime/routes/__tests__/client-history-routes.test.ts +104 -0
  655. package/src/runtime/routes/__tests__/image-generation-routes.test.ts +97 -0
  656. package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +24 -2
  657. package/src/runtime/routes/__tests__/roadmap-routes.test.ts +29 -6
  658. package/src/runtime/routes/browser-context.ts +31 -11
  659. package/src/runtime/routes/browser-routes.ts +43 -4
  660. package/src/runtime/routes/btw-routes.ts +9 -4
  661. package/src/runtime/routes/channel-route-shared.ts +1 -9
  662. package/src/runtime/routes/client-routes.ts +108 -0
  663. package/src/runtime/routes/conversation-management-routes.ts +4 -1
  664. package/src/runtime/routes/conversation-routes.ts +125 -14
  665. package/src/runtime/routes/credential-prompt-routes.ts +4 -3
  666. package/src/runtime/routes/credential-routes.ts +1 -2
  667. package/src/runtime/routes/desktop-setup-routes.test.ts +9 -18
  668. package/src/runtime/routes/desktop-setup-routes.ts +8 -7
  669. package/src/runtime/routes/documents-routes.ts +51 -0
  670. package/src/runtime/routes/email-routes.ts +3 -7
  671. package/src/runtime/routes/events-routes.ts +75 -54
  672. package/src/runtime/routes/guardian-approval-interception.ts +24 -0
  673. package/src/runtime/routes/identity-routes.ts +1 -5
  674. package/src/runtime/routes/image-generation-routes.ts +8 -14
  675. package/src/runtime/routes/inbound-message-handler.ts +37 -41
  676. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.test.ts +168 -0
  677. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.ts +96 -0
  678. package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +39 -10
  679. package/src/runtime/routes/inbound-stages/background-dispatch.ts +9 -9
  680. package/src/runtime/routes/inbound-stages/reaction-intercept.ts +2 -0
  681. package/src/runtime/routes/inference-profile-session-handler.ts +11 -1
  682. package/src/runtime/routes/inference-profiles-routes.ts +9 -0
  683. package/src/runtime/routes/llm-context-normalization.ts +1 -7
  684. package/src/runtime/routes/migration-routes.ts +73 -2
  685. package/src/runtime/routes/notification-routes.ts +10 -0
  686. package/src/runtime/routes/oauth-commands-routes.ts +57 -54
  687. package/src/runtime/routes/oauth-providers.ts +11 -0
  688. package/src/runtime/routes/oauth-request-hints.test.ts +246 -0
  689. package/src/runtime/routes/oauth-request-hints.ts +149 -0
  690. package/src/runtime/routes/plugins-routes.ts +6 -0
  691. package/src/runtime/routes/roadmap-routes.ts +47 -8
  692. package/src/runtime/routes/schedule-routes.ts +2 -0
  693. package/src/runtime/routes/secret-routes.ts +99 -119
  694. package/src/runtime/routes/settings-routes.ts +10 -4
  695. package/src/runtime/routes/watcher-routes.ts +1 -3
  696. package/src/runtime/trust-class.test.ts +7 -6
  697. package/src/runtime/trust-class.ts +9 -17
  698. package/src/schedule/__tests__/worker-control.test.ts +36 -5
  699. package/src/schedule/__tests__/worker-mcp-bootstrap.test.ts +107 -0
  700. package/src/schedule/__tests__/worker-mcp-readiness.test.ts +173 -0
  701. package/src/schedule/__tests__/worker-mcp-tools.test.ts +13 -47
  702. package/src/schedule/run-script.ts +2 -2
  703. package/src/schedule/scheduler.ts +47 -11
  704. package/src/schedule/tool-surface-readiness.ts +65 -0
  705. package/src/schedule/worker-control.ts +15 -5
  706. package/src/schedule/worker-mcp.ts +85 -0
  707. package/src/schedule/worker.ts +9 -31
  708. package/src/security/secure-keys.ts +74 -112
  709. package/src/skills/clawhub.ts +3 -8
  710. package/src/skills/managed-store.ts +1 -5
  711. package/src/stt/types.ts +4 -0
  712. package/src/subagent/notify.ts +54 -34
  713. package/src/subagent/parent-notification.ts +6 -0
  714. package/src/telemetry/__tests__/phone-call-funnel.test.ts +72 -0
  715. package/src/telemetry/phone-call-funnel.ts +144 -0
  716. package/src/tools/__tests__/tool-schema-root-combinator-guard.test.ts +81 -0
  717. package/src/tools/ask-question/ask-question-tool.test.ts +167 -5
  718. package/src/tools/ask-question/ask-question-tool.ts +120 -22
  719. package/src/tools/browser/browser-execution.ts +29 -60
  720. package/src/tools/browser/cdp-client/__tests__/cdp-dom-helpers.test.ts +561 -39
  721. package/src/tools/browser/cdp-client/cdp-dom-helpers.ts +435 -39
  722. package/src/tools/browser/cdp-client/cdp-inspect-client.ts +1 -8
  723. package/src/tools/client-os.ts +15 -0
  724. package/src/tools/computer-use/skill-proxy-bridge.ts +1 -1
  725. package/src/tools/computer-use/target.ts +42 -0
  726. package/src/tools/document/document-tool.ts +13 -18
  727. package/src/tools/execution-target.ts +14 -16
  728. package/src/tools/host-terminal/host-shell.ts +16 -7
  729. package/src/tools/network/__tests__/web-fetch-tinyfish.test.ts +250 -0
  730. package/src/tools/network/__tests__/web-search.test.ts +203 -0
  731. package/src/tools/network/web-fetch.ts +274 -7
  732. package/src/tools/network/web-search.ts +468 -6
  733. package/src/tools/permission-checker.ts +3 -3
  734. package/src/tools/policy-context.ts +3 -1
  735. package/src/tools/shared/filesystem/file-ops-service.ts +1 -31
  736. package/src/tools/shared/shell-output.test.ts +10 -0
  737. package/src/tools/shared/shell-output.ts +14 -2
  738. package/src/tools/skills/load.ts +2 -1
  739. package/src/tools/skills/sandbox-runner.ts +24 -4
  740. package/src/tools/skills/scaffold-managed.ts +2 -1
  741. package/src/tools/skills/skill-script-runner.ts +7 -0
  742. package/src/tools/skills/skill-tool-factory.ts +15 -1
  743. package/src/tools/terminal/__tests__/safe-env.test.ts +44 -4
  744. package/src/tools/terminal/__tests__/sanitized-bash.test.ts +0 -14
  745. package/src/tools/terminal/safe-env.ts +39 -24
  746. package/src/tools/terminal/sanitized-bash.ts +15 -2
  747. package/src/tools/terminal/shell-launch.test.ts +162 -0
  748. package/src/tools/terminal/shell.test.ts +29 -0
  749. package/src/tools/terminal/shell.ts +15 -7
  750. package/src/tools/tool-approval-handler.ts +5 -1
  751. package/src/tools/tool-types.ts +3 -3
  752. package/src/tools/types.ts +8 -2
  753. package/src/util/__tests__/worker-process-command.test.ts +5 -4
  754. package/src/util/abort-reasons.ts +18 -26
  755. package/src/util/browser-human-verification.ts +14 -0
  756. package/src/util/canonicalize-identity.test.ts +21 -0
  757. package/src/util/canonicalize-identity.ts +16 -0
  758. package/src/util/errors.ts +13 -3
  759. package/src/util/host-process.test.ts +17 -1
  760. package/src/util/host-process.ts +24 -0
  761. package/src/util/provider-error-patterns.ts +3 -0
  762. package/src/util/unicode.ts +29 -0
  763. package/src/util/worker-process.ts +77 -35
  764. package/src/workspace/migrations/157-add-desktop-files-launcher.ts +94 -0
  765. package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +4 -10
  766. package/src/workspace/migrations/__tests__/157-add-desktop-files-launcher.test.ts +113 -0
  767. package/src/workspace/migrations/registry.ts +2 -0
  768. package/src/__tests__/computer-use-window-schema.test.ts +0 -24
  769. package/src/api/events/heartbeat-alert.ts +0 -20
  770. package/src/daemon/message-types/web-activity.ts +0 -65
  771. package/src/desktop/desktop-dependencies.test.ts +0 -131
  772. package/src/tools/computer-use/definitions.ts +0 -607
@@ -1089,6 +1089,7 @@ async function drainSingleMessage(
1089
1089
  content: resolvedContent,
1090
1090
  attachments: next.attachments,
1091
1091
  requestId: next.requestId,
1092
+ activeSurfaceId: next.activeSurfaceId,
1092
1093
  metadata: { ...next.metadata, sentAt: next.sentAt },
1093
1094
  displayContent: next.displayContent,
1094
1095
  clientMessageId: next.clientMessageId,
@@ -1155,6 +1156,7 @@ async function drainSingleMessage(
1155
1156
  messageId: userMessageId,
1156
1157
  requestId: next.requestId,
1157
1158
  clientMessageId: next.clientMessageId,
1159
+ modeSession: conversation.modeSessions.getTurnOwner(next.requestId),
1158
1160
  });
1159
1161
  // The row this echo announces is already durably persisted, so advance
1160
1162
  // the snapshot↔stream anchor to the echo's seq (stamped inline by the
@@ -1464,6 +1466,7 @@ async function drainBatch(
1464
1466
  content: qmContent,
1465
1467
  attachments: qm.attachments,
1466
1468
  requestId: qm.requestId,
1469
+ activeSurfaceId: qm.activeSurfaceId,
1467
1470
  metadata: { ...qm.metadata, sentAt: qm.sentAt },
1468
1471
  displayContent: qm.displayContent,
1469
1472
  clientMessageId: qm.clientMessageId,
@@ -1574,6 +1577,7 @@ async function drainBatch(
1574
1577
  messageId: lastUserMessageId,
1575
1578
  requestId: qm.requestId,
1576
1579
  clientMessageId: qm.clientMessageId,
1580
+ modeSession: conversation.modeSessions.getTurnOwner(qm.requestId),
1577
1581
  });
1578
1582
  // Advance the snapshot↔stream anchor to this echo's seq — the batched
1579
1583
  // row persisted just above and the agent loop for the batch has not
@@ -2298,6 +2302,7 @@ export async function processMessage(
2298
2302
  content: resolvedContent,
2299
2303
  attachments,
2300
2304
  requestId,
2305
+ activeSurfaceId,
2301
2306
  displayContent,
2302
2307
  scripted,
2303
2308
  ...(callerMetadata ? { metadata: callerMetadata } : {}),
@@ -8,6 +8,8 @@
8
8
  import { statSync } from "node:fs";
9
9
  import { join } from "node:path";
10
10
 
11
+ import { TrustClassSchema } from "@vellumai/gateway-client";
12
+
11
13
  import {
12
14
  getApp,
13
15
  getAppDirPath,
@@ -81,7 +83,6 @@ import type {
81
83
  import type { ContentBlock, Message } from "../providers/types.js";
82
84
  import type { TrustClass } from "../runtime/actor-trust-resolver.js";
83
85
  import { resolveCapabilities } from "../runtime/capabilities.js";
84
- import { trustClassSchema } from "../runtime/trust-class.js";
85
86
  import type { SubagentState } from "../subagent/types.js";
86
87
  import { TERMINAL_STATUSES } from "../subagent/types.js";
87
88
  import { canonicalizeInboundIdentity } from "../util/canonicalize-identity.js";
@@ -903,6 +904,11 @@ export function buildChannelCapabilityBlock(
903
904
  "- Do NOT use markdown tables — use bullet lists instead. No markdown headers — use **bold** or CAPS for emphasis.",
904
905
  );
905
906
  }
907
+ if (caps.channel === "email") {
908
+ lines.push(
909
+ "- Conversation text is not emailed. To reply, run `assistant email send` (see `assistant email send --help`). Use `--reply-to` to keep the thread. Skip a reply only when none is needed.",
910
+ );
911
+ }
906
912
  }
907
913
 
908
914
  // Inject group chat etiquette only when the chat type indicates a multi-party
@@ -1183,7 +1189,7 @@ function placeholderForBlockType(type: ContentBlock["type"]): string | null {
1183
1189
  function rowToRenderable(row: SlackTranscriptInputRow): RenderableSlackMessage {
1184
1190
  const slackMeta = readSlackMetadataFromMessageMetadata(row.metadata);
1185
1191
  const provenanceTrustClass = row.metadata
1186
- ? trustClassSchema.safeParse(
1192
+ ? TrustClassSchema.safeParse(
1187
1193
  safeParseRecord(row.metadata).provenanceTrustClass,
1188
1194
  ).data
1189
1195
  : undefined;
@@ -2035,20 +2041,6 @@ export async function composeInjectorChain(ctx: TurnContext): Promise<string> {
2035
2041
  */
2036
2042
  const DEFAULT_PLACEMENT: InjectionPlacement = "append-user-tail";
2037
2043
 
2038
- /**
2039
- * Count leading memory-prefix blocks on a user message's `content`.
2040
- *
2041
- * Delegates to {@link countMemoryPrefixBlocks} from
2042
- * `memory/graph/conversation-graph-memory.js` — the canonical state-machine
2043
- * for locating the memory-prefix boundary. Reusing it here keeps the
2044
- * PKB-context / PKB-reminder / NOW splice rules aligned on a single source
2045
- * of truth so their ordering relative to any memory prefix is stable and
2046
- * testable.
2047
- */
2048
- function countMemoryPrefixBlocksOnContent(content: ContentBlock[]): number {
2049
- return countMemoryPrefixBlocks(content);
2050
- }
2051
-
2052
2044
  /**
2053
2045
  * Apply one injector block to a `runMessages` array according to its
2054
2046
  * declared {@link InjectionPlacement}:
@@ -2101,9 +2093,7 @@ function applyInjectionBlock(
2101
2093
  { ...userTail, content: [...userTail.content, textBlock] },
2102
2094
  ];
2103
2095
  case "after-memory-prefix": {
2104
- const memoryPrefixCount = countMemoryPrefixBlocksOnContent(
2105
- userTail.content,
2106
- );
2096
+ const memoryPrefixCount = countMemoryPrefixBlocks(userTail.content);
2107
2097
  return [
2108
2098
  ...runMessages.slice(0, -1),
2109
2099
  {
@@ -2161,7 +2151,7 @@ function stripTailV2DynamicMemoryPrefix(
2161
2151
  if (!last || last.role !== "user") {
2162
2152
  return messages;
2163
2153
  }
2164
- const prefixCount = countMemoryPrefixBlocksOnContent(last.content);
2154
+ const prefixCount = countMemoryPrefixBlocks(last.content);
2165
2155
  if (prefixCount === 0) {
2166
2156
  return messages;
2167
2157
  }
@@ -463,6 +463,7 @@ export function projectSkillTools(
463
463
  skill.directoryPath,
464
464
  currentHash,
465
465
  skill.bundled,
466
+ skill.owner?.kind === "plugin" ? skill.owner.id : undefined,
466
467
  );
467
468
 
468
469
  if (tools.length > 0) {
@@ -550,10 +550,10 @@ export function clearAllActiveConversations(): number {
550
550
  /**
551
551
  * Evict in-memory conversations after a config/prompt reload so the next
552
552
  * turn rebuilds them against the new config. Conversations with in-flight work
553
- * (a live turn, a queued successor, or an active subagent) are marked stale
554
- * and rebuilt by `getOrCreateConversation` once that work finishes. Idle
555
- * conversations are disposed and dropped. Also used when provider credentials
556
- * change.
553
+ * are marked stale and rebuilt by `getOrCreateConversation` once that work
554
+ * finishes. This includes live mode-session sources and process-local
555
+ * structural continuations. Idle conversations are disposed and dropped. Also
556
+ * used when provider credentials change.
557
557
  */
558
558
  export function evictConversationsForReload(): void {
559
559
  const subagentManager = getSubagentManager();
@@ -0,0 +1,135 @@
1
+ /**
2
+ * Canonical conversation surface snapshots for host readers.
3
+ *
4
+ * Combines an unbounded persisted `ui_surface` scan with the loaded
5
+ * conversation's live `surfaceState` and current-turn snapshots. Hits from
6
+ * the persisted scan are never written into `surfaceState`.
7
+ */
8
+
9
+ import { coerceSurfaceDataRecord } from "../api/surfaces.js";
10
+ import { getMessages } from "../persistence/conversation-crud.js";
11
+ import { resolveCapabilities } from "../runtime/capabilities.js";
12
+ import { isPlainObject } from "../util/object.js";
13
+ import { findConversationOrSubagent } from "./conversation-registry.js";
14
+ import { isRowVisibleToUntrustedActor } from "./message-provenance.js";
15
+
16
+ export interface ConversationSurfaceSnapshot {
17
+ surfaceId: string;
18
+ surfaceType: string;
19
+ data: Record<string, unknown>;
20
+ completed: boolean;
21
+ completionSummary?: string;
22
+ }
23
+
24
+ function cloneSurfaceData(value: unknown): Record<string, unknown> {
25
+ const record = coerceSurfaceDataRecord(value);
26
+ try {
27
+ return structuredClone(record);
28
+ } catch {
29
+ return JSON.parse(JSON.stringify(record)) as Record<string, unknown>;
30
+ }
31
+ }
32
+
33
+ function surfaceTypeFromBlock(value: unknown): string {
34
+ return typeof value === "string" && value.trim().length > 0
35
+ ? value
36
+ : "dynamic_page";
37
+ }
38
+
39
+ function rememberOrder(order: string[], surfaceId: string): void {
40
+ if (!order.includes(surfaceId)) {
41
+ order.push(surfaceId);
42
+ }
43
+ }
44
+
45
+ /**
46
+ * Current surface snapshots for a loaded conversation.
47
+ *
48
+ * Fails closed to an empty list when the conversation is not already
49
+ * loaded or the turn-or-resting trust snapshot is missing. Untrusted
50
+ * views reuse {@link isRowVisibleToUntrustedActor} and do not add
51
+ * live-only cards that never appeared in a visible persisted row.
52
+ */
53
+ export function listConversationSurfaceSnapshots(
54
+ conversationId: string,
55
+ ): ConversationSurfaceSnapshot[] {
56
+ const conversation = findConversationOrSubagent(conversationId);
57
+ if (!conversation) {
58
+ return [];
59
+ }
60
+ const trust = conversation.getTurnOrRestingTrust();
61
+ if (!trust) {
62
+ return [];
63
+ }
64
+ const canAccessMemory = resolveCapabilities(trust.trustClass).canAccessMemory;
65
+ const filterByProvenance = !canAccessMemory;
66
+
67
+ const order: string[] = [];
68
+ const records = new Map<string, ConversationSurfaceSnapshot>();
69
+
70
+ const rows = getMessages(conversationId);
71
+ for (const row of rows) {
72
+ if (filterByProvenance && !isRowVisibleToUntrustedActor(row.metadata)) {
73
+ continue;
74
+ }
75
+ if (!Array.isArray(row.content)) {
76
+ continue;
77
+ }
78
+ for (const block of row.content) {
79
+ if (!isPlainObject(block) || block.type !== "ui_surface") {
80
+ continue;
81
+ }
82
+ const surfaceId = block.surfaceId;
83
+ if (typeof surfaceId !== "string" || surfaceId.length === 0) {
84
+ continue;
85
+ }
86
+ const completionSummary =
87
+ typeof block.completionSummary === "string"
88
+ ? block.completionSummary
89
+ : undefined;
90
+ rememberOrder(order, surfaceId);
91
+ records.set(surfaceId, {
92
+ surfaceId,
93
+ surfaceType: surfaceTypeFromBlock(block.surfaceType),
94
+ data: cloneSurfaceData(block.data),
95
+ completed: block.completed === true,
96
+ ...(completionSummary !== undefined ? { completionSummary } : {}),
97
+ });
98
+ }
99
+ }
100
+
101
+ for (const [surfaceId, entry] of conversation.surfaceState.entries()) {
102
+ const existing = records.get(surfaceId);
103
+ if (!existing && filterByProvenance) {
104
+ continue;
105
+ }
106
+ rememberOrder(order, surfaceId);
107
+ records.set(surfaceId, {
108
+ surfaceId,
109
+ surfaceType: entry.surfaceType,
110
+ data: cloneSurfaceData(entry.data),
111
+ completed: existing?.completed === true,
112
+ ...(existing?.completionSummary !== undefined
113
+ ? { completionSummary: existing.completionSummary }
114
+ : {}),
115
+ });
116
+ }
117
+
118
+ for (const surface of conversation.currentTurnSurfaces) {
119
+ const existing = records.get(surface.surfaceId);
120
+ const completionSummary =
121
+ typeof surface.completionSummary === "string"
122
+ ? surface.completionSummary
123
+ : existing?.completionSummary;
124
+ rememberOrder(order, surface.surfaceId);
125
+ records.set(surface.surfaceId, {
126
+ surfaceId: surface.surfaceId,
127
+ surfaceType: surface.surfaceType,
128
+ data: cloneSurfaceData(surface.data),
129
+ completed: surface.completed === true || existing?.completed === true,
130
+ ...(completionSummary !== undefined ? { completionSummary } : {}),
131
+ });
132
+ }
133
+
134
+ return order.map((surfaceId) => records.get(surfaceId)!);
135
+ }
@@ -28,6 +28,8 @@ import {
28
28
  resolveEffectiveAppHtml,
29
29
  updateApp,
30
30
  } from "../apps/app-store.js";
31
+ import { executeDesktopComputerUse } from "../desktop/desktop-computer-use.js";
32
+ import { canUseVirtualDesktop } from "../desktop/virtual-desktop-feature.js";
31
33
  import { recordActivationEvent } from "../onboarding/onboarding-events-store.js";
32
34
  import {
33
35
  getMessages,
@@ -57,6 +59,7 @@ import {
57
59
  import { resolveAppId } from "../tools/apps/resolve-app-id.js";
58
60
  import { formatDesktopAppRequired } from "../tools/capability-offer.js";
59
61
  import { POINT_AT_PROXY_TOOL } from "../tools/computer-use/skill-proxy-bridge.js";
62
+ import { computerUseTarget } from "../tools/computer-use/target.js";
60
63
  import type { ToolExecutionResult } from "../tools/types.js";
61
64
  import { getLogger } from "../util/logger.js";
62
65
  import { isPlainObject } from "../util/object.js";
@@ -70,7 +73,7 @@ import {
70
73
  type SurfaceShowPair,
71
74
  type SurfaceStateEntry,
72
75
  } from "./conversation-surface-state.js";
73
- import type { HostCuProxy } from "./host-cu-proxy.js";
76
+ import { HostCuProxy } from "./host-cu-proxy.js";
74
77
  import { resolveHostCuTarget } from "./host-cu-target.js";
75
78
  import type {
76
79
  AnySurfaceData,
@@ -89,9 +92,11 @@ import type {
89
92
  } from "./message-protocol.js";
90
93
  import { INTERACTIVE_SURFACE_TYPES } from "./message-protocol.js";
91
94
  import { isRowVisibleToUntrustedActor } from "./message-provenance.js";
95
+ import { bestEffortModeSessionTracking } from "./mode-session-tracking.js";
92
96
  import type { TrustContext } from "./trust-context-types.js";
93
97
  import { restingTrust } from "./trust-context-types.js";
94
98
  import { turnActorPrincipalId } from "./turn-actor.js";
99
+ import { virtualDesktopContext } from "./virtual-desktop-context.js";
95
100
  export {
96
101
  buildSurfaceShowPair,
97
102
  type CurrentTurnSurface,
@@ -178,12 +183,61 @@ const NON_BLOCKING_PENDING_SURFACE_TYPES = new Set<SurfaceType>([
178
183
  export function hasBlockingPendingSurface(ctx: {
179
184
  pendingSurfaceActions: Map<string, { surfaceType: SurfaceType }>;
180
185
  }): boolean {
181
- for (const entry of ctx.pendingSurfaceActions.values()) {
186
+ return blockingPendingSurfaceIds(ctx).length > 0;
187
+ }
188
+
189
+ export function blockingPendingSurfaceIds(ctx: {
190
+ pendingSurfaceActions: Map<string, { surfaceType: SurfaceType }>;
191
+ }): string[] {
192
+ const ids: string[] = [];
193
+ for (const [surfaceId, entry] of ctx.pendingSurfaceActions) {
182
194
  if (!NON_BLOCKING_PENDING_SURFACE_TYPES.has(entry.surfaceType)) {
183
- return true;
195
+ ids.push(surfaceId);
184
196
  }
185
197
  }
186
- return false;
198
+ return ids;
199
+ }
200
+
201
+ function acceptModeSessionSurfaceResponse(
202
+ ctx: Conversation,
203
+ requestId: string,
204
+ surfaceId: string,
205
+ ): void {
206
+ bestEffortModeSessionTracking("surface response admission", () =>
207
+ ctx.modeSessions?.acceptTurn(requestId, {
208
+ kind: "surface",
209
+ responseId: surfaceId,
210
+ }),
211
+ );
212
+ }
213
+
214
+ function invalidateModeSessionSurfaceWait(
215
+ ctx: Partial<Pick<Conversation, "modeSessions">>,
216
+ surfaceId: string,
217
+ ): void {
218
+ bestEffortModeSessionTracking("surface wait invalidation", () =>
219
+ ctx.modeSessions?.invalidateStructuralWait({
220
+ kind: "surface",
221
+ responseId: surfaceId,
222
+ }),
223
+ );
224
+ }
225
+
226
+ function settleModeSessionSurfaceWait(
227
+ ctx: Pick<Conversation, "conversationId" | "modeSessions">,
228
+ surfaceId: string,
229
+ ): void {
230
+ try {
231
+ ctx.modeSessions?.settleStructuralWait(
232
+ { kind: "surface", responseId: surfaceId },
233
+ { status: "completed", endReason: "surface_launch_settled" },
234
+ );
235
+ } catch (err) {
236
+ log.warn(
237
+ { err, conversationId: ctx.conversationId, surfaceId },
238
+ "Mode-session launcher settlement failed",
239
+ );
240
+ }
187
241
  }
188
242
 
189
243
  /**
@@ -1292,7 +1346,8 @@ export function cleanupStandaloneSurface(
1292
1346
  | "lastSurfaceAction"
1293
1347
  | "accumulatedSurfaceState"
1294
1348
  | "surfaceUndoStacks"
1295
- >,
1349
+ > &
1350
+ Partial<Pick<Conversation, "modeSessions">>,
1296
1351
  surfaceId: string,
1297
1352
  ): void {
1298
1353
  const entry = ctx.pendingStandaloneSurfaces?.get(surfaceId);
@@ -1305,6 +1360,7 @@ export function cleanupStandaloneSurface(
1305
1360
  ctx.lastSurfaceAction.delete(surfaceId);
1306
1361
  ctx.accumulatedSurfaceState.delete(surfaceId);
1307
1362
  ctx.surfaceUndoStacks.delete(surfaceId);
1363
+ invalidateModeSessionSurfaceWait(ctx, surfaceId);
1308
1364
 
1309
1365
  // Record a tombstone so late client actions are silently dropped.
1310
1366
  if (ctx.recentlyCompletedStandaloneSurfaces) {
@@ -2110,6 +2166,7 @@ export async function handleSurfaceAction(
2110
2166
  ...(anchorMessageId ? { anchorMessageId } : {}),
2111
2167
  ...(originTrustContext ? { originTrustContext } : {}),
2112
2168
  });
2169
+ settleModeSessionSurfaceWait(ctx, surfaceId);
2113
2170
  log.info(
2114
2171
  { originConversationId: ctx.conversationId, conversationId, surfaceId },
2115
2172
  "launch_conversation dispatched inline from surface action",
@@ -2284,6 +2341,8 @@ export async function handleSurfaceAction(
2284
2341
  return QUEUE_FULL_RESULT;
2285
2342
  }
2286
2343
 
2344
+ acceptModeSessionSurfaceResponse(ctx, requestId, surfaceId);
2345
+
2287
2346
  // Terminal user commit accepted — record the activation milestone if this
2288
2347
  // surface was tagged (best-effort, no-op otherwise). Deferred until after
2289
2348
  // the rejection check so a queue-full click doesn't over-report a moment
@@ -2310,6 +2369,7 @@ export async function handleSurfaceAction(
2310
2369
  type: "user_message_echo",
2311
2370
  text: prompt,
2312
2371
  conversationId: ctx.conversationId,
2372
+ modeSession: ctx.modeSessions.getTurnOwner(requestId),
2313
2373
  });
2314
2374
  }
2315
2375
 
@@ -2537,6 +2597,8 @@ export async function handleSurfaceAction(
2537
2597
  return QUEUE_FULL_RESULT;
2538
2598
  }
2539
2599
 
2600
+ acceptModeSessionSurfaceResponse(ctx, requestId, surfaceId);
2601
+
2540
2602
  // Terminal user commit accepted — record the activation milestone if this
2541
2603
  // surface was tagged (best-effort, no-op otherwise). Deferred until after the
2542
2604
  // rejection check so a queue-full click doesn't over-report a moment (and the
@@ -2563,6 +2625,7 @@ export async function handleSurfaceAction(
2563
2625
  type: "user_message_echo",
2564
2626
  text: prompt,
2565
2627
  conversationId: ctx.conversationId,
2628
+ modeSession: ctx.modeSessions.getTurnOwner(requestId),
2566
2629
  });
2567
2630
  }
2568
2631
  if (result.queued) {
@@ -3161,6 +3224,28 @@ export async function surfaceProxyResolver(
3161
3224
  ): Promise<ToolExecutionResult> {
3162
3225
  // Route CU proxy tools (all computer_use_* action tools)
3163
3226
  if (toolName.startsWith("computer_use_")) {
3227
+ const desktopContext = virtualDesktopContext(ctx, signal);
3228
+ if (
3229
+ toolName !== POINT_AT_PROXY_TOOL &&
3230
+ computerUseTarget(input, desktopContext) === "assistant-desktop"
3231
+ ) {
3232
+ if (!canUseVirtualDesktop(desktopContext)) {
3233
+ return {
3234
+ content:
3235
+ "The assistant desktop requires an identified guardian and an enabled platform-hosted assistant.",
3236
+ isError: true,
3237
+ };
3238
+ }
3239
+ if (!ctx.hostCuProxy) {
3240
+ ctx.setHostCuProxy(new HostCuProxy());
3241
+ }
3242
+ return executeDesktopComputerUse(
3243
+ toolName,
3244
+ input,
3245
+ desktopContext,
3246
+ ctx.hostCuProxy!,
3247
+ );
3248
+ }
3164
3249
  const hostCuProxy = ensureHostCuProxy(ctx);
3165
3250
  if (!hostCuProxy || !hostCuProxy.isAvailable()) {
3166
3251
  return {
@@ -3180,6 +3265,15 @@ export async function surfaceProxyResolver(
3180
3265
  : typeof input.answer === "string"
3181
3266
  ? input.answer
3182
3267
  : "Task complete";
3268
+ bestEffortModeSessionTracking("computer completion", () =>
3269
+ ctx.computerUseModeSessions.endTask({
3270
+ turnId: ctx.currentRequestId,
3271
+ source: {
3272
+ sourceId: hostCuProxy.sourceId,
3273
+ generation: hostCuProxy.resetGeneration,
3274
+ },
3275
+ }),
3276
+ );
3183
3277
  hostCuProxy.endTask(ctx.conversationId);
3184
3278
  return { content: summary, isError: false };
3185
3279
  }
@@ -3215,9 +3309,11 @@ export async function surfaceProxyResolver(
3215
3309
  // `maxStepsPerSession` would let a long walkthrough exhaust a budget
3216
3310
  // meant for actions and be told to call `computer_use_done`, which has
3217
3311
  // nothing to do with what it was doing.
3312
+ const activityAt = Date.now();
3218
3313
  if (toolName !== POINT_AT_PROXY_TOOL) {
3219
3314
  hostCuProxy.recordAction(toolName, input, reasoning);
3220
3315
  }
3316
+ const turnId = ctx.currentRequestId;
3221
3317
  return hostCuProxy.request(
3222
3318
  toolName,
3223
3319
  input,
@@ -3227,6 +3323,18 @@ export async function surfaceProxyResolver(
3227
3323
  signal,
3228
3324
  targetClientId,
3229
3325
  sourceActorPrincipalId,
3326
+ toolName !== POINT_AT_PROXY_TOOL && turnId
3327
+ ? () => {
3328
+ ctx.computerUseModeSessions.recordAction({
3329
+ turnId,
3330
+ source: {
3331
+ sourceId: hostCuProxy.sourceId,
3332
+ generation: hostCuProxy.resetGeneration,
3333
+ },
3334
+ at: activityAt,
3335
+ });
3336
+ }
3337
+ : undefined,
3230
3338
  );
3231
3339
  }
3232
3340
 
@@ -19,9 +19,15 @@ import {
19
19
  resolveSendUserMessageActive,
20
20
  SEND_USER_MESSAGE_TOOL_NAME,
21
21
  } from "../config/send-user-message-gate.js";
22
+ import { canUseVirtualDesktop } from "../desktop/virtual-desktop-feature.js";
22
23
  import { supportsChannelReaction } from "../messaging/providers/index.js";
23
24
  import type { PermissionPrompter } from "../permissions/prompter.js";
24
25
  import type { SecretPrompter } from "../permissions/secret-prompter.js";
26
+ import {
27
+ type ConversationToolSurface,
28
+ hashConversationToolSurface,
29
+ recordConversationToolSurface,
30
+ } from "../persistence/conversation-tool-surface.js";
25
31
  import { getBindingByConversation } from "../persistence/external-conversation-store.js";
26
32
  import { getAllDefaultPluginNames } from "../plugins/defaults/main.js";
27
33
  import { isActivationSession } from "../plugins/defaults/memory/activation-session-store.js";
@@ -82,6 +88,7 @@ import {
82
88
  } from "./doordash-steps.js";
83
89
  import { runPostExecutionSideEffects } from "./tool-side-effects.js";
84
90
  import { FALLBACK_TURN_TRUST, resolveTrustClass } from "./trust-context.js";
91
+ import { virtualDesktopContext } from "./virtual-desktop-context.js";
85
92
 
86
93
  const log = getLogger("conversation-tool-setup");
87
94
 
@@ -457,8 +464,7 @@ export function createToolExecutor(
457
464
  sendUserMessageActive: resolveSendUserMessageActive(ctx),
458
465
  toolUseId,
459
466
  isPlatformHosted: getIsPlatform(),
460
- transportInterface: ctx.transportInterface,
461
- clientOs: resolveTurnClientOs(ctx).clientOs,
467
+ ...resolveTurnClientOs(ctx),
462
468
  overrideProfile: ctx.currentTurnOverrideProfile,
463
469
  cronRunId: ctx.currentTurnCronRunId,
464
470
  invokingCallSite: ctx.currentCallSite ?? "mainAgent",
@@ -738,6 +744,7 @@ function isToolSupportedOnClientOs(name: string, ctx: Conversation): boolean {
738
744
  clientOs,
739
745
  transportInterface,
740
746
  sourceActorPrincipalId: ctx.getTurnActorPrincipalId?.(),
747
+ trustClass: virtualDesktopContext(ctx).trustClass,
741
748
  });
742
749
  }
743
750
 
@@ -932,8 +939,12 @@ export function canSpawnSubagentsForTurn(ctx: Conversation): boolean {
932
939
  // executor instead. That is the right answer to "is this tool on the wire"
933
940
  // and the wrong one to "could this turn actually spawn": the memory
934
941
  // retrospective wake runs in execution mode with an allowlist that names
935
- // `skill_load` but neither the dispatcher nor the spawn tool, so the spawn
936
- // is denied after the prompt has already told the model to delegate.
942
+ // `skill_load` but neither the dispatcher nor the spawn tool, so a spawn is
943
+ // denied at execution. A wake replaying its source's recorded surface
944
+ // renders the source's delegation state in place of this answer
945
+ // (`Conversation.delegateIndependentTasksReplay`): a denied spawn attempt
946
+ // costs one tool error, a system prompt that differs from the source's
947
+ // costs the whole cached prefix behind it.
937
948
  const allowlist = ctx.subagentAllowedTools;
938
949
  return SUBAGENT_SPAWN_PATH_TOOL_NAMES.every(
939
950
  (name) =>
@@ -943,6 +954,57 @@ export function canSpawnSubagentsForTurn(ctx: Conversation): boolean {
943
954
  );
944
955
  }
945
956
 
957
+ /**
958
+ * Build the agent loop's `onToolsSent` observer for a conversation: record
959
+ * the tool array each provider call sends, with the delegation-section state
960
+ * the prompt build captured for the prompt that call carries
961
+ * (`Conversation.renderedDelegateIndependentTasks`), so a later fork wake can
962
+ * replay both (`recordConversationToolSurface`). Only the loop's send boundary sees the
963
+ * sent array. The resolver is also consulted out of band (the token count
964
+ * behind `/compact` and `/clean`, compaction estimates), where a read outside
965
+ * any turn resolves a clientless surface that would overwrite the one the
966
+ * conversation's turns actually send.
967
+ *
968
+ * Arrays that are not the conversation's own surface are skipped: a replaying
969
+ * wake sends its source's array, an empty array is a tools-disabled call (a
970
+ * fork replaying it could never call `remember`), and disk-pressure cleanup
971
+ * mode narrows the wire to cleanup tools. Best-effort: a failed write is
972
+ * logged and the surface still counts as recorded, so a persistent failure
973
+ * logs once per distinct surface rather than once per provider call.
974
+ */
975
+ export function createWireToolSurfaceRecorder(
976
+ ctx: Conversation,
977
+ ): (tools: ToolDefinition[]) => void {
978
+ return (tools) => {
979
+ if (
980
+ !ctx.conversationId ||
981
+ ctx.wireToolReplay ||
982
+ tools.length === 0 ||
983
+ ctx.diskPressureCleanupModeActive === true
984
+ ) {
985
+ return;
986
+ }
987
+ const surface: ConversationToolSurface = {
988
+ tools,
989
+ // Unknown before the first prompt build or for a verbatim override.
990
+ delegateIndependentTasks: ctx.renderedDelegateIndependentTasks ?? null,
991
+ };
992
+ try {
993
+ ctx.recordedToolSurfaceHash = recordConversationToolSurface(
994
+ ctx.conversationId,
995
+ surface,
996
+ ctx.recordedToolSurfaceHash,
997
+ );
998
+ } catch (err) {
999
+ log.warn(
1000
+ { err, conversationId: ctx.conversationId },
1001
+ "failed to record the conversation's wire tool surface; continuing",
1002
+ );
1003
+ ctx.recordedToolSurfaceHash = hashConversationToolSurface(surface);
1004
+ }
1005
+ };
1006
+ }
1007
+
946
1008
  /**
947
1009
  * Build a resolveTools callback that merges base tool definitions with
948
1010
  * dynamically projected skill tools on each agent turn. Also updates
@@ -1131,7 +1193,7 @@ export function createResolveToolsCallback(
1131
1193
  definition.name === "bash"
1132
1194
  ? {
1133
1195
  ...definition,
1134
- description: `${definition.description} For browser tasks, use assistant browser navigate --url <url> directly. It installs the virtual desktop if needed, starts Chrome, and completes the action in one call. Use timeout_seconds: ${getConfig().timeouts.shellMaxTimeoutSec} for first use; setup progress is visible in the Virtual desktop panel. Use assistant browser --help for other browser actions. Use this managed path even if saved notes describe manual setup. Do not install packages or launch Chrome, X servers, or screenshot scripts yourself.`,
1196
+ description: `${definition.description} For browser tasks, use assistant browser navigate --url <url> directly. Chrome and desktop components are included in the assistant image. The command starts Chrome and completes the action without installing dependencies. If components are missing, report the image problem. Use assistant browser --help for other browser actions. Use this managed path even if saved notes describe manual setup. Do not install packages or launch Chrome, X servers, or screenshot scripts yourself.`,
1135
1197
  }
1136
1198
  : definition,
1137
1199
  );
@@ -1147,6 +1209,9 @@ export function createResolveToolsCallback(
1147
1209
  const effectivePreactivated = [
1148
1210
  ...DEFAULT_PREACTIVATED_SKILL_IDS,
1149
1211
  ...(ctx.preactivatedSkillIds ?? []),
1212
+ ...(canUseVirtualDesktop(virtualDesktopContext(ctx))
1213
+ ? ["computer-use"]
1214
+ : []),
1150
1215
  ];
1151
1216
  const projection = projectSkillTools(history, {
1152
1217
  preactivatedSkillIds: effectivePreactivated,
@@ -1237,6 +1302,14 @@ export function createResolveToolsCallback(
1237
1302
 
1238
1303
  ctx.allowedToolNames = turnAllowed;
1239
1304
 
1240
- return applyActivityField(allBaseDefs);
1305
+ // A wake replaying its source's recorded surface sends that array
1306
+ // verbatim: the wire tool block is the first tier of the provider cache
1307
+ // prefix (tools → system → messages), so only the same bytes read the
1308
+ // source's cached prefix instead of rewriting it. Execution is unaffected:
1309
+ // `allowedToolNames` above and the executor's allowlist gate still decide
1310
+ // what may run.
1311
+ return ctx.wireToolReplay
1312
+ ? [...ctx.wireToolReplay]
1313
+ : applyActivityField(allBaseDefs);
1241
1314
  };
1242
1315
  }