@vellumai/assistant 0.12.2 → 0.12.3-staging.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (772) hide show
  1. package/.env.example +2 -0
  2. package/AGENTS.md +1 -1
  3. package/ARCHITECTURE.md +43 -29
  4. package/Dockerfile +63 -0
  5. package/docs/architecture/integrations.md +7 -0
  6. package/docs/architecture/memory.md +17 -2
  7. package/docs/architecture/security.md +10 -5
  8. package/docs/architecture/turn-actor.md +2 -0
  9. package/docs/credential-execution-service.md +1 -1
  10. package/docs/desktop-browser-cli.md +50 -5
  11. package/docs/live-voice-task-outcomes.md +100 -0
  12. package/docs/mode-sessions.md +183 -0
  13. package/docs/voice-input-diagnostics.md +58 -0
  14. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  15. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
  16. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  17. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  18. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
  19. package/node_modules/@vellumai/environments/src/shell.test.ts +21 -0
  20. package/node_modules/@vellumai/environments/src/shell.ts +24 -0
  21. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  22. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
  23. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  24. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  25. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
  26. package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +20 -0
  27. package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +8 -2
  28. package/node_modules/@vellumai/gateway-client/src/index.ts +4 -1
  29. package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +38 -7
  30. package/node_modules/@vellumai/gateway-client/src/trust-verdict-contract.ts +5 -6
  31. package/node_modules/@vellumai/service-contracts/package.json +2 -1
  32. package/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
  33. package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  34. package/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  35. package/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
  36. package/node_modules/@vellumai/slack-text/src/index.ts +4 -8
  37. package/openapi.yaml +716 -16
  38. package/package.json +1 -1
  39. package/scripts/smoke-desktop-browser-cli.ts +11 -26
  40. package/scripts/smoke-desktop-window-theme.ts +2 -3
  41. package/scripts/sync-llm-catalog.ts +3 -0
  42. package/scripts/voice-ttft-spike.ts +2 -2
  43. package/src/__tests__/agent-loop-output-hooks.test.ts +150 -1
  44. package/src/__tests__/agent-loop-provider-error-recording.test.ts +87 -19
  45. package/src/__tests__/agent-loop.test.ts +259 -0
  46. package/src/__tests__/agent-wake-delegation-prompt.test.ts +64 -2
  47. package/src/__tests__/annotate-activity-metadata.test.ts +3 -1
  48. package/src/__tests__/annotate-risk-options.test.ts +2 -0
  49. package/src/__tests__/answered-question-persistence.test.ts +2 -0
  50. package/src/__tests__/anthropic-gateway-shared.test.ts +22 -0
  51. package/src/__tests__/anthropic-provider.test.ts +55 -0
  52. package/src/__tests__/approval-interception-trust-gates.test.ts +40 -0
  53. package/src/__tests__/approval-message-composer.test.ts +1 -118
  54. package/src/__tests__/approval-routes-http.test.ts +3 -0
  55. package/src/__tests__/assistant-event-hub-connection-history.test.ts +111 -0
  56. package/src/__tests__/assistant-event-hub-targeted.test.ts +50 -0
  57. package/src/__tests__/assistant-event-targeting.test.ts +186 -0
  58. package/src/__tests__/assistant-stream-state.test.ts +29 -9
  59. package/src/__tests__/attachments-store.test.ts +22 -3
  60. package/src/__tests__/browser-fill-credential.test.ts +11 -22
  61. package/src/__tests__/btw-routes.test.ts +4 -1
  62. package/src/__tests__/build-persisted-content.test.ts +1 -1
  63. package/src/__tests__/call-conversation-messages.test.ts +5 -0
  64. package/src/__tests__/call-recovery.test.ts +7 -2
  65. package/src/__tests__/call-routes-http.test.ts +12 -0
  66. package/src/__tests__/call-store.test.ts +7 -2
  67. package/src/__tests__/channel-approval.test.ts +9 -14
  68. package/src/__tests__/channel-reply-delivery.test.ts +56 -0
  69. package/src/__tests__/channel-retry-sweep.test.ts +9 -0
  70. package/src/__tests__/chat-credential-redaction.test.ts +23 -0
  71. package/src/__tests__/chat-reveal-guard-priming.test.ts +2 -0
  72. package/src/__tests__/checker.test.ts +11 -3
  73. package/src/__tests__/client-os-metadata-persistence.test.ts +35 -1
  74. package/src/__tests__/computer-use-screenshot-attachments.test.ts +411 -0
  75. package/src/__tests__/computer-use-screenshot-selection.test.ts +50 -0
  76. package/src/__tests__/computer-use-skill-manifest-regression.test.ts +0 -25
  77. package/src/__tests__/computer-use-tools.test.ts +1 -233
  78. package/src/__tests__/config-model-image-provider.test.ts +39 -0
  79. package/src/__tests__/config-schema.test.ts +16 -0
  80. package/src/__tests__/config-watcher.test.ts +24 -0
  81. package/src/__tests__/context-search-agent-runner.test.ts +260 -1
  82. package/src/__tests__/context-search-fanout.test.ts +1 -1
  83. package/src/__tests__/context-search-types.test.ts +7 -7
  84. package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +8 -2
  85. package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +2 -0
  86. package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +2 -0
  87. package/src/__tests__/conversation-agent-loop-overflow.test.ts +2 -0
  88. package/src/__tests__/conversation-agent-loop.test.ts +797 -16
  89. package/src/__tests__/conversation-app-control-lifecycle.test.ts +53 -1
  90. package/src/__tests__/conversation-attachments.test.ts +106 -0
  91. package/src/__tests__/conversation-evictor.test.ts +62 -37
  92. package/src/__tests__/conversation-fork-crud.test.ts +191 -0
  93. package/src/__tests__/conversation-history-boundary-repair-failure.test.ts +174 -0
  94. package/src/__tests__/conversation-loop-db-lock-resilience.test.ts +2 -0
  95. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +2 -0
  96. package/src/__tests__/conversation-rendered-delegation-state.test.ts +276 -0
  97. package/src/__tests__/conversation-retry-route.test.ts +22 -0
  98. package/src/__tests__/conversation-routes-disk-view.test.ts +2 -0
  99. package/src/__tests__/conversation-routes-enabled-plugins.test.ts +2 -0
  100. package/src/__tests__/conversation-routes-guardian-reply.test.ts +10 -0
  101. package/src/__tests__/conversation-routes-hidden-queue.test.ts +2 -0
  102. package/src/__tests__/conversation-routes-slash-commands.test.ts +2 -0
  103. package/src/__tests__/conversation-runtime-assembly.test.ts +29 -0
  104. package/src/__tests__/conversation-surface-snapshots.test.ts +543 -0
  105. package/src/__tests__/conversation-surfaces-action-delivery.test.ts +151 -0
  106. package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +63 -1
  107. package/src/__tests__/conversation-tool-setup-attribution.test.ts +23 -0
  108. package/src/__tests__/conversation-undo.test.ts +61 -0
  109. package/src/__tests__/credential-execution-client.test.ts +72 -55
  110. package/src/__tests__/credential-prompt-route.test.ts +35 -3
  111. package/src/__tests__/cu-unified-flow.test.ts +82 -1
  112. package/src/__tests__/db-conversation-tool-surface.test.ts +221 -0
  113. package/src/__tests__/db-mode-sessions-migration.test.ts +175 -0
  114. package/src/__tests__/deterministic-verification-control-plane.test.ts +2 -0
  115. package/src/__tests__/dm-backfill.test.ts +91 -1
  116. package/src/__tests__/document-create-route.test.ts +88 -0
  117. package/src/__tests__/document-sync-tags.test.ts +27 -0
  118. package/src/__tests__/emit-signal-routing-intent.test.ts +142 -2
  119. package/src/__tests__/events-tail-route.test.ts +58 -0
  120. package/src/__tests__/evict-conversations-for-reload.test.ts +57 -8
  121. package/src/__tests__/guardian-action-sweep.test.ts +16 -0
  122. package/src/__tests__/guardian-dispatch.test.ts +13 -0
  123. package/src/__tests__/headless-browser-interactions.test.ts +64 -20
  124. package/src/__tests__/heartbeat-service.test.ts +2 -59
  125. package/src/__tests__/helpers/mock-conversation.ts +21 -0
  126. package/src/__tests__/helpers/native-web-search-harness.ts +2 -0
  127. package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +5 -0
  128. package/src/__tests__/history-repair.test.ts +53 -0
  129. package/src/__tests__/host-cu-proxy.test.ts +46 -2
  130. package/src/__tests__/http-user-message-parity.test.ts +2 -0
  131. package/src/__tests__/image-credentials.test.ts +49 -0
  132. package/src/__tests__/image-service-dispatcher.test.ts +45 -0
  133. package/src/__tests__/inference-profile-session-handler.test.ts +26 -0
  134. package/src/__tests__/interrupt-on-send.test.ts +57 -43
  135. package/src/__tests__/interrupt-turn-note.test.ts +5 -6
  136. package/src/__tests__/list-messages-tool-merge.test.ts +85 -1
  137. package/src/__tests__/llm-catalog-parity.test.ts +27 -4
  138. package/src/__tests__/media-generate-image.test.ts +49 -1
  139. package/src/__tests__/message-provenance-contact-id.test.ts +69 -0
  140. package/src/__tests__/migration-export-to-gcs.test.ts +142 -0
  141. package/src/__tests__/mode-session-history.test.ts +245 -0
  142. package/src/__tests__/mode-session-recovery.test.ts +222 -0
  143. package/src/__tests__/mode-session-wire.test.ts +141 -0
  144. package/src/__tests__/notification-guardian-path.test.ts +7 -0
  145. package/src/__tests__/notification-telegram-adapter.test.ts +2 -2
  146. package/src/__tests__/notification-vellum-adapter.test.ts +34 -0
  147. package/src/__tests__/oauth-apps-routes.test.ts +1 -0
  148. package/src/__tests__/oauth-commands-routes.test.ts +114 -101
  149. package/src/__tests__/oauth-connect-orchestrator.test.ts +2 -0
  150. package/src/__tests__/oauth-provider-serializer.test.ts +13 -0
  151. package/src/__tests__/oauth-providers-routes.test.ts +3 -0
  152. package/src/__tests__/outbound-slack-persistence.test.ts +2 -0
  153. package/src/__tests__/persist-media-references.test.ts +50 -0
  154. package/src/__tests__/persistence-secret-redaction.test.ts +2 -0
  155. package/src/__tests__/plugin-api-provider.test.ts +5 -0
  156. package/src/__tests__/plugin-api-resolve-credential.test.ts +97 -0
  157. package/src/__tests__/plugin-import-boundary-guard.test.ts +9 -1
  158. package/src/__tests__/process-message-display-content.test.ts +56 -0
  159. package/src/__tests__/reaction-persistence.test.ts +5 -0
  160. package/src/__tests__/request-secret-standalone-channel-gate.test.ts +25 -9
  161. package/src/__tests__/run-conversation-turn-persistence.test.ts +138 -1
  162. package/src/__tests__/runtime-events-sse-reconnect.test.ts +1 -1
  163. package/src/__tests__/schedule-routes.test.ts +20 -0
  164. package/src/__tests__/scheduler-result-notification.test.ts +23 -4
  165. package/src/__tests__/script-proxy-certs.test.ts +1 -1
  166. package/src/__tests__/scripted-turn-metadata-persistence.test.ts +73 -0
  167. package/src/__tests__/secret-ingress-http.test.ts +2 -0
  168. package/src/__tests__/secret-routes-platform-proxy.test.ts +86 -31
  169. package/src/__tests__/secret-routes-scrub.test.ts +22 -27
  170. package/src/__tests__/secret-scanner.test.ts +20 -0
  171. package/src/__tests__/secure-keys-managed-failover.test.ts +8 -0
  172. package/src/__tests__/secure-keys.test.ts +50 -4
  173. package/src/__tests__/send-endpoint-busy.test.ts +163 -1
  174. package/src/__tests__/server-history-render.test.ts +91 -1
  175. package/src/__tests__/server-tool-pairing.test.ts +107 -0
  176. package/src/__tests__/settings-routes.test.ts +50 -1
  177. package/src/__tests__/skill-script-runner-sandbox.test.ts +32 -0
  178. package/src/__tests__/skills.test.ts +5 -4
  179. package/src/__tests__/subagent-fork-notifications.test.ts +1 -2
  180. package/src/__tests__/subagent-manager-notify.test.ts +1 -2
  181. package/src/__tests__/subagent-notify-parent.test.ts +146 -39
  182. package/src/__tests__/subagent-notify-stale-rebuild.test.ts +2 -4
  183. package/src/__tests__/subagent-tool-gate-mode.test.ts +226 -0
  184. package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +2 -0
  185. package/src/__tests__/terminal-tools.test.ts +8 -0
  186. package/src/__tests__/thread-backfill.test.ts +53 -0
  187. package/src/__tests__/tool-preview-lifecycle.test.ts +2 -0
  188. package/src/__tests__/tool-result-follow-up.test.ts +99 -0
  189. package/src/__tests__/tool-result-metadata-plumbing.test.ts +66 -1
  190. package/src/__tests__/tool-start-timestamp.test.ts +2 -0
  191. package/src/__tests__/twilio-routes.test.ts +1 -0
  192. package/src/__tests__/unicode.test.ts +36 -0
  193. package/src/__tests__/web-search-backend-failure.test.ts +1 -1
  194. package/src/activation/progress-store.test.ts +12 -7
  195. package/src/agent/loop.ts +136 -53
  196. package/src/agent/tool-result-follow-up.ts +70 -0
  197. package/src/api/attachment-provenance.test.ts +68 -0
  198. package/src/api/computer-use-tool.test.ts +55 -0
  199. package/src/api/computer-use-tool.ts +27 -0
  200. package/src/api/events/assistant-outbound-attachment.ts +3 -0
  201. package/src/api/events/assistant-turn-start.ts +3 -0
  202. package/src/api/events/desktop-activity-changed.ts +10 -0
  203. package/src/api/events/generation-handoff.ts +2 -0
  204. package/src/api/events/message-complete.ts +2 -0
  205. package/src/api/events/notification-conversation-created.ts +14 -16
  206. package/src/api/events/notification-intent.ts +4 -2
  207. package/src/api/events/question-request.ts +1 -0
  208. package/src/api/events/tool-result.ts +85 -0
  209. package/src/api/events/tool-use-start.ts +3 -0
  210. package/src/api/events/user-message-echo.ts +3 -0
  211. package/src/api/index.ts +42 -6
  212. package/src/api/mode-session.test.ts +131 -0
  213. package/src/api/mode-session.ts +148 -0
  214. package/src/api/remember-tool.ts +15 -0
  215. package/src/api/responses/conversation-message.ts +9 -0
  216. package/src/approvals/AGENTS.md +4 -2
  217. package/src/approvals/approval-primitive.ts +5 -2
  218. package/src/approvals/scoped-approval-grants.ts +6 -2
  219. package/src/browser/__tests__/operations.test.ts +29 -0
  220. package/src/browser/operation-meta.ts +2 -1
  221. package/src/browser/operations.ts +29 -0
  222. package/src/browser/virtual-desktop-target.ts +2 -9
  223. package/src/bundler/bundle-signer.ts +1 -3
  224. package/src/calls/__tests__/barge-in-guard.test.ts +3 -0
  225. package/src/calls/__tests__/call-controller.test.ts +94 -0
  226. package/src/calls/__tests__/call-funnel-telemetry.test.ts +204 -0
  227. package/src/calls/__tests__/voice-control-protocol.test.ts +15 -0
  228. package/src/calls/__tests__/voice-escalation-judge.test.ts +209 -0
  229. package/src/calls/__tests__/voice-leg-coordinator.test.ts +168 -0
  230. package/src/{live-voice/__tests__/live-voice-metrics.test.ts → calls/__tests__/voice-metrics.test.ts} +32 -38
  231. package/src/calls/__tests__/voice-session-bridge.test.ts +576 -122
  232. package/src/calls/barge-in-guard.ts +5 -0
  233. package/src/calls/call-controller.ts +196 -1
  234. package/src/calls/call-domain.ts +4 -0
  235. package/src/calls/call-store.ts +87 -2
  236. package/src/calls/guardian-action-sweep.ts +22 -8
  237. package/src/calls/media-stream-audio-transcode.ts +1 -3
  238. package/src/calls/typesafe-noul.ts +156 -0
  239. package/src/calls/voice-control-protocol.ts +6 -0
  240. package/src/calls/voice-escalation-judge.ts +145 -0
  241. package/src/calls/voice-escalation-target.ts +10 -0
  242. package/src/calls/voice-leg-coordinator.ts +121 -3
  243. package/src/{live-voice/live-voice-metrics.ts → calls/voice-metrics.ts} +94 -87
  244. package/src/calls/voice-session-bridge.ts +256 -112
  245. package/src/calls/voice-triage-escalate.ts +7 -0
  246. package/src/cli/__tests__/roadmap-cli.test.ts +4 -41
  247. package/src/cli/commands/__tests__/cli-test-harness.ts +21 -3
  248. package/src/cli/commands/__tests__/notifications.test.ts +35 -10
  249. package/src/cli/commands/__tests__/plugins.test.ts +20 -1
  250. package/src/cli/commands/__tests__/schedules.test.ts +14 -0
  251. package/src/cli/commands/bash.help.ts +4 -3
  252. package/src/cli/commands/browser.help.ts +9 -6
  253. package/src/cli/commands/browser.ts +0 -1
  254. package/src/cli/commands/clients.help.ts +46 -0
  255. package/src/cli/commands/clients.ts +116 -0
  256. package/src/cli/commands/credentials.help.ts +9 -4
  257. package/src/cli/commands/notifications.help.ts +16 -2
  258. package/src/cli/commands/notifications.ts +30 -8
  259. package/src/cli/commands/oauth/index.help.ts +10 -0
  260. package/src/cli/commands/oauth/providers.ts +12 -0
  261. package/src/cli/commands/oauth/request.test.ts +179 -2
  262. package/src/cli/commands/oauth/request.ts +41 -10
  263. package/src/cli/commands/plugins.ts +12 -5
  264. package/src/cli/commands/roadmap.help.ts +18 -21
  265. package/src/cli/commands/roadmap.ts +4 -21
  266. package/src/cli/commands/schedules.ts +2 -0
  267. package/src/cli/lib/__tests__/inspect-plugin.test.ts +54 -0
  268. package/src/cli/lib/__tests__/install-from-github.test.ts +41 -0
  269. package/src/cli/lib/__tests__/list-installed-plugins.test.ts +5 -2
  270. package/src/cli/lib/__tests__/local-plugin-upgrade.test.ts +133 -51
  271. package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +8 -2
  272. package/src/cli/lib/bundled-marketplace.json +906 -1
  273. package/src/cli/lib/bundled-plugin-packages.json +783 -1
  274. package/src/cli/lib/inspect-plugin.ts +11 -4
  275. package/src/cli/lib/plugin-marketplace.ts +8 -0
  276. package/src/cli/lib/publish-plugin.ts +1 -2
  277. package/src/cli/lib/upgrade-plugin.ts +19 -11
  278. package/src/cli/output.ts +18 -4
  279. package/src/config/__tests__/default-provider.test.ts +6 -0
  280. package/src/config/__tests__/memory-retrospective-schema.test.ts +14 -1
  281. package/src/config/bundled-skills/computer-use/SKILL.md +1 -1
  282. package/src/config/bundled-skills/computer-use/TOOLS.json +89 -1
  283. package/src/config/bundled-skills/image-studio/SKILL.md +17 -2
  284. package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +7 -11
  285. package/src/config/bundled-skills/schedule/SKILL.md +6 -6
  286. package/src/config/bundled-skills/sequences/TOOLS.json +1 -5
  287. package/src/config/call-site-defaults.ts +9 -0
  288. package/src/config/feature-flag-registry.json +16 -0
  289. package/src/config/loader.ts +9 -17
  290. package/src/config/platform-identity.ts +8 -8
  291. package/src/config/profile-text-generation.test.ts +51 -0
  292. package/src/config/profile-text-generation.ts +51 -0
  293. package/src/config/schemas/__tests__/memory-v3.test.ts +12 -0
  294. package/src/config/schemas/__tests__/tools.test.ts +31 -0
  295. package/src/config/schemas/call-site-catalog.ts +14 -0
  296. package/src/config/schemas/channels.ts +0 -18
  297. package/src/config/schemas/llm.ts +9 -2
  298. package/src/config/schemas/mcp.ts +6 -1
  299. package/src/config/schemas/memory-retrospective.ts +11 -2
  300. package/src/config/schemas/memory-v3.ts +12 -0
  301. package/src/config/schemas/services.ts +15 -6
  302. package/src/config/schemas/tools.ts +10 -0
  303. package/src/config/session-groups-gate.ts +21 -0
  304. package/src/config/skills.ts +2 -6
  305. package/src/conversations/__tests__/message-consolidation.test.ts +149 -0
  306. package/src/conversations/message-consolidation.ts +102 -5
  307. package/src/credential-execution/ces-connect.test.ts +69 -0
  308. package/src/credential-execution/ces-connect.ts +115 -0
  309. package/src/credential-execution/ces-runtime.ts +55 -93
  310. package/src/credential-execution/executable-discovery.ts +34 -142
  311. package/src/credential-execution/process-manager.test.ts +20 -24
  312. package/src/credential-execution/process-manager.ts +24 -19
  313. package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +10 -1
  314. package/src/daemon/__tests__/conversation-tool-setup.test.ts +43 -0
  315. package/src/daemon/__tests__/inference-profile-notification.test.ts +2 -0
  316. package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +36 -0
  317. package/src/daemon/assistant-attachments.ts +20 -12
  318. package/src/daemon/browser-mode-session.test.ts +402 -0
  319. package/src/daemon/browser-mode-session.ts +186 -0
  320. package/src/daemon/chat-credential-redaction.ts +16 -1
  321. package/src/daemon/computer-use-mode-session.test.ts +210 -0
  322. package/src/daemon/computer-use-mode-session.ts +105 -0
  323. package/src/daemon/config-watcher.ts +10 -0
  324. package/src/daemon/conversation-agent-loop-handlers.ts +132 -13
  325. package/src/daemon/conversation-agent-loop.ts +231 -25
  326. package/src/daemon/conversation-attachments.ts +93 -12
  327. package/src/daemon/conversation-evictor.ts +5 -24
  328. package/src/daemon/conversation-history.ts +10 -0
  329. package/src/daemon/conversation-interrupt-repair.ts +4 -43
  330. package/src/daemon/conversation-interrupt.ts +7 -11
  331. package/src/daemon/conversation-messaging.ts +58 -5
  332. package/src/daemon/conversation-mode-session.test.ts +1667 -0
  333. package/src/daemon/conversation-mode-session.ts +1008 -0
  334. package/src/daemon/conversation-process.ts +5 -0
  335. package/src/daemon/conversation-runtime-assembly.ts +10 -20
  336. package/src/daemon/conversation-skill-tools.ts +1 -0
  337. package/src/daemon/conversation-store.ts +4 -4
  338. package/src/daemon/conversation-surface-snapshots.ts +135 -0
  339. package/src/daemon/conversation-surfaces.ts +113 -5
  340. package/src/daemon/conversation-tool-setup.ts +79 -6
  341. package/src/daemon/conversation-turn-finalize.ts +26 -13
  342. package/src/daemon/conversation.ts +215 -58
  343. package/src/daemon/daemon-control.ts +3 -7
  344. package/src/daemon/handlers/config-model.ts +7 -5
  345. package/src/daemon/handlers/shared.ts +55 -16
  346. package/src/daemon/host-cu-proxy.ts +51 -22
  347. package/src/daemon/lifecycle.ts +30 -13
  348. package/src/daemon/mcp-reload-service.ts +1 -1
  349. package/src/daemon/message-protocol.ts +0 -1
  350. package/src/daemon/message-provenance.ts +28 -4
  351. package/src/daemon/message-types/shared.ts +1 -0
  352. package/src/daemon/mode-session-startup-recovery.ts +23 -0
  353. package/src/daemon/mode-session-tracking.ts +54 -0
  354. package/src/daemon/orphan-reaper.ts +4 -3
  355. package/src/daemon/persist-media-references.ts +20 -3
  356. package/src/daemon/process-message.ts +13 -4
  357. package/src/daemon/tool-setup-types.ts +6 -0
  358. package/src/daemon/virtual-desktop-context.ts +21 -0
  359. package/src/daemon/wake-conversation-ops.ts +50 -15
  360. package/src/desktop/__tests__/fake-desktop.ts +1 -0
  361. package/src/desktop/desktop-accessibility-script.ts +88 -0
  362. package/src/desktop/desktop-accessibility.test.ts +160 -0
  363. package/src/desktop/desktop-accessibility.ts +169 -0
  364. package/src/desktop/desktop-automation-lease.test.ts +302 -64
  365. package/src/desktop/desktop-automation-lease.ts +129 -54
  366. package/src/desktop/desktop-browser-endpoint.ts +4 -0
  367. package/src/desktop/desktop-computer-use-routing.test.ts +330 -0
  368. package/src/desktop/desktop-computer-use.test.ts +449 -0
  369. package/src/desktop/desktop-computer-use.ts +502 -0
  370. package/src/desktop/desktop-dependencies.ts +47 -293
  371. package/src/desktop/desktop-display.ts +2 -2
  372. package/src/desktop/desktop-help.ts +15 -0
  373. package/src/desktop/desktop-panel-config.test.ts +13 -3
  374. package/src/desktop/desktop-panel-config.ts +109 -15
  375. package/src/desktop/desktop-session-manager.test.ts +69 -21
  376. package/src/desktop/desktop-session-manager.ts +76 -22
  377. package/src/desktop/desktop-stream-bridge.test.ts +76 -132
  378. package/src/desktop/desktop-stream-bridge.ts +9 -11
  379. package/src/desktop/desktop-wallpaper-renderer.ts +132 -0
  380. package/src/desktop/desktop-wallpaper-worker.ts +17 -0
  381. package/src/desktop/desktop-wallpaper.test.ts +48 -7
  382. package/src/desktop/desktop-wallpaper.ts +34 -102
  383. package/src/desktop/desktop-window-theme.ts +5 -25
  384. package/src/desktop/virtual-desktop-feature.ts +53 -3
  385. package/src/desktop/virtual-desktop-platform.test.ts +5 -9
  386. package/src/documents/document-store.ts +23 -0
  387. package/src/heartbeat/__tests__/heartbeat-service.test.ts +22 -26
  388. package/src/heartbeat/heartbeat-service.ts +5 -33
  389. package/src/ipc/__tests__/browser-ipc.test.ts +215 -6
  390. package/src/ipc/assistant-server.ts +2 -0
  391. package/src/ipc/routes/__tests__/channel-reply-ipc-routes.test.ts +111 -0
  392. package/src/ipc/routes/channel-reply-ipc-routes.ts +73 -0
  393. package/src/live-voice/__tests__/continuation-judge.test.ts +151 -0
  394. package/src/live-voice/__tests__/live-activity-reporter.test.ts +154 -15
  395. package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +743 -31
  396. package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +191 -10
  397. package/src/live-voice/__tests__/live-voice-session-groups-gate.test.ts +87 -0
  398. package/src/live-voice/__tests__/live-voice-session-manager.test.ts +5 -1
  399. package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +41 -0
  400. package/src/live-voice/__tests__/live-voice-sight-frame.test.ts +1017 -9
  401. package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +224 -1
  402. package/src/live-voice/__tests__/live-voice-tts-session.test.ts +101 -0
  403. package/src/live-voice/__tests__/live-voice-tts.test.ts +44 -0
  404. package/src/live-voice/__tests__/live-voice-vad.test.ts +939 -75
  405. package/src/live-voice/__tests__/microphone-idle-gate.test.ts +87 -0
  406. package/src/live-voice/__tests__/session-controls.test.ts +23 -0
  407. package/src/live-voice/__tests__/task-outcomes.test.ts +117 -0
  408. package/src/live-voice/__tests__/voice-input-diagnostics.test.ts +117 -0
  409. package/src/live-voice/camera-mode-session.test.ts +265 -0
  410. package/src/live-voice/camera-mode-session.ts +277 -0
  411. package/src/live-voice/continuation-judge.ts +123 -0
  412. package/src/live-voice/live-activity-reporter.ts +115 -6
  413. package/src/live-voice/live-voice-manager.ts +36 -1
  414. package/src/live-voice/live-voice-photo.ts +88 -7
  415. package/src/live-voice/live-voice-session-manager.ts +15 -0
  416. package/src/live-voice/live-voice-session.ts +1443 -747
  417. package/src/live-voice/live-voice-tts.ts +22 -4
  418. package/src/live-voice/microphone-idle-gate.ts +56 -0
  419. package/src/live-voice/protocol.ts +171 -4
  420. package/src/live-voice/session-controls.ts +11 -3
  421. package/src/live-voice/task-outcomes.ts +131 -0
  422. package/src/live-voice/voice-input-diagnostics.ts +117 -0
  423. package/src/mcp/__tests__/manager-tool-caps.test.ts +156 -0
  424. package/src/mcp/__tests__/startup.test.ts +30 -7
  425. package/src/mcp/__tests__/tool-caps.test.ts +134 -0
  426. package/src/mcp/manager.ts +171 -101
  427. package/src/mcp/startup.ts +33 -13
  428. package/src/mcp/tool-caps.ts +184 -0
  429. package/src/media/__tests__/image-models.test.ts +58 -0
  430. package/src/media/image-credentials.ts +5 -0
  431. package/src/media/image-models.ts +52 -0
  432. package/src/media/image-service.ts +8 -0
  433. package/src/media/openrouter-image-service.test.ts +133 -0
  434. package/src/media/openrouter-image-service.ts +153 -0
  435. package/src/media/types.ts +1 -1
  436. package/src/messaging/provider-message-metadata.ts +3 -3
  437. package/src/messaging/providers/__tests__/transport-dispatch.test.ts +65 -11
  438. package/src/messaging/providers/discord/send.ts +4 -17
  439. package/src/messaging/providers/discord/transport.ts +2 -2
  440. package/src/messaging/providers/outlook/client.ts +16 -22
  441. package/src/messaging/providers/telegram-bot/send.test.ts +236 -1
  442. package/src/messaging/providers/telegram-bot/send.ts +10 -7
  443. package/src/messaging/providers/telegram-bot/transport.ts +3 -10
  444. package/src/messaging/providers/whatsapp/send.test.ts +169 -4
  445. package/src/monitoring/__tests__/control.test.ts +17 -5
  446. package/src/monitoring/control.ts +15 -5
  447. package/src/notifications/AGENTS.md +1 -1
  448. package/src/notifications/README.md +83 -99
  449. package/src/notifications/__tests__/broadcaster.test.ts +1 -1
  450. package/src/notifications/__tests__/channel-allowlist.test.ts +39 -0
  451. package/src/notifications/__tests__/copy-composer.test.ts +71 -1
  452. package/src/notifications/__tests__/decision-engine.test.ts +79 -2
  453. package/src/notifications/__tests__/deterministic-checks.test.ts +10 -10
  454. package/src/notifications/__tests__/home-feed-side-effect.test.ts +93 -4
  455. package/src/notifications/__tests__/notification-utils.test.ts +18 -0
  456. package/src/notifications/adapters/macos.ts +37 -46
  457. package/src/notifications/broadcaster.ts +5 -6
  458. package/src/notifications/channel-allowlist.ts +44 -0
  459. package/src/notifications/copy-composer.ts +11 -26
  460. package/src/notifications/decision-engine.ts +24 -14
  461. package/src/notifications/deterministic-checks.ts +1 -1
  462. package/src/notifications/emit-signal.ts +123 -37
  463. package/src/notifications/home-feed-side-effect.ts +9 -7
  464. package/src/notifications/notification-utils.ts +19 -0
  465. package/src/notifications/schedule-result-producer.ts +4 -6
  466. package/src/notifications/signal.ts +4 -19
  467. package/src/notifications/types.ts +15 -0
  468. package/src/oauth/AGENTS.md +2 -0
  469. package/src/oauth/__tests__/identity-verifier.test.ts +36 -1
  470. package/src/oauth/connection-resolver.ts +1 -2
  471. package/src/oauth/identity-verifier.ts +24 -0
  472. package/src/oauth/oauth-store.ts +12 -4
  473. package/src/oauth/provider-serializer.ts +8 -0
  474. package/src/oauth/seed-providers.ts +42 -0
  475. package/src/onboarding/onboarding-events-store.ts +99 -12
  476. package/src/permissions/confirmation-guardian-request.test.ts +16 -0
  477. package/src/permissions/confirmation-guardian-request.ts +2 -3
  478. package/src/permissions/question-prompter.test.ts +31 -0
  479. package/src/permissions/question-prompter.ts +2 -0
  480. package/src/persistence/__tests__/client-connection-events-store.test.ts +322 -0
  481. package/src/persistence/attachments-store.ts +84 -44
  482. package/src/persistence/client-connection-events-store.ts +377 -0
  483. package/src/persistence/conversation-crud.ts +166 -26
  484. package/src/persistence/conversation-mode-sessions.test.ts +424 -0
  485. package/src/persistence/conversation-mode-sessions.ts +460 -0
  486. package/src/persistence/conversation-plugin-facade.ts +30 -0
  487. package/src/persistence/conversation-queries.ts +6 -19
  488. package/src/persistence/conversation-tool-surface.ts +135 -0
  489. package/src/persistence/conversation-types.test.ts +32 -0
  490. package/src/persistence/conversation-types.ts +26 -4
  491. package/src/persistence/llm-usage-store.ts +3 -9
  492. package/src/persistence/message-metadata.ts +25 -0
  493. package/src/persistence/migrations/378-create-conversation-tool-surfaces.test.ts +78 -0
  494. package/src/persistence/migrations/378-create-conversation-tool-surfaces.ts +29 -0
  495. package/src/persistence/migrations/379-oauth-providers-response-ok-field.test.ts +92 -0
  496. package/src/persistence/migrations/379-oauth-providers-response-ok-field.ts +32 -0
  497. package/src/persistence/migrations/380-conversation-tool-surfaces-delegate-independent-tasks.test.ts +90 -0
  498. package/src/persistence/migrations/380-conversation-tool-surfaces-delegate-independent-tasks.ts +34 -0
  499. package/src/persistence/migrations/381-create-conversation-mode-sessions.ts +41 -0
  500. package/src/persistence/migrations/382-create-client-connection-events.ts +40 -0
  501. package/src/persistence/migrations/__tests__/382-create-client-connection-events.test.ts +56 -0
  502. package/src/persistence/schema/conversation-tool-surfaces.ts +32 -0
  503. package/src/persistence/schema/conversations.ts +67 -0
  504. package/src/persistence/schema/index.ts +1 -0
  505. package/src/persistence/schema/infrastructure.ts +29 -0
  506. package/src/persistence/schema/oauth.ts +1 -0
  507. package/src/persistence/steps.ts +41 -3
  508. package/src/plugin-api/__tests__/plugin-name-env.test.ts +249 -0
  509. package/src/plugin-api/conversation-turn.ts +31 -7
  510. package/src/plugin-api/effective-context-window.ts +33 -0
  511. package/src/plugin-api/index.ts +35 -6
  512. package/src/plugin-api/plugin-channel-turn-trust.test.ts +133 -0
  513. package/src/plugin-api/plugin-channel-turn-trust.ts +71 -0
  514. package/src/plugin-api/plugin-name-env.ts +125 -0
  515. package/src/plugin-api/resolve-credential.ts +79 -20
  516. package/src/plugin-api/vision-support.test.ts +22 -0
  517. package/src/plugin-api/vision-support.ts +17 -5
  518. package/src/plugins/__tests__/active-plugin-env.test.ts +136 -0
  519. package/src/plugins/active-plugin-env.ts +51 -0
  520. package/src/plugins/defaults/index.ts +24 -0
  521. package/src/plugins/defaults/memory/AGENTS.md +39 -5
  522. package/src/plugins/defaults/memory/__tests__/archive-removal-framing.test.ts +64 -0
  523. package/src/plugins/defaults/memory/__tests__/buffer-file.test.ts +365 -0
  524. package/src/plugins/defaults/memory/__tests__/buffer-format.test.ts +43 -0
  525. package/src/plugins/defaults/memory/__tests__/conversation-memory-purge.test.ts +1 -0
  526. package/src/plugins/defaults/memory/__tests__/db-memory-attach.test.ts +2 -0
  527. package/src/plugins/defaults/memory/__tests__/fixtures/buffer-appender.ts +17 -0
  528. package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +74 -0
  529. package/src/plugins/defaults/memory/__tests__/memory-run-evidence.test.ts +161 -0
  530. package/src/plugins/defaults/memory/__tests__/relocated-memory-test-rows.ts +10 -0
  531. package/src/plugins/defaults/memory/__tests__/worker-control.test.ts +19 -6
  532. package/src/plugins/defaults/memory/buffer-file.ts +354 -0
  533. package/src/plugins/defaults/memory/buffer-format.ts +40 -0
  534. package/src/plugins/defaults/memory/context-search/agent-protocol.ts +9 -12
  535. package/src/plugins/defaults/memory/context-search/agent-runner.ts +77 -22
  536. package/src/plugins/defaults/memory/context-search/format.ts +53 -8
  537. package/src/plugins/defaults/memory/context-search/limits.ts +10 -12
  538. package/src/plugins/defaults/memory/context-search/recall-input.ts +38 -0
  539. package/src/plugins/defaults/memory/context-search/search.ts +8 -14
  540. package/src/plugins/defaults/memory/context-search/sources/conversations.ts +1 -0
  541. package/src/plugins/defaults/memory/context-search/sources/memory-v2.ts +2 -1
  542. package/src/plugins/defaults/memory/context-search/sources/workspace.ts +2 -1
  543. package/src/plugins/defaults/memory/context-search/types.ts +1 -11
  544. package/src/plugins/defaults/memory/conversation-memory-purge.ts +4 -0
  545. package/src/plugins/defaults/memory/graph/__tests__/handle-remember-v2.test.ts +11 -4
  546. package/src/plugins/defaults/memory/graph/capability-seed.ts +1 -2
  547. package/src/plugins/defaults/memory/graph/tool-handlers.ts +13 -63
  548. package/src/plugins/defaults/memory/graph/tools.ts +35 -64
  549. package/src/plugins/defaults/memory/host-utils.ts +0 -10
  550. package/src/plugins/defaults/memory/injectors.ts +4 -3
  551. package/src/plugins/defaults/memory/memory-retrospective-job.ts +72 -183
  552. package/src/plugins/defaults/memory/memory-run-evidence.ts +213 -0
  553. package/src/plugins/defaults/memory/src/memory-eval-routes.ts +2 -3
  554. package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +6 -1
  555. package/src/plugins/defaults/memory/src/memory-item-routes.ts +2 -1
  556. package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts +407 -99
  557. package/src/plugins/defaults/memory/substrate/__tests__/consolidation-prompt-flag-gating-guard.test.ts +10 -0
  558. package/src/plugins/defaults/memory/substrate/__tests__/prompts-consolidation.test.ts +107 -7
  559. package/src/plugins/defaults/memory/substrate/consolidation-job.ts +307 -86
  560. package/src/plugins/defaults/memory/substrate/consolidation-tool-surface.ts +34 -0
  561. package/src/plugins/defaults/memory/substrate/page-index.ts +2 -1
  562. package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +91 -51
  563. package/src/plugins/defaults/memory/substrate/sweep-job.ts +1 -1
  564. package/src/plugins/defaults/memory/tools.test.ts +124 -9
  565. package/src/plugins/defaults/memory/tools.ts +38 -11
  566. package/src/plugins/defaults/memory/v1/graph/consolidation.ts +2 -2
  567. package/src/plugins/defaults/memory/v1/graph/extraction.ts +2 -1
  568. package/src/plugins/defaults/memory/v1/graph/retriever.ts +1 -1
  569. package/src/plugins/defaults/memory/v2/__tests__/migration.test.ts +5 -0
  570. package/src/plugins/defaults/memory/v2/__tests__/reranker.test.ts +5 -2
  571. package/src/plugins/defaults/memory/v2/reranker.ts +2 -1
  572. package/src/plugins/defaults/memory/v3/__tests__/injection.test.ts +81 -1
  573. package/src/plugins/defaults/memory/v3/__tests__/orchestrate.test.ts +87 -0
  574. package/src/plugins/defaults/memory/v3/__tests__/plugin-schema.test.ts +12 -0
  575. package/src/plugins/defaults/memory/v3/__tests__/pool-log-store.test.ts +332 -15
  576. package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +626 -27
  577. package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +148 -28
  578. package/src/plugins/defaults/memory/v3/card.ts +2 -1
  579. package/src/plugins/defaults/memory/v3/injector.ts +212 -178
  580. package/src/plugins/defaults/memory/v3/orchestrate.ts +159 -32
  581. package/src/plugins/defaults/memory/v3/plugin-schema.ts +54 -4
  582. package/src/plugins/defaults/memory/v3/pool-log-store.ts +275 -37
  583. package/src/plugins/defaults/memory/v3/pool-select.test.ts +47 -5
  584. package/src/plugins/defaults/memory/v3/pool-select.ts +710 -59
  585. package/src/plugins/defaults/memory/v3/sections.ts +2 -1
  586. package/src/plugins/defaults/memory/v3/shadow-plugin.ts +41 -14
  587. package/src/plugins/defaults/memory/worker-control.ts +15 -5
  588. package/src/plugins/defaults/task-progress/__tests__/registration.test.ts +70 -0
  589. package/src/plugins/defaults/task-progress/__tests__/task-progress-context.test.ts +380 -0
  590. package/src/plugins/defaults/task-progress/hooks/post-compact.ts +19 -0
  591. package/src/plugins/defaults/task-progress/hooks/user-prompt-submit.ts +24 -0
  592. package/src/plugins/defaults/task-progress/package.json +14 -0
  593. package/src/plugins/defaults/task-progress/src/apply-task-progress-context.ts +93 -0
  594. package/src/plugins/defaults/task-progress/src/format-task-progress-context.ts +67 -0
  595. package/src/plugins/defaults/tool-error/hooks/post-tool-use.ts +4 -1
  596. package/src/plugins/defaults/tool-result-truncate/terminal.ts +1 -46
  597. package/src/prompts/__tests__/parallel-tasks-section.test.ts +25 -0
  598. package/src/prompts/delegation-gate.ts +57 -0
  599. package/src/prompts/system-prompt.ts +14 -32
  600. package/src/providers/__tests__/context-overflow-error.test.ts +10 -0
  601. package/src/providers/__tests__/search-provider-catalog.test.ts +18 -0
  602. package/src/providers/anthropic/client.ts +10 -1
  603. package/src/providers/anthropic-gateway-shared.ts +6 -0
  604. package/src/providers/fetch-provider-catalog.ts +11 -0
  605. package/src/providers/gemini/client.ts +25 -8
  606. package/src/providers/inference/adapter-factory.ts +6 -0
  607. package/src/providers/jev/client.test.ts +270 -0
  608. package/src/providers/jev/client.ts +534 -0
  609. package/src/providers/model-catalog.ts +59 -3
  610. package/src/providers/openai/__tests__/chat-completions-extra-body.test.ts +94 -1
  611. package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +106 -1
  612. package/src/providers/openai/__tests__/chat-template-flatten.test.ts +24 -0
  613. package/src/providers/openai/__tests__/google-thought-signature.test.ts +1 -2
  614. package/src/providers/openai/chat-completions-provider.ts +100 -29
  615. package/src/providers/openai/google-thought-signature.ts +0 -6
  616. package/src/providers/openai/responses-provider.ts +23 -1
  617. package/src/providers/ratelimit.ts +1 -3
  618. package/src/providers/search-provider-catalog.ts +24 -0
  619. package/src/providers/server-tool-pairing.ts +16 -7
  620. package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +24 -0
  621. package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +7 -1
  622. package/src/providers/speech-to-text/__tests__/vellum-managed-realtime.test.ts +50 -1
  623. package/src/providers/speech-to-text/deepgram-flux-frames.ts +11 -0
  624. package/src/providers/speech-to-text/vellum-managed-flux-realtime.ts +13 -0
  625. package/src/providers/types.ts +5 -0
  626. package/src/routes/control.ts +7 -5
  627. package/src/runtime/AGENTS.md +6 -4
  628. package/src/runtime/__tests__/agent-wake.test.ts +94 -1
  629. package/src/runtime/actor-trust-resolver.ts +3 -3
  630. package/src/runtime/agent-wake.ts +32 -5
  631. package/src/runtime/approval-message-composer.ts +3 -69
  632. package/src/runtime/assistant-event-hub.ts +73 -75
  633. package/src/runtime/assistant-event-publish-options.ts +7 -0
  634. package/src/runtime/assistant-event-targeting.ts +79 -0
  635. package/src/runtime/assistant-stream-state.ts +13 -100
  636. package/src/runtime/background-job-runner.ts +5 -5
  637. package/src/runtime/channel-retry-sweep.ts +1 -0
  638. package/src/runtime/guardian-action-message-composer.ts +8 -161
  639. package/src/runtime/guardian-action-service.ts +2 -17
  640. package/src/runtime/guardian-reply-router.ts +29 -40
  641. package/src/runtime/http-server.ts +2 -2
  642. package/src/runtime/http-types.ts +6 -0
  643. package/src/runtime/message-composer-types.ts +3 -61
  644. package/src/runtime/migrations/__tests__/vbundle-import-parity.test.ts +67 -1
  645. package/src/runtime/migrations/__tests__/vbundle-import-policy.test.ts +89 -0
  646. package/src/runtime/migrations/vbundle-builder.ts +24 -1
  647. package/src/runtime/migrations/vbundle-import-analyzer.ts +13 -4
  648. package/src/runtime/migrations/vbundle-import-policy.ts +43 -11
  649. package/src/runtime/migrations/vbundle-importer.ts +4 -1
  650. package/src/runtime/migrations/vbundle-streaming-importer.ts +4 -1
  651. package/src/runtime/migrations/vbundle-validator.ts +1 -0
  652. package/src/runtime/pending-interactions-mode-session.test.ts +62 -0
  653. package/src/runtime/pending-interactions.ts +29 -0
  654. package/src/runtime/routes/__tests__/client-history-routes.test.ts +104 -0
  655. package/src/runtime/routes/__tests__/image-generation-routes.test.ts +97 -0
  656. package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +24 -2
  657. package/src/runtime/routes/__tests__/roadmap-routes.test.ts +29 -6
  658. package/src/runtime/routes/browser-context.ts +31 -11
  659. package/src/runtime/routes/browser-routes.ts +43 -4
  660. package/src/runtime/routes/btw-routes.ts +9 -4
  661. package/src/runtime/routes/channel-route-shared.ts +1 -9
  662. package/src/runtime/routes/client-routes.ts +108 -0
  663. package/src/runtime/routes/conversation-management-routes.ts +4 -1
  664. package/src/runtime/routes/conversation-routes.ts +125 -14
  665. package/src/runtime/routes/credential-prompt-routes.ts +4 -3
  666. package/src/runtime/routes/credential-routes.ts +1 -2
  667. package/src/runtime/routes/desktop-setup-routes.test.ts +9 -18
  668. package/src/runtime/routes/desktop-setup-routes.ts +8 -7
  669. package/src/runtime/routes/documents-routes.ts +51 -0
  670. package/src/runtime/routes/email-routes.ts +3 -7
  671. package/src/runtime/routes/events-routes.ts +75 -54
  672. package/src/runtime/routes/guardian-approval-interception.ts +24 -0
  673. package/src/runtime/routes/identity-routes.ts +1 -5
  674. package/src/runtime/routes/image-generation-routes.ts +8 -14
  675. package/src/runtime/routes/inbound-message-handler.ts +37 -41
  676. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.test.ts +168 -0
  677. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.ts +96 -0
  678. package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +39 -10
  679. package/src/runtime/routes/inbound-stages/background-dispatch.ts +9 -9
  680. package/src/runtime/routes/inbound-stages/reaction-intercept.ts +2 -0
  681. package/src/runtime/routes/inference-profile-session-handler.ts +11 -1
  682. package/src/runtime/routes/inference-profiles-routes.ts +9 -0
  683. package/src/runtime/routes/llm-context-normalization.ts +1 -7
  684. package/src/runtime/routes/migration-routes.ts +73 -2
  685. package/src/runtime/routes/notification-routes.ts +10 -0
  686. package/src/runtime/routes/oauth-commands-routes.ts +57 -54
  687. package/src/runtime/routes/oauth-providers.ts +11 -0
  688. package/src/runtime/routes/oauth-request-hints.test.ts +246 -0
  689. package/src/runtime/routes/oauth-request-hints.ts +149 -0
  690. package/src/runtime/routes/plugins-routes.ts +6 -0
  691. package/src/runtime/routes/roadmap-routes.ts +47 -8
  692. package/src/runtime/routes/schedule-routes.ts +2 -0
  693. package/src/runtime/routes/secret-routes.ts +99 -119
  694. package/src/runtime/routes/settings-routes.ts +10 -4
  695. package/src/runtime/routes/watcher-routes.ts +1 -3
  696. package/src/runtime/trust-class.test.ts +7 -6
  697. package/src/runtime/trust-class.ts +9 -17
  698. package/src/schedule/__tests__/worker-control.test.ts +36 -5
  699. package/src/schedule/__tests__/worker-mcp-bootstrap.test.ts +107 -0
  700. package/src/schedule/__tests__/worker-mcp-readiness.test.ts +173 -0
  701. package/src/schedule/__tests__/worker-mcp-tools.test.ts +13 -47
  702. package/src/schedule/run-script.ts +2 -2
  703. package/src/schedule/scheduler.ts +47 -11
  704. package/src/schedule/tool-surface-readiness.ts +65 -0
  705. package/src/schedule/worker-control.ts +15 -5
  706. package/src/schedule/worker-mcp.ts +85 -0
  707. package/src/schedule/worker.ts +9 -31
  708. package/src/security/secure-keys.ts +74 -112
  709. package/src/skills/clawhub.ts +3 -8
  710. package/src/skills/managed-store.ts +1 -5
  711. package/src/stt/types.ts +4 -0
  712. package/src/subagent/notify.ts +54 -34
  713. package/src/subagent/parent-notification.ts +6 -0
  714. package/src/telemetry/__tests__/phone-call-funnel.test.ts +72 -0
  715. package/src/telemetry/phone-call-funnel.ts +144 -0
  716. package/src/tools/__tests__/tool-schema-root-combinator-guard.test.ts +81 -0
  717. package/src/tools/ask-question/ask-question-tool.test.ts +167 -5
  718. package/src/tools/ask-question/ask-question-tool.ts +120 -22
  719. package/src/tools/browser/browser-execution.ts +29 -60
  720. package/src/tools/browser/cdp-client/__tests__/cdp-dom-helpers.test.ts +561 -39
  721. package/src/tools/browser/cdp-client/cdp-dom-helpers.ts +435 -39
  722. package/src/tools/browser/cdp-client/cdp-inspect-client.ts +1 -8
  723. package/src/tools/client-os.ts +15 -0
  724. package/src/tools/computer-use/skill-proxy-bridge.ts +1 -1
  725. package/src/tools/computer-use/target.ts +42 -0
  726. package/src/tools/document/document-tool.ts +13 -18
  727. package/src/tools/execution-target.ts +14 -16
  728. package/src/tools/host-terminal/host-shell.ts +16 -7
  729. package/src/tools/network/__tests__/web-fetch-tinyfish.test.ts +250 -0
  730. package/src/tools/network/__tests__/web-search.test.ts +203 -0
  731. package/src/tools/network/web-fetch.ts +274 -7
  732. package/src/tools/network/web-search.ts +468 -6
  733. package/src/tools/permission-checker.ts +3 -3
  734. package/src/tools/policy-context.ts +3 -1
  735. package/src/tools/shared/filesystem/file-ops-service.ts +1 -31
  736. package/src/tools/shared/shell-output.test.ts +10 -0
  737. package/src/tools/shared/shell-output.ts +14 -2
  738. package/src/tools/skills/load.ts +2 -1
  739. package/src/tools/skills/sandbox-runner.ts +24 -4
  740. package/src/tools/skills/scaffold-managed.ts +2 -1
  741. package/src/tools/skills/skill-script-runner.ts +7 -0
  742. package/src/tools/skills/skill-tool-factory.ts +15 -1
  743. package/src/tools/terminal/__tests__/safe-env.test.ts +44 -4
  744. package/src/tools/terminal/__tests__/sanitized-bash.test.ts +0 -14
  745. package/src/tools/terminal/safe-env.ts +39 -24
  746. package/src/tools/terminal/sanitized-bash.ts +15 -2
  747. package/src/tools/terminal/shell-launch.test.ts +162 -0
  748. package/src/tools/terminal/shell.test.ts +29 -0
  749. package/src/tools/terminal/shell.ts +15 -7
  750. package/src/tools/tool-approval-handler.ts +5 -1
  751. package/src/tools/tool-types.ts +3 -3
  752. package/src/tools/types.ts +8 -2
  753. package/src/util/__tests__/worker-process-command.test.ts +5 -4
  754. package/src/util/abort-reasons.ts +18 -26
  755. package/src/util/browser-human-verification.ts +14 -0
  756. package/src/util/canonicalize-identity.test.ts +21 -0
  757. package/src/util/canonicalize-identity.ts +16 -0
  758. package/src/util/errors.ts +13 -3
  759. package/src/util/host-process.test.ts +17 -1
  760. package/src/util/host-process.ts +24 -0
  761. package/src/util/provider-error-patterns.ts +3 -0
  762. package/src/util/unicode.ts +29 -0
  763. package/src/util/worker-process.ts +77 -35
  764. package/src/workspace/migrations/157-add-desktop-files-launcher.ts +94 -0
  765. package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +4 -10
  766. package/src/workspace/migrations/__tests__/157-add-desktop-files-launcher.test.ts +113 -0
  767. package/src/workspace/migrations/registry.ts +2 -0
  768. package/src/__tests__/computer-use-window-schema.test.ts +0 -24
  769. package/src/api/events/heartbeat-alert.ts +0 -20
  770. package/src/daemon/message-types/web-activity.ts +0 -65
  771. package/src/desktop/desktop-dependencies.test.ts +0 -131
  772. package/src/tools/computer-use/definitions.ts +0 -607
@@ -57,7 +57,8 @@ function contentBlocks(msg: ServerToolPairingMessage): ReadonlyArray<unknown> {
57
57
  * `tool_use`, no result emitted) and executes it on the next request, placing
58
58
  * the web_search_tool_result at the head of the next assistant message. Both
59
59
  * the deferred tail and the resulting split pair must be sent back verbatim
60
- * for the provider to pair and execute them.
60
+ * for the provider to pair and execute them, and the user message answering
61
+ * a deferred tail must hold tool_result blocks alone.
61
62
  */
62
63
  export interface ServerToolPairing {
63
64
  /**
@@ -67,8 +68,12 @@ export interface ServerToolPairing {
67
68
  resolvedPairIds: Set<string>;
68
69
  /**
69
70
  * Unanswered use ids in the final assistant message of an active tool-loop
70
- * continuation (only user messages after it, carrying tool_result blocks or
71
- * nothing at all). The provider executes these on this request.
71
+ * continuation: nothing follows it, or only user messages made solely of
72
+ * tool_result blocks. The provider executes these on this request. Any
73
+ * other trailing block (guidance text after the results, a queued user
74
+ * message) closes the assistant turn on the provider side, which then
75
+ * rejects the unanswered use as unpaired, so such a use is an orphan to
76
+ * repair rather than a deferral to preserve.
72
77
  */
73
78
  deferredUseIds: Set<string>;
74
79
  /** Ids whose use/result pair spans two messages. */
@@ -118,10 +123,14 @@ export function analyzeServerToolPairing(
118
123
  if (lastAssistantIndex >= 0) {
119
124
  const trailing = messages.slice(lastAssistantIndex + 1);
120
125
  const trailingAllUser = trailing.every((m) => m.role === "user");
121
- const trailingHasToolResult = trailing.some((m) =>
122
- contentBlocks(m).some(isToolResultShapedBlock),
123
- );
124
- if (trailingAllUser && (trailing.length === 0 || trailingHasToolResult)) {
126
+ const trailingBlocks = trailing.flatMap((m) => [...contentBlocks(m)]);
127
+ const trailingIsToolResultsOnly =
128
+ trailingBlocks.length > 0 &&
129
+ trailingBlocks.every(isToolResultShapedBlock);
130
+ if (
131
+ trailingAllUser &&
132
+ (trailing.length === 0 || trailingIsToolResultsOnly)
133
+ ) {
125
134
  for (const block of contentBlocks(messages[lastAssistantIndex])) {
126
135
  if (isServerToolUseBlock(block) && !resolvedPairIds.has(block.id)) {
127
136
  deferredUseIds.add(block.id);
@@ -129,6 +129,30 @@ describe("parseFluxFrame", () => {
129
129
  ]);
130
130
  });
131
131
 
132
+ test.each(["model", "timeout", "manual"])(
133
+ "retains the %s turn-end trigger and decoded audio position",
134
+ (trigger) => {
135
+ expect(
136
+ parseFluxFrame({ ...END_OF_TURN, trigger, audio_window_end: 12.5 })[1],
137
+ ).toMatchObject({
138
+ type: "turn-end",
139
+ confidence: 0.88,
140
+ trigger,
141
+ audioWindowEndSeconds: 12.5,
142
+ });
143
+ },
144
+ );
145
+
146
+ test("omits unknown triggers and malformed audio positions", () => {
147
+ expect(
148
+ parseFluxFrame({
149
+ ...END_OF_TURN,
150
+ trigger: "future-trigger",
151
+ audio_window_end: "invalid",
152
+ }),
153
+ ).toEqual(parseFluxFrame(END_OF_TURN));
154
+ });
155
+
132
156
  test("flat frames without a nested event field are accepted", () => {
133
157
  expect(parseFluxFrame({ type: "StartOfTurn" })).toEqual([
134
158
  { type: "turn-start" },
@@ -334,6 +334,7 @@ describe("DeepgramFluxRealtimeTranscriber", () => {
334
334
  type: "turn-end",
335
335
  text: "what is the weather",
336
336
  confidence: 0.91,
337
+ audioWindowEndSeconds: 1.5,
337
338
  turnIndex: 0,
338
339
  },
339
340
  ]);
@@ -382,7 +383,12 @@ describe("DeepgramFluxRealtimeTranscriber", () => {
382
383
  );
383
384
  expect(events).toEqual([
384
385
  { type: "final", text: "still here" },
385
- { type: "turn-end", text: "still here", turnIndex: 0 },
386
+ {
387
+ type: "turn-end",
388
+ text: "still here",
389
+ turnIndex: 0,
390
+ audioWindowEndSeconds: 1.5,
391
+ },
386
392
  ]);
387
393
  });
388
394
  });
@@ -8,6 +8,7 @@
8
8
  import { afterEach, beforeEach, describe, expect, test } from "bun:test";
9
9
 
10
10
  import { SttError, type SttStreamServerEvent } from "../../../stt/types.js";
11
+ import { VellumManagedFluxRealtimeTranscriber } from "../vellum-managed-flux-realtime.js";
11
12
  import { VellumManagedRealtimeTranscriber } from "../vellum-managed-realtime.js";
12
13
  import type { SpeechRelayConnection } from "../vellum-speech-relay-connection.js";
13
14
 
@@ -99,7 +100,7 @@ const CONNECTION: SpeechRelayConnection = {
99
100
 
100
101
  const tick = () => new Promise<void>((r) => setTimeout(r, 0));
101
102
 
102
- describe("VellumManagedRealtimeTranscriber", () => {
103
+ describe("Vellum managed realtime transcribers", () => {
103
104
  let sockets: MockWebSocket[];
104
105
  let originalWebSocket: unknown;
105
106
  let originalFetch: unknown;
@@ -262,6 +263,54 @@ describe("VellumManagedRealtimeTranscriber", () => {
262
263
  expect(new URL(sockets[1]!.url).searchParams.get("key")).toBe("vk-secret");
263
264
  });
264
265
 
266
+ test("Flux turn indices stay in one sequence across relay reconnects", async () => {
267
+ const adapter = new VellumManagedFluxRealtimeTranscriber(CONNECTION);
268
+ const { events, onEvent } = collector();
269
+ const started = adapter.start(onEvent);
270
+ sockets[0]!.simulateOpen();
271
+ await started;
272
+ try {
273
+ for (let generation = 0; generation < 3; generation += 1) {
274
+ const socket = sockets[generation]!;
275
+ const turnIndex = generation === 0 ? 5 : 0;
276
+ const startFrame = JSON.stringify({
277
+ type: "TurnInfo",
278
+ event: "StartOfTurn",
279
+ turn_index: turnIndex,
280
+ });
281
+ socket.simulateMessage(startFrame);
282
+ socket.simulateMessage(startFrame);
283
+ socket.simulateMessage(
284
+ JSON.stringify({
285
+ type: "TurnInfo",
286
+ event: "EndOfTurn",
287
+ // A later end without its start still advances the sequence.
288
+ turn_index: generation === 1 ? 2 : turnIndex,
289
+ transcript: "hello",
290
+ }),
291
+ );
292
+ if (generation < 2) {
293
+ socket.simulateMessage(velayErrorFrame("session_duration_exceeded"));
294
+ socket.simulateClose(1000, "session_duration_exceeded");
295
+ await tick();
296
+ sockets[generation + 1]!.simulateOpen();
297
+ await tick();
298
+ }
299
+ }
300
+ expect(
301
+ events.flatMap((event) =>
302
+ event.type === "turn-start" || event.type === "turn-end"
303
+ ? [event.turnIndex]
304
+ : [],
305
+ ),
306
+ ).toEqual([5, 5, 5, 6, 6, 8, 9, 9, 9]);
307
+ expect(events.some((event) => event.type === "closed")).toBe(false);
308
+ } finally {
309
+ adapter.stop();
310
+ sockets.at(-1)!.simulateClose();
311
+ }
312
+ });
313
+
265
314
  test("finalize during a re-dial settles immediately", async () => {
266
315
  const { adapter, events } = await startAdapter();
267
316
 
@@ -71,6 +71,7 @@ export interface FluxTurnFrame {
71
71
  transcript?: string;
72
72
  /** Flux's end-of-turn confidence in [0, 1]. */
73
73
  end_of_turn_confidence?: number;
74
+ trigger?: "model" | "manual" | "timeout";
74
75
  }
75
76
 
76
77
  /**
@@ -153,6 +154,8 @@ export function parseFluxFrame(raw: unknown): SttStreamServerEvent[] {
153
154
  case "EndOfTurn": {
154
155
  const text = readTranscript(frame);
155
156
  const confidence = readNumber(frame.end_of_turn_confidence);
157
+ const trigger = frame.trigger;
158
+ const audioWindowEndSeconds = readNumber(frame.audio_window_end);
156
159
  return [
157
160
  { type: "final", text },
158
161
  {
@@ -160,6 +163,14 @@ export function parseFluxFrame(raw: unknown): SttStreamServerEvent[] {
160
163
  text,
161
164
  ...(confidence !== undefined ? { confidence } : {}),
162
165
  ...(turnIndex !== undefined ? { turnIndex } : {}),
166
+ ...(trigger === "model" ||
167
+ trigger === "manual" ||
168
+ trigger === "timeout"
169
+ ? { trigger }
170
+ : {}),
171
+ ...(audioWindowEndSeconds !== undefined
172
+ ? { audioWindowEndSeconds }
173
+ : {}),
163
174
  },
164
175
  ];
165
176
  }
@@ -79,6 +79,9 @@ export class VellumManagedFluxRealtimeTranscriber implements StreamingTranscribe
79
79
  private redialOnClose = false;
80
80
  private stopping = false;
81
81
  private closedEmitted = false;
82
+ // Consumers see one stream across relay reconnects, with one turn sequence.
83
+ private turnIndexOffset = 0;
84
+ private latestTurnIndex = -1;
82
85
 
83
86
  constructor(
84
87
  connection: SpeechRelayConnection,
@@ -225,6 +228,15 @@ export class VellumManagedFluxRealtimeTranscriber implements StreamingTranscribe
225
228
  return;
226
229
  }
227
230
 
231
+ if (
232
+ (event.type === "turn-start" || event.type === "turn-end") &&
233
+ event.turnIndex !== undefined
234
+ ) {
235
+ const turnIndex = this.turnIndexOffset + event.turnIndex;
236
+ this.latestTurnIndex = Math.max(this.latestTurnIndex, turnIndex);
237
+ this.emit({ ...event, turnIndex });
238
+ return;
239
+ }
228
240
  this.emit(event);
229
241
  }
230
242
 
@@ -236,6 +248,7 @@ export class VellumManagedFluxRealtimeTranscriber implements StreamingTranscribe
236
248
  private beginRedial(): void {
237
249
  log.info("Managed Flux relay session hit velay's duration cap, re-dialing");
238
250
  this.inner = null;
251
+ this.turnIndexOffset = this.latestTurnIndex + 1;
239
252
  void (async () => {
240
253
  try {
241
254
  const inner = await this.dial();
@@ -551,6 +551,8 @@ export interface ContextOverflowErrorOptions {
551
551
  cause?: unknown;
552
552
  /** Semantic reason override; defaults to `context_overflow`. */
553
553
  reason?: ProviderErrorReason;
554
+ /** Inspectable SDK/wire request payload for the overflowing call. */
555
+ rawRequest?: unknown;
554
556
  }
555
557
 
556
558
  /**
@@ -581,6 +583,9 @@ export class ContextOverflowError extends ProviderError {
581
583
  super(message, provider, options.statusCode ?? 400, {
582
584
  reason: options.reason ?? "context_overflow",
583
585
  ...(options.cause !== undefined ? { cause: options.cause } : {}),
586
+ ...(options.rawRequest !== undefined
587
+ ? { rawRequest: options.rawRequest }
588
+ : {}),
584
589
  });
585
590
  this.name = "ContextOverflowError";
586
591
  this.actualTokens = options.actualTokens;
@@ -44,13 +44,15 @@ function routeHostPidPath(): string {
44
44
  return getProcPidPath(ROUTE_HOST_PROC_NAME);
45
45
  }
46
46
 
47
+ const ROUTE_HOST_ENTRY = new URL("./worker.ts", import.meta.url);
48
+
47
49
  /**
48
50
  * Read the PID file and report liveness. A missing or malformed file reports
49
- * not_running; a file pointing at a dead process is cleaned up and reported as
50
- * not_running.
51
+ * not_running; a file pointing at a dead process, or at a live process that
52
+ * is not this worker, is cleaned up and reported as not_running.
51
53
  */
52
54
  export function probeRouteHostWorker(): WorkerProcessStatus {
53
- return probeWorkerPidFile(routeHostPidPath());
55
+ return probeWorkerPidFile(routeHostPidPath(), ROUTE_HOST_ENTRY, "routes");
54
56
  }
55
57
 
56
58
  export class RouteHostSpawnError extends WorkerProcessSpawnError {}
@@ -67,7 +69,7 @@ export async function spawnRouteHostWorkerProcess(
67
69
  try {
68
70
  return await spawnWorkerProcess({
69
71
  pidPath: routeHostPidPath(),
70
- entry: new URL("./worker.ts", import.meta.url),
72
+ entry: ROUTE_HOST_ENTRY,
71
73
  packagedEntry: "routes",
72
74
  workerLabel: "Route host",
73
75
  options: opts,
@@ -86,7 +88,7 @@ export async function spawnRouteHostWorkerProcess(
86
88
  * (e.g. EPERM) — a not-running host is a no-op.
87
89
  */
88
90
  export function stopRouteHostWorkerProcess(): WorkerProcessStatus {
89
- return stopWorkerProcess(routeHostPidPath());
91
+ return stopWorkerProcess(routeHostPidPath(), ROUTE_HOST_ENTRY, "routes");
90
92
  }
91
93
 
92
94
  /**
@@ -24,7 +24,7 @@ Queueing remains the fallback, not a legacy path, and every non-`released` outco
24
24
 
25
25
  A repair that fell back to the queue carries its durable requirement onto the drain that runs the queued message, and `pendingInterruptRepair` is what carries it: the drain reads that flag and asks `repairInterruptedToolUseBlocks` for `requireDurable`, so a second persist failure leaves the flag armed and the message queued instead of persisting the prompt behind an unmatched durable `tool_use`. Only a steered drain settles for an in-memory repair, because it has no row of its own to persist behind it.
26
26
 
27
- **The model has to be told its turn was cut off, once.** An interrupt caught mid-tool says so through the synthetic `tool_result` (`PREEMPTED_TOOL_RESULT_TEXT`). An interrupt caught during the provider call has no `tool_use` to answer, so `repairInterruptedToolUseBlocks` reports `preemptedToolResultOnTail: false` and the handover arms `pendingInterruptNote`. The persist of the interrupting user message consumes that flag exactly once: it stamps `interruptedPriorTurn` on the row and appends `INTERRUPTED_TURN_NOTE_TEXT` to the message's LLM-facing content only, so the persisted row stays what the user typed and no client renders the note. `reinjectInterruptTurnNote` rebuilds the identical block at history load, next to the attachment annotations and for the same reason: a reload that rebuilds it differently breaks provider prefix-cache parity. Do not add a second annotation for the same interrupt; one notice per cut-off turn.
27
+ **The two interrupt texts are split by job, and the behavior instruction lives in exactly one of them.** A synthetic `tool_result` (`PREEMPTED_TOOL_RESULT_TEXT`) states the fact about the one call it answers: the call stopped early and may have taken effect anyway. It is repeated once per stopped call in a parallel batch, so it holds no instruction. Every completed handover arms `pendingInterruptNote`, whether or not a tool call was stopped. The persist of the interrupting user message consumes that flag exactly once: it stamps `interruptedPriorTurn` on the row and appends `INTERRUPTED_TURN_NOTE_TEXT` to the message's LLM-facing content only, so the persisted row stays what the user typed and no client renders the note. That note is the single place the behavior after an interrupt is written, it leads with the short acknowledgement the user is waiting for, and it sits after the user's words instead of before them. `reinjectInterruptTurnNote` rebuilds the identical block at history load, next to the attachment annotations and for the same reason: a reload that rebuilds it differently breaks provider prefix-cache parity. Do not add a third annotation, and do not put instructions back in the tool result.
28
28
 
29
29
  Two ordering rules a change here must not break. The synthetic `tool_result` is persisted BEFORE the caller writes the interrupting user row, because a user message between a `tool_use` and its result is a sequence every provider rejects. And the `thinking` / `message_interrupted` activity state is emitted by the agent loop at the head of the replacement turn, never at the handover: activity states are cached in `lastActivityStateMsg` and replayed to reconnecting clients, so a send that fails before its turn starts would otherwise strand every client on a busy conversation that is running nothing.
30
30
 
@@ -163,7 +163,7 @@ In the CDP factory the bridge is the internal `"host-bridge"` candidate kind (`I
163
163
 
164
164
  ### Browser CLI surface defaults
165
165
 
166
- The browser execute and tab routes share `browser/virtual-desktop-target.ts`. Platform-hosted web guardian conversations use installed, enabled virtual desktop Chrome by default. The shared `isVirtualDesktopEnabled` gate requires both `IS_PLATFORM` and `IS_CONTAINERIZED` alongside the `assistant-desktop` flag. Native renderer turns also use the `web` transport, so the frozen turn `clientOs` excludes native apps from automatic streamed-browser selection. Native apps retain their existing backend selection and fallback behavior. Explicit desktop/backend/client targets and existing personal-browser sessions override the surface default. Disabled or uninstalled desktop support retains the existing browser path; selection never triggers installation.
166
+ The browser execute and tab routes share `browser/virtual-desktop-target.ts`. Platform-hosted web guardian conversations use image-provided virtual desktop Chrome by default when enabled. The shared `isVirtualDesktopEnabled` gate requires both `IS_PLATFORM` and `IS_CONTAINERIZED` alongside the `assistant-desktop` flag. Native renderer turns also use the `web` transport, so the frozen turn `clientOs` excludes native apps from automatic streamed-browser selection. Native apps retain their existing backend selection and fallback behavior. Explicit desktop/backend/client targets and existing personal-browser sessions override the surface default. Disabled desktop support retains the existing browser path. Missing image components fail the managed operation without installing dependencies or falling back to a personal browser.
167
167
 
168
168
  ### Per-tool `browser_mode` override
169
169
 
@@ -277,6 +277,8 @@ Verification SESSION state (sessions, secrets, rate limits, validate+consume) AN
277
277
 
278
278
  The daemon relays session lifecycle operations over the `verification_sessions_*` IPC routes via `assistant/src/channels/gateway-verification-sessions.ts` and keeps what is presentation: message composition and channel delivery (`channel-verification-routes.ts`, `verification-outbound-actions.ts`). `channel-verification-service.ts` retains only guardian-delivery reads (`getGuardianBinding`, `isGuardian`, `isGuardianBoundForChannel`).
279
279
 
280
+ The gateway answers a code (or an invite) it consumed at ingress with a reply it composes itself, and delivers it through the daemon's IPC-only `deliver_gateway_reply` method (`ipc/routes/channel-reply-ipc-routes.ts`), which hands it to the channel transport the inbound message's callback URL names. The gateway never sends to a provider itself: the transports are the one outbound path per channel, whoever composed the text. The reply is not recorded in a conversation, since the message it answers never entered one.
281
+
280
282
  The verified outcome is written in-process by the gateway: the HTTP guardian-attest handler calls `ContactStore.markChannelVerified` directly (verifiedVia "manual"); the code-match paths (text and the `verification_sessions_validate_consume` engine route) apply role side effects in-engine — guardian phone binding commits in the same gateway transaction as the consume. The revoke/downgrade outcome is relayed from the daemon via `ipcCallPersistent("mark_channel_revoked", …)` to `ContactStore.markChannelRevoked`.
281
283
 
282
284
  ## Rate Limiting & Diagnostics
@@ -301,6 +303,6 @@ The provider-level rate limiter (`providers/ratelimit.ts`) also logs warnings (m
301
303
 
302
304
  ## HTTP-Only Transport
303
305
 
304
- HTTP is the sole transport for client-daemon communication. The runtime HTTP server (`assistant/src/runtime/http-server.ts`) is the canonical API surface. Clients connect via HTTP for request/response operations and SSE (`GET /v1/events`) for streaming server-to-client events.
306
+ HTTP is the sole transport for browser and desktop clients; the CLI and the gateway reach the daemon over the IPC socket (see `assistant/AGENTS.md`). The runtime HTTP server (`assistant/src/runtime/http-server.ts`) is the canonical API surface. Clients connect via HTTP for request/response operations and SSE (`GET /v1/events`) for streaming server-to-client events.
305
307
 
306
- When writing skills that need to call daemon configuration endpoints, use `curl` with the runtime HTTP API (JWT-authenticated via `Authorization: Bearer <jwt>`). The assistant already knows how to use `curl`.
308
+ A skill never calls the runtime HTTP API directly: every request from a skill targets the gateway, and `gateway-only-guard.test.ts` fails CI on a runtime-port URL in a skill. For configuration reads, use the CLI (`assistant config get`); for control-plane writes with no CLI verb, use `$INTERNAL_GATEWAY_BASE_URL`. See "Gateway-Only API Consumption" in `gateway/AGENTS.md` for the rule and its exceptions.
@@ -18,9 +18,10 @@
18
18
  * route captured calls back to the originating conversation's probe arrays.
19
19
  */
20
20
 
21
- import { beforeEach, describe, expect, mock, test } from "bun:test";
21
+ import { beforeEach, describe, expect, mock, spyOn, test } from "bun:test";
22
22
 
23
23
  import type { DiskPressureStatus } from "../../daemon/disk-pressure-guard.js";
24
+ import { desktopAutomationLease } from "../../desktop/desktop-automation-lease.js";
24
25
 
25
26
  // ── Per-conversation capture registry ────────────────────────────────
26
27
  //
@@ -501,6 +502,13 @@ function makeWakeConversation(options: {
501
502
  `tools:${snapshotAllowedTools()?.join(",") ?? "all"}`,
502
503
  );
503
504
  },
505
+ // Mirrors Conversation.setPreactivatedSkillIds; the wake's tool-scope
506
+ // restore calls it, and a double without it throws inside the restore
507
+ // closure, leaving every field restored after it untouched.
508
+ preactivatedSkillIds: undefined as string[] | undefined,
509
+ setPreactivatedSkillIds(ids: string[] | undefined) {
510
+ this.preactivatedSkillIds = ids;
511
+ },
504
512
  get wakePersonaOverride() {
505
513
  return wakePersonaOverride;
506
514
  },
@@ -1357,6 +1365,50 @@ describe("wakeAgentForOpportunity", () => {
1357
1365
  expect(conversation.toolContextPin).toBeUndefined();
1358
1366
  });
1359
1367
 
1368
+ test("applies wireToolDefinitions and the delegation state alongside the allowlist and restores both after the wake", async () => {
1369
+ const replay = [
1370
+ { name: "remember", description: "Save", input_schema: {} },
1371
+ { name: "bell_jingle", description: "Ring", input_schema: {} },
1372
+ ];
1373
+ let replayDuringRun: unknown;
1374
+ let delegationDuringRun: unknown;
1375
+ const conversation = makeWakeConversation({
1376
+ runImpl: async (input) => {
1377
+ replayDuringRun = conversation.wireToolReplay;
1378
+ delegationDuringRun = conversation.delegateIndependentTasksReplay;
1379
+ return runResult([
1380
+ ...input,
1381
+ { role: "assistant", content: [{ type: "text", text: "Saved." }] },
1382
+ ]);
1383
+ },
1384
+ });
1385
+
1386
+ const result = await wakeAgentForOpportunity(
1387
+ {
1388
+ conversationId: conversation.conversationId,
1389
+ hint: "review for memories",
1390
+ source: "memory-retrospective",
1391
+ allowedTools: ["remember"],
1392
+ toolGateMode: "execution",
1393
+ wireToolDefinitions: replay,
1394
+ delegateIndependentTasks: true,
1395
+ },
1396
+ { resolveTarget: async () => conversation },
1397
+ );
1398
+
1399
+ expect(result).toEqual({ invoked: true, producedToolCalls: false });
1400
+ // The replay array is live on the conversation for the duration of the
1401
+ // run (the tool resolver returns it as the wire array), as is the
1402
+ // delegation-section state the source's prompt rendered (the prompt
1403
+ // build reads it)...
1404
+ expect(replayDuringRun).toEqual(replay);
1405
+ expect(delegationDuringRun).toBe(true);
1406
+ // ...and both are cleared alongside the allowlist + gate mode after the
1407
+ // wake.
1408
+ expect(conversation.wireToolReplay).toBeUndefined();
1409
+ expect(conversation.delegateIndependentTasksReplay).toBeUndefined();
1410
+ });
1411
+
1360
1412
  test("defaults to the wire gate mode when toolGateMode is absent", async () => {
1361
1413
  let gateModeDuringRun: string | undefined;
1362
1414
  const conversation = makeWakeConversation({
@@ -1517,6 +1569,47 @@ describe("wakeAgentForOpportunity", () => {
1517
1569
  expect(conversation.pushedMessages[2]).toEqual(followupAssistant);
1518
1570
  });
1519
1571
 
1572
+ for (const fails of [false, true]) {
1573
+ test(`releases desktop control before a ${fails ? "failed" : "completed"} wake drains the queue`, async () => {
1574
+ const conversation = makeWakeConversation({
1575
+ scriptedAssistant: {
1576
+ role: "assistant",
1577
+ content: [{ type: "text", text: "reply" }],
1578
+ },
1579
+ ...(fails
1580
+ ? {
1581
+ runImpl: async () => {
1582
+ throw new Error("wake failed");
1583
+ },
1584
+ }
1585
+ : {}),
1586
+ });
1587
+ const release = spyOn(desktopAutomationLease, "releaseForConversation");
1588
+ const setProcessing = conversation.setProcessing.bind(conversation);
1589
+ conversation.setProcessing = (processing) => {
1590
+ if (!processing) {
1591
+ expect(release).toHaveBeenCalledWith(conversation.conversationId);
1592
+ }
1593
+ setProcessing(processing);
1594
+ };
1595
+ try {
1596
+ const result = await wakeAgentForOpportunity(
1597
+ {
1598
+ conversationId: conversation.conversationId,
1599
+ hint: "x",
1600
+ source: "unit-test",
1601
+ },
1602
+ { resolveTarget: async () => conversation },
1603
+ );
1604
+ expect(result.invoked).toBe(!fails);
1605
+ expect(release).toHaveBeenCalledTimes(1);
1606
+ expect(conversation.drainQueueCalls).toBe(1);
1607
+ } finally {
1608
+ release.mockRestore();
1609
+ }
1610
+ });
1611
+ }
1612
+
1520
1613
  test("marks processing true during the run and false afterwards", async () => {
1521
1614
  const conversation = makeWakeConversation({
1522
1615
  baseline: [{ role: "user", content: [{ type: "text", text: "hi" }] }],
@@ -42,9 +42,9 @@ const log = getLogger("actor-trust-resolver");
42
42
  // ---------------------------------------------------------------------------
43
43
 
44
44
  /**
45
- * Trust classification for an inbound actor. Defined once in `./trust-class.ts`
46
- * (shared with the persistence metadata schema) and re-exported here, the
47
- * canonical import site for the resolver's consumers.
45
+ * Trust classification for an inbound actor. Declared in
46
+ * `@vellumai/gateway-client` and re-exported here for the resolver's
47
+ * consumers.
48
48
  */
49
49
  export type { TrustClass };
50
50
 
@@ -107,6 +107,7 @@ import {
107
107
  persistWakeTriggerMessage,
108
108
  scopeWakeAllowedTools,
109
109
  } from "../daemon/wake-conversation-ops.js";
110
+ import { desktopAutomationLease } from "../desktop/desktop-automation-lease.js";
110
111
  import {
111
112
  recordCompactionEndBestEffort,
112
113
  recordCompactionStartBestEffort,
@@ -118,7 +119,7 @@ import {
118
119
  setAgentLoopExitReasonOnLatestLog,
119
120
  } from "../persistence/llm-request-log-store.js";
120
121
  import type { SystemPromptPersonaOverride } from "../prompts/system-prompt.js";
121
- import type { Message } from "../providers/types.js";
122
+ import type { Message, ToolDefinition } from "../providers/types.js";
122
123
  import {
123
124
  type UntrustedContentSource,
124
125
  wrapUntrustedContent,
@@ -350,6 +351,27 @@ export interface WakeOptions {
350
351
  * skill-management authoring tools callable directly.
351
352
  */
352
353
  preactivateSkillIds?: readonly string[];
354
+ /**
355
+ * Tool definitions to send verbatim as the wake's wire tool array, in place
356
+ * of the ones the conversation would resolve for itself. Used by fork-based
357
+ * memory retrospectives to replay the SOURCE conversation's recorded surface
358
+ * (`getRecordedConversationToolSurface`) so the provider prompt-cache prefix matches
359
+ * the source's live turns byte for byte. Definitions only: what may execute
360
+ * is still decided by `allowedTools` and the turn's own active set. Applied
361
+ * and restored alongside `allowedTools`; ignored when `allowedTools` is
362
+ * absent.
363
+ */
364
+ wireToolDefinitions?: readonly ToolDefinition[];
365
+ /**
366
+ * Whether the source's recorded surface rendered the system prompt's
367
+ * parallel-delegation section, replayed into the wake's prompt so that
368
+ * tier of the provider cache prefix matches the source's too (the wake's
369
+ * own scope cannot spawn, so deriving it would render the section off where
370
+ * an interactive source rendered it on). Prompt only: a spawn attempt is
371
+ * still rejected at execution. Applied and restored alongside
372
+ * `allowedTools`; ignored when `allowedTools` is absent.
373
+ */
374
+ delegateIndependentTasks?: boolean;
353
375
  /**
354
376
  * Explicit persona/channel slugs for the wake's system-prompt build,
355
377
  * applied to the conversation for the duration of the run and restored
@@ -1090,7 +1112,7 @@ export async function wakeAgentForOpportunity(
1090
1112
  try {
1091
1113
  recordRequestLog(
1092
1114
  conversationId,
1093
- JSON.stringify(record.rawRequest),
1115
+ JSON.stringify(record.rawRequest ?? null),
1094
1116
  JSON.stringify(record.rawResponse),
1095
1117
  undefined,
1096
1118
  record.provider,
@@ -1410,6 +1432,7 @@ export async function wakeAgentForOpportunity(
1410
1432
  * is one function rather than a rebuild bolted onto either half.
1411
1433
  */
1412
1434
  const restoreWakeTurnScope = (): void => {
1435
+ desktopAutomationLease.releaseForConversation(conversationId);
1413
1436
  restoreWakeAllowedTools();
1414
1437
  clearWakePersonaOverride();
1415
1438
  syncWakeLoopSystemPrompt();
@@ -1422,9 +1445,13 @@ export async function wakeAgentForOpportunity(
1422
1445
  restoreWakeToolScope = scopeWakeAllowedTools(
1423
1446
  conversation,
1424
1447
  new Set(opts.allowedTools),
1425
- opts.toolGateMode,
1426
- opts.toolContextPin,
1427
- opts.preactivateSkillIds,
1448
+ {
1449
+ gateMode: opts.toolGateMode,
1450
+ toolContextPin: opts.toolContextPin,
1451
+ preactivateSkillIds: opts.preactivateSkillIds,
1452
+ wireToolDefinitions: opts.wireToolDefinitions,
1453
+ delegateIndependentTasks: opts.delegateIndependentTasks,
1454
+ },
1428
1455
  );
1429
1456
  return true;
1430
1457
  } catch (err) {