@vellumai/assistant 0.12.2 → 0.12.3-staging.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (772) hide show
  1. package/.env.example +2 -0
  2. package/AGENTS.md +1 -1
  3. package/ARCHITECTURE.md +43 -29
  4. package/Dockerfile +63 -0
  5. package/docs/architecture/integrations.md +7 -0
  6. package/docs/architecture/memory.md +17 -2
  7. package/docs/architecture/security.md +10 -5
  8. package/docs/architecture/turn-actor.md +2 -0
  9. package/docs/credential-execution-service.md +1 -1
  10. package/docs/desktop-browser-cli.md +50 -5
  11. package/docs/live-voice-task-outcomes.md +100 -0
  12. package/docs/mode-sessions.md +183 -0
  13. package/docs/voice-input-diagnostics.md +58 -0
  14. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  15. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
  16. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  17. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  18. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
  19. package/node_modules/@vellumai/environments/src/shell.test.ts +21 -0
  20. package/node_modules/@vellumai/environments/src/shell.ts +24 -0
  21. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  22. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
  23. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  24. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  25. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
  26. package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +20 -0
  27. package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +8 -2
  28. package/node_modules/@vellumai/gateway-client/src/index.ts +4 -1
  29. package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +38 -7
  30. package/node_modules/@vellumai/gateway-client/src/trust-verdict-contract.ts +5 -6
  31. package/node_modules/@vellumai/service-contracts/package.json +2 -1
  32. package/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
  33. package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  34. package/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  35. package/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
  36. package/node_modules/@vellumai/slack-text/src/index.ts +4 -8
  37. package/openapi.yaml +716 -16
  38. package/package.json +1 -1
  39. package/scripts/smoke-desktop-browser-cli.ts +11 -26
  40. package/scripts/smoke-desktop-window-theme.ts +2 -3
  41. package/scripts/sync-llm-catalog.ts +3 -0
  42. package/scripts/voice-ttft-spike.ts +2 -2
  43. package/src/__tests__/agent-loop-output-hooks.test.ts +150 -1
  44. package/src/__tests__/agent-loop-provider-error-recording.test.ts +87 -19
  45. package/src/__tests__/agent-loop.test.ts +259 -0
  46. package/src/__tests__/agent-wake-delegation-prompt.test.ts +64 -2
  47. package/src/__tests__/annotate-activity-metadata.test.ts +3 -1
  48. package/src/__tests__/annotate-risk-options.test.ts +2 -0
  49. package/src/__tests__/answered-question-persistence.test.ts +2 -0
  50. package/src/__tests__/anthropic-gateway-shared.test.ts +22 -0
  51. package/src/__tests__/anthropic-provider.test.ts +55 -0
  52. package/src/__tests__/approval-interception-trust-gates.test.ts +40 -0
  53. package/src/__tests__/approval-message-composer.test.ts +1 -118
  54. package/src/__tests__/approval-routes-http.test.ts +3 -0
  55. package/src/__tests__/assistant-event-hub-connection-history.test.ts +111 -0
  56. package/src/__tests__/assistant-event-hub-targeted.test.ts +50 -0
  57. package/src/__tests__/assistant-event-targeting.test.ts +186 -0
  58. package/src/__tests__/assistant-stream-state.test.ts +29 -9
  59. package/src/__tests__/attachments-store.test.ts +22 -3
  60. package/src/__tests__/browser-fill-credential.test.ts +11 -22
  61. package/src/__tests__/btw-routes.test.ts +4 -1
  62. package/src/__tests__/build-persisted-content.test.ts +1 -1
  63. package/src/__tests__/call-conversation-messages.test.ts +5 -0
  64. package/src/__tests__/call-recovery.test.ts +7 -2
  65. package/src/__tests__/call-routes-http.test.ts +12 -0
  66. package/src/__tests__/call-store.test.ts +7 -2
  67. package/src/__tests__/channel-approval.test.ts +9 -14
  68. package/src/__tests__/channel-reply-delivery.test.ts +56 -0
  69. package/src/__tests__/channel-retry-sweep.test.ts +9 -0
  70. package/src/__tests__/chat-credential-redaction.test.ts +23 -0
  71. package/src/__tests__/chat-reveal-guard-priming.test.ts +2 -0
  72. package/src/__tests__/checker.test.ts +11 -3
  73. package/src/__tests__/client-os-metadata-persistence.test.ts +35 -1
  74. package/src/__tests__/computer-use-screenshot-attachments.test.ts +411 -0
  75. package/src/__tests__/computer-use-screenshot-selection.test.ts +50 -0
  76. package/src/__tests__/computer-use-skill-manifest-regression.test.ts +0 -25
  77. package/src/__tests__/computer-use-tools.test.ts +1 -233
  78. package/src/__tests__/config-model-image-provider.test.ts +39 -0
  79. package/src/__tests__/config-schema.test.ts +16 -0
  80. package/src/__tests__/config-watcher.test.ts +24 -0
  81. package/src/__tests__/context-search-agent-runner.test.ts +260 -1
  82. package/src/__tests__/context-search-fanout.test.ts +1 -1
  83. package/src/__tests__/context-search-types.test.ts +7 -7
  84. package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +8 -2
  85. package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +2 -0
  86. package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +2 -0
  87. package/src/__tests__/conversation-agent-loop-overflow.test.ts +2 -0
  88. package/src/__tests__/conversation-agent-loop.test.ts +797 -16
  89. package/src/__tests__/conversation-app-control-lifecycle.test.ts +53 -1
  90. package/src/__tests__/conversation-attachments.test.ts +106 -0
  91. package/src/__tests__/conversation-evictor.test.ts +62 -37
  92. package/src/__tests__/conversation-fork-crud.test.ts +191 -0
  93. package/src/__tests__/conversation-history-boundary-repair-failure.test.ts +174 -0
  94. package/src/__tests__/conversation-loop-db-lock-resilience.test.ts +2 -0
  95. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +2 -0
  96. package/src/__tests__/conversation-rendered-delegation-state.test.ts +276 -0
  97. package/src/__tests__/conversation-retry-route.test.ts +22 -0
  98. package/src/__tests__/conversation-routes-disk-view.test.ts +2 -0
  99. package/src/__tests__/conversation-routes-enabled-plugins.test.ts +2 -0
  100. package/src/__tests__/conversation-routes-guardian-reply.test.ts +10 -0
  101. package/src/__tests__/conversation-routes-hidden-queue.test.ts +2 -0
  102. package/src/__tests__/conversation-routes-slash-commands.test.ts +2 -0
  103. package/src/__tests__/conversation-runtime-assembly.test.ts +29 -0
  104. package/src/__tests__/conversation-surface-snapshots.test.ts +543 -0
  105. package/src/__tests__/conversation-surfaces-action-delivery.test.ts +151 -0
  106. package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +63 -1
  107. package/src/__tests__/conversation-tool-setup-attribution.test.ts +23 -0
  108. package/src/__tests__/conversation-undo.test.ts +61 -0
  109. package/src/__tests__/credential-execution-client.test.ts +72 -55
  110. package/src/__tests__/credential-prompt-route.test.ts +35 -3
  111. package/src/__tests__/cu-unified-flow.test.ts +82 -1
  112. package/src/__tests__/db-conversation-tool-surface.test.ts +221 -0
  113. package/src/__tests__/db-mode-sessions-migration.test.ts +175 -0
  114. package/src/__tests__/deterministic-verification-control-plane.test.ts +2 -0
  115. package/src/__tests__/dm-backfill.test.ts +91 -1
  116. package/src/__tests__/document-create-route.test.ts +88 -0
  117. package/src/__tests__/document-sync-tags.test.ts +27 -0
  118. package/src/__tests__/emit-signal-routing-intent.test.ts +142 -2
  119. package/src/__tests__/events-tail-route.test.ts +58 -0
  120. package/src/__tests__/evict-conversations-for-reload.test.ts +57 -8
  121. package/src/__tests__/guardian-action-sweep.test.ts +16 -0
  122. package/src/__tests__/guardian-dispatch.test.ts +13 -0
  123. package/src/__tests__/headless-browser-interactions.test.ts +64 -20
  124. package/src/__tests__/heartbeat-service.test.ts +2 -59
  125. package/src/__tests__/helpers/mock-conversation.ts +21 -0
  126. package/src/__tests__/helpers/native-web-search-harness.ts +2 -0
  127. package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +5 -0
  128. package/src/__tests__/history-repair.test.ts +53 -0
  129. package/src/__tests__/host-cu-proxy.test.ts +46 -2
  130. package/src/__tests__/http-user-message-parity.test.ts +2 -0
  131. package/src/__tests__/image-credentials.test.ts +49 -0
  132. package/src/__tests__/image-service-dispatcher.test.ts +45 -0
  133. package/src/__tests__/inference-profile-session-handler.test.ts +26 -0
  134. package/src/__tests__/interrupt-on-send.test.ts +57 -43
  135. package/src/__tests__/interrupt-turn-note.test.ts +5 -6
  136. package/src/__tests__/list-messages-tool-merge.test.ts +85 -1
  137. package/src/__tests__/llm-catalog-parity.test.ts +27 -4
  138. package/src/__tests__/media-generate-image.test.ts +49 -1
  139. package/src/__tests__/message-provenance-contact-id.test.ts +69 -0
  140. package/src/__tests__/migration-export-to-gcs.test.ts +142 -0
  141. package/src/__tests__/mode-session-history.test.ts +245 -0
  142. package/src/__tests__/mode-session-recovery.test.ts +222 -0
  143. package/src/__tests__/mode-session-wire.test.ts +141 -0
  144. package/src/__tests__/notification-guardian-path.test.ts +7 -0
  145. package/src/__tests__/notification-telegram-adapter.test.ts +2 -2
  146. package/src/__tests__/notification-vellum-adapter.test.ts +34 -0
  147. package/src/__tests__/oauth-apps-routes.test.ts +1 -0
  148. package/src/__tests__/oauth-commands-routes.test.ts +114 -101
  149. package/src/__tests__/oauth-connect-orchestrator.test.ts +2 -0
  150. package/src/__tests__/oauth-provider-serializer.test.ts +13 -0
  151. package/src/__tests__/oauth-providers-routes.test.ts +3 -0
  152. package/src/__tests__/outbound-slack-persistence.test.ts +2 -0
  153. package/src/__tests__/persist-media-references.test.ts +50 -0
  154. package/src/__tests__/persistence-secret-redaction.test.ts +2 -0
  155. package/src/__tests__/plugin-api-provider.test.ts +5 -0
  156. package/src/__tests__/plugin-api-resolve-credential.test.ts +97 -0
  157. package/src/__tests__/plugin-import-boundary-guard.test.ts +9 -1
  158. package/src/__tests__/process-message-display-content.test.ts +56 -0
  159. package/src/__tests__/reaction-persistence.test.ts +5 -0
  160. package/src/__tests__/request-secret-standalone-channel-gate.test.ts +25 -9
  161. package/src/__tests__/run-conversation-turn-persistence.test.ts +138 -1
  162. package/src/__tests__/runtime-events-sse-reconnect.test.ts +1 -1
  163. package/src/__tests__/schedule-routes.test.ts +20 -0
  164. package/src/__tests__/scheduler-result-notification.test.ts +23 -4
  165. package/src/__tests__/script-proxy-certs.test.ts +1 -1
  166. package/src/__tests__/scripted-turn-metadata-persistence.test.ts +73 -0
  167. package/src/__tests__/secret-ingress-http.test.ts +2 -0
  168. package/src/__tests__/secret-routes-platform-proxy.test.ts +86 -31
  169. package/src/__tests__/secret-routes-scrub.test.ts +22 -27
  170. package/src/__tests__/secret-scanner.test.ts +20 -0
  171. package/src/__tests__/secure-keys-managed-failover.test.ts +8 -0
  172. package/src/__tests__/secure-keys.test.ts +50 -4
  173. package/src/__tests__/send-endpoint-busy.test.ts +163 -1
  174. package/src/__tests__/server-history-render.test.ts +91 -1
  175. package/src/__tests__/server-tool-pairing.test.ts +107 -0
  176. package/src/__tests__/settings-routes.test.ts +50 -1
  177. package/src/__tests__/skill-script-runner-sandbox.test.ts +32 -0
  178. package/src/__tests__/skills.test.ts +5 -4
  179. package/src/__tests__/subagent-fork-notifications.test.ts +1 -2
  180. package/src/__tests__/subagent-manager-notify.test.ts +1 -2
  181. package/src/__tests__/subagent-notify-parent.test.ts +146 -39
  182. package/src/__tests__/subagent-notify-stale-rebuild.test.ts +2 -4
  183. package/src/__tests__/subagent-tool-gate-mode.test.ts +226 -0
  184. package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +2 -0
  185. package/src/__tests__/terminal-tools.test.ts +8 -0
  186. package/src/__tests__/thread-backfill.test.ts +53 -0
  187. package/src/__tests__/tool-preview-lifecycle.test.ts +2 -0
  188. package/src/__tests__/tool-result-follow-up.test.ts +99 -0
  189. package/src/__tests__/tool-result-metadata-plumbing.test.ts +66 -1
  190. package/src/__tests__/tool-start-timestamp.test.ts +2 -0
  191. package/src/__tests__/twilio-routes.test.ts +1 -0
  192. package/src/__tests__/unicode.test.ts +36 -0
  193. package/src/__tests__/web-search-backend-failure.test.ts +1 -1
  194. package/src/activation/progress-store.test.ts +12 -7
  195. package/src/agent/loop.ts +136 -53
  196. package/src/agent/tool-result-follow-up.ts +70 -0
  197. package/src/api/attachment-provenance.test.ts +68 -0
  198. package/src/api/computer-use-tool.test.ts +55 -0
  199. package/src/api/computer-use-tool.ts +27 -0
  200. package/src/api/events/assistant-outbound-attachment.ts +3 -0
  201. package/src/api/events/assistant-turn-start.ts +3 -0
  202. package/src/api/events/desktop-activity-changed.ts +10 -0
  203. package/src/api/events/generation-handoff.ts +2 -0
  204. package/src/api/events/message-complete.ts +2 -0
  205. package/src/api/events/notification-conversation-created.ts +14 -16
  206. package/src/api/events/notification-intent.ts +4 -2
  207. package/src/api/events/question-request.ts +1 -0
  208. package/src/api/events/tool-result.ts +85 -0
  209. package/src/api/events/tool-use-start.ts +3 -0
  210. package/src/api/events/user-message-echo.ts +3 -0
  211. package/src/api/index.ts +42 -6
  212. package/src/api/mode-session.test.ts +131 -0
  213. package/src/api/mode-session.ts +148 -0
  214. package/src/api/remember-tool.ts +15 -0
  215. package/src/api/responses/conversation-message.ts +9 -0
  216. package/src/approvals/AGENTS.md +4 -2
  217. package/src/approvals/approval-primitive.ts +5 -2
  218. package/src/approvals/scoped-approval-grants.ts +6 -2
  219. package/src/browser/__tests__/operations.test.ts +29 -0
  220. package/src/browser/operation-meta.ts +2 -1
  221. package/src/browser/operations.ts +29 -0
  222. package/src/browser/virtual-desktop-target.ts +2 -9
  223. package/src/bundler/bundle-signer.ts +1 -3
  224. package/src/calls/__tests__/barge-in-guard.test.ts +3 -0
  225. package/src/calls/__tests__/call-controller.test.ts +94 -0
  226. package/src/calls/__tests__/call-funnel-telemetry.test.ts +204 -0
  227. package/src/calls/__tests__/voice-control-protocol.test.ts +15 -0
  228. package/src/calls/__tests__/voice-escalation-judge.test.ts +209 -0
  229. package/src/calls/__tests__/voice-leg-coordinator.test.ts +168 -0
  230. package/src/{live-voice/__tests__/live-voice-metrics.test.ts → calls/__tests__/voice-metrics.test.ts} +32 -38
  231. package/src/calls/__tests__/voice-session-bridge.test.ts +576 -122
  232. package/src/calls/barge-in-guard.ts +5 -0
  233. package/src/calls/call-controller.ts +196 -1
  234. package/src/calls/call-domain.ts +4 -0
  235. package/src/calls/call-store.ts +87 -2
  236. package/src/calls/guardian-action-sweep.ts +22 -8
  237. package/src/calls/media-stream-audio-transcode.ts +1 -3
  238. package/src/calls/typesafe-noul.ts +156 -0
  239. package/src/calls/voice-control-protocol.ts +6 -0
  240. package/src/calls/voice-escalation-judge.ts +145 -0
  241. package/src/calls/voice-escalation-target.ts +10 -0
  242. package/src/calls/voice-leg-coordinator.ts +121 -3
  243. package/src/{live-voice/live-voice-metrics.ts → calls/voice-metrics.ts} +94 -87
  244. package/src/calls/voice-session-bridge.ts +256 -112
  245. package/src/calls/voice-triage-escalate.ts +7 -0
  246. package/src/cli/__tests__/roadmap-cli.test.ts +4 -41
  247. package/src/cli/commands/__tests__/cli-test-harness.ts +21 -3
  248. package/src/cli/commands/__tests__/notifications.test.ts +35 -10
  249. package/src/cli/commands/__tests__/plugins.test.ts +20 -1
  250. package/src/cli/commands/__tests__/schedules.test.ts +14 -0
  251. package/src/cli/commands/bash.help.ts +4 -3
  252. package/src/cli/commands/browser.help.ts +9 -6
  253. package/src/cli/commands/browser.ts +0 -1
  254. package/src/cli/commands/clients.help.ts +46 -0
  255. package/src/cli/commands/clients.ts +116 -0
  256. package/src/cli/commands/credentials.help.ts +9 -4
  257. package/src/cli/commands/notifications.help.ts +16 -2
  258. package/src/cli/commands/notifications.ts +30 -8
  259. package/src/cli/commands/oauth/index.help.ts +10 -0
  260. package/src/cli/commands/oauth/providers.ts +12 -0
  261. package/src/cli/commands/oauth/request.test.ts +179 -2
  262. package/src/cli/commands/oauth/request.ts +41 -10
  263. package/src/cli/commands/plugins.ts +12 -5
  264. package/src/cli/commands/roadmap.help.ts +18 -21
  265. package/src/cli/commands/roadmap.ts +4 -21
  266. package/src/cli/commands/schedules.ts +2 -0
  267. package/src/cli/lib/__tests__/inspect-plugin.test.ts +54 -0
  268. package/src/cli/lib/__tests__/install-from-github.test.ts +41 -0
  269. package/src/cli/lib/__tests__/list-installed-plugins.test.ts +5 -2
  270. package/src/cli/lib/__tests__/local-plugin-upgrade.test.ts +133 -51
  271. package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +8 -2
  272. package/src/cli/lib/bundled-marketplace.json +906 -1
  273. package/src/cli/lib/bundled-plugin-packages.json +783 -1
  274. package/src/cli/lib/inspect-plugin.ts +11 -4
  275. package/src/cli/lib/plugin-marketplace.ts +8 -0
  276. package/src/cli/lib/publish-plugin.ts +1 -2
  277. package/src/cli/lib/upgrade-plugin.ts +19 -11
  278. package/src/cli/output.ts +18 -4
  279. package/src/config/__tests__/default-provider.test.ts +6 -0
  280. package/src/config/__tests__/memory-retrospective-schema.test.ts +14 -1
  281. package/src/config/bundled-skills/computer-use/SKILL.md +1 -1
  282. package/src/config/bundled-skills/computer-use/TOOLS.json +89 -1
  283. package/src/config/bundled-skills/image-studio/SKILL.md +17 -2
  284. package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +7 -11
  285. package/src/config/bundled-skills/schedule/SKILL.md +6 -6
  286. package/src/config/bundled-skills/sequences/TOOLS.json +1 -5
  287. package/src/config/call-site-defaults.ts +9 -0
  288. package/src/config/feature-flag-registry.json +16 -0
  289. package/src/config/loader.ts +9 -17
  290. package/src/config/platform-identity.ts +8 -8
  291. package/src/config/profile-text-generation.test.ts +51 -0
  292. package/src/config/profile-text-generation.ts +51 -0
  293. package/src/config/schemas/__tests__/memory-v3.test.ts +12 -0
  294. package/src/config/schemas/__tests__/tools.test.ts +31 -0
  295. package/src/config/schemas/call-site-catalog.ts +14 -0
  296. package/src/config/schemas/channels.ts +0 -18
  297. package/src/config/schemas/llm.ts +9 -2
  298. package/src/config/schemas/mcp.ts +6 -1
  299. package/src/config/schemas/memory-retrospective.ts +11 -2
  300. package/src/config/schemas/memory-v3.ts +12 -0
  301. package/src/config/schemas/services.ts +15 -6
  302. package/src/config/schemas/tools.ts +10 -0
  303. package/src/config/session-groups-gate.ts +21 -0
  304. package/src/config/skills.ts +2 -6
  305. package/src/conversations/__tests__/message-consolidation.test.ts +149 -0
  306. package/src/conversations/message-consolidation.ts +102 -5
  307. package/src/credential-execution/ces-connect.test.ts +69 -0
  308. package/src/credential-execution/ces-connect.ts +115 -0
  309. package/src/credential-execution/ces-runtime.ts +55 -93
  310. package/src/credential-execution/executable-discovery.ts +34 -142
  311. package/src/credential-execution/process-manager.test.ts +20 -24
  312. package/src/credential-execution/process-manager.ts +24 -19
  313. package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +10 -1
  314. package/src/daemon/__tests__/conversation-tool-setup.test.ts +43 -0
  315. package/src/daemon/__tests__/inference-profile-notification.test.ts +2 -0
  316. package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +36 -0
  317. package/src/daemon/assistant-attachments.ts +20 -12
  318. package/src/daemon/browser-mode-session.test.ts +402 -0
  319. package/src/daemon/browser-mode-session.ts +186 -0
  320. package/src/daemon/chat-credential-redaction.ts +16 -1
  321. package/src/daemon/computer-use-mode-session.test.ts +210 -0
  322. package/src/daemon/computer-use-mode-session.ts +105 -0
  323. package/src/daemon/config-watcher.ts +10 -0
  324. package/src/daemon/conversation-agent-loop-handlers.ts +132 -13
  325. package/src/daemon/conversation-agent-loop.ts +231 -25
  326. package/src/daemon/conversation-attachments.ts +93 -12
  327. package/src/daemon/conversation-evictor.ts +5 -24
  328. package/src/daemon/conversation-history.ts +10 -0
  329. package/src/daemon/conversation-interrupt-repair.ts +4 -43
  330. package/src/daemon/conversation-interrupt.ts +7 -11
  331. package/src/daemon/conversation-messaging.ts +58 -5
  332. package/src/daemon/conversation-mode-session.test.ts +1667 -0
  333. package/src/daemon/conversation-mode-session.ts +1008 -0
  334. package/src/daemon/conversation-process.ts +5 -0
  335. package/src/daemon/conversation-runtime-assembly.ts +10 -20
  336. package/src/daemon/conversation-skill-tools.ts +1 -0
  337. package/src/daemon/conversation-store.ts +4 -4
  338. package/src/daemon/conversation-surface-snapshots.ts +135 -0
  339. package/src/daemon/conversation-surfaces.ts +113 -5
  340. package/src/daemon/conversation-tool-setup.ts +79 -6
  341. package/src/daemon/conversation-turn-finalize.ts +26 -13
  342. package/src/daemon/conversation.ts +215 -58
  343. package/src/daemon/daemon-control.ts +3 -7
  344. package/src/daemon/handlers/config-model.ts +7 -5
  345. package/src/daemon/handlers/shared.ts +55 -16
  346. package/src/daemon/host-cu-proxy.ts +51 -22
  347. package/src/daemon/lifecycle.ts +30 -13
  348. package/src/daemon/mcp-reload-service.ts +1 -1
  349. package/src/daemon/message-protocol.ts +0 -1
  350. package/src/daemon/message-provenance.ts +28 -4
  351. package/src/daemon/message-types/shared.ts +1 -0
  352. package/src/daemon/mode-session-startup-recovery.ts +23 -0
  353. package/src/daemon/mode-session-tracking.ts +54 -0
  354. package/src/daemon/orphan-reaper.ts +4 -3
  355. package/src/daemon/persist-media-references.ts +20 -3
  356. package/src/daemon/process-message.ts +13 -4
  357. package/src/daemon/tool-setup-types.ts +6 -0
  358. package/src/daemon/virtual-desktop-context.ts +21 -0
  359. package/src/daemon/wake-conversation-ops.ts +50 -15
  360. package/src/desktop/__tests__/fake-desktop.ts +1 -0
  361. package/src/desktop/desktop-accessibility-script.ts +88 -0
  362. package/src/desktop/desktop-accessibility.test.ts +160 -0
  363. package/src/desktop/desktop-accessibility.ts +169 -0
  364. package/src/desktop/desktop-automation-lease.test.ts +302 -64
  365. package/src/desktop/desktop-automation-lease.ts +129 -54
  366. package/src/desktop/desktop-browser-endpoint.ts +4 -0
  367. package/src/desktop/desktop-computer-use-routing.test.ts +330 -0
  368. package/src/desktop/desktop-computer-use.test.ts +449 -0
  369. package/src/desktop/desktop-computer-use.ts +502 -0
  370. package/src/desktop/desktop-dependencies.ts +47 -293
  371. package/src/desktop/desktop-display.ts +2 -2
  372. package/src/desktop/desktop-help.ts +15 -0
  373. package/src/desktop/desktop-panel-config.test.ts +13 -3
  374. package/src/desktop/desktop-panel-config.ts +109 -15
  375. package/src/desktop/desktop-session-manager.test.ts +69 -21
  376. package/src/desktop/desktop-session-manager.ts +76 -22
  377. package/src/desktop/desktop-stream-bridge.test.ts +76 -132
  378. package/src/desktop/desktop-stream-bridge.ts +9 -11
  379. package/src/desktop/desktop-wallpaper-renderer.ts +132 -0
  380. package/src/desktop/desktop-wallpaper-worker.ts +17 -0
  381. package/src/desktop/desktop-wallpaper.test.ts +48 -7
  382. package/src/desktop/desktop-wallpaper.ts +34 -102
  383. package/src/desktop/desktop-window-theme.ts +5 -25
  384. package/src/desktop/virtual-desktop-feature.ts +53 -3
  385. package/src/desktop/virtual-desktop-platform.test.ts +5 -9
  386. package/src/documents/document-store.ts +23 -0
  387. package/src/heartbeat/__tests__/heartbeat-service.test.ts +22 -26
  388. package/src/heartbeat/heartbeat-service.ts +5 -33
  389. package/src/ipc/__tests__/browser-ipc.test.ts +215 -6
  390. package/src/ipc/assistant-server.ts +2 -0
  391. package/src/ipc/routes/__tests__/channel-reply-ipc-routes.test.ts +111 -0
  392. package/src/ipc/routes/channel-reply-ipc-routes.ts +73 -0
  393. package/src/live-voice/__tests__/continuation-judge.test.ts +151 -0
  394. package/src/live-voice/__tests__/live-activity-reporter.test.ts +154 -15
  395. package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +743 -31
  396. package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +191 -10
  397. package/src/live-voice/__tests__/live-voice-session-groups-gate.test.ts +87 -0
  398. package/src/live-voice/__tests__/live-voice-session-manager.test.ts +5 -1
  399. package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +41 -0
  400. package/src/live-voice/__tests__/live-voice-sight-frame.test.ts +1017 -9
  401. package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +224 -1
  402. package/src/live-voice/__tests__/live-voice-tts-session.test.ts +101 -0
  403. package/src/live-voice/__tests__/live-voice-tts.test.ts +44 -0
  404. package/src/live-voice/__tests__/live-voice-vad.test.ts +939 -75
  405. package/src/live-voice/__tests__/microphone-idle-gate.test.ts +87 -0
  406. package/src/live-voice/__tests__/session-controls.test.ts +23 -0
  407. package/src/live-voice/__tests__/task-outcomes.test.ts +117 -0
  408. package/src/live-voice/__tests__/voice-input-diagnostics.test.ts +117 -0
  409. package/src/live-voice/camera-mode-session.test.ts +265 -0
  410. package/src/live-voice/camera-mode-session.ts +277 -0
  411. package/src/live-voice/continuation-judge.ts +123 -0
  412. package/src/live-voice/live-activity-reporter.ts +115 -6
  413. package/src/live-voice/live-voice-manager.ts +36 -1
  414. package/src/live-voice/live-voice-photo.ts +88 -7
  415. package/src/live-voice/live-voice-session-manager.ts +15 -0
  416. package/src/live-voice/live-voice-session.ts +1443 -747
  417. package/src/live-voice/live-voice-tts.ts +22 -4
  418. package/src/live-voice/microphone-idle-gate.ts +56 -0
  419. package/src/live-voice/protocol.ts +171 -4
  420. package/src/live-voice/session-controls.ts +11 -3
  421. package/src/live-voice/task-outcomes.ts +131 -0
  422. package/src/live-voice/voice-input-diagnostics.ts +117 -0
  423. package/src/mcp/__tests__/manager-tool-caps.test.ts +156 -0
  424. package/src/mcp/__tests__/startup.test.ts +30 -7
  425. package/src/mcp/__tests__/tool-caps.test.ts +134 -0
  426. package/src/mcp/manager.ts +171 -101
  427. package/src/mcp/startup.ts +33 -13
  428. package/src/mcp/tool-caps.ts +184 -0
  429. package/src/media/__tests__/image-models.test.ts +58 -0
  430. package/src/media/image-credentials.ts +5 -0
  431. package/src/media/image-models.ts +52 -0
  432. package/src/media/image-service.ts +8 -0
  433. package/src/media/openrouter-image-service.test.ts +133 -0
  434. package/src/media/openrouter-image-service.ts +153 -0
  435. package/src/media/types.ts +1 -1
  436. package/src/messaging/provider-message-metadata.ts +3 -3
  437. package/src/messaging/providers/__tests__/transport-dispatch.test.ts +65 -11
  438. package/src/messaging/providers/discord/send.ts +4 -17
  439. package/src/messaging/providers/discord/transport.ts +2 -2
  440. package/src/messaging/providers/outlook/client.ts +16 -22
  441. package/src/messaging/providers/telegram-bot/send.test.ts +236 -1
  442. package/src/messaging/providers/telegram-bot/send.ts +10 -7
  443. package/src/messaging/providers/telegram-bot/transport.ts +3 -10
  444. package/src/messaging/providers/whatsapp/send.test.ts +169 -4
  445. package/src/monitoring/__tests__/control.test.ts +17 -5
  446. package/src/monitoring/control.ts +15 -5
  447. package/src/notifications/AGENTS.md +1 -1
  448. package/src/notifications/README.md +83 -99
  449. package/src/notifications/__tests__/broadcaster.test.ts +1 -1
  450. package/src/notifications/__tests__/channel-allowlist.test.ts +39 -0
  451. package/src/notifications/__tests__/copy-composer.test.ts +71 -1
  452. package/src/notifications/__tests__/decision-engine.test.ts +79 -2
  453. package/src/notifications/__tests__/deterministic-checks.test.ts +10 -10
  454. package/src/notifications/__tests__/home-feed-side-effect.test.ts +93 -4
  455. package/src/notifications/__tests__/notification-utils.test.ts +18 -0
  456. package/src/notifications/adapters/macos.ts +37 -46
  457. package/src/notifications/broadcaster.ts +5 -6
  458. package/src/notifications/channel-allowlist.ts +44 -0
  459. package/src/notifications/copy-composer.ts +11 -26
  460. package/src/notifications/decision-engine.ts +24 -14
  461. package/src/notifications/deterministic-checks.ts +1 -1
  462. package/src/notifications/emit-signal.ts +123 -37
  463. package/src/notifications/home-feed-side-effect.ts +9 -7
  464. package/src/notifications/notification-utils.ts +19 -0
  465. package/src/notifications/schedule-result-producer.ts +4 -6
  466. package/src/notifications/signal.ts +4 -19
  467. package/src/notifications/types.ts +15 -0
  468. package/src/oauth/AGENTS.md +2 -0
  469. package/src/oauth/__tests__/identity-verifier.test.ts +36 -1
  470. package/src/oauth/connection-resolver.ts +1 -2
  471. package/src/oauth/identity-verifier.ts +24 -0
  472. package/src/oauth/oauth-store.ts +12 -4
  473. package/src/oauth/provider-serializer.ts +8 -0
  474. package/src/oauth/seed-providers.ts +42 -0
  475. package/src/onboarding/onboarding-events-store.ts +99 -12
  476. package/src/permissions/confirmation-guardian-request.test.ts +16 -0
  477. package/src/permissions/confirmation-guardian-request.ts +2 -3
  478. package/src/permissions/question-prompter.test.ts +31 -0
  479. package/src/permissions/question-prompter.ts +2 -0
  480. package/src/persistence/__tests__/client-connection-events-store.test.ts +322 -0
  481. package/src/persistence/attachments-store.ts +84 -44
  482. package/src/persistence/client-connection-events-store.ts +377 -0
  483. package/src/persistence/conversation-crud.ts +166 -26
  484. package/src/persistence/conversation-mode-sessions.test.ts +424 -0
  485. package/src/persistence/conversation-mode-sessions.ts +460 -0
  486. package/src/persistence/conversation-plugin-facade.ts +30 -0
  487. package/src/persistence/conversation-queries.ts +6 -19
  488. package/src/persistence/conversation-tool-surface.ts +135 -0
  489. package/src/persistence/conversation-types.test.ts +32 -0
  490. package/src/persistence/conversation-types.ts +26 -4
  491. package/src/persistence/llm-usage-store.ts +3 -9
  492. package/src/persistence/message-metadata.ts +25 -0
  493. package/src/persistence/migrations/378-create-conversation-tool-surfaces.test.ts +78 -0
  494. package/src/persistence/migrations/378-create-conversation-tool-surfaces.ts +29 -0
  495. package/src/persistence/migrations/379-oauth-providers-response-ok-field.test.ts +92 -0
  496. package/src/persistence/migrations/379-oauth-providers-response-ok-field.ts +32 -0
  497. package/src/persistence/migrations/380-conversation-tool-surfaces-delegate-independent-tasks.test.ts +90 -0
  498. package/src/persistence/migrations/380-conversation-tool-surfaces-delegate-independent-tasks.ts +34 -0
  499. package/src/persistence/migrations/381-create-conversation-mode-sessions.ts +41 -0
  500. package/src/persistence/migrations/382-create-client-connection-events.ts +40 -0
  501. package/src/persistence/migrations/__tests__/382-create-client-connection-events.test.ts +56 -0
  502. package/src/persistence/schema/conversation-tool-surfaces.ts +32 -0
  503. package/src/persistence/schema/conversations.ts +67 -0
  504. package/src/persistence/schema/index.ts +1 -0
  505. package/src/persistence/schema/infrastructure.ts +29 -0
  506. package/src/persistence/schema/oauth.ts +1 -0
  507. package/src/persistence/steps.ts +41 -3
  508. package/src/plugin-api/__tests__/plugin-name-env.test.ts +249 -0
  509. package/src/plugin-api/conversation-turn.ts +31 -7
  510. package/src/plugin-api/effective-context-window.ts +33 -0
  511. package/src/plugin-api/index.ts +35 -6
  512. package/src/plugin-api/plugin-channel-turn-trust.test.ts +133 -0
  513. package/src/plugin-api/plugin-channel-turn-trust.ts +71 -0
  514. package/src/plugin-api/plugin-name-env.ts +125 -0
  515. package/src/plugin-api/resolve-credential.ts +79 -20
  516. package/src/plugin-api/vision-support.test.ts +22 -0
  517. package/src/plugin-api/vision-support.ts +17 -5
  518. package/src/plugins/__tests__/active-plugin-env.test.ts +136 -0
  519. package/src/plugins/active-plugin-env.ts +51 -0
  520. package/src/plugins/defaults/index.ts +24 -0
  521. package/src/plugins/defaults/memory/AGENTS.md +39 -5
  522. package/src/plugins/defaults/memory/__tests__/archive-removal-framing.test.ts +64 -0
  523. package/src/plugins/defaults/memory/__tests__/buffer-file.test.ts +365 -0
  524. package/src/plugins/defaults/memory/__tests__/buffer-format.test.ts +43 -0
  525. package/src/plugins/defaults/memory/__tests__/conversation-memory-purge.test.ts +1 -0
  526. package/src/plugins/defaults/memory/__tests__/db-memory-attach.test.ts +2 -0
  527. package/src/plugins/defaults/memory/__tests__/fixtures/buffer-appender.ts +17 -0
  528. package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +74 -0
  529. package/src/plugins/defaults/memory/__tests__/memory-run-evidence.test.ts +161 -0
  530. package/src/plugins/defaults/memory/__tests__/relocated-memory-test-rows.ts +10 -0
  531. package/src/plugins/defaults/memory/__tests__/worker-control.test.ts +19 -6
  532. package/src/plugins/defaults/memory/buffer-file.ts +354 -0
  533. package/src/plugins/defaults/memory/buffer-format.ts +40 -0
  534. package/src/plugins/defaults/memory/context-search/agent-protocol.ts +9 -12
  535. package/src/plugins/defaults/memory/context-search/agent-runner.ts +77 -22
  536. package/src/plugins/defaults/memory/context-search/format.ts +53 -8
  537. package/src/plugins/defaults/memory/context-search/limits.ts +10 -12
  538. package/src/plugins/defaults/memory/context-search/recall-input.ts +38 -0
  539. package/src/plugins/defaults/memory/context-search/search.ts +8 -14
  540. package/src/plugins/defaults/memory/context-search/sources/conversations.ts +1 -0
  541. package/src/plugins/defaults/memory/context-search/sources/memory-v2.ts +2 -1
  542. package/src/plugins/defaults/memory/context-search/sources/workspace.ts +2 -1
  543. package/src/plugins/defaults/memory/context-search/types.ts +1 -11
  544. package/src/plugins/defaults/memory/conversation-memory-purge.ts +4 -0
  545. package/src/plugins/defaults/memory/graph/__tests__/handle-remember-v2.test.ts +11 -4
  546. package/src/plugins/defaults/memory/graph/capability-seed.ts +1 -2
  547. package/src/plugins/defaults/memory/graph/tool-handlers.ts +13 -63
  548. package/src/plugins/defaults/memory/graph/tools.ts +35 -64
  549. package/src/plugins/defaults/memory/host-utils.ts +0 -10
  550. package/src/plugins/defaults/memory/injectors.ts +4 -3
  551. package/src/plugins/defaults/memory/memory-retrospective-job.ts +72 -183
  552. package/src/plugins/defaults/memory/memory-run-evidence.ts +213 -0
  553. package/src/plugins/defaults/memory/src/memory-eval-routes.ts +2 -3
  554. package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +6 -1
  555. package/src/plugins/defaults/memory/src/memory-item-routes.ts +2 -1
  556. package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts +407 -99
  557. package/src/plugins/defaults/memory/substrate/__tests__/consolidation-prompt-flag-gating-guard.test.ts +10 -0
  558. package/src/plugins/defaults/memory/substrate/__tests__/prompts-consolidation.test.ts +107 -7
  559. package/src/plugins/defaults/memory/substrate/consolidation-job.ts +307 -86
  560. package/src/plugins/defaults/memory/substrate/consolidation-tool-surface.ts +34 -0
  561. package/src/plugins/defaults/memory/substrate/page-index.ts +2 -1
  562. package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +91 -51
  563. package/src/plugins/defaults/memory/substrate/sweep-job.ts +1 -1
  564. package/src/plugins/defaults/memory/tools.test.ts +124 -9
  565. package/src/plugins/defaults/memory/tools.ts +38 -11
  566. package/src/plugins/defaults/memory/v1/graph/consolidation.ts +2 -2
  567. package/src/plugins/defaults/memory/v1/graph/extraction.ts +2 -1
  568. package/src/plugins/defaults/memory/v1/graph/retriever.ts +1 -1
  569. package/src/plugins/defaults/memory/v2/__tests__/migration.test.ts +5 -0
  570. package/src/plugins/defaults/memory/v2/__tests__/reranker.test.ts +5 -2
  571. package/src/plugins/defaults/memory/v2/reranker.ts +2 -1
  572. package/src/plugins/defaults/memory/v3/__tests__/injection.test.ts +81 -1
  573. package/src/plugins/defaults/memory/v3/__tests__/orchestrate.test.ts +87 -0
  574. package/src/plugins/defaults/memory/v3/__tests__/plugin-schema.test.ts +12 -0
  575. package/src/plugins/defaults/memory/v3/__tests__/pool-log-store.test.ts +332 -15
  576. package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +626 -27
  577. package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +148 -28
  578. package/src/plugins/defaults/memory/v3/card.ts +2 -1
  579. package/src/plugins/defaults/memory/v3/injector.ts +212 -178
  580. package/src/plugins/defaults/memory/v3/orchestrate.ts +159 -32
  581. package/src/plugins/defaults/memory/v3/plugin-schema.ts +54 -4
  582. package/src/plugins/defaults/memory/v3/pool-log-store.ts +275 -37
  583. package/src/plugins/defaults/memory/v3/pool-select.test.ts +47 -5
  584. package/src/plugins/defaults/memory/v3/pool-select.ts +710 -59
  585. package/src/plugins/defaults/memory/v3/sections.ts +2 -1
  586. package/src/plugins/defaults/memory/v3/shadow-plugin.ts +41 -14
  587. package/src/plugins/defaults/memory/worker-control.ts +15 -5
  588. package/src/plugins/defaults/task-progress/__tests__/registration.test.ts +70 -0
  589. package/src/plugins/defaults/task-progress/__tests__/task-progress-context.test.ts +380 -0
  590. package/src/plugins/defaults/task-progress/hooks/post-compact.ts +19 -0
  591. package/src/plugins/defaults/task-progress/hooks/user-prompt-submit.ts +24 -0
  592. package/src/plugins/defaults/task-progress/package.json +14 -0
  593. package/src/plugins/defaults/task-progress/src/apply-task-progress-context.ts +93 -0
  594. package/src/plugins/defaults/task-progress/src/format-task-progress-context.ts +67 -0
  595. package/src/plugins/defaults/tool-error/hooks/post-tool-use.ts +4 -1
  596. package/src/plugins/defaults/tool-result-truncate/terminal.ts +1 -46
  597. package/src/prompts/__tests__/parallel-tasks-section.test.ts +25 -0
  598. package/src/prompts/delegation-gate.ts +57 -0
  599. package/src/prompts/system-prompt.ts +14 -32
  600. package/src/providers/__tests__/context-overflow-error.test.ts +10 -0
  601. package/src/providers/__tests__/search-provider-catalog.test.ts +18 -0
  602. package/src/providers/anthropic/client.ts +10 -1
  603. package/src/providers/anthropic-gateway-shared.ts +6 -0
  604. package/src/providers/fetch-provider-catalog.ts +11 -0
  605. package/src/providers/gemini/client.ts +25 -8
  606. package/src/providers/inference/adapter-factory.ts +6 -0
  607. package/src/providers/jev/client.test.ts +270 -0
  608. package/src/providers/jev/client.ts +534 -0
  609. package/src/providers/model-catalog.ts +59 -3
  610. package/src/providers/openai/__tests__/chat-completions-extra-body.test.ts +94 -1
  611. package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +106 -1
  612. package/src/providers/openai/__tests__/chat-template-flatten.test.ts +24 -0
  613. package/src/providers/openai/__tests__/google-thought-signature.test.ts +1 -2
  614. package/src/providers/openai/chat-completions-provider.ts +100 -29
  615. package/src/providers/openai/google-thought-signature.ts +0 -6
  616. package/src/providers/openai/responses-provider.ts +23 -1
  617. package/src/providers/ratelimit.ts +1 -3
  618. package/src/providers/search-provider-catalog.ts +24 -0
  619. package/src/providers/server-tool-pairing.ts +16 -7
  620. package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +24 -0
  621. package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +7 -1
  622. package/src/providers/speech-to-text/__tests__/vellum-managed-realtime.test.ts +50 -1
  623. package/src/providers/speech-to-text/deepgram-flux-frames.ts +11 -0
  624. package/src/providers/speech-to-text/vellum-managed-flux-realtime.ts +13 -0
  625. package/src/providers/types.ts +5 -0
  626. package/src/routes/control.ts +7 -5
  627. package/src/runtime/AGENTS.md +6 -4
  628. package/src/runtime/__tests__/agent-wake.test.ts +94 -1
  629. package/src/runtime/actor-trust-resolver.ts +3 -3
  630. package/src/runtime/agent-wake.ts +32 -5
  631. package/src/runtime/approval-message-composer.ts +3 -69
  632. package/src/runtime/assistant-event-hub.ts +73 -75
  633. package/src/runtime/assistant-event-publish-options.ts +7 -0
  634. package/src/runtime/assistant-event-targeting.ts +79 -0
  635. package/src/runtime/assistant-stream-state.ts +13 -100
  636. package/src/runtime/background-job-runner.ts +5 -5
  637. package/src/runtime/channel-retry-sweep.ts +1 -0
  638. package/src/runtime/guardian-action-message-composer.ts +8 -161
  639. package/src/runtime/guardian-action-service.ts +2 -17
  640. package/src/runtime/guardian-reply-router.ts +29 -40
  641. package/src/runtime/http-server.ts +2 -2
  642. package/src/runtime/http-types.ts +6 -0
  643. package/src/runtime/message-composer-types.ts +3 -61
  644. package/src/runtime/migrations/__tests__/vbundle-import-parity.test.ts +67 -1
  645. package/src/runtime/migrations/__tests__/vbundle-import-policy.test.ts +89 -0
  646. package/src/runtime/migrations/vbundle-builder.ts +24 -1
  647. package/src/runtime/migrations/vbundle-import-analyzer.ts +13 -4
  648. package/src/runtime/migrations/vbundle-import-policy.ts +43 -11
  649. package/src/runtime/migrations/vbundle-importer.ts +4 -1
  650. package/src/runtime/migrations/vbundle-streaming-importer.ts +4 -1
  651. package/src/runtime/migrations/vbundle-validator.ts +1 -0
  652. package/src/runtime/pending-interactions-mode-session.test.ts +62 -0
  653. package/src/runtime/pending-interactions.ts +29 -0
  654. package/src/runtime/routes/__tests__/client-history-routes.test.ts +104 -0
  655. package/src/runtime/routes/__tests__/image-generation-routes.test.ts +97 -0
  656. package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +24 -2
  657. package/src/runtime/routes/__tests__/roadmap-routes.test.ts +29 -6
  658. package/src/runtime/routes/browser-context.ts +31 -11
  659. package/src/runtime/routes/browser-routes.ts +43 -4
  660. package/src/runtime/routes/btw-routes.ts +9 -4
  661. package/src/runtime/routes/channel-route-shared.ts +1 -9
  662. package/src/runtime/routes/client-routes.ts +108 -0
  663. package/src/runtime/routes/conversation-management-routes.ts +4 -1
  664. package/src/runtime/routes/conversation-routes.ts +125 -14
  665. package/src/runtime/routes/credential-prompt-routes.ts +4 -3
  666. package/src/runtime/routes/credential-routes.ts +1 -2
  667. package/src/runtime/routes/desktop-setup-routes.test.ts +9 -18
  668. package/src/runtime/routes/desktop-setup-routes.ts +8 -7
  669. package/src/runtime/routes/documents-routes.ts +51 -0
  670. package/src/runtime/routes/email-routes.ts +3 -7
  671. package/src/runtime/routes/events-routes.ts +75 -54
  672. package/src/runtime/routes/guardian-approval-interception.ts +24 -0
  673. package/src/runtime/routes/identity-routes.ts +1 -5
  674. package/src/runtime/routes/image-generation-routes.ts +8 -14
  675. package/src/runtime/routes/inbound-message-handler.ts +37 -41
  676. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.test.ts +168 -0
  677. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.ts +96 -0
  678. package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +39 -10
  679. package/src/runtime/routes/inbound-stages/background-dispatch.ts +9 -9
  680. package/src/runtime/routes/inbound-stages/reaction-intercept.ts +2 -0
  681. package/src/runtime/routes/inference-profile-session-handler.ts +11 -1
  682. package/src/runtime/routes/inference-profiles-routes.ts +9 -0
  683. package/src/runtime/routes/llm-context-normalization.ts +1 -7
  684. package/src/runtime/routes/migration-routes.ts +73 -2
  685. package/src/runtime/routes/notification-routes.ts +10 -0
  686. package/src/runtime/routes/oauth-commands-routes.ts +57 -54
  687. package/src/runtime/routes/oauth-providers.ts +11 -0
  688. package/src/runtime/routes/oauth-request-hints.test.ts +246 -0
  689. package/src/runtime/routes/oauth-request-hints.ts +149 -0
  690. package/src/runtime/routes/plugins-routes.ts +6 -0
  691. package/src/runtime/routes/roadmap-routes.ts +47 -8
  692. package/src/runtime/routes/schedule-routes.ts +2 -0
  693. package/src/runtime/routes/secret-routes.ts +99 -119
  694. package/src/runtime/routes/settings-routes.ts +10 -4
  695. package/src/runtime/routes/watcher-routes.ts +1 -3
  696. package/src/runtime/trust-class.test.ts +7 -6
  697. package/src/runtime/trust-class.ts +9 -17
  698. package/src/schedule/__tests__/worker-control.test.ts +36 -5
  699. package/src/schedule/__tests__/worker-mcp-bootstrap.test.ts +107 -0
  700. package/src/schedule/__tests__/worker-mcp-readiness.test.ts +173 -0
  701. package/src/schedule/__tests__/worker-mcp-tools.test.ts +13 -47
  702. package/src/schedule/run-script.ts +2 -2
  703. package/src/schedule/scheduler.ts +47 -11
  704. package/src/schedule/tool-surface-readiness.ts +65 -0
  705. package/src/schedule/worker-control.ts +15 -5
  706. package/src/schedule/worker-mcp.ts +85 -0
  707. package/src/schedule/worker.ts +9 -31
  708. package/src/security/secure-keys.ts +74 -112
  709. package/src/skills/clawhub.ts +3 -8
  710. package/src/skills/managed-store.ts +1 -5
  711. package/src/stt/types.ts +4 -0
  712. package/src/subagent/notify.ts +54 -34
  713. package/src/subagent/parent-notification.ts +6 -0
  714. package/src/telemetry/__tests__/phone-call-funnel.test.ts +72 -0
  715. package/src/telemetry/phone-call-funnel.ts +144 -0
  716. package/src/tools/__tests__/tool-schema-root-combinator-guard.test.ts +81 -0
  717. package/src/tools/ask-question/ask-question-tool.test.ts +167 -5
  718. package/src/tools/ask-question/ask-question-tool.ts +120 -22
  719. package/src/tools/browser/browser-execution.ts +29 -60
  720. package/src/tools/browser/cdp-client/__tests__/cdp-dom-helpers.test.ts +561 -39
  721. package/src/tools/browser/cdp-client/cdp-dom-helpers.ts +435 -39
  722. package/src/tools/browser/cdp-client/cdp-inspect-client.ts +1 -8
  723. package/src/tools/client-os.ts +15 -0
  724. package/src/tools/computer-use/skill-proxy-bridge.ts +1 -1
  725. package/src/tools/computer-use/target.ts +42 -0
  726. package/src/tools/document/document-tool.ts +13 -18
  727. package/src/tools/execution-target.ts +14 -16
  728. package/src/tools/host-terminal/host-shell.ts +16 -7
  729. package/src/tools/network/__tests__/web-fetch-tinyfish.test.ts +250 -0
  730. package/src/tools/network/__tests__/web-search.test.ts +203 -0
  731. package/src/tools/network/web-fetch.ts +274 -7
  732. package/src/tools/network/web-search.ts +468 -6
  733. package/src/tools/permission-checker.ts +3 -3
  734. package/src/tools/policy-context.ts +3 -1
  735. package/src/tools/shared/filesystem/file-ops-service.ts +1 -31
  736. package/src/tools/shared/shell-output.test.ts +10 -0
  737. package/src/tools/shared/shell-output.ts +14 -2
  738. package/src/tools/skills/load.ts +2 -1
  739. package/src/tools/skills/sandbox-runner.ts +24 -4
  740. package/src/tools/skills/scaffold-managed.ts +2 -1
  741. package/src/tools/skills/skill-script-runner.ts +7 -0
  742. package/src/tools/skills/skill-tool-factory.ts +15 -1
  743. package/src/tools/terminal/__tests__/safe-env.test.ts +44 -4
  744. package/src/tools/terminal/__tests__/sanitized-bash.test.ts +0 -14
  745. package/src/tools/terminal/safe-env.ts +39 -24
  746. package/src/tools/terminal/sanitized-bash.ts +15 -2
  747. package/src/tools/terminal/shell-launch.test.ts +162 -0
  748. package/src/tools/terminal/shell.test.ts +29 -0
  749. package/src/tools/terminal/shell.ts +15 -7
  750. package/src/tools/tool-approval-handler.ts +5 -1
  751. package/src/tools/tool-types.ts +3 -3
  752. package/src/tools/types.ts +8 -2
  753. package/src/util/__tests__/worker-process-command.test.ts +5 -4
  754. package/src/util/abort-reasons.ts +18 -26
  755. package/src/util/browser-human-verification.ts +14 -0
  756. package/src/util/canonicalize-identity.test.ts +21 -0
  757. package/src/util/canonicalize-identity.ts +16 -0
  758. package/src/util/errors.ts +13 -3
  759. package/src/util/host-process.test.ts +17 -1
  760. package/src/util/host-process.ts +24 -0
  761. package/src/util/provider-error-patterns.ts +3 -0
  762. package/src/util/unicode.ts +29 -0
  763. package/src/util/worker-process.ts +77 -35
  764. package/src/workspace/migrations/157-add-desktop-files-launcher.ts +94 -0
  765. package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +4 -10
  766. package/src/workspace/migrations/__tests__/157-add-desktop-files-launcher.test.ts +113 -0
  767. package/src/workspace/migrations/registry.ts +2 -0
  768. package/src/__tests__/computer-use-window-schema.test.ts +0 -24
  769. package/src/api/events/heartbeat-alert.ts +0 -20
  770. package/src/daemon/message-types/web-activity.ts +0 -65
  771. package/src/desktop/desktop-dependencies.test.ts +0 -131
  772. package/src/tools/computer-use/definitions.ts +0 -607
@@ -1,3 +1,4 @@
1
+ import type { Conversation } from "../../daemon/conversation.js";
1
2
  import { resolveTurnClientOs } from "../../daemon/conversation-client-surface.js";
2
3
  import { findConversation } from "../../daemon/conversation-registry.js";
3
4
  import type { ToolContext } from "../../tools/types.js";
@@ -13,6 +14,22 @@ export async function resolveBrowserContext(
13
14
  headers: Record<string, string>,
14
15
  abortSignal?: AbortSignal,
15
16
  ): Promise<ToolContext> {
17
+ return (
18
+ await resolveBrowserExecutionContext(
19
+ conversationId,
20
+ sessionId,
21
+ headers,
22
+ abortSignal,
23
+ )
24
+ ).context;
25
+ }
26
+
27
+ export async function resolveBrowserExecutionContext(
28
+ conversationId: string | undefined,
29
+ sessionId: string,
30
+ headers: Record<string, string>,
31
+ abortSignal?: AbortSignal,
32
+ ): Promise<{ context: ToolContext; conversation?: Conversation }> {
16
33
  const conversation = conversationId
17
34
  ? findConversation(conversationId)
18
35
  : undefined;
@@ -24,16 +41,19 @@ export async function resolveBrowserContext(
24
41
  (signal): signal is AbortSignal => signal !== undefined,
25
42
  );
26
43
  return {
27
- workingDir: process.cwd(),
28
- conversationId: conversation
29
- ? conversationId!
30
- : browserCliConversationKey(sessionId),
31
- trustClass: conversation?.trustContext?.trustClass ?? "unknown",
32
- transportInterface: conversation?.transportInterface,
33
- clientOs: conversation
34
- ? resolveTurnClientOs(conversation).clientOs
35
- : undefined,
36
- sourceActorPrincipalId: actor,
37
- signal: signals.length ? AbortSignal.any(signals) : undefined,
44
+ context: {
45
+ workingDir: process.cwd(),
46
+ conversationId: conversation
47
+ ? conversationId!
48
+ : browserCliConversationKey(sessionId),
49
+ trustClass: conversation?.trustContext?.trustClass ?? "unknown",
50
+ transportInterface: conversation?.transportInterface,
51
+ clientOs: conversation
52
+ ? resolveTurnClientOs(conversation).clientOs
53
+ : undefined,
54
+ sourceActorPrincipalId: actor,
55
+ signal: signals.length ? AbortSignal.any(signals) : undefined,
56
+ },
57
+ ...(conversation ? { conversation } : {}),
38
58
  };
39
59
  }
@@ -11,16 +11,20 @@
11
11
 
12
12
  import { z } from "zod";
13
13
 
14
- import { executeBrowserOperation } from "../../browser/operations.js";
14
+ import {
15
+ browserOperationLifecycle,
16
+ executeBrowserOperation,
17
+ } from "../../browser/operations.js";
15
18
  import {
16
19
  BROWSER_OPERATIONS,
17
20
  type BrowserOperation,
18
21
  } from "../../browser/types.js";
19
22
  import { shouldUseVirtualDesktopBrowser } from "../../browser/virtual-desktop-target.js";
23
+ import { bestEffortModeSessionTracking } from "../../daemon/mode-session-tracking.js";
20
24
  import { executeDesktopBrowserOperation } from "../../desktop/desktop-browser-operations.js";
21
25
  import type { ContentBlock } from "../../providers/types.js";
22
26
  import { LOCAL_PRINCIPALS } from "../auth/route-policy.js";
23
- import { resolveBrowserContext } from "./browser-context.js";
27
+ import { resolveBrowserExecutionContext } from "./browser-context.js";
24
28
  export { browserCliConversationKey } from "./browser-context.js";
25
29
  import type { RouteDefinition, RouteHandlerArgs } from "./types.js";
26
30
 
@@ -68,16 +72,51 @@ async function handleBrowserExecute({
68
72
  const { operation, input, sessionId, conversationId, desktop } =
69
73
  BrowserExecuteParams.parse(body);
70
74
 
71
- const context = await resolveBrowserContext(
75
+ const resolved = await resolveBrowserExecutionContext(
72
76
  conversationId,
73
77
  sessionId,
74
78
  headers,
75
79
  abortSignal,
76
80
  );
81
+ const { context, conversation } = resolved;
82
+ const typedOperation = operation as BrowserOperation;
83
+ const operationToken = conversation?.currentRequestId
84
+ ? bestEffortModeSessionTracking("browser admission", () =>
85
+ conversation.browserModeSessions.beginOperation({
86
+ turnId: conversation.currentRequestId!,
87
+ lifecycle: browserOperationLifecycle(typedOperation),
88
+ at: Date.now(),
89
+ }),
90
+ )
91
+ : undefined;
77
92
  const execute = shouldUseVirtualDesktopBrowser(desktop, input, context)
78
93
  ? executeDesktopBrowserOperation
79
94
  : executeBrowserOperation;
80
- const result = await execute(operation as BrowserOperation, input, context);
95
+ const finishOperationTracking = (isError: boolean) => {
96
+ if (!operationToken) {
97
+ return;
98
+ }
99
+ bestEffortModeSessionTracking("browser completion", () =>
100
+ conversation?.browserModeSessions.finishOperation(operationToken, {
101
+ at: Date.now(),
102
+ isError,
103
+ cancelled: context.signal?.aborted === true,
104
+ ...(typedOperation === "close"
105
+ ? { terminalReason: "browser_closed" as const }
106
+ : typedOperation === "detach"
107
+ ? { terminalReason: "browser_detached" as const }
108
+ : {}),
109
+ }),
110
+ );
111
+ };
112
+ let result;
113
+ try {
114
+ result = await execute(typedOperation, input, context);
115
+ } catch (error) {
116
+ finishOperationTracking(true);
117
+ throw error;
118
+ }
119
+ finishOperationTracking(result.isError);
81
120
 
82
121
  const screenshots = extractScreenshots(result.contentBlocks);
83
122
 
@@ -2,9 +2,10 @@
2
2
  * Route handler for the POST /v1/btw SSE-streaming side-chain endpoint.
3
3
  *
4
4
  * Runs an ephemeral LLM call that reuses the conversation's provider, tool
5
- * definitions, and message history for prompt-cache efficiency. Uses the
6
- * conversation's system prompt when a conversation-specific override is active;
7
- * otherwise builds a fresh prompt excluding BOOTSTRAP.md so first-run
5
+ * definitions, and message history for prompt-cache efficiency; the
6
+ * empty-state greeting targets no real conversation and sends no tools. Uses
7
+ * the conversation's system prompt when a conversation-specific override is
8
+ * active; otherwise builds a fresh prompt excluding BOOTSTRAP.md so first-run
8
9
  * onboarding instructions don't leak into cosmetic UI calls like identity
9
10
  * intro generation. The response is streamed as SSE events (`btw_text_delta`,
10
11
  * `btw_complete`, `btw_error`).
@@ -145,7 +146,11 @@ async function handleBtw({
145
146
  const result = await runBtwSidechain({
146
147
  content: effectiveContent,
147
148
  conversation,
148
- tools: getAllToolDefinitions(),
149
+ // The side-chain forces `tool_choice: none`, so tool definitions
150
+ // only earn their tokens as a shared cache prefix with a real
151
+ // conversation. The greeting runs against an ephemeral one with its
152
+ // own system prompt, so it shares nothing and sends no tools.
153
+ tools: isGreeting ? [] : getAllToolDefinitions(),
149
154
  signal: abortSignal,
150
155
  ...(isGreeting ? { callSite: "emptyStateGreeting" as const } : {}),
151
156
  onEvent: (event) => {
@@ -29,13 +29,6 @@ export function requiredDecisionKeywords(
29
29
  // Callback data parser — format: "apr:<requestId>:<action>"
30
30
  // ---------------------------------------------------------------------------
31
31
 
32
- /** Map legacy callback actions to canonical ones for in-flight buttons. */
33
- const LEGACY_CALLBACK_MAP: Record<string, string> = {
34
- approve_10m: "approve_once",
35
- approve_conversation: "approve_once",
36
- approve_always: "approve_once",
37
- };
38
-
39
32
  export function parseCallbackData(
40
33
  data: string,
41
34
  sourceChannel?: string,
@@ -45,8 +38,7 @@ export function parseCallbackData(
45
38
  return null;
46
39
  }
47
40
  const requestId = parts[1];
48
- const rawAction = parts.slice(2).join(":");
49
- const action = LEGACY_CALLBACK_MAP[rawAction] ?? rawAction;
41
+ const action = parts.slice(2).join(":");
50
42
  if (!requestId || !isApprovalAction(action)) {
51
43
  return null;
52
44
  }
@@ -9,6 +9,7 @@ import { z } from "zod";
9
9
 
10
10
  import type { HostProxyCapability } from "../../channels/types.js";
11
11
  import { isHttpAuthDisabled } from "../../config/env.js";
12
+ import { listClientConnectionHistory } from "../../persistence/client-connection-events-store.js";
12
13
  import { datesToISO } from "../../util/json.js";
13
14
  import { getLogger } from "../../util/logger.js";
14
15
  import {
@@ -112,6 +113,8 @@ export const ROUTES: RouteDefinition[] = [
112
113
  interfaceId: c.interfaceId,
113
114
  capabilities: c.capabilities,
114
115
  machineName: c.machineName,
116
+ clientVersion: c.clientVersion,
117
+ sseWatchdog: c.sseWatchdog,
115
118
  connectedAt: c.connectedAt,
116
119
  lastActiveAt: c.lastActiveAt,
117
120
  degraded: isClientDegraded(
@@ -151,6 +154,79 @@ export const ROUTES: RouteDefinition[] = [
151
154
  return { disconnected: count };
152
155
  },
153
156
  },
157
+ {
158
+ operationId: "list_client_history",
159
+ endpoint: "clients/history",
160
+ method: "GET",
161
+ policy: {
162
+ requiredScopes: ["settings.read"],
163
+ allowedPrincipalTypes: ACTOR_PRINCIPALS,
164
+ },
165
+ summary: "List client connection history",
166
+ description:
167
+ "Return persisted client subscribe/dispose events and coalesced sessions. Reconnects shorter than 60 seconds are flaps, not outages.",
168
+ tags: ["clients"],
169
+ queryParams: [
170
+ {
171
+ name: "clientId",
172
+ type: "string",
173
+ required: false,
174
+ description: "Restrict history to one client UUID.",
175
+ },
176
+ {
177
+ name: "interfaceId",
178
+ type: "string",
179
+ required: false,
180
+ description: "Restrict history to one interface (e.g. chrome-extension).",
181
+ },
182
+ {
183
+ name: "since",
184
+ type: "string",
185
+ required: false,
186
+ description:
187
+ "ISO-8601 or epoch-ms lower bound. Sessions already open at this time are included.",
188
+ },
189
+ {
190
+ name: "limit",
191
+ type: "string",
192
+ required: false,
193
+ description: "Max coalesced sessions to return (1-500, default 50).",
194
+ },
195
+ ],
196
+ responseBody: z.object({
197
+ sessions: z.array(z.object({}).passthrough()),
198
+ events: z.array(z.object({}).passthrough()),
199
+ }),
200
+ handler: ({ queryParams, headers }) => {
201
+ const since = parseHistorySince(queryParams?.since);
202
+ const limit = parseHistoryLimit(queryParams?.limit);
203
+ const callerPrincipalId = headers?.["x-vellum-actor-principal-id"];
204
+ const history = listClientConnectionHistory({
205
+ clientId: queryParams?.clientId?.trim() || undefined,
206
+ interfaceId: queryParams?.interfaceId?.trim() || undefined,
207
+ actorPrincipalId: isHttpAuthDisabled()
208
+ ? undefined
209
+ : callerPrincipalId || "__none__",
210
+ since,
211
+ limit,
212
+ });
213
+
214
+ return {
215
+ sessions: history.sessions.map((session) => ({
216
+ ...session,
217
+ startedAt: new Date(session.startedAt).toISOString(),
218
+ endedAt:
219
+ session.endedAt == null
220
+ ? null
221
+ : new Date(session.endedAt).toISOString(),
222
+ })),
223
+ events: history.events.map((event) => ({
224
+ ...event,
225
+ occurredAt: new Date(event.occurredAt).toISOString(),
226
+ })),
227
+ };
228
+ },
229
+ },
154
230
  {
155
231
  operationId: "report_client_presence",
156
232
  endpoint: "clients/presence",
@@ -342,3 +418,35 @@ export const ROUTES: RouteDefinition[] = [
342
418
  },
343
419
  },
344
420
  ];
421
+
422
+ function parseHistorySince(raw: string | undefined): number | undefined {
423
+ if (!raw?.trim()) {
424
+ return undefined;
425
+ }
426
+ const trimmed = raw.trim();
427
+ if (/^\d+$/.test(trimmed)) {
428
+ const epoch = Number(trimmed);
429
+ if (!Number.isFinite(epoch) || epoch < 0) {
430
+ throw new BadRequestError("since must be a non-negative epoch-ms value");
431
+ }
432
+ return epoch;
433
+ }
434
+ const parsed = Date.parse(trimmed);
435
+ if (Number.isNaN(parsed)) {
436
+ throw new BadRequestError(
437
+ "since must be an ISO-8601 timestamp or epoch milliseconds",
438
+ );
439
+ }
440
+ return parsed;
441
+ }
442
+
443
+ function parseHistoryLimit(raw: string | undefined): number | undefined {
444
+ if (!raw?.trim()) {
445
+ return undefined;
446
+ }
447
+ const parsed = Number(raw.trim());
448
+ if (!Number.isInteger(parsed) || parsed < 1) {
449
+ throw new BadRequestError("limit must be a positive integer");
450
+ }
451
+ return parsed;
452
+ }
@@ -19,7 +19,6 @@
19
19
  * POST /v1/conversations/:id/retry — retry the last assistant turn
20
20
  * POST /v1/conversations/reorder — reorder / pin conversations
21
21
  */
22
-
23
22
  import { v7 as uuidv7 } from "uuid";
24
23
  import { z } from "zod";
25
24
 
@@ -43,6 +42,7 @@ import {
43
42
  undoLastMessage,
44
43
  } from "../../daemon/handlers/conversations.js";
45
44
  import { normalizeConversationType } from "../../daemon/message-types/shared.js";
45
+ import { bestEffortModeSessionTracking } from "../../daemon/mode-session-tracking.js";
46
46
  import { stripConversationIds } from "../../home/feed-writer.js";
47
47
  import {
48
48
  archiveConversation,
@@ -732,6 +732,9 @@ async function handleRetryLastAssistantTurn({
732
732
  if (!discarded) {
733
733
  throw new UnprocessableEntityError("No user message to retry from");
734
734
  }
735
+ bestEffortModeSessionTracking("conversation wait invalidation", () =>
736
+ conversation.modeSessions.invalidateAllStructuralWaits(),
737
+ );
735
738
  } catch (err) {
736
739
  if (conversation.abortController === abortController) {
737
740
  conversation.abortController = null;
@@ -18,6 +18,7 @@ import {
18
18
  createUserMessage,
19
19
  } from "../../agent/message-types.js";
20
20
  import type { AssistantEvent } from "../../api/index.js";
21
+ import { ModeSessionDescriptorSchema } from "../../api/mode-session.js";
21
22
  import {
22
23
  BackgroundToolCompletionSchema,
23
24
  type ConversationContentBlock,
@@ -44,10 +45,8 @@ import { isAssistantFeatureFlagEnabled } from "../../config/assistant-feature-fl
44
45
  import { getUserSelectableProfilesForProvider } from "../../config/default-profile-catalog.js";
45
46
  import { isHttpAuthDisabled } from "../../config/env.js";
46
47
  import { getConfig } from "../../config/loader.js";
47
- import {
48
- mergeConsecutiveAssistantMessages,
49
- mergeToolResultsIntoAssistantMessages,
50
- } from "../../conversations/message-consolidation.js";
48
+ import { isModeSessionRecoveryHealthy } from "../../config/session-groups-gate.js";
49
+ import { consolidateMessageRows } from "../../conversations/message-consolidation.js";
51
50
  import { resolveTurnCommitWaitMs } from "../../daemon/abort-watchdog.js";
52
51
  import { createApprovalConversationGenerator } from "../../daemon/approval-generators.js";
53
52
  import type { Conversation } from "../../daemon/conversation.js";
@@ -113,7 +112,7 @@ import { recordOnboardingEvent } from "../../onboarding/onboarding-events-store.
113
112
  import {
114
113
  classifyKind,
115
114
  getAttachmentById,
116
- getAttachmentMetadataForMessage,
115
+ getAttachmentMetadataForMessages,
117
116
  getAttachmentsByIds,
118
117
  resolveAttachmentsForPersist,
119
118
  } from "../../persistence/attachments-store.js";
@@ -132,6 +131,7 @@ import {
132
131
  isSuppressedQueuedMessage,
133
132
  isSystemCardMetadata,
134
133
  type MessageRow,
134
+ parseMessageMetadata,
135
135
  recordConversationPersistedSeq,
136
136
  setConversationInferenceProfile,
137
137
  } from "../../persistence/conversation-crud.js";
@@ -139,12 +139,18 @@ import {
139
139
  getConversationByKey,
140
140
  getOrCreateConversation,
141
141
  } from "../../persistence/conversation-key-store.js";
142
+ import { listConversationModeSessionsByIds } from "../../persistence/conversation-mode-sessions.js";
142
143
  import { searchConversations } from "../../persistence/conversation-queries.js";
143
144
  import {
145
+ computerUseScreenshotAttachmentIdsFromMetadata,
144
146
  isNoResponseMetadata,
145
147
  messageMetadataIsAmbientSightKeep,
146
148
  } from "../../persistence/conversation-types.js";
147
149
  import { linkRequestLogsToMessage } from "../../persistence/llm-request-log-store.js";
150
+ import {
151
+ mergeMessageMetadata,
152
+ readModeSessionMetadata,
153
+ } from "../../persistence/message-metadata.js";
148
154
  import { assistantTextVisibilityOf } from "../../persistence/user-facing-content.js";
149
155
  import { MEMORY_RETROSPECTIVE_FORK_SOURCE } from "../../plugins/defaults/memory/memory-retrospective-constants.js";
150
156
  import { normalizeOnboardingContext } from "../../prompts/normalize-onboarding.js";
@@ -879,10 +885,31 @@ function displayRowFilter(conversationId: string): (m: MessageRow) => boolean {
879
885
  function consolidateDisplayTurns(rows: MessageRow[]): {
880
886
  messages: MessageRow[];
881
887
  mergedIdMap: Map<string, string[]>;
888
+ modeSessionActivityMap: Map<string, { firstAt: number; lastAt: number }>;
882
889
  } {
883
- return mergeConsecutiveAssistantMessages(
884
- mergeToolResultsIntoAssistantMessages(rows),
885
- );
890
+ return consolidateMessageRows(rows);
891
+ }
892
+
893
+ const MAX_REQUESTED_MODE_SESSION_IDS = 100;
894
+ const MAX_MODE_SESSION_ID_LENGTH = 200;
895
+
896
+ function parseRequestedModeSessionIds(raw: unknown): string[] {
897
+ if (raw === undefined) {
898
+ return [];
899
+ }
900
+ if (typeof raw !== "string") {
901
+ throw new BadRequestError(
902
+ "modeSessionIds must be a comma-separated string",
903
+ );
904
+ }
905
+ const ids = [...new Set(raw.split(",").map((id) => id.trim()))];
906
+ if (
907
+ ids.length > MAX_REQUESTED_MODE_SESSION_IDS ||
908
+ ids.some((id) => id.length === 0 || id.length > MAX_MODE_SESSION_ID_LENGTH)
909
+ ) {
910
+ throw new BadRequestError("modeSessionIds contains an invalid session ID");
911
+ }
912
+ return ids;
886
913
  }
887
914
 
888
915
  export async function handleListMessages({
@@ -918,6 +945,9 @@ export async function handleListMessages({
918
945
  const beforeTimestampRaw = queryParams?.beforeTimestamp;
919
946
  const limitRaw = queryParams?.limit;
920
947
  const pageRaw = queryParams?.page;
948
+ const requestedModeSessionIds = parseRequestedModeSessionIds(
949
+ queryParams?.modeSessionIds,
950
+ );
921
951
 
922
952
  // Validate: reject NaN values with 400
923
953
  if (beforeTimestampRaw != null && isNaN(Number(beforeTimestampRaw))) {
@@ -1021,8 +1051,47 @@ export async function handleListMessages({
1021
1051
  // consolidation is deferred to compaction for prefix-cache stability.
1022
1052
  // Merge here at query time so renderHistoryContent produces the same
1023
1053
  // contentOrder shape as streaming (consecutive tool refs grouped together).
1024
- const { messages: consolidatedMessages, mergedIdMap } =
1025
- consolidateDisplayTurns(rawMessages);
1054
+ const pageModeSessionIds = rawMessages.flatMap((message) => {
1055
+ const owner = readModeSessionMetadata(message.metadata);
1056
+ return owner ? [owner.id] : [];
1057
+ });
1058
+ const summaries = listConversationModeSessionsByIds(resolvedConversationId, [
1059
+ ...new Set([...pageModeSessionIds, ...requestedModeSessionIds]),
1060
+ ]);
1061
+ const validOwners = new Map(
1062
+ summaries.map((summary) => [summary.id, summary.mode]),
1063
+ );
1064
+ const projectionMessages = rawMessages.map((message) => {
1065
+ const owner = readModeSessionMetadata(message.metadata);
1066
+ if (!owner || validOwners.get(owner.id) === owner.mode) {
1067
+ return message;
1068
+ }
1069
+ return {
1070
+ ...message,
1071
+ metadata: mergeMessageMetadata(message.metadata, {
1072
+ modeSession: undefined,
1073
+ }),
1074
+ };
1075
+ });
1076
+ const {
1077
+ messages: consolidatedMessages,
1078
+ mergedIdMap,
1079
+ modeSessionActivityMap,
1080
+ } = consolidateDisplayTurns(projectionMessages);
1081
+ const liveConversation = findConversation(resolvedConversationId);
1082
+ const modeSessions = summaries
1083
+ .filter(
1084
+ (summary) =>
1085
+ isModeSessionRecoveryHealthy() || summary.status !== "active",
1086
+ )
1087
+ .flatMap((summary) => {
1088
+ const descriptor = liveConversation?.modeSessions
1089
+ ? liveConversation.modeSessions.describeSummary(summary)
1090
+ : summary.status !== "active"
1091
+ ? { summary }
1092
+ : undefined;
1093
+ return descriptor ? [descriptor] : [];
1094
+ });
1026
1095
  const assistantSlackDisplayName = getAssistantName()?.trim() || undefined;
1027
1096
 
1028
1097
  // Parse each row's stored content and per-message metadata. Rendering is
@@ -1150,6 +1219,7 @@ export async function handleListMessages({
1150
1219
  deletedAt,
1151
1220
  clientMessageId: msg.clientMessageId ?? undefined,
1152
1221
  assistantTextVisibility: assistantTextVisibilityOf(msg.metadata),
1222
+ modeSession: readModeSessionMetadata(msg.metadata),
1153
1223
  // The row's raw stored envelope, carried to the render pass below so it
1154
1224
  // can read the row's own `assistantTextVisibility` marker. Never part of
1155
1225
  // the wire payload, which the serializer builds field by field.
@@ -1170,6 +1240,9 @@ export async function handleListMessages({
1170
1240
  const messages: RuntimeMessagePayload[] = await Promise.all(
1171
1241
  parsed.map(async (m) => {
1172
1242
  const mergedMessageIds = m.id ? (mergedIdMap.get(m.id) ?? []) : [];
1243
+ const modeSessionActivity = m.id
1244
+ ? modeSessionActivityMap.get(m.id)
1245
+ : undefined;
1173
1246
 
1174
1247
  // Hydrate the row's attachments from the DB. A metadata-only query avoids
1175
1248
  // loading large base64 blobs for non-image attachments (documents, audio);
@@ -1180,9 +1253,21 @@ export async function handleListMessages({
1180
1253
  let msgAttachments: RuntimeAttachmentMetadata[] = [];
1181
1254
  if (m.id) {
1182
1255
  const idsToQuery = [m.id, ...mergedMessageIds];
1183
- const linked = idsToQuery.flatMap((id) =>
1184
- getAttachmentMetadataForMessage(id),
1256
+ const linkedRows = getAttachmentMetadataForMessages(idsToQuery);
1257
+ const computerUseScreenshotIds = new Set(
1258
+ linkedRows.flatMap((row) =>
1259
+ computerUseScreenshotAttachmentIdsFromMetadata(
1260
+ parseMessageMetadata(row.messageMetadata),
1261
+ ),
1262
+ ),
1185
1263
  );
1264
+ const linked = linkedRows.map((row) => {
1265
+ return {
1266
+ ...row.attachment,
1267
+ computerUseScreenshot:
1268
+ computerUseScreenshotIds.has(row.attachment.id) || undefined,
1269
+ };
1270
+ });
1186
1271
  if (linked.length > 0) {
1187
1272
  msgAttachments = await Promise.all(
1188
1273
  linked.map(async (a) => {
@@ -1221,6 +1306,9 @@ export async function handleListMessages({
1221
1306
  ? { thumbnailData: a.thumbnailBase64 }
1222
1307
  : {}),
1223
1308
  fileBacked: true,
1309
+ ...(a.computerUseScreenshot
1310
+ ? { computerUseScreenshot: true }
1311
+ : {}),
1224
1312
  };
1225
1313
  }),
1226
1314
  );
@@ -1333,6 +1421,8 @@ export async function handleListMessages({
1333
1421
  return {
1334
1422
  id: m.id ?? "",
1335
1423
  ...(mergedMessageIds.length > 0 ? { mergedMessageIds } : {}),
1424
+ ...(m.modeSession ? { modeSession: m.modeSession } : {}),
1425
+ ...(modeSessionActivity ? { modeSessionActivity } : {}),
1336
1426
  ...(m.clientMessageId ? { clientMessageId: m.clientMessageId } : {}),
1337
1427
  role: m.role,
1338
1428
  timestamp: new Date(displayTimestamp).toISOString(),
@@ -1422,6 +1512,7 @@ export async function handleListMessages({
1422
1512
  oldestMessageId: oldestMessageId ?? null,
1423
1513
  seq: persistedSeq,
1424
1514
  processing,
1515
+ ...(modeSessions.length > 0 ? { modeSessions } : {}),
1425
1516
  };
1426
1517
  }
1427
1518
 
@@ -1432,10 +1523,16 @@ export async function handleListMessages({
1432
1523
  ...(oldestMessageId != null ? { oldestMessageId } : {}),
1433
1524
  seq: persistedSeq,
1434
1525
  processing,
1526
+ ...(modeSessions.length > 0 ? { modeSessions } : {}),
1435
1527
  };
1436
1528
  }
1437
1529
 
1438
- return { messages, seq: persistedSeq, processing };
1530
+ return {
1531
+ messages,
1532
+ seq: persistedSeq,
1533
+ processing,
1534
+ ...(modeSessions.length > 0 ? { modeSessions } : {}),
1535
+ };
1439
1536
  }
1440
1537
 
1441
1538
  /**
@@ -2167,7 +2264,7 @@ export async function handleSendMessage(
2167
2264
  publishConversationMessagesChanged(conversationId, originClientId);
2168
2265
  },
2169
2266
  afterRelease: () => {
2170
- conversation.warmPromptCache();
2267
+ void conversation.warmPromptCache();
2171
2268
  },
2172
2269
  });
2173
2270
 
@@ -3027,6 +3124,7 @@ export async function handleSendMessage(
3027
3124
  messageId,
3028
3125
  requestId,
3029
3126
  clientMessageId,
3127
+ modeSession: conversation.modeSessions.getTurnOwner(requestId),
3030
3128
  });
3031
3129
  // The row this echo announces was durably persisted above, so advance
3032
3130
  // the snapshot↔stream anchor to the echo's seq (stamped inline by
@@ -3651,6 +3749,13 @@ export const ROUTES: RouteDefinition[] = [
3651
3749
  required: false,
3652
3750
  description: "Maximum number of messages to return.",
3653
3751
  },
3752
+ {
3753
+ name: "modeSessionIds",
3754
+ type: "string",
3755
+ required: false,
3756
+ description:
3757
+ "Comma-separated mode-session IDs to include with summaries for loaded active groups outside this page.",
3758
+ },
3654
3759
  ],
3655
3760
  responseBody: z.object({
3656
3761
  messages: z
@@ -3685,6 +3790,12 @@ export const ROUTES: RouteDefinition[] = [
3685
3790
  .describe(
3686
3791
  "Whether the agent is currently mid-turn for this conversation, sourced authoritatively from the persisted `processing_started_at` column. `true` means a turn is in flight; `false` means the conversation is idle. Clients use this to recover from a dropped SSE stream: if a turn appears to be running locally but the server reports `processing: false`, the turn has ended (or died) and the UI should stop waiting rather than spin indefinitely. Absent on older daemons that predate this field.",
3687
3792
  ),
3793
+ modeSessions: z
3794
+ .array(ModeSessionDescriptorSchema)
3795
+ .optional()
3796
+ .describe(
3797
+ "Conversation-owned mode-session summaries referenced by this page or explicitly requested by ID.",
3798
+ ),
3688
3799
  }),
3689
3800
  handler: (args) => handleListMessages(args),
3690
3801
  },
@@ -177,13 +177,14 @@ async function handleCredentialPrompt({ body = {} }: RouteHandlerArgs) {
177
177
  expiresAt: result.collectionExpiresAt,
178
178
  service: validated.service,
179
179
  field: validated.field,
180
- message: `This channel does not support secure input. The credential has NOT been stored yet. Share this one-time link with the user to collect it securely (single-use${expiresInMinutes ? `, expires in ${expiresInMinutes} minutes` : ""}): ${result.collectionUrl}`,
180
+ message: `${validated.conversationId ? "This channel does not support secure input." : "No conversation is attached to this invocation, so no secure prompt can be shown."} The credential has NOT been stored yet. Share this one-time link with the user to collect it securely (single-use${expiresInMinutes ? `, expires in ${expiresInMinutes} minutes` : ""}): ${result.collectionUrl}`,
181
181
  };
182
182
  }
183
183
  return {
184
184
  ok: false,
185
- error:
186
- "This conversation's channel does not support secure credential entry",
185
+ error: validated.conversationId
186
+ ? "This conversation's channel does not support secure credential entry"
187
+ : "No conversation is attached to this invocation, so no secure prompt can be shown, and a one-time collection link could not be created. Ask the user to add the credential in the app under Settings > Credentials. Never ask them to paste it into chat.",
187
188
  };
188
189
  }
189
190
  // An explicit user cancel is a valid flow, not a failure. Keep it distinct
@@ -607,8 +607,7 @@ async function handleCredentialsDelete({ body }: RouteHandlerArgs) {
607
607
  }
608
608
 
609
609
  async function handleCredentialsStatus() {
610
- const info = await getActiveBackendInfoAsync();
611
- return info;
610
+ return getActiveBackendInfoAsync();
612
611
  }
613
612
 
614
613
  // ---------------------------------------------------------------------------