@vellumai/assistant 0.12.2 → 0.12.3-staging.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (772) hide show
  1. package/.env.example +2 -0
  2. package/AGENTS.md +1 -1
  3. package/ARCHITECTURE.md +43 -29
  4. package/Dockerfile +63 -0
  5. package/docs/architecture/integrations.md +7 -0
  6. package/docs/architecture/memory.md +17 -2
  7. package/docs/architecture/security.md +10 -5
  8. package/docs/architecture/turn-actor.md +2 -0
  9. package/docs/credential-execution-service.md +1 -1
  10. package/docs/desktop-browser-cli.md +50 -5
  11. package/docs/live-voice-task-outcomes.md +100 -0
  12. package/docs/mode-sessions.md +183 -0
  13. package/docs/voice-input-diagnostics.md +58 -0
  14. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  15. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
  16. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  17. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  18. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
  19. package/node_modules/@vellumai/environments/src/shell.test.ts +21 -0
  20. package/node_modules/@vellumai/environments/src/shell.ts +24 -0
  21. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  22. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
  23. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  24. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  25. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
  26. package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +20 -0
  27. package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +8 -2
  28. package/node_modules/@vellumai/gateway-client/src/index.ts +4 -1
  29. package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +38 -7
  30. package/node_modules/@vellumai/gateway-client/src/trust-verdict-contract.ts +5 -6
  31. package/node_modules/@vellumai/service-contracts/package.json +2 -1
  32. package/node_modules/@vellumai/service-contracts/src/__tests__/secret-detection.test.ts +1 -0
  33. package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  34. package/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  35. package/node_modules/@vellumai/service-contracts/src/secret-detection.ts +4 -1
  36. package/node_modules/@vellumai/slack-text/src/index.ts +4 -8
  37. package/openapi.yaml +716 -16
  38. package/package.json +1 -1
  39. package/scripts/smoke-desktop-browser-cli.ts +11 -26
  40. package/scripts/smoke-desktop-window-theme.ts +2 -3
  41. package/scripts/sync-llm-catalog.ts +3 -0
  42. package/scripts/voice-ttft-spike.ts +2 -2
  43. package/src/__tests__/agent-loop-output-hooks.test.ts +150 -1
  44. package/src/__tests__/agent-loop-provider-error-recording.test.ts +87 -19
  45. package/src/__tests__/agent-loop.test.ts +259 -0
  46. package/src/__tests__/agent-wake-delegation-prompt.test.ts +64 -2
  47. package/src/__tests__/annotate-activity-metadata.test.ts +3 -1
  48. package/src/__tests__/annotate-risk-options.test.ts +2 -0
  49. package/src/__tests__/answered-question-persistence.test.ts +2 -0
  50. package/src/__tests__/anthropic-gateway-shared.test.ts +22 -0
  51. package/src/__tests__/anthropic-provider.test.ts +55 -0
  52. package/src/__tests__/approval-interception-trust-gates.test.ts +40 -0
  53. package/src/__tests__/approval-message-composer.test.ts +1 -118
  54. package/src/__tests__/approval-routes-http.test.ts +3 -0
  55. package/src/__tests__/assistant-event-hub-connection-history.test.ts +111 -0
  56. package/src/__tests__/assistant-event-hub-targeted.test.ts +50 -0
  57. package/src/__tests__/assistant-event-targeting.test.ts +186 -0
  58. package/src/__tests__/assistant-stream-state.test.ts +29 -9
  59. package/src/__tests__/attachments-store.test.ts +22 -3
  60. package/src/__tests__/browser-fill-credential.test.ts +11 -22
  61. package/src/__tests__/btw-routes.test.ts +4 -1
  62. package/src/__tests__/build-persisted-content.test.ts +1 -1
  63. package/src/__tests__/call-conversation-messages.test.ts +5 -0
  64. package/src/__tests__/call-recovery.test.ts +7 -2
  65. package/src/__tests__/call-routes-http.test.ts +12 -0
  66. package/src/__tests__/call-store.test.ts +7 -2
  67. package/src/__tests__/channel-approval.test.ts +9 -14
  68. package/src/__tests__/channel-reply-delivery.test.ts +56 -0
  69. package/src/__tests__/channel-retry-sweep.test.ts +9 -0
  70. package/src/__tests__/chat-credential-redaction.test.ts +23 -0
  71. package/src/__tests__/chat-reveal-guard-priming.test.ts +2 -0
  72. package/src/__tests__/checker.test.ts +11 -3
  73. package/src/__tests__/client-os-metadata-persistence.test.ts +35 -1
  74. package/src/__tests__/computer-use-screenshot-attachments.test.ts +411 -0
  75. package/src/__tests__/computer-use-screenshot-selection.test.ts +50 -0
  76. package/src/__tests__/computer-use-skill-manifest-regression.test.ts +0 -25
  77. package/src/__tests__/computer-use-tools.test.ts +1 -233
  78. package/src/__tests__/config-model-image-provider.test.ts +39 -0
  79. package/src/__tests__/config-schema.test.ts +16 -0
  80. package/src/__tests__/config-watcher.test.ts +24 -0
  81. package/src/__tests__/context-search-agent-runner.test.ts +260 -1
  82. package/src/__tests__/context-search-fanout.test.ts +1 -1
  83. package/src/__tests__/context-search-types.test.ts +7 -7
  84. package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +8 -2
  85. package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +2 -0
  86. package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +2 -0
  87. package/src/__tests__/conversation-agent-loop-overflow.test.ts +2 -0
  88. package/src/__tests__/conversation-agent-loop.test.ts +797 -16
  89. package/src/__tests__/conversation-app-control-lifecycle.test.ts +53 -1
  90. package/src/__tests__/conversation-attachments.test.ts +106 -0
  91. package/src/__tests__/conversation-evictor.test.ts +62 -37
  92. package/src/__tests__/conversation-fork-crud.test.ts +191 -0
  93. package/src/__tests__/conversation-history-boundary-repair-failure.test.ts +174 -0
  94. package/src/__tests__/conversation-loop-db-lock-resilience.test.ts +2 -0
  95. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +2 -0
  96. package/src/__tests__/conversation-rendered-delegation-state.test.ts +276 -0
  97. package/src/__tests__/conversation-retry-route.test.ts +22 -0
  98. package/src/__tests__/conversation-routes-disk-view.test.ts +2 -0
  99. package/src/__tests__/conversation-routes-enabled-plugins.test.ts +2 -0
  100. package/src/__tests__/conversation-routes-guardian-reply.test.ts +10 -0
  101. package/src/__tests__/conversation-routes-hidden-queue.test.ts +2 -0
  102. package/src/__tests__/conversation-routes-slash-commands.test.ts +2 -0
  103. package/src/__tests__/conversation-runtime-assembly.test.ts +29 -0
  104. package/src/__tests__/conversation-surface-snapshots.test.ts +543 -0
  105. package/src/__tests__/conversation-surfaces-action-delivery.test.ts +151 -0
  106. package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +63 -1
  107. package/src/__tests__/conversation-tool-setup-attribution.test.ts +23 -0
  108. package/src/__tests__/conversation-undo.test.ts +61 -0
  109. package/src/__tests__/credential-execution-client.test.ts +72 -55
  110. package/src/__tests__/credential-prompt-route.test.ts +35 -3
  111. package/src/__tests__/cu-unified-flow.test.ts +82 -1
  112. package/src/__tests__/db-conversation-tool-surface.test.ts +221 -0
  113. package/src/__tests__/db-mode-sessions-migration.test.ts +175 -0
  114. package/src/__tests__/deterministic-verification-control-plane.test.ts +2 -0
  115. package/src/__tests__/dm-backfill.test.ts +91 -1
  116. package/src/__tests__/document-create-route.test.ts +88 -0
  117. package/src/__tests__/document-sync-tags.test.ts +27 -0
  118. package/src/__tests__/emit-signal-routing-intent.test.ts +142 -2
  119. package/src/__tests__/events-tail-route.test.ts +58 -0
  120. package/src/__tests__/evict-conversations-for-reload.test.ts +57 -8
  121. package/src/__tests__/guardian-action-sweep.test.ts +16 -0
  122. package/src/__tests__/guardian-dispatch.test.ts +13 -0
  123. package/src/__tests__/headless-browser-interactions.test.ts +64 -20
  124. package/src/__tests__/heartbeat-service.test.ts +2 -59
  125. package/src/__tests__/helpers/mock-conversation.ts +21 -0
  126. package/src/__tests__/helpers/native-web-search-harness.ts +2 -0
  127. package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +5 -0
  128. package/src/__tests__/history-repair.test.ts +53 -0
  129. package/src/__tests__/host-cu-proxy.test.ts +46 -2
  130. package/src/__tests__/http-user-message-parity.test.ts +2 -0
  131. package/src/__tests__/image-credentials.test.ts +49 -0
  132. package/src/__tests__/image-service-dispatcher.test.ts +45 -0
  133. package/src/__tests__/inference-profile-session-handler.test.ts +26 -0
  134. package/src/__tests__/interrupt-on-send.test.ts +57 -43
  135. package/src/__tests__/interrupt-turn-note.test.ts +5 -6
  136. package/src/__tests__/list-messages-tool-merge.test.ts +85 -1
  137. package/src/__tests__/llm-catalog-parity.test.ts +27 -4
  138. package/src/__tests__/media-generate-image.test.ts +49 -1
  139. package/src/__tests__/message-provenance-contact-id.test.ts +69 -0
  140. package/src/__tests__/migration-export-to-gcs.test.ts +142 -0
  141. package/src/__tests__/mode-session-history.test.ts +245 -0
  142. package/src/__tests__/mode-session-recovery.test.ts +222 -0
  143. package/src/__tests__/mode-session-wire.test.ts +141 -0
  144. package/src/__tests__/notification-guardian-path.test.ts +7 -0
  145. package/src/__tests__/notification-telegram-adapter.test.ts +2 -2
  146. package/src/__tests__/notification-vellum-adapter.test.ts +34 -0
  147. package/src/__tests__/oauth-apps-routes.test.ts +1 -0
  148. package/src/__tests__/oauth-commands-routes.test.ts +114 -101
  149. package/src/__tests__/oauth-connect-orchestrator.test.ts +2 -0
  150. package/src/__tests__/oauth-provider-serializer.test.ts +13 -0
  151. package/src/__tests__/oauth-providers-routes.test.ts +3 -0
  152. package/src/__tests__/outbound-slack-persistence.test.ts +2 -0
  153. package/src/__tests__/persist-media-references.test.ts +50 -0
  154. package/src/__tests__/persistence-secret-redaction.test.ts +2 -0
  155. package/src/__tests__/plugin-api-provider.test.ts +5 -0
  156. package/src/__tests__/plugin-api-resolve-credential.test.ts +97 -0
  157. package/src/__tests__/plugin-import-boundary-guard.test.ts +9 -1
  158. package/src/__tests__/process-message-display-content.test.ts +56 -0
  159. package/src/__tests__/reaction-persistence.test.ts +5 -0
  160. package/src/__tests__/request-secret-standalone-channel-gate.test.ts +25 -9
  161. package/src/__tests__/run-conversation-turn-persistence.test.ts +138 -1
  162. package/src/__tests__/runtime-events-sse-reconnect.test.ts +1 -1
  163. package/src/__tests__/schedule-routes.test.ts +20 -0
  164. package/src/__tests__/scheduler-result-notification.test.ts +23 -4
  165. package/src/__tests__/script-proxy-certs.test.ts +1 -1
  166. package/src/__tests__/scripted-turn-metadata-persistence.test.ts +73 -0
  167. package/src/__tests__/secret-ingress-http.test.ts +2 -0
  168. package/src/__tests__/secret-routes-platform-proxy.test.ts +86 -31
  169. package/src/__tests__/secret-routes-scrub.test.ts +22 -27
  170. package/src/__tests__/secret-scanner.test.ts +20 -0
  171. package/src/__tests__/secure-keys-managed-failover.test.ts +8 -0
  172. package/src/__tests__/secure-keys.test.ts +50 -4
  173. package/src/__tests__/send-endpoint-busy.test.ts +163 -1
  174. package/src/__tests__/server-history-render.test.ts +91 -1
  175. package/src/__tests__/server-tool-pairing.test.ts +107 -0
  176. package/src/__tests__/settings-routes.test.ts +50 -1
  177. package/src/__tests__/skill-script-runner-sandbox.test.ts +32 -0
  178. package/src/__tests__/skills.test.ts +5 -4
  179. package/src/__tests__/subagent-fork-notifications.test.ts +1 -2
  180. package/src/__tests__/subagent-manager-notify.test.ts +1 -2
  181. package/src/__tests__/subagent-notify-parent.test.ts +146 -39
  182. package/src/__tests__/subagent-notify-stale-rebuild.test.ts +2 -4
  183. package/src/__tests__/subagent-tool-gate-mode.test.ts +226 -0
  184. package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +2 -0
  185. package/src/__tests__/terminal-tools.test.ts +8 -0
  186. package/src/__tests__/thread-backfill.test.ts +53 -0
  187. package/src/__tests__/tool-preview-lifecycle.test.ts +2 -0
  188. package/src/__tests__/tool-result-follow-up.test.ts +99 -0
  189. package/src/__tests__/tool-result-metadata-plumbing.test.ts +66 -1
  190. package/src/__tests__/tool-start-timestamp.test.ts +2 -0
  191. package/src/__tests__/twilio-routes.test.ts +1 -0
  192. package/src/__tests__/unicode.test.ts +36 -0
  193. package/src/__tests__/web-search-backend-failure.test.ts +1 -1
  194. package/src/activation/progress-store.test.ts +12 -7
  195. package/src/agent/loop.ts +136 -53
  196. package/src/agent/tool-result-follow-up.ts +70 -0
  197. package/src/api/attachment-provenance.test.ts +68 -0
  198. package/src/api/computer-use-tool.test.ts +55 -0
  199. package/src/api/computer-use-tool.ts +27 -0
  200. package/src/api/events/assistant-outbound-attachment.ts +3 -0
  201. package/src/api/events/assistant-turn-start.ts +3 -0
  202. package/src/api/events/desktop-activity-changed.ts +10 -0
  203. package/src/api/events/generation-handoff.ts +2 -0
  204. package/src/api/events/message-complete.ts +2 -0
  205. package/src/api/events/notification-conversation-created.ts +14 -16
  206. package/src/api/events/notification-intent.ts +4 -2
  207. package/src/api/events/question-request.ts +1 -0
  208. package/src/api/events/tool-result.ts +85 -0
  209. package/src/api/events/tool-use-start.ts +3 -0
  210. package/src/api/events/user-message-echo.ts +3 -0
  211. package/src/api/index.ts +42 -6
  212. package/src/api/mode-session.test.ts +131 -0
  213. package/src/api/mode-session.ts +148 -0
  214. package/src/api/remember-tool.ts +15 -0
  215. package/src/api/responses/conversation-message.ts +9 -0
  216. package/src/approvals/AGENTS.md +4 -2
  217. package/src/approvals/approval-primitive.ts +5 -2
  218. package/src/approvals/scoped-approval-grants.ts +6 -2
  219. package/src/browser/__tests__/operations.test.ts +29 -0
  220. package/src/browser/operation-meta.ts +2 -1
  221. package/src/browser/operations.ts +29 -0
  222. package/src/browser/virtual-desktop-target.ts +2 -9
  223. package/src/bundler/bundle-signer.ts +1 -3
  224. package/src/calls/__tests__/barge-in-guard.test.ts +3 -0
  225. package/src/calls/__tests__/call-controller.test.ts +94 -0
  226. package/src/calls/__tests__/call-funnel-telemetry.test.ts +204 -0
  227. package/src/calls/__tests__/voice-control-protocol.test.ts +15 -0
  228. package/src/calls/__tests__/voice-escalation-judge.test.ts +209 -0
  229. package/src/calls/__tests__/voice-leg-coordinator.test.ts +168 -0
  230. package/src/{live-voice/__tests__/live-voice-metrics.test.ts → calls/__tests__/voice-metrics.test.ts} +32 -38
  231. package/src/calls/__tests__/voice-session-bridge.test.ts +576 -122
  232. package/src/calls/barge-in-guard.ts +5 -0
  233. package/src/calls/call-controller.ts +196 -1
  234. package/src/calls/call-domain.ts +4 -0
  235. package/src/calls/call-store.ts +87 -2
  236. package/src/calls/guardian-action-sweep.ts +22 -8
  237. package/src/calls/media-stream-audio-transcode.ts +1 -3
  238. package/src/calls/typesafe-noul.ts +156 -0
  239. package/src/calls/voice-control-protocol.ts +6 -0
  240. package/src/calls/voice-escalation-judge.ts +145 -0
  241. package/src/calls/voice-escalation-target.ts +10 -0
  242. package/src/calls/voice-leg-coordinator.ts +121 -3
  243. package/src/{live-voice/live-voice-metrics.ts → calls/voice-metrics.ts} +94 -87
  244. package/src/calls/voice-session-bridge.ts +256 -112
  245. package/src/calls/voice-triage-escalate.ts +7 -0
  246. package/src/cli/__tests__/roadmap-cli.test.ts +4 -41
  247. package/src/cli/commands/__tests__/cli-test-harness.ts +21 -3
  248. package/src/cli/commands/__tests__/notifications.test.ts +35 -10
  249. package/src/cli/commands/__tests__/plugins.test.ts +20 -1
  250. package/src/cli/commands/__tests__/schedules.test.ts +14 -0
  251. package/src/cli/commands/bash.help.ts +4 -3
  252. package/src/cli/commands/browser.help.ts +9 -6
  253. package/src/cli/commands/browser.ts +0 -1
  254. package/src/cli/commands/clients.help.ts +46 -0
  255. package/src/cli/commands/clients.ts +116 -0
  256. package/src/cli/commands/credentials.help.ts +9 -4
  257. package/src/cli/commands/notifications.help.ts +16 -2
  258. package/src/cli/commands/notifications.ts +30 -8
  259. package/src/cli/commands/oauth/index.help.ts +10 -0
  260. package/src/cli/commands/oauth/providers.ts +12 -0
  261. package/src/cli/commands/oauth/request.test.ts +179 -2
  262. package/src/cli/commands/oauth/request.ts +41 -10
  263. package/src/cli/commands/plugins.ts +12 -5
  264. package/src/cli/commands/roadmap.help.ts +18 -21
  265. package/src/cli/commands/roadmap.ts +4 -21
  266. package/src/cli/commands/schedules.ts +2 -0
  267. package/src/cli/lib/__tests__/inspect-plugin.test.ts +54 -0
  268. package/src/cli/lib/__tests__/install-from-github.test.ts +41 -0
  269. package/src/cli/lib/__tests__/list-installed-plugins.test.ts +5 -2
  270. package/src/cli/lib/__tests__/local-plugin-upgrade.test.ts +133 -51
  271. package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts +8 -2
  272. package/src/cli/lib/bundled-marketplace.json +906 -1
  273. package/src/cli/lib/bundled-plugin-packages.json +783 -1
  274. package/src/cli/lib/inspect-plugin.ts +11 -4
  275. package/src/cli/lib/plugin-marketplace.ts +8 -0
  276. package/src/cli/lib/publish-plugin.ts +1 -2
  277. package/src/cli/lib/upgrade-plugin.ts +19 -11
  278. package/src/cli/output.ts +18 -4
  279. package/src/config/__tests__/default-provider.test.ts +6 -0
  280. package/src/config/__tests__/memory-retrospective-schema.test.ts +14 -1
  281. package/src/config/bundled-skills/computer-use/SKILL.md +1 -1
  282. package/src/config/bundled-skills/computer-use/TOOLS.json +89 -1
  283. package/src/config/bundled-skills/image-studio/SKILL.md +17 -2
  284. package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +7 -11
  285. package/src/config/bundled-skills/schedule/SKILL.md +6 -6
  286. package/src/config/bundled-skills/sequences/TOOLS.json +1 -5
  287. package/src/config/call-site-defaults.ts +9 -0
  288. package/src/config/feature-flag-registry.json +16 -0
  289. package/src/config/loader.ts +9 -17
  290. package/src/config/platform-identity.ts +8 -8
  291. package/src/config/profile-text-generation.test.ts +51 -0
  292. package/src/config/profile-text-generation.ts +51 -0
  293. package/src/config/schemas/__tests__/memory-v3.test.ts +12 -0
  294. package/src/config/schemas/__tests__/tools.test.ts +31 -0
  295. package/src/config/schemas/call-site-catalog.ts +14 -0
  296. package/src/config/schemas/channels.ts +0 -18
  297. package/src/config/schemas/llm.ts +9 -2
  298. package/src/config/schemas/mcp.ts +6 -1
  299. package/src/config/schemas/memory-retrospective.ts +11 -2
  300. package/src/config/schemas/memory-v3.ts +12 -0
  301. package/src/config/schemas/services.ts +15 -6
  302. package/src/config/schemas/tools.ts +10 -0
  303. package/src/config/session-groups-gate.ts +21 -0
  304. package/src/config/skills.ts +2 -6
  305. package/src/conversations/__tests__/message-consolidation.test.ts +149 -0
  306. package/src/conversations/message-consolidation.ts +102 -5
  307. package/src/credential-execution/ces-connect.test.ts +69 -0
  308. package/src/credential-execution/ces-connect.ts +115 -0
  309. package/src/credential-execution/ces-runtime.ts +55 -93
  310. package/src/credential-execution/executable-discovery.ts +34 -142
  311. package/src/credential-execution/process-manager.test.ts +20 -24
  312. package/src/credential-execution/process-manager.ts +24 -19
  313. package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +10 -1
  314. package/src/daemon/__tests__/conversation-tool-setup.test.ts +43 -0
  315. package/src/daemon/__tests__/inference-profile-notification.test.ts +2 -0
  316. package/src/daemon/__tests__/turn-tail-deleted-conversation.test.ts +36 -0
  317. package/src/daemon/assistant-attachments.ts +20 -12
  318. package/src/daemon/browser-mode-session.test.ts +402 -0
  319. package/src/daemon/browser-mode-session.ts +186 -0
  320. package/src/daemon/chat-credential-redaction.ts +16 -1
  321. package/src/daemon/computer-use-mode-session.test.ts +210 -0
  322. package/src/daemon/computer-use-mode-session.ts +105 -0
  323. package/src/daemon/config-watcher.ts +10 -0
  324. package/src/daemon/conversation-agent-loop-handlers.ts +132 -13
  325. package/src/daemon/conversation-agent-loop.ts +231 -25
  326. package/src/daemon/conversation-attachments.ts +93 -12
  327. package/src/daemon/conversation-evictor.ts +5 -24
  328. package/src/daemon/conversation-history.ts +10 -0
  329. package/src/daemon/conversation-interrupt-repair.ts +4 -43
  330. package/src/daemon/conversation-interrupt.ts +7 -11
  331. package/src/daemon/conversation-messaging.ts +58 -5
  332. package/src/daemon/conversation-mode-session.test.ts +1667 -0
  333. package/src/daemon/conversation-mode-session.ts +1008 -0
  334. package/src/daemon/conversation-process.ts +5 -0
  335. package/src/daemon/conversation-runtime-assembly.ts +10 -20
  336. package/src/daemon/conversation-skill-tools.ts +1 -0
  337. package/src/daemon/conversation-store.ts +4 -4
  338. package/src/daemon/conversation-surface-snapshots.ts +135 -0
  339. package/src/daemon/conversation-surfaces.ts +113 -5
  340. package/src/daemon/conversation-tool-setup.ts +79 -6
  341. package/src/daemon/conversation-turn-finalize.ts +26 -13
  342. package/src/daemon/conversation.ts +215 -58
  343. package/src/daemon/daemon-control.ts +3 -7
  344. package/src/daemon/handlers/config-model.ts +7 -5
  345. package/src/daemon/handlers/shared.ts +55 -16
  346. package/src/daemon/host-cu-proxy.ts +51 -22
  347. package/src/daemon/lifecycle.ts +30 -13
  348. package/src/daemon/mcp-reload-service.ts +1 -1
  349. package/src/daemon/message-protocol.ts +0 -1
  350. package/src/daemon/message-provenance.ts +28 -4
  351. package/src/daemon/message-types/shared.ts +1 -0
  352. package/src/daemon/mode-session-startup-recovery.ts +23 -0
  353. package/src/daemon/mode-session-tracking.ts +54 -0
  354. package/src/daemon/orphan-reaper.ts +4 -3
  355. package/src/daemon/persist-media-references.ts +20 -3
  356. package/src/daemon/process-message.ts +13 -4
  357. package/src/daemon/tool-setup-types.ts +6 -0
  358. package/src/daemon/virtual-desktop-context.ts +21 -0
  359. package/src/daemon/wake-conversation-ops.ts +50 -15
  360. package/src/desktop/__tests__/fake-desktop.ts +1 -0
  361. package/src/desktop/desktop-accessibility-script.ts +88 -0
  362. package/src/desktop/desktop-accessibility.test.ts +160 -0
  363. package/src/desktop/desktop-accessibility.ts +169 -0
  364. package/src/desktop/desktop-automation-lease.test.ts +302 -64
  365. package/src/desktop/desktop-automation-lease.ts +129 -54
  366. package/src/desktop/desktop-browser-endpoint.ts +4 -0
  367. package/src/desktop/desktop-computer-use-routing.test.ts +330 -0
  368. package/src/desktop/desktop-computer-use.test.ts +449 -0
  369. package/src/desktop/desktop-computer-use.ts +502 -0
  370. package/src/desktop/desktop-dependencies.ts +47 -293
  371. package/src/desktop/desktop-display.ts +2 -2
  372. package/src/desktop/desktop-help.ts +15 -0
  373. package/src/desktop/desktop-panel-config.test.ts +13 -3
  374. package/src/desktop/desktop-panel-config.ts +109 -15
  375. package/src/desktop/desktop-session-manager.test.ts +69 -21
  376. package/src/desktop/desktop-session-manager.ts +76 -22
  377. package/src/desktop/desktop-stream-bridge.test.ts +76 -132
  378. package/src/desktop/desktop-stream-bridge.ts +9 -11
  379. package/src/desktop/desktop-wallpaper-renderer.ts +132 -0
  380. package/src/desktop/desktop-wallpaper-worker.ts +17 -0
  381. package/src/desktop/desktop-wallpaper.test.ts +48 -7
  382. package/src/desktop/desktop-wallpaper.ts +34 -102
  383. package/src/desktop/desktop-window-theme.ts +5 -25
  384. package/src/desktop/virtual-desktop-feature.ts +53 -3
  385. package/src/desktop/virtual-desktop-platform.test.ts +5 -9
  386. package/src/documents/document-store.ts +23 -0
  387. package/src/heartbeat/__tests__/heartbeat-service.test.ts +22 -26
  388. package/src/heartbeat/heartbeat-service.ts +5 -33
  389. package/src/ipc/__tests__/browser-ipc.test.ts +215 -6
  390. package/src/ipc/assistant-server.ts +2 -0
  391. package/src/ipc/routes/__tests__/channel-reply-ipc-routes.test.ts +111 -0
  392. package/src/ipc/routes/channel-reply-ipc-routes.ts +73 -0
  393. package/src/live-voice/__tests__/continuation-judge.test.ts +151 -0
  394. package/src/live-voice/__tests__/live-activity-reporter.test.ts +154 -15
  395. package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +743 -31
  396. package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +191 -10
  397. package/src/live-voice/__tests__/live-voice-session-groups-gate.test.ts +87 -0
  398. package/src/live-voice/__tests__/live-voice-session-manager.test.ts +5 -1
  399. package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +41 -0
  400. package/src/live-voice/__tests__/live-voice-sight-frame.test.ts +1017 -9
  401. package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +224 -1
  402. package/src/live-voice/__tests__/live-voice-tts-session.test.ts +101 -0
  403. package/src/live-voice/__tests__/live-voice-tts.test.ts +44 -0
  404. package/src/live-voice/__tests__/live-voice-vad.test.ts +939 -75
  405. package/src/live-voice/__tests__/microphone-idle-gate.test.ts +87 -0
  406. package/src/live-voice/__tests__/session-controls.test.ts +23 -0
  407. package/src/live-voice/__tests__/task-outcomes.test.ts +117 -0
  408. package/src/live-voice/__tests__/voice-input-diagnostics.test.ts +117 -0
  409. package/src/live-voice/camera-mode-session.test.ts +265 -0
  410. package/src/live-voice/camera-mode-session.ts +277 -0
  411. package/src/live-voice/continuation-judge.ts +123 -0
  412. package/src/live-voice/live-activity-reporter.ts +115 -6
  413. package/src/live-voice/live-voice-manager.ts +36 -1
  414. package/src/live-voice/live-voice-photo.ts +88 -7
  415. package/src/live-voice/live-voice-session-manager.ts +15 -0
  416. package/src/live-voice/live-voice-session.ts +1443 -747
  417. package/src/live-voice/live-voice-tts.ts +22 -4
  418. package/src/live-voice/microphone-idle-gate.ts +56 -0
  419. package/src/live-voice/protocol.ts +171 -4
  420. package/src/live-voice/session-controls.ts +11 -3
  421. package/src/live-voice/task-outcomes.ts +131 -0
  422. package/src/live-voice/voice-input-diagnostics.ts +117 -0
  423. package/src/mcp/__tests__/manager-tool-caps.test.ts +156 -0
  424. package/src/mcp/__tests__/startup.test.ts +30 -7
  425. package/src/mcp/__tests__/tool-caps.test.ts +134 -0
  426. package/src/mcp/manager.ts +171 -101
  427. package/src/mcp/startup.ts +33 -13
  428. package/src/mcp/tool-caps.ts +184 -0
  429. package/src/media/__tests__/image-models.test.ts +58 -0
  430. package/src/media/image-credentials.ts +5 -0
  431. package/src/media/image-models.ts +52 -0
  432. package/src/media/image-service.ts +8 -0
  433. package/src/media/openrouter-image-service.test.ts +133 -0
  434. package/src/media/openrouter-image-service.ts +153 -0
  435. package/src/media/types.ts +1 -1
  436. package/src/messaging/provider-message-metadata.ts +3 -3
  437. package/src/messaging/providers/__tests__/transport-dispatch.test.ts +65 -11
  438. package/src/messaging/providers/discord/send.ts +4 -17
  439. package/src/messaging/providers/discord/transport.ts +2 -2
  440. package/src/messaging/providers/outlook/client.ts +16 -22
  441. package/src/messaging/providers/telegram-bot/send.test.ts +236 -1
  442. package/src/messaging/providers/telegram-bot/send.ts +10 -7
  443. package/src/messaging/providers/telegram-bot/transport.ts +3 -10
  444. package/src/messaging/providers/whatsapp/send.test.ts +169 -4
  445. package/src/monitoring/__tests__/control.test.ts +17 -5
  446. package/src/monitoring/control.ts +15 -5
  447. package/src/notifications/AGENTS.md +1 -1
  448. package/src/notifications/README.md +83 -99
  449. package/src/notifications/__tests__/broadcaster.test.ts +1 -1
  450. package/src/notifications/__tests__/channel-allowlist.test.ts +39 -0
  451. package/src/notifications/__tests__/copy-composer.test.ts +71 -1
  452. package/src/notifications/__tests__/decision-engine.test.ts +79 -2
  453. package/src/notifications/__tests__/deterministic-checks.test.ts +10 -10
  454. package/src/notifications/__tests__/home-feed-side-effect.test.ts +93 -4
  455. package/src/notifications/__tests__/notification-utils.test.ts +18 -0
  456. package/src/notifications/adapters/macos.ts +37 -46
  457. package/src/notifications/broadcaster.ts +5 -6
  458. package/src/notifications/channel-allowlist.ts +44 -0
  459. package/src/notifications/copy-composer.ts +11 -26
  460. package/src/notifications/decision-engine.ts +24 -14
  461. package/src/notifications/deterministic-checks.ts +1 -1
  462. package/src/notifications/emit-signal.ts +123 -37
  463. package/src/notifications/home-feed-side-effect.ts +9 -7
  464. package/src/notifications/notification-utils.ts +19 -0
  465. package/src/notifications/schedule-result-producer.ts +4 -6
  466. package/src/notifications/signal.ts +4 -19
  467. package/src/notifications/types.ts +15 -0
  468. package/src/oauth/AGENTS.md +2 -0
  469. package/src/oauth/__tests__/identity-verifier.test.ts +36 -1
  470. package/src/oauth/connection-resolver.ts +1 -2
  471. package/src/oauth/identity-verifier.ts +24 -0
  472. package/src/oauth/oauth-store.ts +12 -4
  473. package/src/oauth/provider-serializer.ts +8 -0
  474. package/src/oauth/seed-providers.ts +42 -0
  475. package/src/onboarding/onboarding-events-store.ts +99 -12
  476. package/src/permissions/confirmation-guardian-request.test.ts +16 -0
  477. package/src/permissions/confirmation-guardian-request.ts +2 -3
  478. package/src/permissions/question-prompter.test.ts +31 -0
  479. package/src/permissions/question-prompter.ts +2 -0
  480. package/src/persistence/__tests__/client-connection-events-store.test.ts +322 -0
  481. package/src/persistence/attachments-store.ts +84 -44
  482. package/src/persistence/client-connection-events-store.ts +377 -0
  483. package/src/persistence/conversation-crud.ts +166 -26
  484. package/src/persistence/conversation-mode-sessions.test.ts +424 -0
  485. package/src/persistence/conversation-mode-sessions.ts +460 -0
  486. package/src/persistence/conversation-plugin-facade.ts +30 -0
  487. package/src/persistence/conversation-queries.ts +6 -19
  488. package/src/persistence/conversation-tool-surface.ts +135 -0
  489. package/src/persistence/conversation-types.test.ts +32 -0
  490. package/src/persistence/conversation-types.ts +26 -4
  491. package/src/persistence/llm-usage-store.ts +3 -9
  492. package/src/persistence/message-metadata.ts +25 -0
  493. package/src/persistence/migrations/378-create-conversation-tool-surfaces.test.ts +78 -0
  494. package/src/persistence/migrations/378-create-conversation-tool-surfaces.ts +29 -0
  495. package/src/persistence/migrations/379-oauth-providers-response-ok-field.test.ts +92 -0
  496. package/src/persistence/migrations/379-oauth-providers-response-ok-field.ts +32 -0
  497. package/src/persistence/migrations/380-conversation-tool-surfaces-delegate-independent-tasks.test.ts +90 -0
  498. package/src/persistence/migrations/380-conversation-tool-surfaces-delegate-independent-tasks.ts +34 -0
  499. package/src/persistence/migrations/381-create-conversation-mode-sessions.ts +41 -0
  500. package/src/persistence/migrations/382-create-client-connection-events.ts +40 -0
  501. package/src/persistence/migrations/__tests__/382-create-client-connection-events.test.ts +56 -0
  502. package/src/persistence/schema/conversation-tool-surfaces.ts +32 -0
  503. package/src/persistence/schema/conversations.ts +67 -0
  504. package/src/persistence/schema/index.ts +1 -0
  505. package/src/persistence/schema/infrastructure.ts +29 -0
  506. package/src/persistence/schema/oauth.ts +1 -0
  507. package/src/persistence/steps.ts +41 -3
  508. package/src/plugin-api/__tests__/plugin-name-env.test.ts +249 -0
  509. package/src/plugin-api/conversation-turn.ts +31 -7
  510. package/src/plugin-api/effective-context-window.ts +33 -0
  511. package/src/plugin-api/index.ts +35 -6
  512. package/src/plugin-api/plugin-channel-turn-trust.test.ts +133 -0
  513. package/src/plugin-api/plugin-channel-turn-trust.ts +71 -0
  514. package/src/plugin-api/plugin-name-env.ts +125 -0
  515. package/src/plugin-api/resolve-credential.ts +79 -20
  516. package/src/plugin-api/vision-support.test.ts +22 -0
  517. package/src/plugin-api/vision-support.ts +17 -5
  518. package/src/plugins/__tests__/active-plugin-env.test.ts +136 -0
  519. package/src/plugins/active-plugin-env.ts +51 -0
  520. package/src/plugins/defaults/index.ts +24 -0
  521. package/src/plugins/defaults/memory/AGENTS.md +39 -5
  522. package/src/plugins/defaults/memory/__tests__/archive-removal-framing.test.ts +64 -0
  523. package/src/plugins/defaults/memory/__tests__/buffer-file.test.ts +365 -0
  524. package/src/plugins/defaults/memory/__tests__/buffer-format.test.ts +43 -0
  525. package/src/plugins/defaults/memory/__tests__/conversation-memory-purge.test.ts +1 -0
  526. package/src/plugins/defaults/memory/__tests__/db-memory-attach.test.ts +2 -0
  527. package/src/plugins/defaults/memory/__tests__/fixtures/buffer-appender.ts +17 -0
  528. package/src/plugins/defaults/memory/__tests__/memory-retrospective-job.test.ts +74 -0
  529. package/src/plugins/defaults/memory/__tests__/memory-run-evidence.test.ts +161 -0
  530. package/src/plugins/defaults/memory/__tests__/relocated-memory-test-rows.ts +10 -0
  531. package/src/plugins/defaults/memory/__tests__/worker-control.test.ts +19 -6
  532. package/src/plugins/defaults/memory/buffer-file.ts +354 -0
  533. package/src/plugins/defaults/memory/buffer-format.ts +40 -0
  534. package/src/plugins/defaults/memory/context-search/agent-protocol.ts +9 -12
  535. package/src/plugins/defaults/memory/context-search/agent-runner.ts +77 -22
  536. package/src/plugins/defaults/memory/context-search/format.ts +53 -8
  537. package/src/plugins/defaults/memory/context-search/limits.ts +10 -12
  538. package/src/plugins/defaults/memory/context-search/recall-input.ts +38 -0
  539. package/src/plugins/defaults/memory/context-search/search.ts +8 -14
  540. package/src/plugins/defaults/memory/context-search/sources/conversations.ts +1 -0
  541. package/src/plugins/defaults/memory/context-search/sources/memory-v2.ts +2 -1
  542. package/src/plugins/defaults/memory/context-search/sources/workspace.ts +2 -1
  543. package/src/plugins/defaults/memory/context-search/types.ts +1 -11
  544. package/src/plugins/defaults/memory/conversation-memory-purge.ts +4 -0
  545. package/src/plugins/defaults/memory/graph/__tests__/handle-remember-v2.test.ts +11 -4
  546. package/src/plugins/defaults/memory/graph/capability-seed.ts +1 -2
  547. package/src/plugins/defaults/memory/graph/tool-handlers.ts +13 -63
  548. package/src/plugins/defaults/memory/graph/tools.ts +35 -64
  549. package/src/plugins/defaults/memory/host-utils.ts +0 -10
  550. package/src/plugins/defaults/memory/injectors.ts +4 -3
  551. package/src/plugins/defaults/memory/memory-retrospective-job.ts +72 -183
  552. package/src/plugins/defaults/memory/memory-run-evidence.ts +213 -0
  553. package/src/plugins/defaults/memory/src/memory-eval-routes.ts +2 -3
  554. package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +6 -1
  555. package/src/plugins/defaults/memory/src/memory-item-routes.ts +2 -1
  556. package/src/plugins/defaults/memory/substrate/__tests__/consolidation-job.test.ts +407 -99
  557. package/src/plugins/defaults/memory/substrate/__tests__/consolidation-prompt-flag-gating-guard.test.ts +10 -0
  558. package/src/plugins/defaults/memory/substrate/__tests__/prompts-consolidation.test.ts +107 -7
  559. package/src/plugins/defaults/memory/substrate/consolidation-job.ts +307 -86
  560. package/src/plugins/defaults/memory/substrate/consolidation-tool-surface.ts +34 -0
  561. package/src/plugins/defaults/memory/substrate/page-index.ts +2 -1
  562. package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +91 -51
  563. package/src/plugins/defaults/memory/substrate/sweep-job.ts +1 -1
  564. package/src/plugins/defaults/memory/tools.test.ts +124 -9
  565. package/src/plugins/defaults/memory/tools.ts +38 -11
  566. package/src/plugins/defaults/memory/v1/graph/consolidation.ts +2 -2
  567. package/src/plugins/defaults/memory/v1/graph/extraction.ts +2 -1
  568. package/src/plugins/defaults/memory/v1/graph/retriever.ts +1 -1
  569. package/src/plugins/defaults/memory/v2/__tests__/migration.test.ts +5 -0
  570. package/src/plugins/defaults/memory/v2/__tests__/reranker.test.ts +5 -2
  571. package/src/plugins/defaults/memory/v2/reranker.ts +2 -1
  572. package/src/plugins/defaults/memory/v3/__tests__/injection.test.ts +81 -1
  573. package/src/plugins/defaults/memory/v3/__tests__/orchestrate.test.ts +87 -0
  574. package/src/plugins/defaults/memory/v3/__tests__/plugin-schema.test.ts +12 -0
  575. package/src/plugins/defaults/memory/v3/__tests__/pool-log-store.test.ts +332 -15
  576. package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +626 -27
  577. package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +148 -28
  578. package/src/plugins/defaults/memory/v3/card.ts +2 -1
  579. package/src/plugins/defaults/memory/v3/injector.ts +212 -178
  580. package/src/plugins/defaults/memory/v3/orchestrate.ts +159 -32
  581. package/src/plugins/defaults/memory/v3/plugin-schema.ts +54 -4
  582. package/src/plugins/defaults/memory/v3/pool-log-store.ts +275 -37
  583. package/src/plugins/defaults/memory/v3/pool-select.test.ts +47 -5
  584. package/src/plugins/defaults/memory/v3/pool-select.ts +710 -59
  585. package/src/plugins/defaults/memory/v3/sections.ts +2 -1
  586. package/src/plugins/defaults/memory/v3/shadow-plugin.ts +41 -14
  587. package/src/plugins/defaults/memory/worker-control.ts +15 -5
  588. package/src/plugins/defaults/task-progress/__tests__/registration.test.ts +70 -0
  589. package/src/plugins/defaults/task-progress/__tests__/task-progress-context.test.ts +380 -0
  590. package/src/plugins/defaults/task-progress/hooks/post-compact.ts +19 -0
  591. package/src/plugins/defaults/task-progress/hooks/user-prompt-submit.ts +24 -0
  592. package/src/plugins/defaults/task-progress/package.json +14 -0
  593. package/src/plugins/defaults/task-progress/src/apply-task-progress-context.ts +93 -0
  594. package/src/plugins/defaults/task-progress/src/format-task-progress-context.ts +67 -0
  595. package/src/plugins/defaults/tool-error/hooks/post-tool-use.ts +4 -1
  596. package/src/plugins/defaults/tool-result-truncate/terminal.ts +1 -46
  597. package/src/prompts/__tests__/parallel-tasks-section.test.ts +25 -0
  598. package/src/prompts/delegation-gate.ts +57 -0
  599. package/src/prompts/system-prompt.ts +14 -32
  600. package/src/providers/__tests__/context-overflow-error.test.ts +10 -0
  601. package/src/providers/__tests__/search-provider-catalog.test.ts +18 -0
  602. package/src/providers/anthropic/client.ts +10 -1
  603. package/src/providers/anthropic-gateway-shared.ts +6 -0
  604. package/src/providers/fetch-provider-catalog.ts +11 -0
  605. package/src/providers/gemini/client.ts +25 -8
  606. package/src/providers/inference/adapter-factory.ts +6 -0
  607. package/src/providers/jev/client.test.ts +270 -0
  608. package/src/providers/jev/client.ts +534 -0
  609. package/src/providers/model-catalog.ts +59 -3
  610. package/src/providers/openai/__tests__/chat-completions-extra-body.test.ts +94 -1
  611. package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +106 -1
  612. package/src/providers/openai/__tests__/chat-template-flatten.test.ts +24 -0
  613. package/src/providers/openai/__tests__/google-thought-signature.test.ts +1 -2
  614. package/src/providers/openai/chat-completions-provider.ts +100 -29
  615. package/src/providers/openai/google-thought-signature.ts +0 -6
  616. package/src/providers/openai/responses-provider.ts +23 -1
  617. package/src/providers/ratelimit.ts +1 -3
  618. package/src/providers/search-provider-catalog.ts +24 -0
  619. package/src/providers/server-tool-pairing.ts +16 -7
  620. package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +24 -0
  621. package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +7 -1
  622. package/src/providers/speech-to-text/__tests__/vellum-managed-realtime.test.ts +50 -1
  623. package/src/providers/speech-to-text/deepgram-flux-frames.ts +11 -0
  624. package/src/providers/speech-to-text/vellum-managed-flux-realtime.ts +13 -0
  625. package/src/providers/types.ts +5 -0
  626. package/src/routes/control.ts +7 -5
  627. package/src/runtime/AGENTS.md +6 -4
  628. package/src/runtime/__tests__/agent-wake.test.ts +94 -1
  629. package/src/runtime/actor-trust-resolver.ts +3 -3
  630. package/src/runtime/agent-wake.ts +32 -5
  631. package/src/runtime/approval-message-composer.ts +3 -69
  632. package/src/runtime/assistant-event-hub.ts +73 -75
  633. package/src/runtime/assistant-event-publish-options.ts +7 -0
  634. package/src/runtime/assistant-event-targeting.ts +79 -0
  635. package/src/runtime/assistant-stream-state.ts +13 -100
  636. package/src/runtime/background-job-runner.ts +5 -5
  637. package/src/runtime/channel-retry-sweep.ts +1 -0
  638. package/src/runtime/guardian-action-message-composer.ts +8 -161
  639. package/src/runtime/guardian-action-service.ts +2 -17
  640. package/src/runtime/guardian-reply-router.ts +29 -40
  641. package/src/runtime/http-server.ts +2 -2
  642. package/src/runtime/http-types.ts +6 -0
  643. package/src/runtime/message-composer-types.ts +3 -61
  644. package/src/runtime/migrations/__tests__/vbundle-import-parity.test.ts +67 -1
  645. package/src/runtime/migrations/__tests__/vbundle-import-policy.test.ts +89 -0
  646. package/src/runtime/migrations/vbundle-builder.ts +24 -1
  647. package/src/runtime/migrations/vbundle-import-analyzer.ts +13 -4
  648. package/src/runtime/migrations/vbundle-import-policy.ts +43 -11
  649. package/src/runtime/migrations/vbundle-importer.ts +4 -1
  650. package/src/runtime/migrations/vbundle-streaming-importer.ts +4 -1
  651. package/src/runtime/migrations/vbundle-validator.ts +1 -0
  652. package/src/runtime/pending-interactions-mode-session.test.ts +62 -0
  653. package/src/runtime/pending-interactions.ts +29 -0
  654. package/src/runtime/routes/__tests__/client-history-routes.test.ts +104 -0
  655. package/src/runtime/routes/__tests__/image-generation-routes.test.ts +97 -0
  656. package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +24 -2
  657. package/src/runtime/routes/__tests__/roadmap-routes.test.ts +29 -6
  658. package/src/runtime/routes/browser-context.ts +31 -11
  659. package/src/runtime/routes/browser-routes.ts +43 -4
  660. package/src/runtime/routes/btw-routes.ts +9 -4
  661. package/src/runtime/routes/channel-route-shared.ts +1 -9
  662. package/src/runtime/routes/client-routes.ts +108 -0
  663. package/src/runtime/routes/conversation-management-routes.ts +4 -1
  664. package/src/runtime/routes/conversation-routes.ts +125 -14
  665. package/src/runtime/routes/credential-prompt-routes.ts +4 -3
  666. package/src/runtime/routes/credential-routes.ts +1 -2
  667. package/src/runtime/routes/desktop-setup-routes.test.ts +9 -18
  668. package/src/runtime/routes/desktop-setup-routes.ts +8 -7
  669. package/src/runtime/routes/documents-routes.ts +51 -0
  670. package/src/runtime/routes/email-routes.ts +3 -7
  671. package/src/runtime/routes/events-routes.ts +75 -54
  672. package/src/runtime/routes/guardian-approval-interception.ts +24 -0
  673. package/src/runtime/routes/identity-routes.ts +1 -5
  674. package/src/runtime/routes/image-generation-routes.ts +8 -14
  675. package/src/runtime/routes/inbound-message-handler.ts +37 -41
  676. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.test.ts +168 -0
  677. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.ts +96 -0
  678. package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +39 -10
  679. package/src/runtime/routes/inbound-stages/background-dispatch.ts +9 -9
  680. package/src/runtime/routes/inbound-stages/reaction-intercept.ts +2 -0
  681. package/src/runtime/routes/inference-profile-session-handler.ts +11 -1
  682. package/src/runtime/routes/inference-profiles-routes.ts +9 -0
  683. package/src/runtime/routes/llm-context-normalization.ts +1 -7
  684. package/src/runtime/routes/migration-routes.ts +73 -2
  685. package/src/runtime/routes/notification-routes.ts +10 -0
  686. package/src/runtime/routes/oauth-commands-routes.ts +57 -54
  687. package/src/runtime/routes/oauth-providers.ts +11 -0
  688. package/src/runtime/routes/oauth-request-hints.test.ts +246 -0
  689. package/src/runtime/routes/oauth-request-hints.ts +149 -0
  690. package/src/runtime/routes/plugins-routes.ts +6 -0
  691. package/src/runtime/routes/roadmap-routes.ts +47 -8
  692. package/src/runtime/routes/schedule-routes.ts +2 -0
  693. package/src/runtime/routes/secret-routes.ts +99 -119
  694. package/src/runtime/routes/settings-routes.ts +10 -4
  695. package/src/runtime/routes/watcher-routes.ts +1 -3
  696. package/src/runtime/trust-class.test.ts +7 -6
  697. package/src/runtime/trust-class.ts +9 -17
  698. package/src/schedule/__tests__/worker-control.test.ts +36 -5
  699. package/src/schedule/__tests__/worker-mcp-bootstrap.test.ts +107 -0
  700. package/src/schedule/__tests__/worker-mcp-readiness.test.ts +173 -0
  701. package/src/schedule/__tests__/worker-mcp-tools.test.ts +13 -47
  702. package/src/schedule/run-script.ts +2 -2
  703. package/src/schedule/scheduler.ts +47 -11
  704. package/src/schedule/tool-surface-readiness.ts +65 -0
  705. package/src/schedule/worker-control.ts +15 -5
  706. package/src/schedule/worker-mcp.ts +85 -0
  707. package/src/schedule/worker.ts +9 -31
  708. package/src/security/secure-keys.ts +74 -112
  709. package/src/skills/clawhub.ts +3 -8
  710. package/src/skills/managed-store.ts +1 -5
  711. package/src/stt/types.ts +4 -0
  712. package/src/subagent/notify.ts +54 -34
  713. package/src/subagent/parent-notification.ts +6 -0
  714. package/src/telemetry/__tests__/phone-call-funnel.test.ts +72 -0
  715. package/src/telemetry/phone-call-funnel.ts +144 -0
  716. package/src/tools/__tests__/tool-schema-root-combinator-guard.test.ts +81 -0
  717. package/src/tools/ask-question/ask-question-tool.test.ts +167 -5
  718. package/src/tools/ask-question/ask-question-tool.ts +120 -22
  719. package/src/tools/browser/browser-execution.ts +29 -60
  720. package/src/tools/browser/cdp-client/__tests__/cdp-dom-helpers.test.ts +561 -39
  721. package/src/tools/browser/cdp-client/cdp-dom-helpers.ts +435 -39
  722. package/src/tools/browser/cdp-client/cdp-inspect-client.ts +1 -8
  723. package/src/tools/client-os.ts +15 -0
  724. package/src/tools/computer-use/skill-proxy-bridge.ts +1 -1
  725. package/src/tools/computer-use/target.ts +42 -0
  726. package/src/tools/document/document-tool.ts +13 -18
  727. package/src/tools/execution-target.ts +14 -16
  728. package/src/tools/host-terminal/host-shell.ts +16 -7
  729. package/src/tools/network/__tests__/web-fetch-tinyfish.test.ts +250 -0
  730. package/src/tools/network/__tests__/web-search.test.ts +203 -0
  731. package/src/tools/network/web-fetch.ts +274 -7
  732. package/src/tools/network/web-search.ts +468 -6
  733. package/src/tools/permission-checker.ts +3 -3
  734. package/src/tools/policy-context.ts +3 -1
  735. package/src/tools/shared/filesystem/file-ops-service.ts +1 -31
  736. package/src/tools/shared/shell-output.test.ts +10 -0
  737. package/src/tools/shared/shell-output.ts +14 -2
  738. package/src/tools/skills/load.ts +2 -1
  739. package/src/tools/skills/sandbox-runner.ts +24 -4
  740. package/src/tools/skills/scaffold-managed.ts +2 -1
  741. package/src/tools/skills/skill-script-runner.ts +7 -0
  742. package/src/tools/skills/skill-tool-factory.ts +15 -1
  743. package/src/tools/terminal/__tests__/safe-env.test.ts +44 -4
  744. package/src/tools/terminal/__tests__/sanitized-bash.test.ts +0 -14
  745. package/src/tools/terminal/safe-env.ts +39 -24
  746. package/src/tools/terminal/sanitized-bash.ts +15 -2
  747. package/src/tools/terminal/shell-launch.test.ts +162 -0
  748. package/src/tools/terminal/shell.test.ts +29 -0
  749. package/src/tools/terminal/shell.ts +15 -7
  750. package/src/tools/tool-approval-handler.ts +5 -1
  751. package/src/tools/tool-types.ts +3 -3
  752. package/src/tools/types.ts +8 -2
  753. package/src/util/__tests__/worker-process-command.test.ts +5 -4
  754. package/src/util/abort-reasons.ts +18 -26
  755. package/src/util/browser-human-verification.ts +14 -0
  756. package/src/util/canonicalize-identity.test.ts +21 -0
  757. package/src/util/canonicalize-identity.ts +16 -0
  758. package/src/util/errors.ts +13 -3
  759. package/src/util/host-process.test.ts +17 -1
  760. package/src/util/host-process.ts +24 -0
  761. package/src/util/provider-error-patterns.ts +3 -0
  762. package/src/util/unicode.ts +29 -0
  763. package/src/util/worker-process.ts +77 -35
  764. package/src/workspace/migrations/157-add-desktop-files-launcher.ts +94 -0
  765. package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +4 -10
  766. package/src/workspace/migrations/__tests__/157-add-desktop-files-launcher.test.ts +113 -0
  767. package/src/workspace/migrations/registry.ts +2 -0
  768. package/src/__tests__/computer-use-window-schema.test.ts +0 -24
  769. package/src/api/events/heartbeat-alert.ts +0 -20
  770. package/src/daemon/message-types/web-activity.ts +0 -65
  771. package/src/desktop/desktop-dependencies.test.ts +0 -131
  772. package/src/tools/computer-use/definitions.ts +0 -607
@@ -0,0 +1,377 @@
1
+ /**
2
+ * Durable client connection history.
3
+ *
4
+ * The event hub writes one row per client subscribe/dispose. Readers
5
+ * coalesce short reconnects into flaps so a watchdog cycle is not an
6
+ * outage. Writes are best-effort: a missing or unready database never
7
+ * throws back into the hub.
8
+ */
9
+
10
+ import { and, asc, eq, lt, lte, type SQL } from "drizzle-orm";
11
+ import { v4 as uuid } from "uuid";
12
+
13
+ import type { InterfaceId } from "../channels/types.js";
14
+ import { getLogger } from "../util/logger.js";
15
+ import type { DrizzleDb } from "./db-connection.js";
16
+ import * as dbConnection from "./db-connection.js";
17
+ import { clientConnectionEvents } from "./schema/index.js";
18
+
19
+ const log = getLogger("client-connection-events-store");
20
+
21
+ /** How long a reconnect gap can be and still count as a flap, not an outage. */
22
+ export const CLIENT_CONNECTION_FLAP_WINDOW_MS = 60_000;
23
+
24
+ /** Rows older than this are deleted on each successful write. */
25
+ export const CLIENT_CONNECTION_EVENT_RETENTION_MS = 90 * 24 * 60 * 60 * 1000;
26
+
27
+ export const CLIENT_CONNECTION_EVENT_REASONS = [
28
+ "sse_open",
29
+ "sse_close",
30
+ "stale_replaced",
31
+ "shed_backpressure",
32
+ "force_disconnect",
33
+ "cap_evicted",
34
+ ] as const;
35
+
36
+ export type ClientConnectionEventReason =
37
+ (typeof CLIENT_CONNECTION_EVENT_REASONS)[number];
38
+
39
+ /** Session-only close when a later open arrives with no recorded close. */
40
+ export type ClientConnectionSessionCloseReason =
41
+ | ClientConnectionEventReason
42
+ | "implicit_replaced";
43
+
44
+ const CLOSE_REASONS = new Set<ClientConnectionEventReason>([
45
+ "sse_close",
46
+ "stale_replaced",
47
+ "shed_backpressure",
48
+ "force_disconnect",
49
+ "cap_evicted",
50
+ ]);
51
+
52
+ export interface ClientConnectionEventInput {
53
+ clientId: string;
54
+ interfaceId: InterfaceId | string;
55
+ connectionId: string;
56
+ reason: ClientConnectionEventReason;
57
+ occurredAt?: number;
58
+ actorPrincipalId?: string;
59
+ clientVersion?: string;
60
+ sseWatchdog?: boolean;
61
+ machineName?: string;
62
+ }
63
+
64
+ export interface ClientConnectionEvent {
65
+ id: string;
66
+ clientId: string;
67
+ interfaceId: string;
68
+ connectionId: string;
69
+ reason: ClientConnectionEventReason;
70
+ occurredAt: number;
71
+ actorPrincipalId: string | null;
72
+ clientVersion: string | null;
73
+ sseWatchdog: boolean | null;
74
+ machineName: string | null;
75
+ }
76
+
77
+ export interface ClientConnectionSession {
78
+ clientId: string;
79
+ interfaceId: string;
80
+ startedAt: number;
81
+ endedAt: number | null;
82
+ durationMs: number | null;
83
+ flapCount: number;
84
+ openReason: ClientConnectionEventReason;
85
+ closeReason: ClientConnectionSessionCloseReason | null;
86
+ clientVersion: string | null;
87
+ sseWatchdog: boolean | null;
88
+ machineName: string | null;
89
+ connectionIds: string[];
90
+ }
91
+
92
+ export interface ListClientConnectionHistoryParams {
93
+ clientId?: string;
94
+ interfaceId?: string;
95
+ actorPrincipalId?: string;
96
+ since?: number;
97
+ until?: number;
98
+ /** Max sessions to return after coalescing. Clamped to 1-500. */
99
+ limit?: number;
100
+ now?: number;
101
+ }
102
+
103
+ export interface ClientConnectionHistory {
104
+ events: ClientConnectionEvent[];
105
+ sessions: ClientConnectionSession[];
106
+ }
107
+
108
+ const DEFAULT_SESSION_LIMIT = 50;
109
+ const MAX_SESSION_LIMIT = 500;
110
+
111
+ function resolveDb(database?: DrizzleDb): DrizzleDb | null {
112
+ if (database) {
113
+ return database;
114
+ }
115
+ // Namespace import so a test mock of db-connection that omits `isDbOpen`
116
+ // still loads. Incomplete mocks skip the probe and try `getDb()`.
117
+ if (
118
+ typeof dbConnection.isDbOpen === "function" &&
119
+ !dbConnection.isDbOpen()
120
+ ) {
121
+ return null;
122
+ }
123
+ try {
124
+ return dbConnection.getDb();
125
+ } catch (err) {
126
+ log.debug({ err }, "client connection history skipped: database unready");
127
+ return null;
128
+ }
129
+ }
130
+
131
+ function rowToEvent(
132
+ row: typeof clientConnectionEvents.$inferSelect,
133
+ ): ClientConnectionEvent {
134
+ return {
135
+ id: row.id,
136
+ clientId: row.clientId,
137
+ interfaceId: row.interfaceId,
138
+ connectionId: row.connectionId,
139
+ reason: row.reason as ClientConnectionEventReason,
140
+ occurredAt: row.occurredAt,
141
+ actorPrincipalId: row.actorPrincipalId ?? null,
142
+ clientVersion: row.clientVersion ?? null,
143
+ sseWatchdog: row.sseWatchdog ?? null,
144
+ machineName: row.machineName ?? null,
145
+ };
146
+ }
147
+
148
+ function pruneExpired(db: DrizzleDb, now: number): void {
149
+ const cutoff = now - CLIENT_CONNECTION_EVENT_RETENTION_MS;
150
+ db.delete(clientConnectionEvents)
151
+ .where(lt(clientConnectionEvents.occurredAt, cutoff))
152
+ .run();
153
+ }
154
+
155
+ /**
156
+ * Append one hub lifecycle event. Returns null when the main database is
157
+ * not open or the write fails. Never throws.
158
+ */
159
+ export function recordClientConnectionEvent(
160
+ input: ClientConnectionEventInput,
161
+ database?: DrizzleDb,
162
+ ): ClientConnectionEvent | null {
163
+ const db = resolveDb(database);
164
+ if (!db) {
165
+ return null;
166
+ }
167
+ const now = input.occurredAt ?? Date.now();
168
+ const row = {
169
+ id: uuid(),
170
+ clientId: input.clientId,
171
+ interfaceId: input.interfaceId,
172
+ connectionId: input.connectionId,
173
+ reason: input.reason,
174
+ occurredAt: now,
175
+ actorPrincipalId: input.actorPrincipalId ?? null,
176
+ clientVersion: input.clientVersion ?? null,
177
+ sseWatchdog: input.sseWatchdog ?? null,
178
+ machineName: input.machineName ?? null,
179
+ };
180
+ try {
181
+ db.insert(clientConnectionEvents).values(row).run();
182
+ pruneExpired(db, now);
183
+ return rowToEvent(row);
184
+ } catch (err) {
185
+ log.debug({ err }, "failed to record client connection event");
186
+ return null;
187
+ }
188
+ }
189
+
190
+ function clampLimit(limit: number | undefined): number {
191
+ if (limit == null || !Number.isFinite(limit)) {
192
+ return DEFAULT_SESSION_LIMIT;
193
+ }
194
+ return Math.min(MAX_SESSION_LIMIT, Math.max(1, Math.floor(limit)));
195
+ }
196
+
197
+ function startSession(event: ClientConnectionEvent): ClientConnectionSession {
198
+ return {
199
+ clientId: event.clientId,
200
+ interfaceId: event.interfaceId,
201
+ startedAt: event.occurredAt,
202
+ endedAt: event.reason === "sse_open" ? null : event.occurredAt,
203
+ durationMs: event.reason === "sse_open" ? null : 0,
204
+ flapCount: 0,
205
+ openReason: event.reason,
206
+ closeReason: event.reason === "sse_open" ? null : event.reason,
207
+ clientVersion: event.clientVersion,
208
+ sseWatchdog: event.sseWatchdog,
209
+ machineName: event.machineName,
210
+ connectionIds: [event.connectionId],
211
+ };
212
+ }
213
+
214
+ /**
215
+ * Coalesce raw events into sessions. A close followed by an open within
216
+ * {@link CLIENT_CONNECTION_FLAP_WINDOW_MS} continues the same session and
217
+ * increments `flapCount`.
218
+ */
219
+ export function coalesceClientConnectionSessions(
220
+ events: ClientConnectionEvent[],
221
+ now: number = Date.now(),
222
+ ): ClientConnectionSession[] {
223
+ const byClient = new Map<string, ClientConnectionEvent[]>();
224
+ for (const event of events) {
225
+ const list = byClient.get(event.clientId) ?? [];
226
+ list.push(event);
227
+ byClient.set(event.clientId, list);
228
+ }
229
+
230
+ const sessions: ClientConnectionSession[] = [];
231
+ for (const clientEvents of byClient.values()) {
232
+ clientEvents.sort((a, b) => a.occurredAt - b.occurredAt);
233
+ let current: ClientConnectionSession | null = null;
234
+
235
+ const finalize = () => {
236
+ if (!current) {
237
+ return;
238
+ }
239
+ if (current.endedAt == null) {
240
+ current.durationMs = Math.max(0, now - current.startedAt);
241
+ } else {
242
+ current.durationMs = Math.max(0, current.endedAt - current.startedAt);
243
+ }
244
+ sessions.push(current);
245
+ current = null;
246
+ };
247
+
248
+ for (const event of clientEvents) {
249
+ if (event.reason === "sse_open") {
250
+ if (
251
+ current &&
252
+ current.endedAt != null &&
253
+ event.occurredAt - current.endedAt < CLIENT_CONNECTION_FLAP_WINDOW_MS
254
+ ) {
255
+ current.endedAt = null;
256
+ current.closeReason = null;
257
+ current.flapCount += 1;
258
+ current.connectionIds.push(event.connectionId);
259
+ current.clientVersion = event.clientVersion ?? current.clientVersion;
260
+ current.sseWatchdog = event.sseWatchdog ?? current.sseWatchdog;
261
+ current.machineName = event.machineName ?? current.machineName;
262
+ continue;
263
+ }
264
+ if (current && current.endedAt == null) {
265
+ current.endedAt = event.occurredAt;
266
+ current.closeReason = "implicit_replaced";
267
+ }
268
+ finalize();
269
+ current = startSession(event);
270
+ continue;
271
+ }
272
+
273
+ if (!CLOSE_REASONS.has(event.reason)) {
274
+ continue;
275
+ }
276
+
277
+ if (!current) {
278
+ current = startSession(event);
279
+ continue;
280
+ }
281
+
282
+ current.endedAt = event.occurredAt;
283
+ current.closeReason = event.reason;
284
+ if (!current.connectionIds.includes(event.connectionId)) {
285
+ current.connectionIds.push(event.connectionId);
286
+ }
287
+ }
288
+ finalize();
289
+ }
290
+
291
+ sessions.sort((a, b) => b.startedAt - a.startedAt);
292
+ return sessions;
293
+ }
294
+
295
+ /**
296
+ * Read raw events and coalesced sessions. Returns empty lists when the
297
+ * database is not open.
298
+ */
299
+ export function listClientConnectionHistory(
300
+ params: ListClientConnectionHistoryParams = {},
301
+ database?: DrizzleDb,
302
+ ): ClientConnectionHistory {
303
+ const db = resolveDb(database);
304
+ if (!db) {
305
+ return { events: [], sessions: [] };
306
+ }
307
+
308
+ const now = params.now ?? Date.now();
309
+ const limit = clampLimit(params.limit);
310
+ const filters: SQL[] = [];
311
+ if (params.clientId) {
312
+ filters.push(eq(clientConnectionEvents.clientId, params.clientId));
313
+ }
314
+ if (params.interfaceId) {
315
+ filters.push(eq(clientConnectionEvents.interfaceId, params.interfaceId));
316
+ }
317
+ if (params.actorPrincipalId) {
318
+ filters.push(
319
+ eq(clientConnectionEvents.actorPrincipalId, params.actorPrincipalId),
320
+ );
321
+ }
322
+ if (params.until != null) {
323
+ filters.push(lte(clientConnectionEvents.occurredAt, params.until));
324
+ }
325
+
326
+ try {
327
+ const query = db
328
+ .select()
329
+ .from(clientConnectionEvents)
330
+ .orderBy(asc(clientConnectionEvents.occurredAt));
331
+ const rows =
332
+ filters.length > 0 ? query.where(and(...filters)).all() : query.all();
333
+ const events = rows.map(rowToEvent);
334
+ const sessions = coalesceClientConnectionSessions(events, now)
335
+ .filter((session) => {
336
+ return sessionOverlapsWindow(session, params.since, params.until);
337
+ })
338
+ .slice(0, limit);
339
+ return {
340
+ events: events.filter((event) => {
341
+ return eventInWindow(event, params.since, params.until);
342
+ }),
343
+ sessions,
344
+ };
345
+ } catch (err) {
346
+ log.debug({ err }, "failed to list client connection history");
347
+ return { events: [], sessions: [] };
348
+ }
349
+ }
350
+
351
+ function sessionOverlapsWindow(
352
+ session: ClientConnectionSession,
353
+ since?: number,
354
+ until?: number,
355
+ ): boolean {
356
+ if (since != null && session.endedAt != null && session.endedAt < since) {
357
+ return false;
358
+ }
359
+ if (until != null && session.startedAt > until) {
360
+ return false;
361
+ }
362
+ return true;
363
+ }
364
+
365
+ function eventInWindow(
366
+ event: ClientConnectionEvent,
367
+ since?: number,
368
+ until?: number,
369
+ ): boolean {
370
+ if (since != null && event.occurredAt < since) {
371
+ return false;
372
+ }
373
+ if (until != null && event.occurredAt > until) {
374
+ return false;
375
+ }
376
+ return true;
377
+ }
@@ -1,5 +1,6 @@
1
1
  import { mkdirSync, rmSync } from "node:fs";
2
2
 
3
+ import { TrustClassSchema } from "@vellumai/gateway-client";
3
4
  import {
4
5
  and,
5
6
  asc,
@@ -26,6 +27,7 @@ import {
26
27
  forgetActivationConversation,
27
28
  forgetAllActivationConversations,
28
29
  } from "../activation/progress-store.js";
30
+ import { TolerantModeSessionSchema } from "../api/mode-session.js";
29
31
  import type { ChannelId, InterfaceId } from "../channels/types.js";
30
32
  import { parseChannelId, parseInterfaceId } from "../channels/types.js";
31
33
  import { CHANNEL_IDS, isChannelId } from "../channels/types.js";
@@ -47,7 +49,6 @@ import { runHook } from "../plugins/pipeline.js";
47
49
  import type { ContentBlock } from "../providers/types.js";
48
50
  import { getCurrentSeq } from "../runtime/assistant-stream-state.js";
49
51
  import { publishSyncInvalidation } from "../runtime/sync/sync-publisher.js";
50
- import { trustClassSchema } from "../runtime/trust-class.js";
51
52
  import { UserError } from "../util/errors.js";
52
53
  import { getLogger } from "../util/logger.js";
53
54
  import { getLogsDbPath } from "../util/logs-db-path.js";
@@ -91,9 +92,12 @@ import {
91
92
  REFERENTIAL_FORK_STRATEGY,
92
93
  resolveConversationLineage,
93
94
  } from "./conversation-lineage.js";
95
+ import { repairConversationModeSessionBoundaries } from "./conversation-mode-sessions.js";
94
96
  import { deleteConversationRowsInBatches } from "./conversation-row-batch-delete.js";
95
97
  import {
96
98
  BACKGROUND_CONVERSATION_TYPES,
99
+ COMPUTER_USE_SCREENSHOT_ATTACHMENT_IDS_KEY,
100
+ computerUseScreenshotAttachmentIdsFromMetadata,
97
101
  type ConversationCreateType,
98
102
  type ConversationOrigin,
99
103
  isHiddenMessageMetadata,
@@ -331,6 +335,8 @@ const backgroundToolCompletionMetadataSchema = z.object({
331
335
 
332
336
  export const messageMetadataSchema = z
333
337
  .object({
338
+ /** Immutable ownership of this transcript row by a recorded mode session. */
339
+ modeSession: TolerantModeSessionSchema,
334
340
  /**
335
341
  * Epoch ms the content actually happened, when that differs from when
336
342
  * the row was written. Set wherever persistence lags the event: a queued
@@ -381,7 +387,7 @@ export const messageMetadataSchema = z
381
387
  * trust status changes later. Used by the memory write gate (indexer)
382
388
  * and read gate (conversation history loading) to enforce trust-aware access.
383
389
  */
384
- provenanceTrustClass: trustClassSchema.optional(),
390
+ provenanceTrustClass: TrustClassSchema.optional(),
385
391
  /**
386
392
  * Model that actually served this assistant row, carried on the agent
387
393
  * loop's `message_complete` event (the provider's `response.model`, the
@@ -396,6 +402,22 @@ export const messageMetadataSchema = z
396
402
  provenanceSourceChannel: channelIdSchema.optional(),
397
403
  provenanceGuardianExternalUserId: z.string().optional(),
398
404
  provenanceRequesterIdentifier: z.string().optional(),
405
+ /**
406
+ * Contact id of the person who wrote this row, from the gateway trust
407
+ * verdict at persist time. Stamped only on a person's own message or
408
+ * reaction (`actorAuthorProvenance`), never on rows the assistant writes
409
+ * during their turn: the other `provenance*` fields describe the turn,
410
+ * this one the author. Absent when the author resolved to no contact.
411
+ */
412
+ provenanceContactId: z.string().optional(),
413
+ /**
414
+ * Set on a backfilled row whose sender was looked up but no usable gateway
415
+ * verdict came back, so its trust class is the guardian-address fallback
416
+ * and it names no author. Distinguishes that row from a sender the gateway
417
+ * resolved as a stranger, so the lookup can be re-run later. Live ingress
418
+ * never persists such a row: it denies a turn whose verdict failed.
419
+ */
420
+ provenanceLookupFailed: z.boolean().optional(),
399
421
  automated: z.boolean().optional(),
400
422
  /**
401
423
  * Transcript-suppression flag: the row is a machine signal (e.g. the
@@ -413,6 +435,9 @@ export const messageMetadataSchema = z
413
435
  * channel/interface fields cannot stand in for it.
414
436
  */
415
437
  voiceSessionTurn: z.boolean().optional(),
438
+ [COMPUTER_USE_SCREENSHOT_ATTACHMENT_IDS_KEY]: z
439
+ .array(z.string())
440
+ .optional(),
416
441
  /**
417
442
  * Discriminates daemon-authored rows from ordinary turns.
418
443
  * `"system_card"` marks pre-composed status cards (the /compact, /clean,
@@ -464,10 +489,10 @@ export const messageMetadataSchema = z
464
489
  */
465
490
  attachmentStoredPaths: z.record(z.string(), z.string()).optional(),
466
491
  /**
467
- * Marks a role-`"user"` row whose arrival interrupted a turn that had made
468
- * no tool call yet. `loadFromDb` rebuilds the LLM-facing
469
- * `<interrupted_turn>` note from it; the row's own content is exactly what
470
- * the user sent, so clients render nothing extra.
492
+ * Marks a role-`"user"` row whose arrival interrupted a running turn.
493
+ * `loadFromDb` rebuilds the LLM-facing `<interrupted_turn>` note from it;
494
+ * the row's own content is exactly what the user sent, so clients render
495
+ * nothing extra.
471
496
  */
472
497
  interruptedPriorTurn: z.boolean().optional(),
473
498
  memoryInjectedBlock: z.string().optional(),
@@ -572,6 +597,12 @@ export function isProviderErrorMetadata(
572
597
  * assistant rows, and turn grouping closes on them, so display merging and
573
598
  * the turn resolver agree on boundaries. Takes the raw persisted `metadata`
574
599
  * JSON string; malformed JSON and non-assistant roles are never standalone.
600
+ *
601
+ * The web folds adjacent assistant rows again after pagination and reads the
602
+ * same rule off the wire projection in its own `isStandaloneAssistantMessage`
603
+ * (clients/web/src/domains/chat/utils/is-standalone-assistant-message.ts). A
604
+ * kind added here without a matching flag and check there merges on the
605
+ * client anyway.
575
606
  */
576
607
  export function isStandaloneAssistantMessage(
577
608
  role: string,
@@ -1849,7 +1880,25 @@ function populateForkContentsInProcess(args: PopulateForkContentsArgs): void {
1849
1880
  });
1850
1881
  }
1851
1882
 
1852
- widenForkSightFrameTags(messagesToCopy, forkedMessageIds, attachmentIdMap);
1883
+ remapForkWorkspaceAttachmentRefs(
1884
+ messagesToCopy,
1885
+ forkedMessageIds,
1886
+ attachmentIdMap,
1887
+ );
1888
+ widenForkAttachmentIdTags(
1889
+ messagesToCopy,
1890
+ forkedMessageIds,
1891
+ attachmentIdMap,
1892
+ SIGHT_FRAME_ATTACHMENT_IDS_KEY,
1893
+ sightFrameAttachmentIdsFromMetadata,
1894
+ );
1895
+ widenForkAttachmentIdTags(
1896
+ messagesToCopy,
1897
+ forkedMessageIds,
1898
+ attachmentIdMap,
1899
+ COMPUTER_USE_SCREENSHOT_ATTACHMENT_IDS_KEY,
1900
+ computerUseScreenshotAttachmentIdsFromMetadata,
1901
+ );
1853
1902
 
1854
1903
  // Set lastMessageAt to the max createdAt of copied messages so the
1855
1904
  // forked conversation sorts correctly by message recency.
@@ -1895,31 +1944,99 @@ function populateForkContentsInProcess(args: PopulateForkContentsArgs): void {
1895
1944
  }
1896
1945
 
1897
1946
  /**
1898
- * Extend the copied rows' camera-frame tags to name the fork's cloned
1899
- * attachment ids alongside the source ids they were written with.
1947
+ * Remap copied workspace references to the fork-scoped attachment ids created
1948
+ * by the relink loop.
1900
1949
  *
1901
- * A fork leaves its rows describing their attachments two different ways:
1902
- * `messages.content` is copied byte for byte and still names the SOURCE
1903
- * attachment ids, while `message_attachments` is re-linked to freshly CLONED
1904
- * rows under new ids. Readers split along that seam. Camera-frame retention
1905
- * matches the tag against the ids in the content blocks (source ids), and the
1906
- * compactor builds its image manifest from the links (cloned ids) and stamps
1907
- * those onto the frames it rebuilds. A tag naming only one vocabulary goes
1908
- * blind on the other, so it names both.
1950
+ * The same source attachment can be linked to several copied messages. The
1951
+ * shared map keeps every copied reference and message link on one cloned row,
1952
+ * while references to attachments outside the copied window stay unchanged.
1953
+ */
1954
+ function remapForkWorkspaceAttachmentRefs(
1955
+ messagesToCopy: MessageRow[],
1956
+ forkedMessageIds: Map<string, string>,
1957
+ attachmentIdMap: Map<string, string>,
1958
+ ): void {
1959
+ if (attachmentIdMap.size === 0) {
1960
+ return;
1961
+ }
1962
+
1963
+ const db = getDb();
1964
+ for (const message of messagesToCopy) {
1965
+ const forkedMessageId = forkedMessageIds.get(message.id);
1966
+ if (!forkedMessageId) {
1967
+ continue;
1968
+ }
1969
+
1970
+ const remappedContent = remapWorkspaceAttachmentRefs(
1971
+ message.content,
1972
+ attachmentIdMap,
1973
+ );
1974
+ if (remappedContent === message.content) {
1975
+ continue;
1976
+ }
1977
+
1978
+ db.update(messages)
1979
+ .set({ content: JSON.stringify(remappedContent) })
1980
+ .where(eq(messages.id, forkedMessageId))
1981
+ .run();
1982
+ }
1983
+ }
1984
+
1985
+ function remapWorkspaceAttachmentRefs(
1986
+ value: unknown,
1987
+ attachmentIdMap: ReadonlyMap<string, string>,
1988
+ ): unknown {
1989
+ if (Array.isArray(value)) {
1990
+ let changed = false;
1991
+ const remapped = value.map((entry) => {
1992
+ const next = remapWorkspaceAttachmentRefs(entry, attachmentIdMap);
1993
+ changed ||= next !== entry;
1994
+ return next;
1995
+ });
1996
+ return changed ? remapped : value;
1997
+ }
1998
+
1999
+ if (value == null || typeof value !== "object") {
2000
+ return value;
2001
+ }
2002
+
2003
+ const record = value as Record<string, unknown>;
2004
+ let changed = false;
2005
+ const remapped: Record<string, unknown> = {};
2006
+ for (const [key, entry] of Object.entries(record)) {
2007
+ let next = remapWorkspaceAttachmentRefs(entry, attachmentIdMap);
2008
+ if (
2009
+ key === "attachmentId" &&
2010
+ record.type === "workspace_ref" &&
2011
+ typeof entry === "string"
2012
+ ) {
2013
+ next = attachmentIdMap.get(entry) ?? entry;
2014
+ }
2015
+ changed ||= next !== entry;
2016
+ remapped[key] = next;
2017
+ }
2018
+ return changed ? remapped : value;
2019
+ }
2020
+
2021
+ /**
2022
+ * Extend a copied row's attachment-id metadata to name cloned ids alongside
2023
+ * the source ids it was written with.
1909
2024
  *
1910
- * Widening rather than remapping is deliberate: replacing the source ids would
1911
- * fix the compactor's frames by breaking every frame the fork holds directly,
1912
- * which is the common case. Extra ids are inert, since an id no block carries
1913
- * simply never matches.
2025
+ * Copied message content uses the fork-scoped ids, while metadata begins as a
2026
+ * copy of the source row. Naming both ids preserves compatibility with readers
2027
+ * of either vocabulary. Extra ids are inert because an id no block or linked
2028
+ * attachment carries simply never matches.
1914
2029
  *
1915
2030
  * Runs after the attachment loop because that loop is what produces the id map.
1916
2031
  * Only rows that actually carry a tag are rewritten, so an ordinary fork does
1917
2032
  * no extra writes.
1918
2033
  */
1919
- function widenForkSightFrameTags(
2034
+ function widenForkAttachmentIdTags(
1920
2035
  messagesToCopy: MessageRow[],
1921
2036
  forkedMessageIds: Map<string, string>,
1922
2037
  attachmentIdMap: Map<string, string>,
2038
+ metadataKey: string,
2039
+ readIds: (metadata: Record<string, unknown> | null | undefined) => string[],
1923
2040
  ): void {
1924
2041
  if (attachmentIdMap.size === 0) {
1925
2042
  return;
@@ -1931,7 +2048,7 @@ function widenForkSightFrameTags(
1931
2048
  continue;
1932
2049
  }
1933
2050
  const sourceMetadata = parseMessageMetadata(message.metadata);
1934
- const sourceIds = sightFrameAttachmentIdsFromMetadata(sourceMetadata);
2051
+ const sourceIds = readIds(sourceMetadata);
1935
2052
  if (sourceIds.length === 0) {
1936
2053
  continue;
1937
2054
  }
@@ -1955,7 +2072,7 @@ function widenForkSightFrameTags(
1955
2072
  .set({
1956
2073
  metadata: JSON.stringify({
1957
2074
  ...(forkedMetadata ?? {}),
1958
- [SIGHT_FRAME_ATTACHMENT_IDS_KEY]: [...widened],
2075
+ [metadataKey]: [...widened],
1959
2076
  }),
1960
2077
  })
1961
2078
  .where(eq(messages.id, forkedMessageId))
@@ -2806,8 +2923,8 @@ export interface ConversationAttachmentListing {
2806
2923
  * Driven from `messages` so the lineage predicate rides
2807
2924
  * `idx_messages_conversation_created_at`. An attachment linked to more than
2808
2925
  * one row is listed once, on the newest row that carries it. Tool-result rows
2809
- * are left out: the transcript never shows them, and the assistant row carries
2810
- * the promoted copy of every image a tool produced.
2926
+ * are left out: reply-linked output belongs in Files, while tool-result-only
2927
+ * media stays available through tool history.
2811
2928
  *
2812
2929
  * The lineage-wide select still reads every linked row: an exact `total` and
2813
2930
  * the metadata-derived flags are only known after the role and visibility
@@ -4246,6 +4363,24 @@ export function purgeConversationSegments(
4246
4363
  return segmentIds;
4247
4364
  }
4248
4365
 
4366
+ /** Repair derived mode-session boundaries without changing delete success. */
4367
+ function repairModeSessionBoundariesAfterDelete(conversationId: string): void {
4368
+ const maxAttempts = 2;
4369
+ for (let attempt = 1; attempt <= maxAttempts; attempt++) {
4370
+ try {
4371
+ repairConversationModeSessionBoundaries(conversationId);
4372
+ return;
4373
+ } catch (err) {
4374
+ if (attempt === maxAttempts) {
4375
+ log.warn(
4376
+ { err, conversationId, attempts: maxAttempts },
4377
+ "Failed to repair mode session boundaries after message deletion; continuing",
4378
+ );
4379
+ }
4380
+ }
4381
+ }
4382
+ }
4383
+
4249
4384
  export function deleteLastExchange(conversationId: string): number {
4250
4385
  const db = getDb();
4251
4386
 
@@ -4350,6 +4485,10 @@ export function deleteLastExchange(conversationId: string): number {
4350
4485
  } satisfies MessageDeletedInputContext);
4351
4486
  }
4352
4487
 
4488
+ if (deleted > 0) {
4489
+ repairModeSessionBoundariesAfterDelete(conversationId);
4490
+ }
4491
+
4353
4492
  return deleted;
4354
4493
  }
4355
4494
 
@@ -4660,6 +4799,7 @@ export function deleteMessageById(
4660
4799
  messageId,
4661
4800
  createdAt: msgRow.createdAt,
4662
4801
  } satisfies MessageDeletedInputContext);
4802
+ repairModeSessionBoundariesAfterDelete(msgRow.conversationId);
4663
4803
  }
4664
4804
 
4665
4805
  return result;