@vellumai/assistant 0.12.3-staging.1 → 0.12.3-staging.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (560) hide show
  1. package/ARCHITECTURE.md +43 -29
  2. package/Dockerfile +63 -0
  3. package/docs/architecture/security.md +10 -5
  4. package/docs/architecture/turn-actor.md +2 -0
  5. package/docs/desktop-browser-cli.md +45 -4
  6. package/docs/live-voice-task-outcomes.md +100 -0
  7. package/docs/mode-sessions.md +183 -0
  8. package/docs/voice-input-diagnostics.md +58 -0
  9. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  10. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  11. package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  12. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/package.json +2 -1
  13. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  14. package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  15. package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +20 -0
  16. package/node_modules/@vellumai/gateway-client/src/index.ts +4 -1
  17. package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +38 -7
  18. package/node_modules/@vellumai/gateway-client/src/trust-verdict-contract.ts +5 -6
  19. package/node_modules/@vellumai/service-contracts/package.json +2 -1
  20. package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  21. package/node_modules/@vellumai/service-contracts/src/integration-categories.ts +36 -0
  22. package/node_modules/@vellumai/slack-text/src/index.ts +4 -8
  23. package/openapi.yaml +665 -14
  24. package/package.json +1 -1
  25. package/scripts/smoke-desktop-browser-cli.ts +10 -26
  26. package/scripts/smoke-desktop-window-theme.ts +2 -3
  27. package/src/__tests__/agent-loop-output-hooks.test.ts +150 -1
  28. package/src/__tests__/agent-loop-provider-error-recording.test.ts +87 -19
  29. package/src/__tests__/annotate-activity-metadata.test.ts +3 -1
  30. package/src/__tests__/annotate-risk-options.test.ts +2 -0
  31. package/src/__tests__/answered-question-persistence.test.ts +2 -0
  32. package/src/__tests__/anthropic-gateway-shared.test.ts +22 -0
  33. package/src/__tests__/approval-message-composer.test.ts +1 -118
  34. package/src/__tests__/approval-routes-http.test.ts +3 -0
  35. package/src/__tests__/assistant-event-hub-connection-history.test.ts +111 -0
  36. package/src/__tests__/assistant-event-hub-targeted.test.ts +50 -0
  37. package/src/__tests__/assistant-event-targeting.test.ts +186 -0
  38. package/src/__tests__/assistant-stream-state.test.ts +29 -9
  39. package/src/__tests__/browser-fill-credential.test.ts +11 -22
  40. package/src/__tests__/btw-routes.test.ts +4 -1
  41. package/src/__tests__/build-persisted-content.test.ts +1 -1
  42. package/src/__tests__/call-conversation-messages.test.ts +5 -0
  43. package/src/__tests__/call-recovery.test.ts +7 -2
  44. package/src/__tests__/call-routes-http.test.ts +12 -0
  45. package/src/__tests__/call-store.test.ts +7 -2
  46. package/src/__tests__/channel-retry-sweep.test.ts +9 -0
  47. package/src/__tests__/chat-reveal-guard-priming.test.ts +2 -0
  48. package/src/__tests__/checker.test.ts +11 -3
  49. package/src/__tests__/client-os-metadata-persistence.test.ts +35 -1
  50. package/src/__tests__/computer-use-skill-manifest-regression.test.ts +0 -25
  51. package/src/__tests__/computer-use-tools.test.ts +1 -233
  52. package/src/__tests__/config-model-image-provider.test.ts +39 -0
  53. package/src/__tests__/config-schema.test.ts +16 -0
  54. package/src/__tests__/config-watcher.test.ts +24 -0
  55. package/src/__tests__/context-search-agent-runner.test.ts +260 -1
  56. package/src/__tests__/context-search-fanout.test.ts +1 -1
  57. package/src/__tests__/context-search-types.test.ts +7 -7
  58. package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +8 -2
  59. package/src/__tests__/conversation-agent-loop-fatal-cleanup.test.ts +2 -0
  60. package/src/__tests__/conversation-agent-loop.test.ts +546 -16
  61. package/src/__tests__/conversation-app-control-lifecycle.test.ts +53 -1
  62. package/src/__tests__/conversation-evictor.test.ts +62 -37
  63. package/src/__tests__/conversation-history-boundary-repair-failure.test.ts +174 -0
  64. package/src/__tests__/conversation-loop-db-lock-resilience.test.ts +2 -0
  65. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +2 -0
  66. package/src/__tests__/conversation-retry-route.test.ts +22 -0
  67. package/src/__tests__/conversation-routes-disk-view.test.ts +2 -0
  68. package/src/__tests__/conversation-routes-enabled-plugins.test.ts +2 -0
  69. package/src/__tests__/conversation-routes-guardian-reply.test.ts +10 -0
  70. package/src/__tests__/conversation-routes-hidden-queue.test.ts +2 -0
  71. package/src/__tests__/conversation-routes-slash-commands.test.ts +2 -0
  72. package/src/__tests__/conversation-runtime-assembly.test.ts +29 -0
  73. package/src/__tests__/conversation-surface-snapshots.test.ts +543 -0
  74. package/src/__tests__/conversation-surfaces-action-delivery.test.ts +151 -0
  75. package/src/__tests__/conversation-surfaces-point-at-budget.test.ts +63 -1
  76. package/src/__tests__/conversation-tool-setup-attribution.test.ts +23 -0
  77. package/src/__tests__/conversation-undo.test.ts +61 -0
  78. package/src/__tests__/credential-execution-client.test.ts +6 -6
  79. package/src/__tests__/credential-prompt-route.test.ts +35 -3
  80. package/src/__tests__/cu-unified-flow.test.ts +82 -1
  81. package/src/__tests__/db-mode-sessions-migration.test.ts +175 -0
  82. package/src/__tests__/deterministic-verification-control-plane.test.ts +2 -0
  83. package/src/__tests__/dm-backfill.test.ts +91 -1
  84. package/src/__tests__/document-create-route.test.ts +88 -0
  85. package/src/__tests__/document-sync-tags.test.ts +27 -0
  86. package/src/__tests__/emit-signal-routing-intent.test.ts +142 -2
  87. package/src/__tests__/events-tail-route.test.ts +25 -0
  88. package/src/__tests__/evict-conversations-for-reload.test.ts +57 -8
  89. package/src/__tests__/guardian-action-sweep.test.ts +16 -0
  90. package/src/__tests__/guardian-dispatch.test.ts +13 -0
  91. package/src/__tests__/headless-browser-interactions.test.ts +64 -20
  92. package/src/__tests__/heartbeat-service.test.ts +2 -59
  93. package/src/__tests__/helpers/mock-conversation.ts +21 -0
  94. package/src/__tests__/helpers/native-web-search-harness.ts +2 -0
  95. package/src/__tests__/helpers/verification-sessions-ipc-sim.ts +5 -0
  96. package/src/__tests__/host-cu-proxy.test.ts +46 -2
  97. package/src/__tests__/http-user-message-parity.test.ts +2 -0
  98. package/src/__tests__/image-credentials.test.ts +49 -0
  99. package/src/__tests__/image-service-dispatcher.test.ts +45 -0
  100. package/src/__tests__/inference-profile-session-handler.test.ts +1 -1
  101. package/src/__tests__/interrupt-on-send.test.ts +57 -43
  102. package/src/__tests__/interrupt-turn-note.test.ts +5 -6
  103. package/src/__tests__/llm-catalog-parity.test.ts +2 -2
  104. package/src/__tests__/media-generate-image.test.ts +49 -1
  105. package/src/__tests__/message-provenance-contact-id.test.ts +69 -0
  106. package/src/__tests__/migration-export-to-gcs.test.ts +142 -0
  107. package/src/__tests__/mode-session-history.test.ts +245 -0
  108. package/src/__tests__/mode-session-recovery.test.ts +222 -0
  109. package/src/__tests__/mode-session-wire.test.ts +141 -0
  110. package/src/__tests__/notification-guardian-path.test.ts +7 -0
  111. package/src/__tests__/notification-telegram-adapter.test.ts +2 -2
  112. package/src/__tests__/notification-vellum-adapter.test.ts +34 -0
  113. package/src/__tests__/oauth-provider-serializer.test.ts +12 -0
  114. package/src/__tests__/oauth-providers-routes.test.ts +1 -0
  115. package/src/__tests__/outbound-slack-persistence.test.ts +2 -0
  116. package/src/__tests__/persistence-secret-redaction.test.ts +2 -0
  117. package/src/__tests__/plugin-api-provider.test.ts +5 -0
  118. package/src/__tests__/plugin-api-resolve-credential.test.ts +97 -0
  119. package/src/__tests__/plugin-import-boundary-guard.test.ts +9 -0
  120. package/src/__tests__/process-message-display-content.test.ts +56 -0
  121. package/src/__tests__/reaction-persistence.test.ts +5 -0
  122. package/src/__tests__/request-secret-standalone-channel-gate.test.ts +25 -9
  123. package/src/__tests__/runtime-events-sse-reconnect.test.ts +1 -1
  124. package/src/__tests__/scripted-turn-metadata-persistence.test.ts +73 -0
  125. package/src/__tests__/secret-ingress-http.test.ts +2 -0
  126. package/src/__tests__/secure-keys.test.ts +43 -1
  127. package/src/__tests__/send-endpoint-busy.test.ts +163 -1
  128. package/src/__tests__/server-history-render.test.ts +91 -1
  129. package/src/__tests__/settings-routes.test.ts +50 -1
  130. package/src/__tests__/skill-script-runner-sandbox.test.ts +32 -0
  131. package/src/__tests__/subagent-fork-notifications.test.ts +1 -2
  132. package/src/__tests__/subagent-manager-notify.test.ts +1 -2
  133. package/src/__tests__/subagent-notify-parent.test.ts +146 -39
  134. package/src/__tests__/subagent-notify-stale-rebuild.test.ts +2 -4
  135. package/src/__tests__/surface-completion-in-flight-snapshot.test.ts +2 -0
  136. package/src/__tests__/thread-backfill.test.ts +53 -0
  137. package/src/__tests__/tool-preview-lifecycle.test.ts +2 -0
  138. package/src/__tests__/tool-result-metadata-plumbing.test.ts +3 -1
  139. package/src/__tests__/tool-start-timestamp.test.ts +2 -0
  140. package/src/__tests__/twilio-routes.test.ts +1 -0
  141. package/src/__tests__/web-search-backend-failure.test.ts +1 -1
  142. package/src/activation/progress-store.test.ts +12 -7
  143. package/src/agent/loop.ts +99 -41
  144. package/src/api/events/assistant-turn-start.ts +3 -0
  145. package/src/api/events/generation-handoff.ts +2 -0
  146. package/src/api/events/message-complete.ts +2 -0
  147. package/src/api/events/notification-conversation-created.ts +14 -16
  148. package/src/api/events/notification-intent.ts +4 -2
  149. package/src/api/events/tool-result.ts +85 -0
  150. package/src/api/events/tool-use-start.ts +3 -0
  151. package/src/api/events/user-message-echo.ts +3 -0
  152. package/src/api/index.ts +31 -6
  153. package/src/api/mode-session.test.ts +131 -0
  154. package/src/api/mode-session.ts +148 -0
  155. package/src/api/remember-tool.ts +15 -0
  156. package/src/api/responses/conversation-message.ts +6 -0
  157. package/src/approvals/AGENTS.md +4 -2
  158. package/src/browser/__tests__/operations.test.ts +29 -0
  159. package/src/browser/operation-meta.ts +2 -1
  160. package/src/browser/operations.ts +29 -0
  161. package/src/browser/virtual-desktop-target.ts +2 -8
  162. package/src/bundler/bundle-signer.ts +1 -3
  163. package/src/calls/__tests__/barge-in-guard.test.ts +3 -0
  164. package/src/calls/__tests__/call-controller.test.ts +94 -0
  165. package/src/calls/__tests__/call-funnel-telemetry.test.ts +204 -0
  166. package/src/calls/__tests__/voice-control-protocol.test.ts +15 -0
  167. package/src/calls/__tests__/voice-escalation-judge.test.ts +209 -0
  168. package/src/calls/__tests__/voice-leg-coordinator.test.ts +168 -0
  169. package/src/{live-voice/__tests__/live-voice-metrics.test.ts → calls/__tests__/voice-metrics.test.ts} +32 -38
  170. package/src/calls/__tests__/voice-session-bridge.test.ts +576 -122
  171. package/src/calls/barge-in-guard.ts +5 -0
  172. package/src/calls/call-controller.ts +196 -1
  173. package/src/calls/call-domain.ts +4 -0
  174. package/src/calls/call-store.ts +87 -2
  175. package/src/calls/guardian-action-sweep.ts +22 -8
  176. package/src/calls/media-stream-audio-transcode.ts +1 -3
  177. package/src/calls/typesafe-noul.ts +156 -0
  178. package/src/calls/voice-control-protocol.ts +6 -0
  179. package/src/calls/voice-escalation-judge.ts +145 -0
  180. package/src/calls/voice-escalation-target.ts +10 -0
  181. package/src/calls/voice-leg-coordinator.ts +121 -3
  182. package/src/{live-voice/live-voice-metrics.ts → calls/voice-metrics.ts} +94 -87
  183. package/src/calls/voice-session-bridge.ts +256 -112
  184. package/src/calls/voice-triage-escalate.ts +7 -0
  185. package/src/cli/__tests__/roadmap-cli.test.ts +4 -41
  186. package/src/cli/commands/__tests__/notifications.test.ts +35 -10
  187. package/src/cli/commands/browser.help.ts +4 -5
  188. package/src/cli/commands/browser.ts +0 -1
  189. package/src/cli/commands/clients.help.ts +46 -0
  190. package/src/cli/commands/clients.ts +116 -0
  191. package/src/cli/commands/credentials.help.ts +6 -1
  192. package/src/cli/commands/notifications.help.ts +16 -2
  193. package/src/cli/commands/notifications.ts +30 -8
  194. package/src/cli/commands/roadmap.help.ts +18 -21
  195. package/src/cli/commands/roadmap.ts +4 -21
  196. package/src/cli/lib/__tests__/list-installed-plugins.test.ts +5 -2
  197. package/src/cli/lib/bundled-marketplace.json +84 -34
  198. package/src/cli/lib/plugin-marketplace.ts +8 -0
  199. package/src/cli/lib/publish-plugin.ts +1 -2
  200. package/src/config/__tests__/default-provider.test.ts +3 -1
  201. package/src/config/__tests__/memory-retrospective-schema.test.ts +14 -1
  202. package/src/config/bundled-skills/computer-use/SKILL.md +1 -1
  203. package/src/config/bundled-skills/computer-use/TOOLS.json +89 -1
  204. package/src/config/bundled-skills/image-studio/SKILL.md +17 -2
  205. package/src/config/bundled-skills/image-studio/tools/media-generate-image.ts +7 -11
  206. package/src/config/bundled-skills/sequences/TOOLS.json +1 -5
  207. package/src/config/call-site-defaults.ts +9 -0
  208. package/src/config/feature-flag-registry.json +16 -0
  209. package/src/config/profile-text-generation.test.ts +2 -2
  210. package/src/config/schemas/__tests__/tools.test.ts +31 -0
  211. package/src/config/schemas/call-site-catalog.ts +14 -0
  212. package/src/config/schemas/channels.ts +0 -18
  213. package/src/config/schemas/llm.ts +3 -1
  214. package/src/config/schemas/mcp.ts +2 -1
  215. package/src/config/schemas/memory-retrospective.ts +11 -2
  216. package/src/config/schemas/services.ts +15 -6
  217. package/src/config/schemas/tools.ts +10 -0
  218. package/src/config/session-groups-gate.ts +21 -0
  219. package/src/config/skills.ts +2 -6
  220. package/src/conversations/__tests__/message-consolidation.test.ts +149 -0
  221. package/src/conversations/message-consolidation.ts +102 -5
  222. package/src/credential-execution/ces-connect.test.ts +69 -0
  223. package/src/credential-execution/ces-connect.ts +115 -0
  224. package/src/credential-execution/ces-runtime.ts +55 -93
  225. package/src/credential-execution/executable-discovery.ts +1 -11
  226. package/src/daemon/__tests__/conversation-surfaces-launch.test.ts +10 -1
  227. package/src/daemon/__tests__/inference-profile-notification.test.ts +2 -0
  228. package/src/daemon/browser-mode-session.test.ts +402 -0
  229. package/src/daemon/browser-mode-session.ts +186 -0
  230. package/src/daemon/computer-use-mode-session.test.ts +210 -0
  231. package/src/daemon/computer-use-mode-session.ts +105 -0
  232. package/src/daemon/config-watcher.ts +10 -0
  233. package/src/daemon/conversation-agent-loop-handlers.ts +58 -10
  234. package/src/daemon/conversation-agent-loop.ts +219 -25
  235. package/src/daemon/conversation-evictor.ts +5 -24
  236. package/src/daemon/conversation-history.ts +10 -0
  237. package/src/daemon/conversation-interrupt-repair.ts +4 -43
  238. package/src/daemon/conversation-interrupt.ts +7 -11
  239. package/src/daemon/conversation-messaging.ts +58 -5
  240. package/src/daemon/conversation-mode-session.test.ts +1667 -0
  241. package/src/daemon/conversation-mode-session.ts +1008 -0
  242. package/src/daemon/conversation-process.ts +5 -0
  243. package/src/daemon/conversation-runtime-assembly.ts +10 -20
  244. package/src/daemon/conversation-skill-tools.ts +1 -0
  245. package/src/daemon/conversation-store.ts +4 -4
  246. package/src/daemon/conversation-surface-snapshots.ts +135 -0
  247. package/src/daemon/conversation-surfaces.ts +113 -5
  248. package/src/daemon/conversation-tool-setup.ts +8 -3
  249. package/src/daemon/conversation.ts +143 -41
  250. package/src/daemon/handlers/config-model.ts +7 -5
  251. package/src/daemon/handlers/shared.ts +55 -16
  252. package/src/daemon/host-cu-proxy.ts +51 -22
  253. package/src/daemon/lifecycle.ts +30 -13
  254. package/src/daemon/message-protocol.ts +0 -1
  255. package/src/daemon/message-provenance.ts +28 -4
  256. package/src/daemon/mode-session-startup-recovery.ts +23 -0
  257. package/src/daemon/mode-session-tracking.ts +54 -0
  258. package/src/daemon/process-message.ts +13 -4
  259. package/src/daemon/virtual-desktop-context.ts +21 -0
  260. package/src/desktop/__tests__/fake-desktop.ts +1 -0
  261. package/src/desktop/desktop-accessibility-script.ts +88 -0
  262. package/src/desktop/desktop-accessibility.test.ts +160 -0
  263. package/src/desktop/desktop-accessibility.ts +169 -0
  264. package/src/desktop/desktop-automation-lease.test.ts +76 -68
  265. package/src/desktop/desktop-automation-lease.ts +41 -50
  266. package/src/desktop/desktop-browser-endpoint.ts +4 -0
  267. package/src/desktop/desktop-computer-use-routing.test.ts +330 -0
  268. package/src/desktop/desktop-computer-use.test.ts +449 -0
  269. package/src/desktop/desktop-computer-use.ts +502 -0
  270. package/src/desktop/desktop-dependencies.ts +47 -293
  271. package/src/desktop/desktop-display.ts +2 -2
  272. package/src/desktop/desktop-panel-config.test.ts +13 -3
  273. package/src/desktop/desktop-panel-config.ts +109 -15
  274. package/src/desktop/desktop-session-manager.test.ts +69 -21
  275. package/src/desktop/desktop-session-manager.ts +76 -22
  276. package/src/desktop/desktop-stream-bridge.test.ts +11 -122
  277. package/src/desktop/desktop-stream-bridge.ts +6 -8
  278. package/src/desktop/desktop-window-theme.ts +5 -25
  279. package/src/desktop/virtual-desktop-feature.ts +52 -2
  280. package/src/desktop/virtual-desktop-platform.test.ts +5 -9
  281. package/src/documents/document-store.ts +23 -0
  282. package/src/heartbeat/__tests__/heartbeat-service.test.ts +22 -26
  283. package/src/heartbeat/heartbeat-service.ts +5 -33
  284. package/src/ipc/__tests__/browser-ipc.test.ts +215 -6
  285. package/src/ipc/assistant-server.ts +2 -0
  286. package/src/ipc/routes/__tests__/channel-reply-ipc-routes.test.ts +111 -0
  287. package/src/ipc/routes/channel-reply-ipc-routes.ts +73 -0
  288. package/src/live-voice/__tests__/continuation-judge.test.ts +151 -0
  289. package/src/live-voice/__tests__/live-activity-reporter.test.ts +154 -15
  290. package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +743 -31
  291. package/src/live-voice/__tests__/live-voice-look-follow-up.test.ts +191 -10
  292. package/src/live-voice/__tests__/live-voice-session-groups-gate.test.ts +87 -0
  293. package/src/live-voice/__tests__/live-voice-session-manager.test.ts +5 -1
  294. package/src/live-voice/__tests__/live-voice-sight-frame-echo.test.ts +41 -0
  295. package/src/live-voice/__tests__/live-voice-sight-frame.test.ts +1017 -9
  296. package/src/live-voice/__tests__/live-voice-triage-escalate.test.ts +224 -1
  297. package/src/live-voice/__tests__/live-voice-tts-session.test.ts +101 -0
  298. package/src/live-voice/__tests__/live-voice-tts.test.ts +44 -0
  299. package/src/live-voice/__tests__/live-voice-vad.test.ts +939 -75
  300. package/src/live-voice/__tests__/microphone-idle-gate.test.ts +87 -0
  301. package/src/live-voice/__tests__/session-controls.test.ts +23 -0
  302. package/src/live-voice/__tests__/task-outcomes.test.ts +117 -0
  303. package/src/live-voice/__tests__/voice-input-diagnostics.test.ts +117 -0
  304. package/src/live-voice/camera-mode-session.test.ts +265 -0
  305. package/src/live-voice/camera-mode-session.ts +277 -0
  306. package/src/live-voice/continuation-judge.ts +123 -0
  307. package/src/live-voice/live-activity-reporter.ts +115 -6
  308. package/src/live-voice/live-voice-manager.ts +36 -1
  309. package/src/live-voice/live-voice-photo.ts +88 -7
  310. package/src/live-voice/live-voice-session-manager.ts +15 -0
  311. package/src/live-voice/live-voice-session.ts +1443 -747
  312. package/src/live-voice/live-voice-tts.ts +22 -4
  313. package/src/live-voice/microphone-idle-gate.ts +56 -0
  314. package/src/live-voice/protocol.ts +171 -4
  315. package/src/live-voice/session-controls.ts +11 -3
  316. package/src/live-voice/task-outcomes.ts +131 -0
  317. package/src/live-voice/voice-input-diagnostics.ts +117 -0
  318. package/src/mcp/__tests__/manager-tool-caps.test.ts +63 -18
  319. package/src/mcp/__tests__/tool-caps.test.ts +28 -1
  320. package/src/mcp/manager.ts +3 -0
  321. package/src/mcp/tool-caps.ts +12 -4
  322. package/src/media/__tests__/image-models.test.ts +58 -0
  323. package/src/media/image-credentials.ts +5 -0
  324. package/src/media/image-models.ts +52 -0
  325. package/src/media/image-service.ts +8 -0
  326. package/src/media/openrouter-image-service.test.ts +133 -0
  327. package/src/media/openrouter-image-service.ts +153 -0
  328. package/src/media/types.ts +1 -1
  329. package/src/messaging/providers/__tests__/transport-dispatch.test.ts +65 -11
  330. package/src/messaging/providers/discord/send.ts +4 -17
  331. package/src/messaging/providers/discord/transport.ts +2 -2
  332. package/src/messaging/providers/outlook/client.ts +16 -22
  333. package/src/messaging/providers/telegram-bot/send.test.ts +236 -1
  334. package/src/messaging/providers/telegram-bot/send.ts +10 -7
  335. package/src/messaging/providers/telegram-bot/transport.ts +3 -10
  336. package/src/messaging/providers/whatsapp/send.test.ts +169 -4
  337. package/src/monitoring/__tests__/control.test.ts +17 -5
  338. package/src/monitoring/control.ts +15 -5
  339. package/src/notifications/AGENTS.md +1 -1
  340. package/src/notifications/README.md +83 -99
  341. package/src/notifications/__tests__/broadcaster.test.ts +1 -1
  342. package/src/notifications/__tests__/channel-allowlist.test.ts +39 -0
  343. package/src/notifications/__tests__/copy-composer.test.ts +1 -1
  344. package/src/notifications/__tests__/decision-engine.test.ts +79 -2
  345. package/src/notifications/__tests__/deterministic-checks.test.ts +10 -10
  346. package/src/notifications/__tests__/home-feed-side-effect.test.ts +93 -4
  347. package/src/notifications/__tests__/notification-utils.test.ts +18 -0
  348. package/src/notifications/adapters/macos.ts +37 -46
  349. package/src/notifications/broadcaster.ts +5 -6
  350. package/src/notifications/channel-allowlist.ts +44 -0
  351. package/src/notifications/copy-composer.ts +0 -23
  352. package/src/notifications/decision-engine.ts +24 -14
  353. package/src/notifications/deterministic-checks.ts +1 -1
  354. package/src/notifications/emit-signal.ts +123 -37
  355. package/src/notifications/home-feed-side-effect.ts +9 -7
  356. package/src/notifications/notification-utils.ts +19 -0
  357. package/src/notifications/signal.ts +4 -19
  358. package/src/notifications/types.ts +15 -0
  359. package/src/oauth/connection-resolver.ts +1 -2
  360. package/src/oauth/oauth-store.ts +1 -3
  361. package/src/oauth/provider-serializer.ts +7 -0
  362. package/src/oauth/seed-providers.ts +30 -0
  363. package/src/onboarding/onboarding-events-store.ts +99 -12
  364. package/src/persistence/__tests__/client-connection-events-store.test.ts +322 -0
  365. package/src/persistence/client-connection-events-store.ts +377 -0
  366. package/src/persistence/conversation-crud.ts +55 -6
  367. package/src/persistence/conversation-mode-sessions.test.ts +424 -0
  368. package/src/persistence/conversation-mode-sessions.ts +460 -0
  369. package/src/persistence/conversation-queries.ts +6 -19
  370. package/src/persistence/llm-usage-store.ts +3 -9
  371. package/src/persistence/message-metadata.ts +25 -0
  372. package/src/persistence/migrations/381-create-conversation-mode-sessions.ts +41 -0
  373. package/src/persistence/migrations/382-create-client-connection-events.ts +40 -0
  374. package/src/persistence/migrations/__tests__/382-create-client-connection-events.test.ts +56 -0
  375. package/src/persistence/schema/conversations.ts +67 -0
  376. package/src/persistence/schema/infrastructure.ts +29 -0
  377. package/src/persistence/steps.ts +17 -0
  378. package/src/plugin-api/__tests__/plugin-name-env.test.ts +249 -0
  379. package/src/plugin-api/effective-context-window.ts +33 -0
  380. package/src/plugin-api/index.ts +24 -5
  381. package/src/plugin-api/plugin-name-env.ts +125 -0
  382. package/src/plugin-api/resolve-credential.ts +79 -20
  383. package/src/plugin-api/vision-support.test.ts +22 -0
  384. package/src/plugin-api/vision-support.ts +17 -5
  385. package/src/plugins/__tests__/active-plugin-env.test.ts +136 -0
  386. package/src/plugins/active-plugin-env.ts +51 -0
  387. package/src/plugins/defaults/index.ts +24 -0
  388. package/src/plugins/defaults/memory/__tests__/archive-removal-framing.test.ts +64 -0
  389. package/src/plugins/defaults/memory/__tests__/worker-control.test.ts +19 -6
  390. package/src/plugins/defaults/memory/context-search/agent-protocol.ts +9 -12
  391. package/src/plugins/defaults/memory/context-search/agent-runner.ts +78 -22
  392. package/src/plugins/defaults/memory/context-search/format.ts +51 -7
  393. package/src/plugins/defaults/memory/context-search/limits.ts +10 -12
  394. package/src/plugins/defaults/memory/context-search/recall-input.ts +38 -0
  395. package/src/plugins/defaults/memory/context-search/search.ts +8 -14
  396. package/src/plugins/defaults/memory/context-search/sources/conversations.ts +1 -0
  397. package/src/plugins/defaults/memory/context-search/types.ts +1 -11
  398. package/src/plugins/defaults/memory/graph/__tests__/handle-remember-v2.test.ts +11 -4
  399. package/src/plugins/defaults/memory/graph/tool-handlers.ts +12 -21
  400. package/src/plugins/defaults/memory/graph/tools.ts +35 -64
  401. package/src/plugins/defaults/memory/src/memory-eval-routes.ts +2 -3
  402. package/src/plugins/defaults/memory/src/memory-item-routes.test.ts +5 -0
  403. package/src/plugins/defaults/memory/src/memory-item-routes.ts +2 -1
  404. package/src/plugins/defaults/memory/substrate/prompts/consolidation.ts +2 -2
  405. package/src/plugins/defaults/memory/tools.test.ts +124 -9
  406. package/src/plugins/defaults/memory/tools.ts +38 -11
  407. package/src/plugins/defaults/memory/v3/__tests__/pool-log-store.test.ts +165 -27
  408. package/src/plugins/defaults/memory/v3/__tests__/pool-select.test.ts +626 -27
  409. package/src/plugins/defaults/memory/v3/__tests__/shadow-plugin.test.ts +46 -27
  410. package/src/plugins/defaults/memory/v3/orchestrate.ts +78 -38
  411. package/src/plugins/defaults/memory/v3/pool-log-store.ts +65 -79
  412. package/src/plugins/defaults/memory/v3/pool-select.test.ts +14 -5
  413. package/src/plugins/defaults/memory/v3/pool-select.ts +683 -44
  414. package/src/plugins/defaults/memory/worker-control.ts +15 -5
  415. package/src/plugins/defaults/task-progress/__tests__/registration.test.ts +70 -0
  416. package/src/plugins/defaults/task-progress/__tests__/task-progress-context.test.ts +380 -0
  417. package/src/plugins/defaults/task-progress/hooks/post-compact.ts +19 -0
  418. package/src/plugins/defaults/task-progress/hooks/user-prompt-submit.ts +24 -0
  419. package/src/plugins/defaults/task-progress/package.json +14 -0
  420. package/src/plugins/defaults/task-progress/src/apply-task-progress-context.ts +93 -0
  421. package/src/plugins/defaults/task-progress/src/format-task-progress-context.ts +67 -0
  422. package/src/providers/__tests__/context-overflow-error.test.ts +10 -0
  423. package/src/providers/__tests__/search-provider-catalog.test.ts +18 -0
  424. package/src/providers/anthropic/client.ts +10 -1
  425. package/src/providers/anthropic-gateway-shared.ts +6 -0
  426. package/src/providers/fetch-provider-catalog.ts +11 -0
  427. package/src/providers/gemini/client.ts +25 -8
  428. package/src/providers/inference/adapter-factory.ts +1 -1
  429. package/src/providers/jev/client.test.ts +10 -0
  430. package/src/providers/jev/client.ts +20 -4
  431. package/src/providers/model-catalog.ts +4 -3
  432. package/src/providers/openai/__tests__/chat-completions-extra-body.test.ts +94 -1
  433. package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +106 -1
  434. package/src/providers/openai/__tests__/chat-template-flatten.test.ts +24 -0
  435. package/src/providers/openai/__tests__/google-thought-signature.test.ts +1 -2
  436. package/src/providers/openai/chat-completions-provider.ts +100 -29
  437. package/src/providers/openai/google-thought-signature.ts +0 -6
  438. package/src/providers/openai/responses-provider.ts +23 -1
  439. package/src/providers/ratelimit.ts +1 -3
  440. package/src/providers/search-provider-catalog.ts +24 -0
  441. package/src/providers/speech-to-text/__tests__/deepgram-flux-frames.test.ts +24 -0
  442. package/src/providers/speech-to-text/__tests__/deepgram-flux-realtime.test.ts +7 -1
  443. package/src/providers/speech-to-text/__tests__/vellum-managed-realtime.test.ts +50 -1
  444. package/src/providers/speech-to-text/deepgram-flux-frames.ts +11 -0
  445. package/src/providers/speech-to-text/vellum-managed-flux-realtime.ts +13 -0
  446. package/src/providers/types.ts +5 -0
  447. package/src/routes/control.ts +7 -5
  448. package/src/runtime/AGENTS.md +4 -2
  449. package/src/runtime/actor-trust-resolver.ts +3 -3
  450. package/src/runtime/agent-wake.ts +1 -1
  451. package/src/runtime/approval-message-composer.ts +3 -69
  452. package/src/runtime/assistant-event-hub.ts +73 -75
  453. package/src/runtime/assistant-event-publish-options.ts +7 -0
  454. package/src/runtime/assistant-event-targeting.ts +79 -0
  455. package/src/runtime/assistant-stream-state.ts +13 -100
  456. package/src/runtime/background-job-runner.ts +5 -5
  457. package/src/runtime/channel-retry-sweep.ts +1 -0
  458. package/src/runtime/guardian-action-message-composer.ts +8 -161
  459. package/src/runtime/guardian-reply-router.ts +28 -32
  460. package/src/runtime/http-types.ts +6 -0
  461. package/src/runtime/message-composer-types.ts +3 -61
  462. package/src/runtime/migrations/__tests__/vbundle-import-parity.test.ts +67 -1
  463. package/src/runtime/migrations/vbundle-builder.ts +24 -1
  464. package/src/runtime/migrations/vbundle-import-analyzer.ts +13 -4
  465. package/src/runtime/migrations/vbundle-import-policy.ts +10 -0
  466. package/src/runtime/migrations/vbundle-importer.ts +4 -1
  467. package/src/runtime/migrations/vbundle-streaming-importer.ts +4 -1
  468. package/src/runtime/migrations/vbundle-validator.ts +1 -0
  469. package/src/runtime/pending-interactions-mode-session.test.ts +62 -0
  470. package/src/runtime/pending-interactions.ts +29 -0
  471. package/src/runtime/routes/__tests__/client-history-routes.test.ts +104 -0
  472. package/src/runtime/routes/__tests__/image-generation-routes.test.ts +97 -0
  473. package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +1 -1
  474. package/src/runtime/routes/__tests__/roadmap-routes.test.ts +29 -6
  475. package/src/runtime/routes/browser-context.ts +31 -11
  476. package/src/runtime/routes/browser-routes.ts +43 -4
  477. package/src/runtime/routes/btw-routes.ts +9 -4
  478. package/src/runtime/routes/client-routes.ts +108 -0
  479. package/src/runtime/routes/conversation-management-routes.ts +4 -1
  480. package/src/runtime/routes/conversation-routes.ts +113 -27
  481. package/src/runtime/routes/credential-prompt-routes.ts +4 -3
  482. package/src/runtime/routes/credential-routes.ts +1 -2
  483. package/src/runtime/routes/desktop-setup-routes.test.ts +9 -18
  484. package/src/runtime/routes/desktop-setup-routes.ts +4 -6
  485. package/src/runtime/routes/documents-routes.ts +51 -0
  486. package/src/runtime/routes/email-routes.ts +3 -7
  487. package/src/runtime/routes/events-routes.ts +75 -54
  488. package/src/runtime/routes/identity-routes.ts +1 -5
  489. package/src/runtime/routes/image-generation-routes.ts +8 -14
  490. package/src/runtime/routes/inbound-message-handler.ts +35 -38
  491. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.test.ts +168 -0
  492. package/src/runtime/routes/inbound-stages/backfill-sender-provenance.ts +96 -0
  493. package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +38 -9
  494. package/src/runtime/routes/inbound-stages/background-dispatch.ts +2 -5
  495. package/src/runtime/routes/inbound-stages/reaction-intercept.ts +2 -0
  496. package/src/runtime/routes/llm-context-normalization.ts +1 -7
  497. package/src/runtime/routes/migration-routes.ts +73 -2
  498. package/src/runtime/routes/notification-routes.ts +10 -0
  499. package/src/runtime/routes/oauth-providers.ts +7 -0
  500. package/src/runtime/routes/plugins-routes.ts +6 -0
  501. package/src/runtime/routes/roadmap-routes.ts +47 -8
  502. package/src/runtime/routes/secret-routes.ts +1 -1
  503. package/src/runtime/routes/settings-routes.ts +10 -4
  504. package/src/runtime/routes/watcher-routes.ts +1 -3
  505. package/src/runtime/trust-class.test.ts +7 -6
  506. package/src/runtime/trust-class.ts +9 -17
  507. package/src/schedule/__tests__/worker-control.test.ts +36 -5
  508. package/src/schedule/worker-control.ts +15 -5
  509. package/src/security/secure-keys.ts +53 -103
  510. package/src/skills/clawhub.ts +3 -8
  511. package/src/skills/managed-store.ts +1 -5
  512. package/src/stt/types.ts +4 -0
  513. package/src/subagent/notify.ts +54 -34
  514. package/src/subagent/parent-notification.ts +6 -0
  515. package/src/telemetry/__tests__/phone-call-funnel.test.ts +72 -0
  516. package/src/telemetry/phone-call-funnel.ts +144 -0
  517. package/src/tools/__tests__/tool-schema-root-combinator-guard.test.ts +81 -0
  518. package/src/tools/ask-question/ask-question-tool.ts +6 -4
  519. package/src/tools/browser/browser-execution.ts +22 -59
  520. package/src/tools/browser/cdp-client/__tests__/cdp-dom-helpers.test.ts +561 -39
  521. package/src/tools/browser/cdp-client/cdp-dom-helpers.ts +435 -39
  522. package/src/tools/browser/cdp-client/cdp-inspect-client.ts +1 -8
  523. package/src/tools/client-os.ts +15 -0
  524. package/src/tools/computer-use/skill-proxy-bridge.ts +1 -1
  525. package/src/tools/computer-use/target.ts +42 -0
  526. package/src/tools/document/document-tool.ts +13 -18
  527. package/src/tools/execution-target.ts +14 -16
  528. package/src/tools/host-terminal/host-shell.ts +4 -1
  529. package/src/tools/network/__tests__/web-fetch-tinyfish.test.ts +250 -0
  530. package/src/tools/network/__tests__/web-search.test.ts +203 -0
  531. package/src/tools/network/web-fetch.ts +274 -7
  532. package/src/tools/network/web-search.ts +468 -6
  533. package/src/tools/permission-checker.ts +3 -3
  534. package/src/tools/policy-context.ts +3 -1
  535. package/src/tools/skills/load.ts +2 -1
  536. package/src/tools/skills/sandbox-runner.ts +11 -2
  537. package/src/tools/skills/skill-script-runner.ts +7 -0
  538. package/src/tools/skills/skill-tool-factory.ts +15 -1
  539. package/src/tools/terminal/__tests__/safe-env.test.ts +11 -0
  540. package/src/tools/terminal/safe-env.ts +4 -0
  541. package/src/tools/terminal/shell.ts +2 -0
  542. package/src/tools/tool-approval-handler.ts +5 -1
  543. package/src/tools/tool-types.ts +3 -3
  544. package/src/tools/types.ts +8 -2
  545. package/src/util/__tests__/worker-process-command.test.ts +5 -4
  546. package/src/util/abort-reasons.ts +18 -26
  547. package/src/util/canonicalize-identity.test.ts +21 -0
  548. package/src/util/canonicalize-identity.ts +16 -0
  549. package/src/util/errors.ts +13 -3
  550. package/src/util/provider-error-patterns.ts +3 -0
  551. package/src/util/worker-process.ts +77 -35
  552. package/src/workspace/migrations/157-add-desktop-files-launcher.ts +94 -0
  553. package/src/workspace/migrations/__tests__/156-extract-workspace-mcp-json.test.ts +4 -10
  554. package/src/workspace/migrations/__tests__/157-add-desktop-files-launcher.test.ts +113 -0
  555. package/src/workspace/migrations/registry.ts +2 -0
  556. package/src/__tests__/computer-use-window-schema.test.ts +0 -24
  557. package/src/api/events/heartbeat-alert.ts +0 -20
  558. package/src/daemon/message-types/web-activity.ts +0 -65
  559. package/src/desktop/desktop-dependencies.test.ts +0 -131
  560. package/src/tools/computer-use/definitions.ts +0 -607
package/src/agent/loop.ts CHANGED
@@ -1,4 +1,7 @@
1
1
  import type { AnsweredQuestion } from "../api/events/question-answered.js";
2
+ import type { ToolActivityMetadata } from "../api/events/tool-result.js";
3
+ import { resolveCallSiteConfig } from "../config/llm-resolver.js";
4
+ import { getConfig } from "../config/loader.js";
2
5
  import type { LLMCallSite } from "../config/schemas/llm.js";
3
6
  import { SEND_USER_MESSAGE_TOOL_NAME } from "../config/send-user-message-constants.js";
4
7
  import { recordEstimate } from "../context/estimator-calibration.js";
@@ -11,7 +14,6 @@ import {
11
14
  getCalibrationProviderKey,
12
15
  } from "../context/token-estimator.js";
13
16
  import { spoolAndStubOversizedToolResults } from "../context/tool-result-spool.js";
14
- import type { ToolActivityMetadata } from "../daemon/message-types/web-activity.js";
15
17
  import { parseActualTokensFromError } from "../daemon/parse-actual-tokens-from-error.js";
16
18
  import type { TrustContext } from "../daemon/trust-context-types.js";
17
19
  import type {
@@ -443,25 +445,26 @@ export type AgentEvent =
443
445
  /**
444
446
  * Emitted when the provider call throws — i.e. the provider
445
447
  * rejected the request before returning a usable response. Carries
446
- * the loop-level raw request we attempted to send (messages, tools,
447
- * system prompt, provider-agnostic config) plus the thrown error.
448
+ * the wire request we attempted to send plus the thrown error.
448
449
  * Consumers (`handleProviderError` in the daemon handlers, the
449
450
  * `onEvent` in `agent-wake`) persist these as `llm_request_logs`
450
451
  * rows so failed calls are queryable in the LLM inspector instead
451
452
  * of only surfacing in pino logs.
452
453
  *
453
- * `rawRequest` is the loop-level abstract shape rather than the
454
- * provider-specific payload (which the provider builds internally
455
- * and never returns when it throws). `actualProvider` echoes the
456
- * `ProviderError.provider` tag when available so the persisted row
457
- * has the same `provider` column value as a successful `usage` row.
454
+ * `rawRequest` is `ProviderError.rawRequest` when the throw carried
455
+ * an inspectable SDK/wire payload (including extra body fields such
456
+ * as `directions`). The loop does not invent a substitute snapshot
457
+ * of messages/tools/systemPrompt; a missing payload stays missing.
458
+ * `actualProvider` echoes `ProviderError.provider` so a routed
459
+ * invocation (e.g. Vellum via a Fireworks default wrapper) is
460
+ * attributed to the transport that actually ran.
458
461
  *
459
462
  * Re-thrown by the inner LLM-call try/catch after emission so the
460
463
  * outer agent-loop catch still handles abort, the existing `error`
461
464
  * event, and the loop break.
462
465
  */
463
466
  type: "provider_error";
464
- rawRequest: unknown;
467
+ rawRequest?: unknown;
465
468
  error: Error;
466
469
  actualProvider?: string;
467
470
  }
@@ -663,6 +666,18 @@ type AgentLoopContextWindowResolver = () => {
663
666
  overflowRecovery: { enabled: boolean; safetyMarginRatio: number };
664
667
  };
665
668
 
669
+ /** Final request surface after the pre-model hook has settled. */
670
+ export interface PreparedModelCall {
671
+ callSite?: LLMCallSite;
672
+ overrideProfile?: string;
673
+ forceOverrideProfile: boolean;
674
+ /** Present when the finalized route opts out of prompt caching. */
675
+ disableCache?: true;
676
+ signal?: AbortSignal;
677
+ systemPrompt: string | null;
678
+ tools: ToolDefinition[];
679
+ }
680
+
666
681
  interface AgentLoopRunOptionsBase {
667
682
  /** Input history the run starts from; the loop appends its output onto a copy. */
668
683
  messages: Message[];
@@ -690,7 +705,10 @@ interface AgentLoopRunOptionsBase {
690
705
  onCheckpoint?: (
691
706
  checkpoint: CheckpointInfo,
692
707
  ) => CheckpointDecision | Promise<CheckpointDecision>;
708
+ /** Semantic call site exposed to hooks, events, and loop behavior. */
693
709
  callSite?: LLMCallSite;
710
+ /** Provider-resolution call site when it differs from turn semantics. */
711
+ inferenceCallSite?: LLMCallSite;
694
712
  /**
695
713
  * Route this run's user-facing text through the `send_user_message` tool
696
714
  * instead of streamed assistant text. The daemon sets it for main-agent runs
@@ -740,6 +758,8 @@ interface AgentLoopRunOptionsBase {
740
758
  */
741
759
  forceOverrideProfile?: boolean;
742
760
  resolveOverrideProfile?: () => string | undefined;
761
+ /** Observe a finalized model request without delaying provider dispatch. */
762
+ onModelCallPrepared?: (prepared: PreparedModelCall) => void;
743
763
  /**
744
764
  * When `true`, the loop owns turn-start and mid-loop compaction. The pre-call
745
765
  * budget gate runs before the very first provider call — subsuming the
@@ -1429,18 +1449,21 @@ export class AgentLoop {
1429
1449
  requestId,
1430
1450
  onCheckpoint,
1431
1451
  callSite,
1452
+ inferenceCallSite,
1432
1453
  suppressAssistantText = false,
1433
1454
  supportsDynamicUi = true,
1434
1455
  trust,
1435
1456
  overrideProfile,
1436
1457
  forceOverrideProfile = false,
1437
1458
  resolveOverrideProfile,
1459
+ onModelCallPrepared,
1438
1460
  compactInPlace = false,
1439
1461
  isNonInteractive = false,
1440
1462
  model: runModel,
1441
1463
  latencyTracker,
1442
1464
  injectionLedgerResets,
1443
1465
  } = options;
1466
+ const providerCallSite = inferenceCallSite ?? callSite;
1444
1467
  // Snapshot the system prompt once per run. The instance field is mutable
1445
1468
  // (the conversation may update it between turns), but a single run must
1446
1469
  // use one consistent prompt — an aborted run left detached after the
@@ -1886,7 +1909,7 @@ export class AgentLoop {
1886
1909
  // unexecutable client tool. The advisor consult's `advisorProfile` can
1887
1910
  // route `subagentSpawn` to a provider/model whose native-search support
1888
1911
  // differs from the construction-time default, so the gate resolves the
1889
- // routed target (callSite + overrideProfile) via
1912
+ // routed target (providerCallSite + overrideProfile) via
1890
1913
  // `supportsNativeWebSearchFor` rather than the static
1891
1914
  // `this.provider.supportsNativeWebSearch` snapshot; providers without
1892
1915
  // the routing-aware probe fall back to the static flag. This is a SERVER
@@ -1898,7 +1921,7 @@ export class AgentLoop {
1898
1921
  .supportsNativeWebSearchFor
1899
1922
  ? this.provider.supportsNativeWebSearchFor(
1900
1923
  buildNativeWebSearchProbeOptions(
1901
- callSite,
1924
+ providerCallSite,
1902
1925
  resolveEffectiveOverrideProfile(),
1903
1926
  forceOverrideProfile,
1904
1927
  this.conversationId,
@@ -1917,11 +1940,11 @@ export class AgentLoop {
1917
1940
  // 1. Per-run explicit (`runModel`)
1918
1941
  // 2. Call-site resolved values (filled by
1919
1942
  // `RetryProvider.normalizeSendMessageOptions` from
1920
- // `resolveCallSiteConfig(callSite, llm)`)
1943
+ // `resolveCallSiteConfig(providerCallSite, llm)`)
1921
1944
  // 3. Conversation defaults (`this.config.*`, from the resolved
1922
1945
  // default call-site config)
1923
1946
  //
1924
- // When `callSite` is present we deliberately leave
1947
+ // When `providerCallSite` is present we deliberately leave
1925
1948
  // `max_tokens`/`thinking`/`effort`/`speed` *unset* in `providerConfig`
1926
1949
  // so the normalizer can fill them from the call-site resolution. The
1927
1950
  // normalizer only writes these fields when they're undefined; if we
@@ -1929,10 +1952,10 @@ export class AgentLoop {
1929
1952
  // for these knobs is silently ignored.
1930
1953
  //
1931
1954
  // `toolChoice` and `cacheTtl` are not part of the call-site schema, so
1932
- // they always come from `this.config` regardless of `callSite`.
1955
+ // they always come from `this.config` regardless of `providerCallSite`.
1933
1956
  const providerConfig: Record<string, unknown> = {};
1934
1957
 
1935
- if (!callSite) {
1958
+ if (!providerCallSite) {
1936
1959
  providerConfig.max_tokens = this.config.maxTokens;
1937
1960
  }
1938
1961
 
@@ -1940,7 +1963,7 @@ export class AgentLoop {
1940
1963
  providerConfig.model = runModel;
1941
1964
  }
1942
1965
 
1943
- if (!callSite) {
1966
+ if (!providerCallSite) {
1944
1967
  const thinking = normalizeThinkingConfigForWire(this.config.thinking);
1945
1968
  if (thinking !== undefined) {
1946
1969
  providerConfig.thinking = thinking;
@@ -1979,9 +2002,9 @@ export class AgentLoop {
1979
2002
  // defaults when absent).
1980
2003
  // User-initiated conversation turns default to `mainAgent` in the
1981
2004
  // agent loop's caller; other invocation contexts (heartbeat, filing,
1982
- // analyze, etc.) pass their own `callSite`.
1983
- if (callSite) {
1984
- providerConfig.callSite = callSite;
2005
+ // analyze, etc.) pass their own provider-resolution site.
2006
+ if (providerCallSite) {
2007
+ providerConfig.callSite = providerCallSite;
1985
2008
  providerConfig.usageTracking = "manual";
1986
2009
  // Per-conversation seed for deterministic `mix`-profile expansion.
1987
2010
  // Sourced from the loop's own conversation id so every LLM call in a
@@ -2009,7 +2032,7 @@ export class AgentLoop {
2009
2032
  // `activeProfile` and any call-site named profile. Threading it on
2010
2033
  // every send (rather than once at construction) keeps subagents that
2011
2034
  // share an `AgentLoop` instance but ought to inherit a different
2012
- // profile correct — and matches how `callSite` is plumbed.
2035
+ // profile correct, matching how the provider call site is plumbed.
2013
2036
  const effectiveOverrideProfile = resolveEffectiveOverrideProfile();
2014
2037
  if (effectiveOverrideProfile) {
2015
2038
  providerConfig.overrideProfile = effectiveOverrideProfile;
@@ -2233,6 +2256,49 @@ export class AgentLoop {
2233
2256
  );
2234
2257
  }
2235
2258
 
2259
+ if (onModelCallPrepared && !signal?.aborted) {
2260
+ const preparedOverrideProfile =
2261
+ typeof providerConfig.overrideProfile === "string" &&
2262
+ providerConfig.overrideProfile.length > 0
2263
+ ? providerConfig.overrideProfile
2264
+ : undefined;
2265
+ try {
2266
+ const preparedForceOverrideProfile =
2267
+ providerConfig.forceOverrideProfile === true;
2268
+ const disableCache =
2269
+ providerCallSite !== undefined &&
2270
+ resolveCallSiteConfig(providerCallSite, getConfig().llm, {
2271
+ ...(preparedOverrideProfile !== undefined
2272
+ ? { overrideProfile: preparedOverrideProfile }
2273
+ : {}),
2274
+ ...(preparedForceOverrideProfile
2275
+ ? { forceOverrideProfile: true }
2276
+ : {}),
2277
+ ...(this.conversationId !== undefined
2278
+ ? { selectionSeed: this.conversationId }
2279
+ : {}),
2280
+ }).disableCache === true;
2281
+ onModelCallPrepared({
2282
+ ...(providerCallSite !== undefined
2283
+ ? { callSite: providerCallSite }
2284
+ : {}),
2285
+ ...(preparedOverrideProfile !== undefined
2286
+ ? { overrideProfile: preparedOverrideProfile }
2287
+ : {}),
2288
+ forceOverrideProfile: preparedForceOverrideProfile,
2289
+ ...(disableCache ? { disableCache: true as const } : {}),
2290
+ ...(signal !== undefined ? { signal } : {}),
2291
+ systemPrompt: providerOptions.systemPrompt ?? null,
2292
+ tools: currentTools,
2293
+ });
2294
+ } catch (preparedError) {
2295
+ rlog.warn(
2296
+ { err: preparedError },
2297
+ "Prepared model-call observer failed; continuing with provider dispatch",
2298
+ );
2299
+ }
2300
+ }
2301
+
2236
2302
  // Announce the LLM-call boundary so downstream handlers (the
2237
2303
  // daemon's persistence pipeline) can reserve an empty assistant row
2238
2304
  // and stamp the resulting `messageId` onto every streaming event the
@@ -2250,10 +2316,10 @@ export class AgentLoop {
2250
2316
  // turn body (tool execution, plugin pipelines, checkpoints), so
2251
2317
  // recording there would risk mis-attributing tool/plugin throws as
2252
2318
  // provider rejections. On provider failure we emit `provider_error`
2253
- // with the loop-level raw request so consumers can persist it as an
2254
- // `llm_request_logs` row, then re-throw so the existing outer catch
2255
- // continues to handle abort sync, the `error` event, and the loop
2256
- // break unchanged.
2319
+ // with the inspectable wire request from the throw (when present)
2320
+ // so consumers can persist it as an `llm_request_logs` row, then
2321
+ // re-throw so the existing outer catch continues to handle abort
2322
+ // sync, the `error` event, and the loop break unchanged.
2257
2323
  // Latency: the request is about to leave for the provider. The span
2258
2324
  // from here to the first streamed token is time-to-first-token.
2259
2325
  latencyTracker?.mark("request_sent");
@@ -2276,26 +2342,18 @@ export class AgentLoop {
2276
2342
  llmCallError instanceof Error
2277
2343
  ? llmCallError
2278
2344
  : new Error(String(llmCallError));
2279
- // Strip non-serializable / runtime-only fields from `options`
2280
- // before snapshotting. `onEvent` is a closure with side effects
2281
- // and `signal` is an AbortSignal — neither is meaningful in a
2282
- // persisted log row, and `JSON.stringify` would silently drop or
2283
- // misrepresent both.
2284
- const rawRequest = {
2285
- provider: this.provider.name,
2286
- messages: sanitizedHistory,
2287
- tools: providerOptions.tools,
2288
- systemPrompt: providerOptions.systemPrompt,
2289
- config: providerOptions.config,
2290
- };
2345
+ const invocationProvider =
2346
+ errInstance instanceof ProviderError
2347
+ ? errInstance.provider
2348
+ : this.provider.name;
2291
2349
  onEvent({
2292
2350
  type: "provider_error",
2293
- rawRequest,
2351
+ ...(errInstance instanceof ProviderError &&
2352
+ errInstance.rawRequest !== undefined
2353
+ ? { rawRequest: errInstance.rawRequest }
2354
+ : {}),
2294
2355
  error: errInstance,
2295
- actualProvider:
2296
- errInstance instanceof ProviderError
2297
- ? errInstance.provider
2298
- : this.provider.name,
2356
+ actualProvider: invocationProvider,
2299
2357
  });
2300
2358
  }
2301
2359
  providerCallError = llmCallError;
@@ -20,10 +20,13 @@
20
20
 
21
21
  import { z } from "zod";
22
22
 
23
+ import { ModeSessionSchema } from "../mode-session.js";
24
+
23
25
  export const AssistantTurnStartEventSchema = z.object({
24
26
  type: z.literal("assistant_turn_start"),
25
27
  messageId: z.string(),
26
28
  conversationId: z.string().optional(),
29
+ modeSession: ModeSessionSchema.optional(),
27
30
  });
28
31
 
29
32
  export type AssistantTurnStartEvent = z.infer<
@@ -18,6 +18,7 @@
18
18
 
19
19
  import { z } from "zod";
20
20
 
21
+ import { ModeSessionSchema } from "../mode-session.js";
21
22
  import { AssistantOutboundAttachmentSchema } from "./assistant-outbound-attachment.js";
22
23
 
23
24
  export const GenerationHandoffEventSchema = z.object({
@@ -30,6 +31,7 @@ export const GenerationHandoffEventSchema = z.object({
30
31
  queuedCount: z.number(),
31
32
  /** Database row id of the just-finished assistant turn. */
32
33
  messageId: z.string().optional(),
34
+ modeSession: ModeSessionSchema.optional(),
33
35
  attachments: z.array(AssistantOutboundAttachmentSchema).optional(),
34
36
  attachmentWarnings: z.array(z.string()).optional(),
35
37
  });
@@ -24,12 +24,14 @@
24
24
 
25
25
  import { z } from "zod";
26
26
 
27
+ import { ModeSessionSchema } from "../mode-session.js";
27
28
  import { AssistantOutboundAttachmentSchema } from "./assistant-outbound-attachment.js";
28
29
 
29
30
  export const MessageCompleteEventSchema = z.object({
30
31
  type: z.literal("message_complete"),
31
32
  messageId: z.string().optional(),
32
33
  conversationId: z.string().optional(),
34
+ modeSession: ModeSessionSchema.optional(),
33
35
  source: z.enum(["main", "aux"]).optional(),
34
36
  attachments: z.array(AssistantOutboundAttachmentSchema).optional(),
35
37
  /** Soft warnings produced while resolving attachments (e.g. format
@@ -2,10 +2,10 @@
2
2
  * `notification_conversation_created` SSE event.
3
3
  *
4
4
  * Server → client broadcast emitted when an incoming notification
5
- * creates a new vellum conversation. Clients use it to place the
6
- * conversation in the sidebar (`groupId`, `source`) and decide whether
7
- * to raise a fallback OS banner (`silent`). `targetGuardianPrincipalId`
8
- * scopes guardian-sensitive conversations to a bound identity.
5
+ * creates a new vellum conversation. First-party clients ignore it; the
6
+ * web client lists it as a no-op. A guardian-sensitive
7
+ * conversation is announced only to connections authenticated as the
8
+ * guardian named in `targetGuardianPrincipalId`.
9
9
  *
10
10
  * Canonical wire-contract source. Daemon code imports the type
11
11
  * directly from this file; external consumers import via
@@ -21,27 +21,25 @@ export const NotificationConversationCreatedEventSchema = z.object({
21
21
  sourceEventName: z.string(),
22
22
  /**
23
23
  * When set, this conversation was created for a guardian-sensitive
24
- * notification and should only be surfaced by clients bound to this
25
- * guardian identity.
24
+ * notification. The daemon delivers the event only to connections whose
25
+ * verified principal is this guardian, so every client that receives it
26
+ * is one it was meant for.
26
27
  */
27
28
  targetGuardianPrincipalId: z.string().optional(),
28
29
  /**
29
- * Conversation group identifier propagated from the signal producer.
30
- * Clients use this to place the conversation in the correct sidebar
31
- * folder (e.g. "system:scheduled" for schedule completion threads).
30
+ * Conversation group identifier propagated from the signal producer:
31
+ * the sidebar folder the conversation belongs in (e.g.
32
+ * "system:scheduled" for schedule completion threads).
32
33
  */
33
34
  groupId: z.string().optional(),
34
35
  /**
35
- * Semantic source of the conversation (e.g. "schedule", "reminder").
36
- * Allows clients to override the default "notification" source so the
37
- * conversation is attributed correctly.
36
+ * Semantic source of the conversation (e.g. "schedule", "reminder"),
37
+ * overriding the default "notification" source.
38
38
  */
39
39
  source: z.string().optional(),
40
40
  /**
41
- * Mirrors `NotificationIntent.silent`. When true the client must not
42
- * post a fallback OS banner for this conversation — the sidebar entry
43
- * still appears, but the always-on inbox is the only surfaced channel.
44
- * Derived from the originating signal's `attentionHints.urgency`.
41
+ * Mirrors `NotificationIntent.silent`, which the server sets for
42
+ * low- and medium-urgency signals. Clients act on the intent's copy.
45
43
  */
46
44
  silent: z.boolean().optional(),
47
45
  });
@@ -2,8 +2,10 @@
2
2
  * `notification_intent` SSE event.
3
3
  *
4
4
  * Broadcast when a notification should be displayed. Clients turn it
5
- * into a local OS / in-app notification, filter guardian-scoped intents
6
- * to matching devices, and ack delivery back to the daemon.
5
+ * into a local OS / in-app notification and ack delivery back to the
6
+ * daemon. A guardian-scoped intent (`targetGuardianPrincipalId` set) is
7
+ * delivered only to connections whose verified principal is that
8
+ * guardian, so every client that receives one is meant to show it.
7
9
  *
8
10
  * `silent` tells the client not to post the intent to the OS
9
11
  * notification surface (non-banner side effects still run); the server
@@ -29,6 +29,7 @@
29
29
 
30
30
  import { z } from "zod";
31
31
 
32
+ import { ModeSessionSchema } from "../mode-session.js";
32
33
  import {
33
34
  AllowlistOptionSchema,
34
35
  ConfirmationDiffSchema,
@@ -51,6 +52,8 @@ export const WebSearchProviderIdSchema = z.enum([
51
52
  "keenable",
52
53
  "firecrawl",
53
54
  "fastcrw",
55
+ "searxng",
56
+ "tinyfish",
54
57
  ]);
55
58
 
56
59
  export type WebSearchProviderId = z.infer<typeof WebSearchProviderIdSchema>;
@@ -59,18 +62,24 @@ export const WebFetchProviderIdSchema = z.enum([
59
62
  "default",
60
63
  "firecrawl",
61
64
  "fastcrw",
65
+ "tinyfish",
62
66
  ]);
63
67
 
64
68
  export type WebFetchProviderId = z.infer<typeof WebFetchProviderIdSchema>;
65
69
 
66
70
  export const WebSearchResultItemSchema = z.object({
71
+ /** 1-indexed. */
67
72
  rank: z.number(),
68
73
  title: z.string(),
69
74
  url: z.string(),
75
+ /** The lowercased host. */
70
76
  domain: z.string(),
71
77
  faviconUrl: z.string().optional(),
78
+ /** Absent for `anthropic-native`, whose content is encrypted. */
72
79
  snippet: z.string().optional(),
80
+ /** A freshness hint; Brave only. */
73
81
  age: z.string().optional(),
82
+ /** Tavily only. */
74
83
  score: z.number().optional(),
75
84
  });
76
85
 
@@ -107,9 +116,84 @@ export const WebFetchMetadataSchema = z.object({
107
116
 
108
117
  export type WebFetchMetadata = z.infer<typeof WebFetchMetadataSchema>;
109
118
 
119
+ /** A place `recall` searches. */
120
+ export const RecallSourceSchema = z.enum([
121
+ "memory",
122
+ "conversations",
123
+ "workspace",
124
+ ]);
125
+
126
+ export type RecallSource = z.infer<typeof RecallSourceSchema>;
127
+
128
+ /** How hard `recall` searches: more depth, more rounds across the sources. */
129
+ export const RecallDepthSchema = z.enum(["fast", "standard", "deep"]);
130
+
131
+ export type RecallDepth = z.infer<typeof RecallDepthSchema>;
132
+
133
+ /** One piece of evidence a `recall` result stands on. */
134
+ export const RecallEvidenceItemSchema = z.object({
135
+ source: RecallSourceSchema,
136
+ title: z.string(),
137
+ /** Where in its source: a memory page, a conversation and when, a file and line. */
138
+ locator: z.string(),
139
+ /** The excerpt, collapsed to one line and cut to a few hundred characters. */
140
+ excerpt: z.string(),
141
+ timestampMs: z.number().optional(),
142
+ /**
143
+ * The workspace-relative file the item was found in, for a workspace file or
144
+ * a memory page, so a client can open it.
145
+ */
146
+ path: z.string().optional(),
147
+ /** The conversation the item was found in, so a client can open it. */
148
+ conversationId: z.string().optional(),
149
+ /** The message in that conversation, so a client can open it there. */
150
+ messageId: z.string().optional(),
151
+ });
152
+
153
+ export type RecallEvidenceItem = z.infer<typeof RecallEvidenceItemSchema>;
154
+
155
+ /** How one source fared: searched, or degraded with the reason. */
156
+ export const RecallSearchedSourceSchema = z.object({
157
+ source: RecallSourceSchema,
158
+ status: z.enum(["searched", "degraded"]),
159
+ evidenceCount: z.number(),
160
+ error: z.string().optional(),
161
+ });
162
+
163
+ export type RecallSearchedSource = z.infer<typeof RecallSearchedSourceSchema>;
164
+
165
+ /**
166
+ * A `recall` call as structured data: what it searched for and how, the answer
167
+ * when one was written, the evidence it stands on, and how each source fared.
168
+ * The same result the tool's text gives the model, for a client to lay out.
169
+ */
170
+ export const RecallMetadataSchema = z.object({
171
+ query: z.string(),
172
+ depth: RecallDepthSchema,
173
+ sources: z.array(RecallSourceSchema),
174
+ /**
175
+ * The answer written from the evidence. Absent when recall fell back to
176
+ * listing what it found, or found nothing.
177
+ */
178
+ answer: z.string().optional(),
179
+ evidence: z.array(RecallEvidenceItemSchema),
180
+ searchedSources: z.array(RecallSearchedSourceSchema),
181
+ });
182
+
183
+ export type RecallMetadata = z.infer<typeof RecallMetadataSchema>;
184
+
185
+ /** A `remember` call as structured data: the facts it saved, as written. */
186
+ export const RememberMetadataSchema = z.object({
187
+ facts: z.array(z.string()),
188
+ });
189
+
190
+ export type RememberMetadata = z.infer<typeof RememberMetadataSchema>;
191
+
110
192
  export const ToolActivityMetadataSchema = z.object({
111
193
  webSearch: WebSearchMetadataSchema.optional(),
112
194
  webFetch: WebFetchMetadataSchema.optional(),
195
+ recall: RecallMetadataSchema.optional(),
196
+ remember: RememberMetadataSchema.optional(),
113
197
  });
114
198
 
115
199
  export type ToolActivityMetadata = z.infer<typeof ToolActivityMetadataSchema>;
@@ -126,6 +210,7 @@ export const ToolResultEventSchema = z.object({
126
210
  imageDataList: z.array(z.string()).optional(),
127
211
  toolUseId: z.string().optional(),
128
212
  messageId: z.string().optional(),
213
+ modeSession: ModeSessionSchema.optional(),
129
214
  riskLevel: z.string().optional(),
130
215
  riskReason: z.string().optional(),
131
216
  matchedTrustRuleId: z.string().optional(),
@@ -18,6 +18,8 @@
18
18
 
19
19
  import { z } from "zod";
20
20
 
21
+ import { ModeSessionSchema } from "../mode-session.js";
22
+
21
23
  export const ToolUseStartEventSchema = z.object({
22
24
  type: z.literal("tool_use_start"),
23
25
  toolName: z.string(),
@@ -25,6 +27,7 @@ export const ToolUseStartEventSchema = z.object({
25
27
  toolUseId: z.string().optional(),
26
28
  messageId: z.string().optional(),
27
29
  conversationId: z.string().optional(),
30
+ modeSession: ModeSessionSchema.optional(),
28
31
  /**
29
32
  * Unix ms when the daemon began executing the tool. Lets clients anchor a
30
33
  * live elapsed-time counter to the server clock instead of the moment the
@@ -26,6 +26,8 @@
26
26
 
27
27
  import { z } from "zod";
28
28
 
29
+ import { ModeSessionSchema } from "../mode-session.js";
30
+
29
31
  export const UserMessageEchoEventSchema = z
30
32
  .object({
31
33
  type: z.literal("user_message_echo"),
@@ -35,6 +37,7 @@ export const UserMessageEchoEventSchema = z
35
37
  requestId: z.string().optional(),
36
38
  clientMessageId: z.string().optional(),
37
39
  cameraFrame: z.literal(true).optional(),
40
+ modeSession: ModeSessionSchema.optional(),
38
41
  })
39
42
  .strict();
40
43
 
package/src/api/index.ts CHANGED
@@ -4,6 +4,7 @@ export {
4
4
  isComputerUseToolCall,
5
5
  resolveComputerUseToolName,
6
6
  } from "./computer-use-tool.js";
7
+ export { type RememberInput, RememberInputSchema } from "./remember-tool.js";
7
8
 
8
9
  import { AcpAuthRequiredEventSchema } from "./events/acp-auth-required.js";
9
10
  import { AcpSessionCompletedEventSchema } from "./events/acp-session-completed.js";
@@ -50,7 +51,6 @@ import { DocumentEditorUpdateEventSchema } from "./events/document-editor-update
50
51
  import { ErrorEventSchema } from "./events/error.js";
51
52
  import { GenerationCancelledEventSchema } from "./events/generation-cancelled.js";
52
53
  import { GenerationHandoffEventSchema } from "./events/generation-handoff.js";
53
- import { HeartbeatAlertEventSchema } from "./events/heartbeat-alert.js";
54
54
  import { HeartbeatConversationCreatedEventSchema } from "./events/heartbeat-conversation-created.js";
55
55
  import { HomeFeedUpdatedEventSchema } from "./events/home-feed-updated.js";
56
56
  import { HookEventSchema } from "./events/hook-event.js";
@@ -395,10 +395,6 @@ export {
395
395
  type GenerationHandoffEvent,
396
396
  GenerationHandoffEventSchema,
397
397
  } from "./events/generation-handoff.js";
398
- export {
399
- type HeartbeatAlertEvent,
400
- HeartbeatAlertEventSchema,
401
- } from "./events/heartbeat-alert.js";
402
398
  export {
403
399
  type HeartbeatConversationCreatedEvent,
404
400
  HeartbeatConversationCreatedEventSchema,
@@ -647,6 +643,18 @@ export {
647
643
  ToolOutputChunkSubTypeSchema,
648
644
  } from "./events/tool-output-chunk.js";
649
645
  export {
646
+ type RecallDepth,
647
+ RecallDepthSchema,
648
+ type RecallEvidenceItem,
649
+ RecallEvidenceItemSchema,
650
+ type RecallMetadata,
651
+ RecallMetadataSchema,
652
+ type RecallSearchedSource,
653
+ RecallSearchedSourceSchema,
654
+ type RecallSource,
655
+ RecallSourceSchema,
656
+ type RememberMetadata,
657
+ RememberMetadataSchema,
650
658
  type RiskScopeOption,
651
659
  RiskScopeOptionSchema,
652
660
  type ToolActivityMetadata,
@@ -734,6 +742,24 @@ export {
734
742
  type WorkflowStartedEvent,
735
743
  WorkflowStartedEventSchema,
736
744
  } from "./events/workflow-started.js";
745
+ export {
746
+ type ModeSession,
747
+ type ModeSessionActivity,
748
+ ModeSessionActivitySchema,
749
+ type ModeSessionDescriptor,
750
+ ModeSessionDescriptorSchema,
751
+ type ModeSessionMode,
752
+ ModeSessionModeSchema,
753
+ type ModeSessionRuntimeState,
754
+ ModeSessionRuntimeStateSchema,
755
+ ModeSessionSchema,
756
+ type ModeSessionStatus,
757
+ ModeSessionStatusSchema,
758
+ type ModeSessionSummary,
759
+ ModeSessionSummarySchema,
760
+ parseModeSession,
761
+ TolerantModeSessionSchema,
762
+ } from "./mode-session.js";
737
763
  export {
738
764
  type DictationContext,
739
765
  DictationContextSchema,
@@ -1021,7 +1047,6 @@ export const AssistantEventSchema = z.discriminatedUnion("type", [
1021
1047
  ErrorEventSchema,
1022
1048
  GenerationCancelledEventSchema,
1023
1049
  GenerationHandoffEventSchema,
1024
- HeartbeatAlertEventSchema,
1025
1050
  HeartbeatConversationCreatedEventSchema,
1026
1051
  HomeFeedUpdatedEventSchema,
1027
1052
  HookEventSchema,