@vellumai/assistant 0.10.2 → 0.10.3-dev.202606252046.9075fd5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (546) hide show
  1. package/bun.lock +20 -0
  2. package/docs/workspace-tools.md +42 -33
  3. package/eslint-rules/cli-no-daemon-internals.js +6 -0
  4. package/node_modules/@vellumai/gateway-client/src/__tests__/guardian-delivery-contract.test.ts +91 -0
  5. package/node_modules/@vellumai/gateway-client/src/__tests__/trust-verdict-contract.test.ts +31 -0
  6. package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +44 -0
  7. package/node_modules/@vellumai/gateway-client/src/guardian-delivery-contract.ts +48 -0
  8. package/node_modules/@vellumai/gateway-client/src/index.ts +14 -0
  9. package/node_modules/@vellumai/gateway-client/src/trust-verdict-contract.ts +17 -0
  10. package/node_modules/@vellumai/service-contracts/package.json +1 -0
  11. package/node_modules/@vellumai/service-contracts/src/__tests__/channels.test.ts +28 -0
  12. package/node_modules/@vellumai/service-contracts/src/channels.ts +41 -0
  13. package/node_modules/@vellumai/service-contracts/src/index.ts +1 -0
  14. package/openapi.yaml +196 -56
  15. package/package.json +4 -1
  16. package/scripts/test.sh +36 -15
  17. package/src/__tests__/actor-token-service.test.ts +36 -14
  18. package/src/__tests__/actor-trust-resolver-address-fallback.test.ts +83 -31
  19. package/src/__tests__/agent-loop-override-profile.test.ts +1 -0
  20. package/src/__tests__/agent-wake-disk-pressure-callsite.test.ts +2 -0
  21. package/src/__tests__/agent-wake-override-profile.test.ts +2 -0
  22. package/src/__tests__/annotate-activity-metadata.test.ts +2 -0
  23. package/src/__tests__/annotate-risk-options.test.ts +2 -0
  24. package/src/__tests__/approval-cascade.test.ts +2 -0
  25. package/src/__tests__/assistant-attachments.test.ts +42 -0
  26. package/src/__tests__/background-workers-disk-pressure.test.ts +2 -0
  27. package/src/__tests__/btw-routes.test.ts +2 -0
  28. package/src/__tests__/build-persisted-content.test.ts +2 -0
  29. package/src/__tests__/call-controller.test.ts +19 -0
  30. package/src/__tests__/channel-approval-routes.test.ts +21 -26
  31. package/src/__tests__/channel-delivery-store.test.ts +28 -0
  32. package/src/__tests__/channel-guardian.test.ts +164 -78
  33. package/src/__tests__/channel-inbound-disk-pressure.test.ts +11 -19
  34. package/src/__tests__/channel-reply-delivery.test.ts +2 -0
  35. package/src/__tests__/code-search-tool.test.ts +585 -0
  36. package/src/__tests__/compaction-events.test.ts +2 -0
  37. package/src/__tests__/compaction-ledger-store.test.ts +128 -0
  38. package/src/__tests__/compaction.benchmark.test.ts +2 -0
  39. package/src/__tests__/compactor-call-site-logging.test.ts +2 -0
  40. package/src/__tests__/compactor-low-watermark-cut.test.ts +2 -0
  41. package/src/__tests__/compactor-preserved-tail-count.test.ts +2 -0
  42. package/src/__tests__/compactor-summary-call-truncation.test.ts +2 -0
  43. package/src/__tests__/compactor-web-search-strip.test.ts +2 -0
  44. package/src/__tests__/config-loader-backfill.test.ts +123 -10
  45. package/src/__tests__/config-schema.test.ts +1 -0
  46. package/src/__tests__/confirmation-request-guardian-bridge.test.ts +31 -29
  47. package/src/__tests__/consult-deadline.test.ts +60 -0
  48. package/src/__tests__/contact-store-interaction-info.test.ts +156 -0
  49. package/src/__tests__/contact-store-user-file.test.ts +7 -10
  50. package/src/__tests__/contacts-relay-reads.test.ts +19 -24
  51. package/src/__tests__/contacts-write.test.ts +0 -2
  52. package/src/__tests__/conversation-abort-tool-results.test.ts +2 -0
  53. package/src/__tests__/conversation-agent-loop-disk-pressure.test.ts +2 -0
  54. package/src/__tests__/conversation-agent-loop-inference-profile.test.ts +2 -0
  55. package/src/__tests__/conversation-agent-loop-overflow.test.ts +2 -0
  56. package/src/__tests__/conversation-agent-loop.test.ts +134 -0
  57. package/src/__tests__/conversation-analysis-routes.test.ts +2 -0
  58. package/src/__tests__/conversation-app-control-lifecycle.test.ts +2 -0
  59. package/src/__tests__/conversation-attention-telegram.test.ts +9 -11
  60. package/src/__tests__/conversation-confirmation-signals.test.ts +2 -0
  61. package/src/__tests__/conversation-fork-crud.test.ts +354 -24
  62. package/src/__tests__/conversation-history-web-search.test.ts +2 -0
  63. package/src/__tests__/conversation-load-history-repair.test.ts +2 -0
  64. package/src/__tests__/conversation-load-history-stripped.test.ts +2 -0
  65. package/src/__tests__/conversation-pairing.test.ts +2 -0
  66. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +2 -0
  67. package/src/__tests__/conversation-process-callsite.test.ts +2 -0
  68. package/src/__tests__/conversation-provider-retry-repair.test.ts +2 -0
  69. package/src/__tests__/conversation-queue.test.ts +91 -0
  70. package/src/__tests__/conversation-routes-guardian-reply.test.ts +14 -0
  71. package/src/__tests__/conversation-routes-slash-commands.test.ts +14 -0
  72. package/src/__tests__/conversation-slash-queue.test.ts +2 -0
  73. package/src/__tests__/conversation-slash-unknown.test.ts +2 -0
  74. package/src/__tests__/conversation-speed-override.test.ts +2 -0
  75. package/src/__tests__/conversation-surfaces-task-progress.test.ts +29 -0
  76. package/src/__tests__/conversation-title-service.test.ts +2 -0
  77. package/src/__tests__/conversation-tool-setup-attribution.test.ts +47 -0
  78. package/src/__tests__/conversation-usage.test.ts +2 -0
  79. package/src/__tests__/conversation-workspace-cache-state.test.ts +2 -0
  80. package/src/__tests__/conversation-workspace-injection.test.ts +2 -0
  81. package/src/__tests__/conversation-workspace-tool-tracking.test.ts +2 -0
  82. package/src/__tests__/credential-security-invariants.test.ts +1 -1
  83. package/src/__tests__/db-compaction-events-migration.test.ts +129 -0
  84. package/src/__tests__/db-migration-rollback.test.ts +205 -171
  85. package/src/__tests__/db-test-helpers.ts +5 -4
  86. package/src/__tests__/delete-propagation.test.ts +5 -3
  87. package/src/__tests__/deterministic-verification-control-plane.test.ts +4 -2
  88. package/src/__tests__/disk-pressure-guard.test.ts +41 -0
  89. package/src/__tests__/dm-backfill.test.ts +6 -4
  90. package/src/__tests__/dm-persistence.test.ts +2 -0
  91. package/src/__tests__/emit-signal-routing-intent.test.ts +7 -6
  92. package/src/__tests__/events-dev-bypass-actor.test.ts +7 -1
  93. package/src/__tests__/exploration-drift-hook.test.ts +3 -2
  94. package/src/__tests__/filing-service.test.ts +2 -0
  95. package/src/__tests__/guardian-binding-drift-heal.test.ts +104 -19
  96. package/src/__tests__/guardian-dispatch.test.ts +140 -1
  97. package/src/__tests__/guardian-expiry-notifier.test.ts +282 -0
  98. package/src/__tests__/guardian-outbound-http.test.ts +13 -0
  99. package/src/__tests__/guardian-routing-state.test.ts +6 -10
  100. package/src/__tests__/heartbeat-disk-pressure.test.ts +2 -0
  101. package/src/__tests__/heartbeat-service.test.ts +2 -0
  102. package/src/__tests__/helpers/channel-test-adapter.ts +46 -19
  103. package/src/__tests__/helpers/create-guardian-binding.ts +15 -23
  104. package/src/__tests__/helpers/mock-logger.ts +1 -0
  105. package/src/__tests__/helpers/seed-contact-channel.ts +96 -0
  106. package/src/__tests__/host-app-control-routes.test.ts +24 -30
  107. package/src/__tests__/host-bash-routes.test.ts +31 -41
  108. package/src/__tests__/host-browser-routes.test.ts +26 -32
  109. package/src/__tests__/host-cu-routes-targeted.test.ts +25 -33
  110. package/src/__tests__/host-file-routes-targeted.test.ts +40 -52
  111. package/src/__tests__/host-transfer-routes-targeted.test.ts +31 -43
  112. package/src/__tests__/http-user-message-parity.test.ts +290 -8
  113. package/src/__tests__/inbound-invite-redemption.test.ts +115 -10
  114. package/src/__tests__/inbound-slack-persistence.test.ts +2 -0
  115. package/src/__tests__/invite-redemption-service.test.ts +471 -53
  116. package/src/__tests__/invite-routes-http.test.ts +34 -0
  117. package/src/__tests__/invite-service-ipc.test.ts +65 -2
  118. package/src/__tests__/llm-context-normalization.test.ts +105 -0
  119. package/src/__tests__/llm-request-log-error-payload.test.ts +71 -9
  120. package/src/__tests__/llm-usage-store.test.ts +25 -0
  121. package/src/__tests__/mcp-config-secret-boundary.test.ts +392 -0
  122. package/src/__tests__/mcp-health-check.test.ts +2 -1
  123. package/src/__tests__/media-stream-server-integration.test.ts +127 -0
  124. package/src/__tests__/memory-retrieval-hook.test.ts +2 -0
  125. package/src/__tests__/messaging-send-tool.test.ts +2 -0
  126. package/src/__tests__/migration-import-from-url.test.ts +2 -2
  127. package/src/__tests__/mtime-cache.test.ts +275 -5
  128. package/src/__tests__/native-web-search.test.ts +2 -0
  129. package/src/__tests__/non-member-access-request.test.ts +191 -17
  130. package/src/__tests__/notification-broadcaster.test.ts +4 -0
  131. package/src/__tests__/notification-decision-recipient-context.test.ts +33 -32
  132. package/src/__tests__/notification-deep-link.test.ts +6 -0
  133. package/src/__tests__/notification-guardian-path.test.ts +19 -0
  134. package/src/__tests__/onboarding-persona-write.test.ts +52 -22
  135. package/src/__tests__/openai-provider.test.ts +22 -12
  136. package/src/__tests__/openai-responses-provider.test.ts +12 -2
  137. package/src/__tests__/outbound-slack-persistence.test.ts +2 -0
  138. package/src/__tests__/pending-interactions-resolved-event.test.ts +7 -4
  139. package/src/__tests__/persist-onboarding-artifacts.test.ts +1 -0
  140. package/src/__tests__/persistence-secret-redaction.test.ts +2 -0
  141. package/src/__tests__/persona-resolver.test.ts +75 -45
  142. package/src/__tests__/plugin-bootstrap.test.ts +21 -83
  143. package/src/__tests__/plugin-disabled-state.test.ts +190 -0
  144. package/src/__tests__/plugin-pipeline.test.ts +96 -0
  145. package/src/__tests__/plugin-route-contribution.test.ts +6 -19
  146. package/src/__tests__/plugin-tool-contribution.test.ts +5 -20
  147. package/src/__tests__/plugin-types.test.ts +2 -4
  148. package/src/__tests__/process-message-background-slack.test.ts +2 -0
  149. package/src/__tests__/process-message-display-content.test.ts +2 -0
  150. package/src/__tests__/provider-error-scenarios.test.ts +5 -4
  151. package/src/__tests__/provider-usage-tracking.test.ts +39 -0
  152. package/src/__tests__/reaction-intercept-cold-cache-warm.test.ts +135 -0
  153. package/src/__tests__/reaction-intercept-member-verdict-warm.test.ts +158 -0
  154. package/src/__tests__/reaction-persistence.test.ts +51 -4
  155. package/src/__tests__/regenerate-fire-and-forget-trace.test.ts +2 -0
  156. package/src/__tests__/registry.test.ts +4 -1
  157. package/src/__tests__/relay-server.test.ts +758 -32
  158. package/src/__tests__/runtime-attachment-metadata.test.ts +9 -12
  159. package/src/__tests__/secret-ingress-http.test.ts +14 -0
  160. package/src/__tests__/send-endpoint-busy.test.ts +30 -8
  161. package/src/__tests__/settings-routes.test.ts +32 -0
  162. package/src/__tests__/skills.test.ts +44 -0
  163. package/src/__tests__/slack-inbound-verification.test.ts +47 -2
  164. package/src/__tests__/sse-actor-principal-guardian-source.test.ts +79 -0
  165. package/src/__tests__/steer-on-enqueue-question.test.ts +181 -0
  166. package/src/__tests__/stt-hints.test.ts +49 -15
  167. package/src/__tests__/subagent-detail.test.ts +27 -0
  168. package/src/__tests__/subagent-disposal.test.ts +65 -0
  169. package/src/__tests__/subagent-fork-prompt-role.test.ts +195 -0
  170. package/src/__tests__/subagent-fork-spawn.test.ts +6 -7
  171. package/src/__tests__/subagent-notify-parent.test.ts +2 -0
  172. package/src/__tests__/subagent-role-registry.test.ts +24 -6
  173. package/src/__tests__/subagent-spawn-and-await.test.ts +546 -0
  174. package/src/__tests__/subagent-spawn-tool-fork.test.ts +2 -0
  175. package/src/__tests__/subagent-tools.test.ts +398 -1
  176. package/src/__tests__/suggestion-routes.test.ts +2 -0
  177. package/src/__tests__/thread-backfill.test.ts +3 -3
  178. package/src/__tests__/title-generate-hook.test.ts +2 -0
  179. package/src/__tests__/tool-executor-lifecycle-events.test.ts +2 -0
  180. package/src/__tests__/tool-executor.test.ts +16 -11
  181. package/src/__tests__/tool-preview-lifecycle.test.ts +2 -0
  182. package/src/__tests__/tool-result-metadata-plumbing.test.ts +2 -0
  183. package/src/__tests__/tool-start-timestamp.test.ts +2 -0
  184. package/src/__tests__/trusted-contact-approval-notifier.test.ts +37 -51
  185. package/src/__tests__/trusted-contact-inline-approval-integration.test.ts +12 -12
  186. package/src/__tests__/trusted-contact-lifecycle-notifications.test.ts +9 -7
  187. package/src/__tests__/trusted-contact-multichannel.test.ts +16 -7
  188. package/src/__tests__/trusted-contact-verification.test.ts +79 -54
  189. package/src/__tests__/twilio-routes.test.ts +96 -0
  190. package/src/__tests__/ui-file-upload-surface.test.ts +86 -0
  191. package/src/__tests__/verification-control-plane-policy.test.ts +2 -0
  192. package/src/__tests__/voice-guardian-cold-cache-warm.test.ts +137 -0
  193. package/src/__tests__/voice-invite-redemption.test.ts +216 -20
  194. package/src/__tests__/web-search-backend-failure.test.ts +2 -0
  195. package/src/__tests__/workspace-migration-102-preserve-heartbeat-enabled-for-existing-workspaces.test.ts +3 -3
  196. package/src/__tests__/workspace-migration-111-prune-seeded-callsite-defaults.test.ts +208 -0
  197. package/src/__tests__/workspace-migration-112-remove-advisor-callsite-override.test.ts +170 -0
  198. package/src/__tests__/workspace-migration-drop-user-md.test.ts +196 -238
  199. package/src/__tests__/workspace-migration-remove-hooks.test.ts +14 -35
  200. package/src/__tests__/workspace-tool-loader.test.ts +195 -2
  201. package/src/a2a/__tests__/e2e-a2a-channel.test.ts +35 -47
  202. package/src/agent/loop-exclusive-tool.test.ts +154 -0
  203. package/src/agent/loop-native-web-search.test.ts +200 -0
  204. package/src/agent/loop.ts +164 -1
  205. package/src/api/constants/sse-replay.ts +41 -0
  206. package/src/api/events/conversation-notice.ts +26 -0
  207. package/src/api/index.ts +19 -1
  208. package/src/api/responses/llm-request-log-entry.ts +29 -0
  209. package/src/api/responses/subagent-detail.ts +17 -0
  210. package/src/api/surfaces.ts +39 -3
  211. package/src/approvals/guardian-channel-delivery.ts +30 -0
  212. package/src/approvals/guardian-expiry-notifier.ts +148 -0
  213. package/src/approvals/guardian-request-resolvers.ts +17 -15
  214. package/src/calls/__tests__/relay-setup-router.test.ts +267 -17
  215. package/src/calls/call-domain.ts +3 -3
  216. package/src/calls/guardian-dispatch.ts +14 -9
  217. package/src/calls/inbound-trust-reader.ts +24 -2
  218. package/src/calls/media-stream-server.ts +21 -0
  219. package/src/calls/relay-access-wait.ts +6 -6
  220. package/src/calls/relay-server.ts +188 -51
  221. package/src/calls/relay-setup-router.ts +47 -17
  222. package/src/calls/relay-verification.ts +4 -4
  223. package/src/calls/stt-hints.ts +9 -12
  224. package/src/calls/twilio-routes.ts +14 -4
  225. package/src/channels/types.ts +10 -20
  226. package/src/cli/commands/__tests__/cache.test.ts +8 -1
  227. package/src/cli/commands/__tests__/conversations-slack.test.ts +1 -0
  228. package/src/cli/commands/cache.ts +194 -181
  229. package/src/cli/commands/contacts.ts +10 -7
  230. package/src/cli/commands/db/__tests__/repair.test.ts +6 -5
  231. package/src/cli/commands/db/status.ts +37 -1
  232. package/src/cli/commands/mcp.ts +252 -218
  233. package/src/cli/commands/memory/__tests__/worker.test.ts +432 -0
  234. package/src/cli/commands/memory/index.ts +2 -0
  235. package/src/cli/commands/memory/worker.ts +242 -0
  236. package/src/cli/commands/plugins.ts +125 -13
  237. package/src/cli/lib/__tests__/install-from-github.test.ts +102 -0
  238. package/src/cli/lib/__tests__/list-installed-plugins.test.ts +160 -1
  239. package/src/cli/lib/__tests__/publish-plugin.test.ts +404 -0
  240. package/src/cli/lib/list-installed-plugins.ts +179 -1
  241. package/src/cli/lib/publish-plugin.ts +633 -0
  242. package/src/config/__tests__/loader-callsite-strip-fallback.test.ts +143 -0
  243. package/src/config/__tests__/sync-gated-profiles.test.ts +16 -10
  244. package/src/config/bundled-skills/contacts/tools/contact-merge.ts +27 -17
  245. package/src/config/bundled-skills/contacts/tools/contact-search.ts +13 -3
  246. package/src/config/bundled-skills/subagent/SKILL.md +17 -2
  247. package/src/config/bundled-skills/subagent/TOOLS.json +5 -4
  248. package/src/config/call-site-defaults.ts +0 -6
  249. package/src/config/feature-flag-registry.json +5 -13
  250. package/src/config/llm-resolver.ts +0 -3
  251. package/src/config/loader.ts +38 -5
  252. package/src/config/prune-seeded-callsite-defaults.ts +110 -0
  253. package/src/config/schemas/__tests__/memory-v3.test.ts +1 -0
  254. package/src/config/schemas/call-site-catalog.ts +0 -7
  255. package/src/config/schemas/heartbeat.ts +2 -5
  256. package/src/config/schemas/llm.ts +3 -12
  257. package/src/config/schemas/memory-lifecycle.ts +12 -0
  258. package/src/config/schemas/memory-v3.ts +7 -0
  259. package/src/config/schemas/memory.ts +4 -0
  260. package/src/config/schemas/timeouts.ts +8 -0
  261. package/src/config/seed-inference-profiles.ts +30 -34
  262. package/src/config/skills.ts +27 -5
  263. package/src/config/sync-gated-profiles.ts +12 -16
  264. package/src/contacts/__tests__/contacts-write-revoke-relay.test.ts +128 -0
  265. package/src/contacts/__tests__/guardian-delivery-reader.test.ts +312 -0
  266. package/src/contacts/__tests__/member-write-relay.test.ts +226 -0
  267. package/src/contacts/contact-store.ts +27 -237
  268. package/src/contacts/contacts-write.ts +18 -55
  269. package/src/contacts/gateway-channel-read.ts +51 -0
  270. package/src/contacts/guardian-delivery-reader.ts +223 -0
  271. package/src/contacts/member-write-relay.ts +183 -0
  272. package/src/contacts/types.ts +3 -15
  273. package/src/daemon/__tests__/conversation-tool-setup-exclude.test.ts +35 -0
  274. package/src/daemon/__tests__/conversation-tool-setup.test.ts +0 -44
  275. package/src/daemon/assistant-attachments.ts +27 -4
  276. package/src/daemon/conversation-agent-loop.ts +28 -0
  277. package/src/daemon/conversation-notices.ts +60 -0
  278. package/src/daemon/conversation-process.ts +35 -16
  279. package/src/daemon/conversation-surfaces.ts +111 -38
  280. package/src/daemon/conversation-tool-setup.ts +71 -30
  281. package/src/daemon/conversation.ts +23 -1
  282. package/src/daemon/disk-pressure-guard.ts +12 -2
  283. package/src/daemon/event-loop-watchdog.ts +28 -1
  284. package/src/daemon/external-plugins-bootstrap.ts +17 -41
  285. package/src/daemon/handlers/__tests__/config-a2a-accept.test.ts +0 -1
  286. package/src/daemon/handlers/__tests__/config-a2a-complete.test.ts +0 -2
  287. package/src/daemon/handlers/__tests__/config-a2a-redeem.test.ts +25 -2
  288. package/src/daemon/handlers/__tests__/config-channels.test.ts +220 -0
  289. package/src/daemon/handlers/config-a2a.ts +6 -14
  290. package/src/daemon/handlers/config-channels.ts +67 -26
  291. package/src/daemon/handlers/conversations.ts +77 -0
  292. package/src/daemon/lifecycle.ts +16 -0
  293. package/src/daemon/mcp-reload-service.ts +10 -0
  294. package/src/daemon/memory-v2-startup.test.ts +72 -0
  295. package/src/daemon/memory-v2-startup.ts +87 -19
  296. package/src/daemon/message-types/conversations.ts +2 -0
  297. package/src/daemon/message-types/surfaces.ts +12 -12
  298. package/src/daemon/server.ts +0 -4
  299. package/src/daemon/shutdown-handlers.ts +20 -0
  300. package/src/daemon/tool-setup-types.ts +9 -0
  301. package/src/heartbeat/heartbeat-service.ts +5 -0
  302. package/src/home/relationship-state-writer.ts +5 -0
  303. package/src/hooks/hook-loader.ts +341 -0
  304. package/src/ipc/__tests__/clients-list-ipc.test.ts +1 -1
  305. package/src/ipc/assistant-server.ts +2 -2
  306. package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +1 -0
  307. package/src/mcp/client.ts +15 -1
  308. package/src/mcp/mcp-auth-orchestrator.ts +6 -1
  309. package/src/mcp/mcp-header-store.ts +134 -0
  310. package/src/mcp/mcp-oauth-provider.ts +19 -8
  311. package/src/memory/__tests__/301-create-watchdog-events.test.ts +110 -0
  312. package/src/memory/__tests__/memory-retrospective-job.test.ts +8 -0
  313. package/src/memory/__tests__/prompt-override.test.ts +192 -0
  314. package/src/memory/__tests__/watchdog-events-store.test.ts +161 -0
  315. package/src/memory/compaction-ledger-store.ts +107 -0
  316. package/src/memory/conversation-crud.ts +129 -61
  317. package/src/memory/db-connection.ts +22 -3
  318. package/src/memory/db-init.ts +37 -503
  319. package/src/memory/db-singleton.ts +6 -4
  320. package/src/memory/jobs-worker.ts +104 -0
  321. package/src/memory/llm-request-log-store.ts +26 -1
  322. package/src/memory/llm-usage-store.ts +48 -20
  323. package/src/memory/memory-retrospective-job.ts +14 -8
  324. package/src/memory/migrations/209-strip-thinking-from-consolidated.ts +152 -56
  325. package/src/memory/migrations/300-add-processing-started-at.ts +30 -0
  326. package/src/memory/migrations/301-create-watchdog-events.ts +45 -0
  327. package/src/memory/migrations/302-create-compaction-events.ts +107 -0
  328. package/src/memory/migrations/__tests__/209-strip-thinking-from-consolidated.test.ts +297 -0
  329. package/src/memory/migrations/__tests__/run-migrations.test.ts +2 -2
  330. package/src/memory/migrations/run-migrations.ts +106 -10
  331. package/src/memory/migrations/validate-migration-state.ts +105 -67
  332. package/src/memory/prompt-override.ts +129 -0
  333. package/src/memory/schema/contacts.ts +6 -2
  334. package/src/memory/schema/conversations.ts +37 -0
  335. package/src/memory/schema/infrastructure.ts +20 -0
  336. package/src/memory/steps.ts +1294 -0
  337. package/src/memory/v2/__tests__/cli-command-store.test.ts +25 -0
  338. package/src/memory/v2/__tests__/skill-store.test.ts +80 -0
  339. package/src/memory/v2/cli-command-store.ts +75 -38
  340. package/src/memory/v2/prompts/consolidation.ts +13 -82
  341. package/src/memory/v2/prompts/router.ts +21 -93
  342. package/src/memory/v2/skill-store.ts +68 -31
  343. package/src/memory/watchdog-events-store.ts +87 -0
  344. package/src/memory/worker-control.ts +204 -0
  345. package/src/memory/worker-process.ts +89 -0
  346. package/src/messaging/channel-binding-metadata.ts +31 -0
  347. package/src/messaging/provider-types.ts +8 -0
  348. package/src/messaging/providers/slack/binding-metadata.ts +60 -0
  349. package/src/notifications/__tests__/broadcaster.test.ts +8 -8
  350. package/src/notifications/__tests__/connected-channels.test.ts +86 -0
  351. package/src/notifications/__tests__/decision-engine.test.ts +78 -9
  352. package/src/notifications/__tests__/destination-resolver.test.ts +151 -0
  353. package/src/notifications/broadcaster.ts +8 -1
  354. package/src/notifications/decision-engine.ts +15 -7
  355. package/src/notifications/destination-resolver.ts +53 -25
  356. package/src/notifications/emit-signal.ts +34 -15
  357. package/src/onboarding/checkin-event.test.ts +222 -0
  358. package/src/onboarding/checkin-event.ts +321 -0
  359. package/src/onboarding/schedule-checkin.ts +190 -0
  360. package/src/permissions/question-prompter.test.ts +1 -1
  361. package/src/permissions/question-prompter.ts +7 -4
  362. package/src/plugin-api/index.ts +12 -12
  363. package/src/plugin-api/types.ts +12 -14
  364. package/src/plugin-api/vision-support.test.ts +28 -4
  365. package/src/plugin-api/vision-support.ts +66 -31
  366. package/src/plugins/defaults/empty-response/hooks/post-model-call.ts +2 -2
  367. package/src/plugins/defaults/empty-response/hooks/stop.ts +2 -2
  368. package/src/plugins/defaults/exploration-drift/hooks/post-tool-use.ts +4 -3
  369. package/src/plugins/defaults/history-repair/hooks/post-model-call.ts +2 -2
  370. package/src/plugins/defaults/history-repair/hooks/stop.ts +2 -2
  371. package/src/plugins/defaults/history-repair/hooks/user-prompt-submit.ts +2 -2
  372. package/src/plugins/defaults/image-fallback/__tests__/image-fallback.test.ts +47 -7
  373. package/src/plugins/defaults/image-fallback/hooks/post-tool-use.ts +12 -13
  374. package/src/plugins/defaults/image-fallback/hooks/user-prompt-submit.ts +14 -22
  375. package/src/plugins/defaults/image-fallback/src/caption-blocks.ts +42 -11
  376. package/src/plugins/defaults/image-recovery/hooks/post-model-call.ts +2 -2
  377. package/src/plugins/defaults/image-recovery/hooks/stop.ts +2 -2
  378. package/src/plugins/defaults/index.ts +0 -35
  379. package/src/plugins/defaults/max-tokens-continue/hooks/post-model-call.ts +2 -2
  380. package/src/plugins/defaults/max-tokens-continue/hooks/stop.ts +2 -2
  381. package/src/plugins/defaults/memory-retrieval/hooks/post-compact.ts +2 -2
  382. package/src/plugins/defaults/memory-retrieval/hooks/user-prompt-submit.ts +2 -2
  383. package/src/plugins/defaults/memory-v3-shadow/__tests__/injection.test.ts +33 -3
  384. package/src/plugins/defaults/memory-v3-shadow/__tests__/pool-select.test.ts +48 -4
  385. package/src/plugins/defaults/memory-v3-shadow/__tests__/shadow-plugin.test.ts +4 -8
  386. package/src/plugins/defaults/memory-v3-shadow/hooks/post-compact.ts +2 -2
  387. package/src/plugins/defaults/memory-v3-shadow/hooks/user-prompt-submit.ts +2 -2
  388. package/src/plugins/defaults/memory-v3-shadow/injector.ts +43 -15
  389. package/src/plugins/defaults/memory-v3-shadow/orchestrate.ts +11 -2
  390. package/src/plugins/defaults/memory-v3-shadow/pool-select.test.ts +146 -0
  391. package/src/plugins/defaults/memory-v3-shadow/pool-select.ts +77 -13
  392. package/src/plugins/defaults/memory-v3-shadow/shadow-plugin.ts +12 -11
  393. package/src/plugins/defaults/surface-completion-nudge/hooks/post-model-call.ts +2 -2
  394. package/src/plugins/defaults/surface-completion-nudge/hooks/stop.ts +2 -2
  395. package/src/plugins/defaults/task-progress-nudge/hooks/post-tool-use.ts +2 -2
  396. package/src/plugins/defaults/title-generate/hooks/stop.ts +2 -2
  397. package/src/plugins/defaults/title-generate/hooks/user-prompt-submit.ts +2 -2
  398. package/src/plugins/defaults/tool-error/hooks/post-tool-use.ts +2 -2
  399. package/src/plugins/defaults/tool-result-truncate/hooks/post-tool-use.ts +2 -2
  400. package/src/plugins/disabled-state.ts +31 -0
  401. package/src/plugins/external-plugin-loader.ts +2 -2
  402. package/src/plugins/mtime-cache.ts +186 -330
  403. package/src/plugins/pipeline.ts +111 -13
  404. package/src/plugins/registry.ts +59 -16
  405. package/src/plugins/surface-import.ts +121 -0
  406. package/src/plugins/types.ts +7 -9
  407. package/src/prompts/persona-resolver.ts +43 -11
  408. package/src/providers/anthropic/client.ts +5 -0
  409. package/src/providers/call-site-routing.ts +45 -0
  410. package/src/providers/model-catalog.ts +16 -0
  411. package/src/providers/openai/__tests__/api-error-normalization.test.ts +321 -0
  412. package/src/providers/openai/api-error-normalization.ts +270 -0
  413. package/src/providers/openai/chat-completions-provider.ts +37 -83
  414. package/src/providers/openai/responses-provider.ts +50 -46
  415. package/src/providers/openrouter/client.ts +5 -0
  416. package/src/providers/provider-send-message.ts +10 -0
  417. package/src/providers/ratelimit.ts +10 -0
  418. package/src/providers/registry.ts +1 -1
  419. package/src/providers/retry.ts +10 -0
  420. package/src/providers/types.ts +22 -0
  421. package/src/providers/usage-tracking.ts +10 -0
  422. package/src/runtime/__tests__/channel-verification-service.test.ts +133 -0
  423. package/src/runtime/__tests__/guardian-vellum-migration.test.ts +184 -0
  424. package/src/runtime/__tests__/is-guardian-bound-for-channel.test.ts +66 -0
  425. package/src/runtime/__tests__/local-principal-trust.test.ts +162 -0
  426. package/src/runtime/__tests__/member-verdict-cache.test.ts +119 -0
  427. package/src/runtime/__tests__/trust-verdict-consumer.test.ts +402 -123
  428. package/src/runtime/access-request-helper.ts +18 -39
  429. package/src/runtime/actor-trust-resolver.ts +46 -19
  430. package/src/runtime/anchored-guardian.test.ts +109 -0
  431. package/src/runtime/anchored-guardian.ts +86 -0
  432. package/src/runtime/assistant-event-hub.ts +1 -1
  433. package/src/runtime/assistant-stream-state.ts +9 -2
  434. package/src/runtime/auth/__tests__/require-bound-guardian.test.ts +99 -0
  435. package/src/runtime/auth/require-bound-guardian.ts +21 -11
  436. package/src/runtime/channel-verification-service.ts +56 -31
  437. package/src/runtime/confirmation-request-guardian-bridge.ts +3 -3
  438. package/src/runtime/guardian-vellum-migration.ts +69 -8
  439. package/src/runtime/invite-redemption-service.ts +213 -187
  440. package/src/runtime/local-actor-identity.test.ts +108 -0
  441. package/src/runtime/local-actor-identity.ts +85 -13
  442. package/src/runtime/local-principal-trust.ts +52 -0
  443. package/src/runtime/member-verdict-cache.ts +0 -0
  444. package/src/runtime/pending-interactions.ts +11 -1
  445. package/src/runtime/routes/__tests__/channel-verification-revoke.test.ts +56 -5
  446. package/src/runtime/routes/__tests__/channel-verification-routes.test.ts +1 -1
  447. package/src/runtime/routes/__tests__/contact-routes.test.ts +305 -0
  448. package/src/runtime/routes/__tests__/global-search-routes.test.ts +92 -0
  449. package/src/runtime/routes/__tests__/http-adapter-actor-header.test.ts +129 -0
  450. package/src/runtime/routes/__tests__/surface-action-routes.test.ts +188 -0
  451. package/src/runtime/routes/browser-routes.ts +1 -1
  452. package/src/runtime/routes/canonical-guardian-expiry-sweep.ts +13 -5
  453. package/src/runtime/routes/channel-verification-routes.ts +3 -3
  454. package/src/runtime/routes/contact-routes.ts +48 -57
  455. package/src/runtime/routes/conversation-cli-routes.ts +4 -5
  456. package/src/runtime/routes/conversation-list-routes.ts +4 -7
  457. package/src/runtime/routes/conversation-query-routes.ts +72 -0
  458. package/src/runtime/routes/conversation-routes.ts +84 -85
  459. package/src/runtime/routes/events-routes.ts +2 -2
  460. package/src/runtime/routes/global-search-routes.ts +3 -1
  461. package/src/runtime/routes/guardian-action-routes.ts +4 -5
  462. package/src/runtime/routes/host-app-control-routes.ts +5 -4
  463. package/src/runtime/routes/host-bash-routes.ts +5 -4
  464. package/src/runtime/routes/host-browser-routes.ts +9 -11
  465. package/src/runtime/routes/host-cu-routes.ts +5 -4
  466. package/src/runtime/routes/host-file-routes.ts +5 -4
  467. package/src/runtime/routes/host-transfer-routes.ts +6 -6
  468. package/src/runtime/routes/http-adapter.ts +16 -1
  469. package/src/runtime/routes/identity-routes.ts +3 -2
  470. package/src/runtime/routes/inbound-message-handler.ts +5 -5
  471. package/src/runtime/routes/inbound-stages/acl-enforcement.test.ts +97 -5
  472. package/src/runtime/routes/inbound-stages/acl-enforcement.ts +61 -59
  473. package/src/runtime/routes/inbound-stages/background-dispatch.ts +13 -5
  474. package/src/runtime/routes/inbound-stages/escalation-intercept.ts +7 -7
  475. package/src/runtime/routes/inbound-stages/guardian-activation-intercept.test.ts +21 -8
  476. package/src/runtime/routes/inbound-stages/guardian-activation-intercept.ts +14 -3
  477. package/src/runtime/routes/inbound-stages/reaction-intercept.ts +19 -0
  478. package/src/runtime/routes/index.ts +2 -0
  479. package/src/runtime/routes/llm-context-normalization.ts +83 -0
  480. package/src/runtime/routes/mcp-auth-routes.ts +171 -19
  481. package/src/runtime/routes/migration-rollback-routes.ts +4 -3
  482. package/src/runtime/routes/migration-routes.ts +4 -1
  483. package/src/runtime/routes/onboarding-checkin-routes.ts +86 -0
  484. package/src/runtime/routes/settings-routes.ts +8 -3
  485. package/src/runtime/routes/subagents-routes.ts +5 -0
  486. package/src/runtime/routes/surface-action-routes.ts +42 -56
  487. package/src/runtime/services/__tests__/conversation-serializer.test.ts +1 -0
  488. package/src/runtime/services/conversation-serializer.ts +13 -58
  489. package/src/runtime/tool-grant-request-helper.ts +3 -3
  490. package/src/runtime/trust-verdict-consumer.ts +111 -40
  491. package/src/runtime/verification-outbound-actions.ts +18 -18
  492. package/src/signals/user-message.ts +16 -0
  493. package/src/subagent/__tests__/consult-prompt.test.ts +35 -0
  494. package/src/{plugins/defaults/advisor/__tests__/transcript.test.ts → subagent/__tests__/consult-transcript.test.ts} +47 -10
  495. package/src/{plugins/defaults/advisor/steering.ts → subagent/consult-prompt.ts} +22 -32
  496. package/src/{plugins/defaults/advisor/transcript.ts → subagent/consult-transcript.ts} +18 -8
  497. package/src/subagent/index.ts +1 -1
  498. package/src/subagent/manager.ts +254 -33
  499. package/src/subagent/types.ts +11 -4
  500. package/src/telemetry/types.ts +34 -1
  501. package/src/telemetry/usage-telemetry-reporter.test.ts +3 -2
  502. package/src/telemetry/usage-telemetry-reporter.ts +87 -3
  503. package/src/tools/ask-question/ask-question-tool.test.ts +29 -0
  504. package/src/tools/ask-question/ask-question-tool.ts +13 -0
  505. package/src/tools/executor.ts +4 -4
  506. package/src/tools/filesystem/search.ts +543 -0
  507. package/src/tools/registry.ts +38 -0
  508. package/src/tools/shared/filesystem/path-policy.ts +12 -5
  509. package/src/tools/subagent/consult-deadline.ts +49 -0
  510. package/src/tools/subagent/spawn.ts +234 -5
  511. package/src/tools/tool-approval-handler.ts +1 -1
  512. package/src/tools/tool-defaults.ts +9 -2
  513. package/src/tools/tool-manifest.ts +3 -0
  514. package/src/tools/types.ts +131 -25
  515. package/src/tools/workspace-tools/loader.ts +348 -244
  516. package/src/util/errors.ts +26 -1
  517. package/src/util/logger.ts +9 -0
  518. package/src/util/platform.ts +19 -0
  519. package/src/util/telemetry-db-path.ts +24 -0
  520. package/src/workflows/library.test.ts +140 -0
  521. package/src/workflows/library.ts +82 -28
  522. package/src/workspace/migrations/017-seed-persona-dirs.ts +3 -34
  523. package/src/workspace/migrations/019-scope-journal-to-guardian.ts +3 -24
  524. package/src/workspace/migrations/031-drop-user-md.ts +232 -148
  525. package/src/workspace/migrations/048-remove-workspace-hooks.ts +14 -66
  526. package/src/workspace/migrations/111-prune-seeded-callsite-defaults.ts +134 -0
  527. package/src/workspace/migrations/112-remove-advisor-callsite-override.ts +64 -0
  528. package/src/workspace/migrations/registry.ts +4 -0
  529. package/src/__tests__/workspace-tools-watcher-flag.test.ts +0 -70
  530. package/src/daemon/workspace-tools-watcher.ts +0 -328
  531. package/src/memory/migrations/registry.ts +0 -573
  532. package/src/plugins/defaults/advisor/__tests__/advisor-gate.test.ts +0 -56
  533. package/src/plugins/defaults/advisor/__tests__/advisor-state-store.test.ts +0 -43
  534. package/src/plugins/defaults/advisor/__tests__/agent-loop-integration.test.ts +0 -137
  535. package/src/plugins/defaults/advisor/__tests__/consult.test.ts +0 -153
  536. package/src/plugins/defaults/advisor/__tests__/hooks.test.ts +0 -138
  537. package/src/plugins/defaults/advisor/advisor-gate.ts +0 -29
  538. package/src/plugins/defaults/advisor/advisor-state-store.ts +0 -94
  539. package/src/plugins/defaults/advisor/config.ts +0 -21
  540. package/src/plugins/defaults/advisor/consult.ts +0 -93
  541. package/src/plugins/defaults/advisor/hooks/post-model-call.ts +0 -34
  542. package/src/plugins/defaults/advisor/hooks/pre-model-call.ts +0 -30
  543. package/src/plugins/defaults/advisor/hooks/user-prompt-submit.ts +0 -19
  544. package/src/plugins/defaults/advisor/package.json +0 -14
  545. package/src/plugins/defaults/advisor/tools/advisor.ts +0 -65
  546. package/src/providers/openai/__tests__/api-error-detail.test.ts +0 -120
@@ -1,10 +1,11 @@
1
1
  /**
2
2
  * Holds the assistant DB connection singletons and their close callbacks.
3
3
  *
4
- * Three connections live here under the same `globalThis.vellumAssistant`
4
+ * Four connections live here under the same `globalThis.vellumAssistant`
5
5
  * namespace, keyed by slot: the `main` daemon connection, the `logs`
6
- * connection (`assistant-logs.db`), and the `memory` connection
7
- * (`assistant-memory.db`). Each is opened lazily by `db-connection.ts`.
6
+ * connection (`assistant-logs.db`), the `memory` connection
7
+ * (`assistant-memory.db`), and the `telemetry` connection
8
+ * (`assistant-telemetry.db`). Each is opened lazily by `db-connection.ts`.
8
9
  *
9
10
  * Lives in its own module (rather than alongside the resolvers in
10
11
  * `db-connection.ts`) so test code can reset the singletons without
@@ -33,7 +34,7 @@
33
34
  */
34
35
 
35
36
  /** Which connection a slot holds. */
36
- export type DbSlotKey = "main" | "logs" | "memory";
37
+ export type DbSlotKey = "main" | "logs" | "memory" | "telemetry";
37
38
 
38
39
  type DbSlot = {
39
40
  db: unknown;
@@ -57,6 +58,7 @@ function slots(): DbSlots {
57
58
  main: emptySlot(),
58
59
  logs: emptySlot(),
59
60
  memory: emptySlot(),
61
+ telemetry: emptySlot(),
60
62
  });
61
63
  }
62
64
 
@@ -93,6 +93,11 @@ import {
93
93
  memoryV2ConsolidateJob,
94
94
  } from "./v2/consolidation-job.js";
95
95
  import { memoryV2SweepJob } from "./v2/sweep-job.js";
96
+ import {
97
+ removeSyncRunnerMarker,
98
+ spawnMemoryWorkerProcess,
99
+ writeSyncRunnerMarker,
100
+ } from "./worker-control.js";
96
101
 
97
102
  const log = getLogger("memory-jobs-worker");
98
103
 
@@ -161,7 +166,74 @@ export interface MemoryJobsWorker {
161
166
  stop(): void;
162
167
  }
163
168
 
169
+ /**
170
+ * Start the daemon's memory jobs worker supervisor.
171
+ *
172
+ * The daemon always runs the in-process supervisor returned here. The
173
+ * supervisor owns the synchronous in-process runner and reconciles to
174
+ * `memory.worker.enabled` on every poll, re-reading the flag from disk so a
175
+ * runtime change takes effect without a restart:
176
+ * - flag off (default): drain the queue in-process and publish the
177
+ * sync-runner marker so `status` reports the synchronous runner as going.
178
+ * - flag on: stand down (the out-of-process worker owns the queue) and clear
179
+ * the marker.
180
+ * Gating on the flag — rather than on the worker process actually being present
181
+ * — keeps exactly one drainer active and avoids a boot race: when the flag is
182
+ * on the supervisor never processes, so it can't claim jobs that the spawning
183
+ * worker's startup recovery would then reset out from under it.
184
+ *
185
+ * `memory.worker.enabled` is also the persisted boot preference: when set, the
186
+ * out-of-process worker is spawned here at startup so it is running
187
+ * immediately. The CLI `memory worker start`/`stop` commands flip the flag (and
188
+ * spawn/stop the worker process), so the supervisor switches the running daemon
189
+ * between synchronous and out-of-process modes within one poll. When the flag
190
+ * is on but no worker process is running, neither drainer processes — `status`
191
+ * surfaces this (worker not running, synchronous runner not running).
192
+ *
193
+ * This dispatcher must not be used as the standalone worker process's entry —
194
+ * that would recurse and fork-bomb, and the flag-on worker process would stand
195
+ * itself down. `worker-process.ts` calls {@link startInProcessMemoryJobsWorker}
196
+ * directly with no options.
197
+ */
164
198
  export function startMemoryJobsWorker(): MemoryJobsWorker {
199
+ if (getConfig().memory.worker?.enabled === true) {
200
+ void spawnMemoryWorkerProcess()
201
+ .then(({ pid, alreadyRunning }) =>
202
+ log.info(
203
+ { pid, alreadyRunning },
204
+ alreadyRunning
205
+ ? "Memory worker process already running — reusing it"
206
+ : "Memory worker process started",
207
+ ),
208
+ )
209
+ .catch((err) =>
210
+ log.warn(
211
+ { err },
212
+ "Failed to start memory worker process — the in-process supervisor will drain the queue instead",
213
+ ),
214
+ );
215
+ }
216
+
217
+ return startInProcessMemoryJobsWorker({ standDownForWorkerProcess: true });
218
+ }
219
+
220
+ /**
221
+ * Run the memory jobs worker in-process on the caller's event loop: poll for
222
+ * claimable jobs with adaptive backoff until {@link MemoryJobsWorker.stop} is
223
+ * called. This is the worker loop itself — used by the daemon supervisor (with
224
+ * `standDownForWorkerProcess`) and by the standalone worker process (without).
225
+ *
226
+ * When `standDownForWorkerProcess` is set the loop acts as the daemon's
227
+ * synchronous-runner supervisor: each tick it skips processing while
228
+ * `memory.worker.enabled` is on (clearing the sync-runner marker), and
229
+ * publishes the marker while it owns processing. The standalone worker process
230
+ * must NOT set this — it runs precisely when the flag is on and would otherwise
231
+ * stand itself down forever.
232
+ */
233
+ export function startInProcessMemoryJobsWorker(
234
+ opts: { standDownForWorkerProcess?: boolean } = {},
235
+ ): MemoryJobsWorker {
236
+ const standDownForWorkerProcess = opts.standDownForWorkerProcess === true;
165
237
  const recovered = resetRunningJobsToPending();
166
238
  if (recovered > 0) {
167
239
  log.info({ recovered }, "Recovered stale running memory jobs");
@@ -184,11 +256,39 @@ export function startMemoryJobsWorker(): MemoryJobsWorker {
184
256
  let tickRunning = false;
185
257
  let timer: ReturnType<typeof setTimeout>;
186
258
  let currentIntervalMs = POLL_INTERVAL_MIN_MS;
259
+ // Tracks whether this supervisor currently owns processing (and so has
260
+ // published the sync-runner marker). Only meaningful when
261
+ // `standDownForWorkerProcess` is set.
262
+ let syncRunnerMarked = false;
187
263
 
188
264
  const tick = async () => {
189
265
  if (stopped || tickRunning) return;
190
266
  tickRunning = true;
191
267
  try {
268
+ if (
269
+ standDownForWorkerProcess &&
270
+ getConfig().memory.worker?.enabled === true
271
+ ) {
272
+ // The out-of-process worker owns the queue — stand the synchronous
273
+ // runner down so jobs aren't processed twice, and retract the marker.
274
+ if (syncRunnerMarked) {
275
+ removeSyncRunnerMarker();
276
+ syncRunnerMarked = false;
277
+ }
278
+ // Switching modes is a rare operator action, so poll at the slow cap
279
+ // while standing down: it still picks up a `memory worker stop` (which
280
+ // flips the flag back off) within one interval, without waking every
281
+ // couple seconds for the whole time the worker owns the queue.
282
+ currentIntervalMs = POLL_INTERVAL_MAX_MS;
283
+ return;
284
+ }
285
+ if (standDownForWorkerProcess && !syncRunnerMarked) {
286
+ // The flag is off — this in-process runner owns processing. Publish the
287
+ // marker so `memory worker status` reports the synchronous runner as
288
+ // going.
289
+ writeSyncRunnerMarker(process.pid);
290
+ syncRunnerMarked = true;
291
+ }
192
292
  const processed = await runMemoryJobsOnce({
193
293
  enableScheduledCleanup: true,
194
294
  });
@@ -237,6 +337,10 @@ export function startMemoryJobsWorker(): MemoryJobsWorker {
237
337
  stop(): void {
238
338
  stopped = true;
239
339
  clearTimeout(timer);
340
+ if (syncRunnerMarked) {
341
+ removeSyncRunnerMarker();
342
+ syncRunnerMarked = false;
343
+ }
240
344
  },
241
345
  };
242
346
  }
@@ -94,14 +94,21 @@ export type CompactionAgentLogRow = LogMetaRow & {
94
94
  * can either stringify it directly (daemon-path `recordRequestLog`) or
95
95
  * store it on a pending-log queue that stringifies later (wake-path
96
96
  * `PendingLog.rawResponse`), without double-encoding.
97
+ *
98
+ * When the provider captured the verbatim upstream body, it is attached as a
99
+ * top-level `rawResponse` sibling so the inspector's Raw tab can show the
100
+ * actual provider payload (parsed JSON, or the raw string for non-JSON error
101
+ * pages) — matching the success path, where Raw shows the real provider JSON.
97
102
  */
98
103
  export function buildProviderErrorResponsePayload(err: Error): {
99
104
  error: Record<string, unknown>;
105
+ rawResponse?: unknown;
100
106
  } {
101
107
  const payload: Record<string, unknown> = {
102
108
  name: err.name,
103
109
  message: err.message,
104
110
  };
111
+ let rawResponse: unknown;
105
112
  if (err instanceof ProviderError) {
106
113
  payload.code = err.code;
107
114
  payload.provider = err.provider;
@@ -111,10 +118,28 @@ export function buildProviderErrorResponsePayload(err: Error): {
111
118
  if (err.retryAfterMs !== undefined) {
112
119
  payload.retryAfterMs = err.retryAfterMs;
113
120
  }
121
+ if (err.apiErrorCode !== undefined) payload.apiErrorCode = err.apiErrorCode;
122
+ if (err.apiErrorType !== undefined) payload.apiErrorType = err.apiErrorType;
123
+ if (err.apiErrorParam !== undefined)
124
+ payload.apiErrorParam = err.apiErrorParam;
125
+ if (err.requestId !== undefined) payload.requestId = err.requestId;
126
+ if (err.rawBody !== undefined) rawResponse = parseRawBody(err.rawBody);
114
127
  } else if (err instanceof AssistantError) {
115
128
  payload.code = err.code;
116
129
  }
117
- return { error: payload };
130
+ return rawResponse !== undefined
131
+ ? { error: payload, rawResponse }
132
+ : { error: payload };
133
+ }
134
+
135
+ /** Parse a captured upstream body as JSON, falling back to the raw string for
136
+ * non-JSON error pages (HTML, plain text) or a truncated/invalid prefix. */
137
+ function parseRawBody(rawBody: string): unknown {
138
+ try {
139
+ return JSON.parse(rawBody);
140
+ } catch {
141
+ return rawBody;
142
+ }
118
143
  }
119
144
 
120
145
  export function recordRequestLog(
@@ -525,7 +525,32 @@ export function getUsageTotals(
525
525
  };
526
526
  }
527
527
 
528
- /** Fetch raw events in a time range for in-memory bucketing. */
528
+ /**
529
+ * Width of the SQL pre-aggregation bucket used by the time-series read paths,
530
+ * in milliseconds. The series/daily endpoints bucket events into local-day or
531
+ * local-hour buckets in JavaScript (SQLite's `strftime` is UTC-only and can't
532
+ * honor an IANA timezone). To avoid materializing one JS object per usage event
533
+ * — which on a 90-day window can mean hundreds of thousands of rows on the
534
+ * daemon's main thread — we first roll events up in SQL into fixed UTC buckets,
535
+ * then feed those far-fewer rows into the same JS bucketing logic.
536
+ *
537
+ * 15 minutes is the finest quantum every real-world IANA UTC offset divides
538
+ * into (whole-hour offsets, plus the :30 and :45 zones like Asia/Kolkata and
539
+ * Asia/Kathmandu). Because every local-day and local-hour boundary therefore
540
+ * lands on a 15-minute UTC boundary, no pre-aggregation bucket can straddle a
541
+ * local bucket boundary — so rolling up to 15-minute UTC buckets and then
542
+ * re-bucketing in local time is exactly equal to bucketing each raw event.
543
+ * DST fall-back hours stay distinct because their instants fall in different
544
+ * 15-minute UTC buckets (and the JS layer disambiguates them by UTC offset).
545
+ */
546
+ const USAGE_PREAGG_BUCKET_MS = 15 * 60 * 1000;
547
+
548
+ /**
549
+ * Fetch usage rows for a time range, pre-aggregated into {@link
550
+ * USAGE_PREAGG_BUCKET_MS} UTC buckets, for in-memory local-time bucketing. Each
551
+ * returned row's `created_at` is its UTC bucket start, which the JS bucketing
552
+ * maps to the same local bucket every raw event in that window would map to.
553
+ */
529
554
  function fetchRawBucketRows(
530
555
  range: UsageTimeRange,
531
556
  filter?: UsageAggregationFilter,
@@ -534,13 +559,14 @@ function fetchRawBucketRows(
534
559
  return rawAll<UsageEventBucketRow>(
535
560
  /*sql*/ `
536
561
  SELECT
537
- created_at,
538
- input_tokens,
539
- output_tokens,
540
- estimated_cost_usd,
541
- llm_call_count
562
+ (created_at / ${USAGE_PREAGG_BUCKET_MS}) * ${USAGE_PREAGG_BUCKET_MS} AS created_at,
563
+ COALESCE(SUM(input_tokens), 0) AS input_tokens,
564
+ COALESCE(SUM(output_tokens), 0) AS output_tokens,
565
+ SUM(estimated_cost_usd) AS estimated_cost_usd,
566
+ SUM(COALESCE(llm_call_count, 1)) AS llm_call_count
542
567
  FROM llm_usage_events
543
568
  WHERE ${where.sql}
569
+ GROUP BY (created_at / ${USAGE_PREAGG_BUCKET_MS})
544
570
  ORDER BY created_at ASC
545
571
  `,
546
572
  ...where.params,
@@ -829,17 +855,18 @@ export function getUsageGroupedSeries(
829
855
  WHERE ${where.sql}
830
856
  )
831
857
  SELECT
832
- schedule_usage.created_at,
833
- schedule_usage.input_tokens,
834
- schedule_usage.output_tokens,
835
- schedule_usage.estimated_cost_usd,
836
- schedule_usage.llm_call_count,
837
- schedule_usage.group_key,
838
- schedule_group_jobs.name AS group_label
858
+ (schedule_usage.created_at / ${USAGE_PREAGG_BUCKET_MS}) * ${USAGE_PREAGG_BUCKET_MS} AS created_at,
859
+ COALESCE(SUM(schedule_usage.input_tokens), 0) AS input_tokens,
860
+ COALESCE(SUM(schedule_usage.output_tokens), 0) AS output_tokens,
861
+ SUM(schedule_usage.estimated_cost_usd) AS estimated_cost_usd,
862
+ SUM(COALESCE(schedule_usage.llm_call_count, 1)) AS llm_call_count,
863
+ schedule_usage.group_key AS group_key,
864
+ MAX(schedule_group_jobs.name) AS group_label
839
865
  FROM schedule_usage
840
866
  LEFT JOIN cron_jobs schedule_group_jobs
841
867
  ON schedule_group_jobs.id = schedule_usage.group_key
842
- ORDER BY schedule_usage.created_at ASC
868
+ GROUP BY (schedule_usage.created_at / ${USAGE_PREAGG_BUCKET_MS}), schedule_usage.group_key
869
+ ORDER BY created_at ASC
843
870
  `,
844
871
  ...groupKeySubquery.params,
845
872
  ...where.params,
@@ -850,15 +877,16 @@ export function getUsageGroupedSeries(
850
877
  rows = rawAll<UsageGroupedBucketRow>(
851
878
  /*sql*/ `
852
879
  SELECT
853
- e.created_at,
854
- e.input_tokens,
855
- e.output_tokens,
856
- e.estimated_cost_usd,
857
- e.llm_call_count,
880
+ (e.created_at / ${USAGE_PREAGG_BUCKET_MS}) * ${USAGE_PREAGG_BUCKET_MS} AS created_at,
881
+ COALESCE(SUM(e.input_tokens), 0) AS input_tokens,
882
+ COALESCE(SUM(e.output_tokens), 0) AS output_tokens,
883
+ SUM(e.estimated_cost_usd) AS estimated_cost_usd,
884
+ SUM(COALESCE(e.llm_call_count, 1)) AS llm_call_count,
858
885
  e.${column} AS group_key
859
886
  FROM llm_usage_events e
860
887
  WHERE ${where.sql}
861
- ORDER BY e.created_at ASC
888
+ GROUP BY (e.created_at / ${USAGE_PREAGG_BUCKET_MS}), e.${column}
889
+ ORDER BY created_at ASC
862
890
  `,
863
891
  ...where.params,
864
892
  );
@@ -43,8 +43,8 @@ import {
43
43
  parseInterfaceId,
44
44
  } from "../channels/types.js";
45
45
  import type { AssistantConfig } from "../config/types.js";
46
+ import { getGuardianDelivery } from "../contacts/guardian-delivery-reader.js";
46
47
  import { extractTurnContextTimestamp } from "../context/compactor.js";
47
- import { findConversation } from "../daemon/conversation-registry.js";
48
48
  import {
49
49
  formatLocalTimestamp,
50
50
  resolveTurnTimezoneContext,
@@ -63,6 +63,7 @@ import {
63
63
  forkConversationForRetrospective,
64
64
  getConversation,
65
65
  getMessagesAfter,
66
+ isConversationProcessing,
66
67
  resolveOverrideProfile,
67
68
  } from "./conversation-crud.js";
68
69
  import {
@@ -145,14 +146,15 @@ export async function runForkBasedRetrospective(
145
146
 
146
147
  // Forking mid-turn would capture a half-finished display turn — incremental
147
148
  // checkpoint persistence writes complete tool turns to the DB while the
148
- // agent loop is still running. Peek the in-memory registry only (an
149
- // unloaded conversation is by definition not processing); never load the
150
- // conversation just to check. Bump `lastRunAt` so the cooldown gate
151
- // applies, leave `lastProcessedMessageId` untouched so the next
152
- // interval/message-count trigger re-processes the same messages — nothing
153
- // is lost. Returning (not throwing) keeps the jobs-worker from
149
+ // agent loop is still running. Check the persisted `processing_started_at`
150
+ // column (the cross-process source of truth) instead of the in-memory
151
+ // registry, so this guard works even when running in a separate CLI
152
+ // process with an empty conversation registry. Bump `lastRunAt` so the
153
+ // cooldown gate applies, leave `lastProcessedMessageId` untouched so the
154
+ // next interval/message-count trigger re-processes the same messages —
155
+ // nothing is lost. Returning (not throwing) keeps the jobs-worker from
154
156
  // retry-with-backoff.
155
- if (findConversation(sourceConversationId)?.isProcessing()) {
157
+ if (isConversationProcessing(sourceConversationId)) {
156
158
  bumpRetrospectiveLastRunAt(sourceConversationId, Date.now());
157
159
  log.info(
158
160
  { sourceConversationId },
@@ -289,6 +291,10 @@ export async function runForkBasedRetrospective(
289
291
  // parity — the fork always runs execution gate mode below, so the source's
290
292
  // full tool surface stays on the wire while the allowlist holds at
291
293
  // execution time.
294
+ // Warm the vellum guardian-delivery cache so the sync slug resolution inside
295
+ // resolveSourceParityPins (resolveUserSlug(undefined)) hits a fresh key
296
+ // instead of falling back to "default" on a cold/TTL-expired cache.
297
+ await getGuardianDelivery({ channelTypes: ["vellum"] });
292
298
  const { personaOverride, toolContextPin } = resolveSourceParityPins(
293
299
  sourceConversation,
294
300
  newMessages,
@@ -1,6 +1,105 @@
1
+ import { getLogger } from "../../util/logger.js";
2
+ import { getDbPath } from "../../util/platform.js";
3
+ import { runAsyncSqlite } from "../db-async-query.js";
1
4
  import type { DrizzleDb } from "../db-connection.js";
2
5
  import { getSqliteFrom } from "../db-connection.js";
3
6
 
7
+ const log = getLogger("db-init");
8
+
9
+ /**
10
+ * Checkpoint key holding the highest `messages.rowid` already swept by this
11
+ * migration. Persisted after every window so an interrupted run resumes from
12
+ * where it left off instead of restarting the whole table scan. The value is a
13
+ * plain integer string — deliberately not `started`/`rolling_back`, so
14
+ * `recoverCrashedMigrations` never mistakes it for a stalled step and clears
15
+ * it, and not `step:`-prefixed, so `validateMigrationState` ignores it.
16
+ */
17
+ const WATERMARK_KEY = "migration_209_strip_thinking_watermark";
18
+
19
+ /**
20
+ * Number of `rowid` values swept per `runAsyncSqlite` dispatch. Each window is
21
+ * one off-thread subprocess transaction. The size stays well below the row
22
+ * count of a typical `messages` table so the whole table is never swept in a
23
+ * single subprocess: a window must finish inside {@link WINDOW_TIMEOUT_MS} for
24
+ * the per-window watermark to advance, and only an advancing watermark lets an
25
+ * interrupted run resume from the last completed window instead of re-running
26
+ * the same window forever. A smaller window also bounds WAL growth per statement
27
+ * and how long a single write lock is held, at the cost of more (cheap)
28
+ * subprocess spawns.
29
+ *
30
+ * Exported for the regression test that asserts the table is swept across
31
+ * multiple bounded windows rather than one table-sized sweep.
32
+ */
33
+ export const ROWID_WINDOW = 2_000;
34
+
35
+ /**
36
+ * Per-window wall-clock cap for the sweep subprocess. Set well above the time a
37
+ * {@link ROWID_WINDOW}-sized window needs even on a multi-GB table with large
38
+ * content blobs, so it trips only on a genuinely stuck subprocess (e.g. one
39
+ * blocked on a stale write lock) rather than on legitimately slow progress.
40
+ * Far below `runAsyncSqlite`'s one-hour whole-process default so a stuck window
41
+ * surfaces in minutes and the runner retries from the last completed window on
42
+ * the next boot.
43
+ */
44
+ export const WINDOW_TIMEOUT_MS = 15 * 60 * 1000;
45
+
46
+ /** SQL predicate: this `json_each` element is an internal reasoning block. */
47
+ const IS_THINKING = `json_extract(value, '$.type') IN ('thinking', 'redacted_thinking')`;
48
+
49
+ /**
50
+ * SQL predicate: this `json_each` element is kept. `IS NOT` (not `!=`) so a
51
+ * block with a NULL/absent `type` is preserved, matching the JS filter the
52
+ * original migration used (`b.type !== 'thinking'`).
53
+ */
54
+ const IS_KEPT = `json_extract(value, '$.type') IS NOT 'thinking' AND json_extract(value, '$.type') IS NOT 'redacted_thinking'`;
55
+
56
+ /**
57
+ * Build the two set-based UPDATEs that sweep one rowid window `(lo, hi]`.
58
+ *
59
+ * - Statement A strips thinking/redacted_thinking blocks from rows that retain
60
+ * at least one kept block, rebuilding the content array in place. `json(value)`
61
+ * re-embeds each surviving element as JSON rather than a quoted string.
62
+ * - Statement B replaces all-thinking rows (no kept block survives) with the
63
+ * null-byte placeholder sentinel so the message isn't left empty. `char(0)`
64
+ * is evaluated inside SQLite, so the literal NUL is produced in the database
65
+ * and never written into the SQL piped to the sqlite3 CLI.
66
+ *
67
+ * Both are gated by a cheap `content LIKE '%thinking%'` substring prefilter
68
+ * (covers both `thinking` and `redacted_thinking`) so the expensive `json_each`
69
+ * work only runs for rows that could possibly contain a reasoning block. The
70
+ * two statements touch disjoint row sets (A requires a kept block, B requires
71
+ * none), so their order within the window is immaterial.
72
+ */
73
+ function windowSql(lo: number, hi: number): string {
74
+ const scope = `role = 'assistant'
75
+ AND rowid > ${lo} AND rowid <= ${hi}
76
+ AND content LIKE '%thinking%'
77
+ AND json_valid(content)
78
+ AND json_type(content) = 'array'`;
79
+
80
+ const stripMixed = /*sql*/ `
81
+ UPDATE messages
82
+ SET content = (
83
+ SELECT json_group_array(json(value))
84
+ FROM json_each(messages.content)
85
+ WHERE ${IS_KEPT}
86
+ )
87
+ WHERE ${scope}
88
+ AND EXISTS (SELECT 1 FROM json_each(messages.content) WHERE ${IS_THINKING})
89
+ AND EXISTS (SELECT 1 FROM json_each(messages.content) WHERE ${IS_KEPT});`;
90
+
91
+ const placeholderOnly = /*sql*/ `
92
+ UPDATE messages
93
+ SET content = json_array(
94
+ json_object('type', 'text', 'text', char(0) || '__PLACEHOLDER__[internal blocks omitted]')
95
+ )
96
+ WHERE ${scope}
97
+ AND EXISTS (SELECT 1 FROM json_each(messages.content) WHERE ${IS_THINKING})
98
+ AND NOT EXISTS (SELECT 1 FROM json_each(messages.content) WHERE ${IS_KEPT});`;
99
+
100
+ return `${stripMixed}\n${placeholderOnly}`;
101
+ }
102
+
4
103
  /**
5
104
  * Strip thinking and redacted_thinking blocks from all assistant messages.
6
105
  *
@@ -11,68 +110,65 @@ import { getSqliteFrom } from "../db-connection.js";
11
110
  * provider no longer needs to strip, enabling append-only conversation
12
111
  * history and stable prefix caching.
13
112
  *
14
- * Idempotent — safe to re-run.
113
+ * The rewrite runs entirely inside SQLite (via JSON1), dispatched through
114
+ * {@link runAsyncSqlite} a rowid window at a time. On a host with the `sqlite3`
115
+ * CLI each window executes in a subprocess, leaving the daemon's event loop
116
+ * free to answer `/healthz` while a multi-GB table is rewritten — the original
117
+ * synchronous in-process loop blocked the loop for minutes and tripped the
118
+ * startup probe into a restart loop. Progress is checkpointed per window, so an
119
+ * interrupted run resumes instead of rescanning from the start.
120
+ *
121
+ * Idempotent — safe to re-run. Already-cleaned rows no longer contain a
122
+ * thinking block, so the substring prefilter skips them.
15
123
  */
16
- export function migrateStripThinkingFromConsolidated(
124
+ export async function migrateStripThinkingFromConsolidated(
17
125
  database: DrizzleDb,
18
- ): void {
126
+ ): Promise<void> {
19
127
  const raw = getSqliteFrom(database);
128
+ const dbPath = getDbPath();
20
129
 
21
- const BATCH_SIZE = 100;
22
- let lastRowid = 0;
23
-
24
- for (;;) {
25
- const rows = raw
26
- .query(
27
- `SELECT rowid, id, content FROM messages
28
- WHERE role = 'assistant'
29
- AND rowid > ?
30
- ORDER BY rowid
31
- LIMIT ?`,
32
- )
33
- .all(lastRowid, BATCH_SIZE) as Array<{
34
- rowid: number;
35
- id: string;
36
- content: string;
37
- }>;
38
-
39
- if (rows.length === 0) break;
40
-
41
- for (const row of rows) {
42
- lastRowid = row.rowid;
43
-
44
- let blocks: Array<{ type: string }>;
45
- try {
46
- const parsed = JSON.parse(row.content);
47
- if (!Array.isArray(parsed)) continue;
48
- blocks = parsed;
49
- } catch {
50
- continue;
51
- }
52
-
53
- const hasThinking = blocks.some(
54
- (b) => b.type === "thinking" || b.type === "redacted_thinking",
55
- );
56
- if (!hasThinking) continue;
130
+ const maxRow = (
131
+ raw.query(`SELECT MAX(rowid) AS m FROM messages`).get() as {
132
+ m: number | null;
133
+ }
134
+ ).m;
135
+ if (maxRow == null) return; // empty table — nothing to sweep
57
136
 
58
- const stripped = blocks.filter(
59
- (b) => b.type !== "thinking" && b.type !== "redacted_thinking",
60
- );
137
+ const watermarkRow = raw
138
+ .query(`SELECT value FROM memory_checkpoints WHERE key = ?`)
139
+ .get(WATERMARK_KEY) as { value: string } | undefined;
140
+ let lo = watermarkRow ? Number.parseInt(watermarkRow.value, 10) || 0 : 0;
141
+
142
+ const setWatermark = raw.query(
143
+ `INSERT OR REPLACE INTO memory_checkpoints (key, value, updated_at) VALUES (?, ?, ?)`,
144
+ );
145
+
146
+ while (lo < maxRow) {
147
+ const hi = Math.min(lo + ROWID_WINDOW, maxRow);
61
148
 
62
- // Preserve at least one block so the message isn't empty.
63
- const finalContent =
64
- stripped.length > 0
65
- ? stripped
66
- : [
67
- {
68
- type: "text" as const,
69
- text: "\x00__PLACEHOLDER__[internal blocks omitted]",
70
- },
71
- ];
72
-
73
- raw
74
- .query(`UPDATE messages SET content = ? WHERE id = ?`)
75
- .run(JSON.stringify(finalContent), row.id);
149
+ const res = await runAsyncSqlite(windowSql(lo, hi), {
150
+ dbPath,
151
+ timeoutMs: WINDOW_TIMEOUT_MS,
152
+ });
153
+ if (!res.ok) {
154
+ // Leave the watermark at the last completed window; throwing reports the
155
+ // step failed so the runner retries it (from the watermark) next boot
156
+ // rather than checkpointing it as done.
157
+ throw new Error(
158
+ `strip-thinking window (${lo}, ${hi}] failed: ${res.error}`,
159
+ );
76
160
  }
161
+
162
+ lo = hi;
163
+ setWatermark.run(WATERMARK_KEY, String(lo), Date.now());
77
164
  }
165
+
166
+ // Bound WAL growth left by the windowed rewrites, then drop the watermark so
167
+ // a future re-run (e.g. after a rollback) starts clean.
168
+ await runAsyncSqlite(`PRAGMA wal_checkpoint(TRUNCATE);`, { dbPath });
169
+ raw.query(`DELETE FROM memory_checkpoints WHERE key = ?`).run(WATERMARK_KEY);
170
+
171
+ log.info(
172
+ "Migration 209: stripped thinking blocks from consolidated messages",
173
+ );
78
174
  }
@@ -0,0 +1,30 @@
1
+ import type { DrizzleDb } from "../db-connection.js";
2
+ import { tableHasColumn } from "./schema-introspection.js";
3
+
4
+ const COLUMN_NAME = "processing_started_at";
5
+ const COLUMN_DEFINITION = "processing_started_at INTEGER";
6
+
7
+ /**
8
+ * Add `processing_started_at` column to the `conversations` table.
9
+ *
10
+ * The column is a nullable epoch-ms timestamp: non-NULL means the agent loop
11
+ * is mid-turn for this conversation, NULL means idle. This is the
12
+ * cross-process source of truth for processing state — the in-memory
13
+ * `Conversation._processing` flag is the hot-path read for resident
14
+ * conversations, but out-of-process callers (e.g. the retrospective CLI)
15
+ * read this column directly via `isConversationProcessing()`.
16
+ *
17
+ * No backfill is needed — all existing rows default to NULL (not processing),
18
+ * which is correct for any conversation that isn't actively mid-turn at
19
+ * migration time.
20
+ *
21
+ * Idempotent: guarded with `tableHasColumn` so a crash between the `ALTER
22
+ * TABLE` and the checkpoint write doesn't cause a duplicate-column error on
23
+ * the next boot.
24
+ */
25
+ export function migrateAddProcessingStartedAt(database: DrizzleDb): void {
26
+ if (tableHasColumn(database, "conversations", COLUMN_NAME)) {
27
+ return;
28
+ }
29
+ database.run(`ALTER TABLE conversations ADD COLUMN ${COLUMN_DEFINITION}`);
30
+ }