@vellumai/assistant 0.5.6 → 0.5.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (442) hide show
  1. package/.env.example +16 -2
  2. package/ARCHITECTURE.md +6 -75
  3. package/Dockerfile +3 -2
  4. package/README.md +0 -2
  5. package/bun.lock +0 -414
  6. package/docker-entrypoint.sh +9 -0
  7. package/docs/architecture/keychain-broker.md +45 -240
  8. package/docs/architecture/memory.md +13 -11
  9. package/docs/architecture/security.md +0 -17
  10. package/docs/credential-execution-service.md +2 -2
  11. package/node_modules/@vellumai/ces-contracts/package.json +1 -0
  12. package/node_modules/@vellumai/ces-contracts/src/error.ts +1 -1
  13. package/node_modules/@vellumai/ces-contracts/src/grants.ts +1 -1
  14. package/node_modules/@vellumai/ces-contracts/src/handles.ts +1 -1
  15. package/node_modules/@vellumai/ces-contracts/src/index.ts +1 -1
  16. package/node_modules/@vellumai/ces-contracts/src/rpc.ts +120 -1
  17. package/node_modules/@vellumai/credential-storage/package.json +1 -0
  18. package/node_modules/@vellumai/egress-proxy/package.json +1 -0
  19. package/package.json +2 -3
  20. package/src/__tests__/actor-token-service.test.ts +0 -114
  21. package/src/__tests__/approval-cascade.test.ts +0 -1
  22. package/src/__tests__/assistant-feature-flags-integration.test.ts +30 -29
  23. package/src/__tests__/browser-fill-credential.test.ts +1 -1
  24. package/src/__tests__/browser-skill-endstate.test.ts +6 -5
  25. package/src/__tests__/btw-routes.test.ts +0 -39
  26. package/src/__tests__/call-controller.test.ts +0 -1
  27. package/src/__tests__/call-domain.test.ts +0 -128
  28. package/src/__tests__/ces-rpc-credential-backend.test.ts +199 -0
  29. package/src/__tests__/ces-startup-timeout.test.ts +40 -0
  30. package/src/__tests__/channel-approval-routes.test.ts +0 -5
  31. package/src/__tests__/channel-readiness-service.test.ts +1 -60
  32. package/src/__tests__/checker.test.ts +4 -2
  33. package/src/__tests__/cli-command-risk-guard.test.ts +112 -0
  34. package/src/__tests__/config-schema-cmd.test.ts +0 -2
  35. package/src/__tests__/config-schema.test.ts +3 -1
  36. package/src/__tests__/conversation-abort-tool-results.test.ts +0 -1
  37. package/src/__tests__/conversation-agent-loop-overflow.test.ts +0 -2
  38. package/src/__tests__/conversation-agent-loop.test.ts +2 -4
  39. package/src/__tests__/conversation-attention-telegram.test.ts +0 -5
  40. package/src/__tests__/conversation-confirmation-signals.test.ts +0 -1
  41. package/src/__tests__/conversation-error.test.ts +15 -1
  42. package/src/__tests__/conversation-init.benchmark.test.ts +0 -2
  43. package/src/__tests__/conversation-messaging-secret-redirect.test.ts +1 -1
  44. package/src/__tests__/conversation-pre-run-repair.test.ts +0 -1
  45. package/src/__tests__/conversation-provider-retry-repair.test.ts +0 -1
  46. package/src/__tests__/conversation-queue.test.ts +0 -1
  47. package/src/__tests__/conversation-skill-tools.test.ts +0 -54
  48. package/src/__tests__/conversation-slash-queue.test.ts +0 -1
  49. package/src/__tests__/conversation-slash-unknown.test.ts +0 -1
  50. package/src/__tests__/conversation-title-service.test.ts +87 -0
  51. package/src/__tests__/conversation-workspace-injection.test.ts +0 -1
  52. package/src/__tests__/conversation-workspace-tool-tracking.test.ts +0 -1
  53. package/src/__tests__/credential-execution-client.test.ts +5 -2
  54. package/src/__tests__/credential-execution-feature-gates.test.ts +59 -30
  55. package/src/__tests__/credential-execution-managed-contract.test.ts +35 -20
  56. package/src/__tests__/credential-security-e2e.test.ts +1 -67
  57. package/src/__tests__/credential-security-invariants.test.ts +6 -50
  58. package/src/__tests__/credentials-cli.test.ts +82 -3
  59. package/src/__tests__/daemon-credential-client.test.ts +123 -0
  60. package/src/__tests__/db-migration-rollback.test.ts +2015 -1
  61. package/src/__tests__/deterministic-verification-control-plane.test.ts +1 -0
  62. package/src/__tests__/docker-signing-key-bootstrap.test.ts +34 -143
  63. package/src/__tests__/dynamic-skill-workflow-prompt.test.ts +6 -4
  64. package/src/__tests__/gateway-client-managed-outbound.test.ts +79 -1
  65. package/src/__tests__/guardian-routing-state.test.ts +0 -5
  66. package/src/__tests__/host-shell-tool.test.ts +6 -7
  67. package/src/__tests__/http-user-message-parity.test.ts +3 -103
  68. package/src/__tests__/inbound-invite-redemption.test.ts +0 -4
  69. package/src/__tests__/inline-skill-load-permissions.test.ts +6 -8
  70. package/src/__tests__/intent-routing.test.ts +0 -13
  71. package/src/__tests__/jobs-store-qdrant-breaker.test.ts +178 -0
  72. package/src/__tests__/journal-context.test.ts +335 -0
  73. package/src/__tests__/keychain-broker-client.test.ts +161 -22
  74. package/src/__tests__/memory-context-benchmark.benchmark.test.ts +0 -3
  75. package/src/__tests__/memory-jobs-worker-backoff.test.ts +150 -0
  76. package/src/__tests__/memory-lifecycle-e2e.test.ts +70 -25
  77. package/src/__tests__/memory-recall-quality.test.ts +48 -17
  78. package/src/__tests__/memory-regressions.test.ts +408 -363
  79. package/src/__tests__/memory-retrieval.benchmark.test.ts +0 -3
  80. package/src/__tests__/migration-export-http.test.ts +2 -2
  81. package/src/__tests__/migration-import-commit-http.test.ts +2 -2
  82. package/src/__tests__/migration-import-preflight-http.test.ts +2 -2
  83. package/src/__tests__/migration-validate-http.test.ts +2 -2
  84. package/src/__tests__/non-member-access-request.test.ts +2 -7
  85. package/src/__tests__/notification-decision-fallback.test.ts +4 -0
  86. package/src/__tests__/notification-decision-identity.test.ts +4 -0
  87. package/src/__tests__/notification-decision-strategy.test.ts +71 -0
  88. package/src/__tests__/oauth-cli.test.ts +5 -1
  89. package/src/__tests__/permission-types.test.ts +1 -0
  90. package/src/__tests__/provider-commit-message-generator.test.ts +0 -37
  91. package/src/__tests__/provider-error-scenarios.test.ts +0 -267
  92. package/src/__tests__/provider-managed-proxy-integration.test.ts +5 -6
  93. package/src/__tests__/provider-streaming.benchmark.test.ts +2 -81
  94. package/src/__tests__/qdrant-manager.test.ts +28 -2
  95. package/src/__tests__/registry.test.ts +0 -6
  96. package/src/__tests__/relay-server.test.ts +1 -2
  97. package/src/__tests__/runtime-attachment-metadata.test.ts +0 -4
  98. package/src/__tests__/script-proxy-injection-runtime.test.ts +1 -1
  99. package/src/__tests__/secret-onetime-send.test.ts +1 -1
  100. package/src/__tests__/secret-routes-managed-proxy.test.ts +0 -4
  101. package/src/__tests__/secure-keys.test.ts +95 -272
  102. package/src/__tests__/shell-identity.test.ts +96 -6
  103. package/src/__tests__/skill-feature-flags-integration.test.ts +22 -14
  104. package/src/__tests__/skill-feature-flags.test.ts +46 -45
  105. package/src/__tests__/skill-load-feature-flag.test.ts +7 -10
  106. package/src/__tests__/skill-load-inline-command.test.ts +8 -12
  107. package/src/__tests__/skill-load-inline-includes.test.ts +6 -10
  108. package/src/__tests__/skill-load-tool.test.ts +0 -2
  109. package/src/__tests__/skill-memory.test.ts +17 -3
  110. package/src/__tests__/skill-projection-feature-flag.test.ts +33 -29
  111. package/src/__tests__/skills.test.ts +0 -2
  112. package/src/__tests__/slack-inbound-verification.test.ts +0 -4
  113. package/src/__tests__/stale-approval-dedup.test.ts +171 -0
  114. package/src/__tests__/stt-hints.test.ts +437 -0
  115. package/src/__tests__/suggestion-routes.test.ts +1 -32
  116. package/src/__tests__/system-prompt.test.ts +0 -1
  117. package/src/__tests__/task-memory-cleanup.test.ts +14 -0
  118. package/src/__tests__/tool-executor-shell-integration.test.ts +5 -3
  119. package/src/__tests__/trusted-contact-lifecycle-notifications.test.ts +0 -5
  120. package/src/__tests__/trusted-contact-multichannel.test.ts +0 -4
  121. package/src/__tests__/twilio-routes-twiml.test.ts +139 -1
  122. package/src/__tests__/update-bulletin.test.ts +0 -2
  123. package/src/__tests__/vellum-self-knowledge-inline-command.test.ts +6 -9
  124. package/src/__tests__/voice-quality.test.ts +58 -0
  125. package/src/__tests__/voice-scoped-grant-consumer.test.ts +0 -7
  126. package/src/__tests__/workspace-migration-015-migrate-credentials-to-keychain.test.ts +252 -0
  127. package/src/__tests__/workspace-migration-016-migrate-credentials-from-keychain.test.ts +220 -0
  128. package/src/__tests__/workspace-migration-down-functions.test.ts +1009 -0
  129. package/src/__tests__/workspace-migrations-runner.test.ts +114 -0
  130. package/src/acp/agent-process.ts +9 -1
  131. package/src/agent/loop.ts +1 -1
  132. package/src/approvals/guardian-request-resolvers.ts +164 -38
  133. package/src/calls/__tests__/tts-text-sanitizer.test.ts +254 -0
  134. package/src/calls/audio-store.test.ts +97 -0
  135. package/src/calls/audio-store.ts +205 -0
  136. package/src/calls/call-controller.ts +90 -8
  137. package/src/calls/call-domain.ts +3 -0
  138. package/src/calls/call-store.ts +10 -3
  139. package/src/calls/fish-audio-client.ts +129 -0
  140. package/src/calls/relay-server.ts +27 -0
  141. package/src/calls/stt-hints.ts +189 -0
  142. package/src/calls/tts-text-sanitizer.ts +61 -0
  143. package/src/calls/twilio-routes.ts +34 -5
  144. package/src/calls/types.ts +1 -0
  145. package/src/calls/voice-ingress-preflight.ts +0 -42
  146. package/src/calls/voice-quality.ts +38 -5
  147. package/src/calls/voice-session-bridge.ts +7 -12
  148. package/src/cli/commands/avatar.ts +2 -2
  149. package/src/cli/commands/config.ts +1 -4
  150. package/src/cli/commands/credentials.ts +128 -82
  151. package/src/cli/commands/doctor.ts +2 -2
  152. package/src/cli/commands/keys.ts +7 -7
  153. package/src/cli/commands/memory.ts +1 -1
  154. package/src/cli/commands/oauth/connections.ts +11 -29
  155. package/src/cli/commands/oauth/index.ts +7 -0
  156. package/src/cli/commands/oauth/platform.ts +525 -0
  157. package/src/cli/commands/platform.ts +3 -3
  158. package/src/cli/lib/daemon-credential-client.ts +284 -0
  159. package/src/cli.ts +1 -1
  160. package/src/config/assistant-feature-flags.ts +186 -5
  161. package/src/config/bundled-skills/AGENTS.md +34 -0
  162. package/src/config/bundled-skills/acp/SKILL.md +10 -0
  163. package/src/config/bundled-skills/app-builder/SKILL.md +0 -4
  164. package/src/config/bundled-skills/messaging/SKILL.md +5 -5
  165. package/src/config/bundled-skills/messaging/tools/messaging-analyze-style.ts +2 -2
  166. package/src/config/bundled-skills/phone-calls/TOOLS.json +4 -0
  167. package/src/config/bundled-skills/playbooks/tools/playbook-create.ts +1 -0
  168. package/src/config/bundled-skills/playbooks/tools/playbook-update.ts +1 -0
  169. package/src/config/bundled-skills/settings/SKILL.md +15 -2
  170. package/src/config/bundled-skills/settings/TOOLS.json +47 -2
  171. package/src/config/bundled-skills/settings/tools/avatar-remove.ts +59 -0
  172. package/src/config/bundled-skills/settings/tools/avatar-update.ts +80 -0
  173. package/src/config/bundled-skills/settings/tools/voice-config-update.ts +42 -0
  174. package/src/config/bundled-skills/slack/SKILL.md +1 -1
  175. package/src/config/bundled-tool-registry.ts +5 -11
  176. package/src/config/defaults.ts +0 -2
  177. package/src/config/env-registry.ts +5 -5
  178. package/src/config/env.ts +21 -14
  179. package/src/config/feature-flag-registry.json +49 -9
  180. package/src/config/loader.ts +106 -42
  181. package/src/config/schema.ts +9 -29
  182. package/src/config/schemas/calls.ts +30 -0
  183. package/src/config/schemas/fish-audio.ts +39 -0
  184. package/src/config/schemas/inference.ts +2 -2
  185. package/src/config/schemas/journal.ts +16 -0
  186. package/src/config/schemas/memory-processing.ts +2 -2
  187. package/src/config/schemas/security.ts +0 -4
  188. package/src/config/types.ts +1 -1
  189. package/src/contacts/contact-store.ts +39 -0
  190. package/src/contacts/types.ts +2 -0
  191. package/src/credential-execution/approval-bridge.ts +1 -0
  192. package/src/credential-execution/executable-discovery.ts +28 -4
  193. package/src/credential-execution/feature-gates.ts +16 -0
  194. package/src/credential-execution/process-manager.ts +38 -0
  195. package/src/credential-execution/startup-timeout.ts +36 -0
  196. package/src/daemon/approval-generators.ts +3 -9
  197. package/src/daemon/assistant-attachments.ts +9 -0
  198. package/src/daemon/config-watcher.ts +5 -0
  199. package/src/daemon/conversation-error.ts +13 -1
  200. package/src/daemon/conversation-memory.ts +1 -2
  201. package/src/daemon/conversation-process.ts +18 -1
  202. package/src/daemon/conversation-surfaces.ts +30 -1
  203. package/src/daemon/conversation-tool-setup.ts +0 -105
  204. package/src/daemon/conversation.ts +21 -1
  205. package/src/daemon/guardian-action-generators.ts +3 -9
  206. package/src/daemon/handlers/config-vercel.ts +92 -0
  207. package/src/daemon/handlers/skills.ts +2 -15
  208. package/src/daemon/install-symlink.ts +195 -0
  209. package/src/daemon/lifecycle.ts +234 -51
  210. package/src/daemon/message-types/conversations.ts +4 -4
  211. package/src/daemon/message-types/diagnostics.ts +3 -22
  212. package/src/daemon/message-types/messages.ts +0 -2
  213. package/src/daemon/message-types/upgrades.ts +8 -0
  214. package/src/daemon/server.ts +32 -95
  215. package/src/events/domain-events.ts +2 -1
  216. package/src/inbound/platform-callback-registration.ts +3 -3
  217. package/src/instrument.ts +8 -5
  218. package/src/memory/app-store.ts +31 -0
  219. package/src/memory/conversation-title-service.ts +50 -1
  220. package/src/memory/db-init.ts +16 -0
  221. package/src/memory/indexer.ts +19 -10
  222. package/src/memory/items-extractor.ts +328 -321
  223. package/src/memory/job-handlers/conversation-starters.ts +4 -1
  224. package/src/memory/job-handlers/summarization.ts +26 -16
  225. package/src/memory/jobs-store.ts +63 -6
  226. package/src/memory/jobs-worker.ts +31 -7
  227. package/src/memory/journal-memory.ts +214 -0
  228. package/src/memory/migrations/001-job-deferrals.ts +19 -0
  229. package/src/memory/migrations/004-entity-relation-dedup.ts +10 -0
  230. package/src/memory/migrations/005-fingerprint-scope-unique.ts +76 -0
  231. package/src/memory/migrations/006-scope-salted-fingerprints.ts +50 -0
  232. package/src/memory/migrations/007-assistant-id-to-self.ts +10 -0
  233. package/src/memory/migrations/008-remove-assistant-id-columns.ts +34 -0
  234. package/src/memory/migrations/009-llm-usage-events-drop-assistant-id.ts +26 -0
  235. package/src/memory/migrations/014-backfill-inbox-thread-state.ts +10 -0
  236. package/src/memory/migrations/015-drop-active-search-index.ts +17 -0
  237. package/src/memory/migrations/019-notification-tables-schema-migration.ts +12 -0
  238. package/src/memory/migrations/020-rename-macos-ios-channel-to-vellum.ts +121 -0
  239. package/src/memory/migrations/024-embedding-vector-blob.ts +74 -0
  240. package/src/memory/migrations/026a-embeddings-nullable-vector-json.ts +82 -0
  241. package/src/memory/migrations/036-normalize-phone-identities.ts +11 -0
  242. package/src/memory/migrations/116-messages-fts.ts +106 -1
  243. package/src/memory/migrations/126-backfill-guardian-principal-id.ts +52 -0
  244. package/src/memory/migrations/127-guardian-principal-id-not-null.ts +77 -0
  245. package/src/memory/migrations/134-contacts-notes-column.ts +13 -0
  246. package/src/memory/migrations/135-backfill-contact-interaction-stats.ts +20 -0
  247. package/src/memory/migrations/136-drop-assistant-id-columns.ts +52 -0
  248. package/src/memory/migrations/140-backfill-usage-cache-accounting.ts +13 -0
  249. package/src/memory/migrations/141-rename-verification-table.ts +54 -0
  250. package/src/memory/migrations/142-rename-verification-session-id-column.ts +25 -0
  251. package/src/memory/migrations/143-rename-guardian-verification-values.ts +35 -0
  252. package/src/memory/migrations/144-rename-voice-to-phone.ts +136 -0
  253. package/src/memory/migrations/145-drop-accounts-table.ts +32 -0
  254. package/src/memory/migrations/147-migrate-reminders-to-schedules.ts +14 -1
  255. package/src/memory/migrations/148-drop-reminders-table.ts +35 -1
  256. package/src/memory/migrations/150-oauth-apps-client-secret-path.ts +69 -1
  257. package/src/memory/migrations/162-guardian-timestamps-epoch-ms.ts +290 -0
  258. package/src/memory/migrations/169-rename-gmail-provider-key-to-google.ts +51 -1
  259. package/src/memory/migrations/174-rename-thread-starters-table.ts +47 -1
  260. package/src/memory/migrations/176-drop-capability-card-state.ts +13 -0
  261. package/src/memory/migrations/180-backfill-inline-attachments-to-disk.ts +16 -0
  262. package/src/memory/migrations/181-rename-thread-starters-checkpoints.ts +28 -1
  263. package/src/memory/migrations/190-call-session-skip-disclosure.ts +15 -0
  264. package/src/memory/migrations/191-backfill-audio-attachment-mime-types.ts +64 -0
  265. package/src/memory/migrations/192-contacts-user-file-column.ts +15 -0
  266. package/src/memory/migrations/193-add-source-type-columns.ts +81 -0
  267. package/src/memory/migrations/index.ts +5 -0
  268. package/src/memory/migrations/registry.ts +98 -0
  269. package/src/memory/migrations/validate-migration-state.ts +137 -11
  270. package/src/memory/qdrant-circuit-breaker.ts +9 -0
  271. package/src/memory/qdrant-manager.ts +64 -7
  272. package/src/memory/retriever.test.ts +37 -25
  273. package/src/memory/retriever.ts +24 -49
  274. package/src/memory/schema/calls.ts +1 -0
  275. package/src/memory/schema/contacts.ts +1 -0
  276. package/src/memory/schema/memory-core.ts +2 -0
  277. package/src/memory/search/formatting.ts +7 -44
  278. package/src/memory/search/staleness.ts +4 -0
  279. package/src/memory/search/tier-classifier.ts +10 -2
  280. package/src/memory/search/types.ts +2 -5
  281. package/src/memory/task-memory-cleanup.ts +4 -3
  282. package/src/notifications/adapters/slack.ts +168 -6
  283. package/src/notifications/broadcaster.ts +1 -0
  284. package/src/notifications/copy-composer.ts +59 -2
  285. package/src/notifications/decision-engine.ts +4 -1
  286. package/src/notifications/signal.ts +2 -0
  287. package/src/notifications/types.ts +2 -0
  288. package/src/oauth/connection-resolver.ts +6 -4
  289. package/src/permissions/checker.ts +0 -38
  290. package/src/permissions/shell-identity.ts +76 -22
  291. package/src/permissions/types.ts +4 -2
  292. package/src/platform/client.ts +35 -7
  293. package/src/prompts/journal-context.ts +133 -0
  294. package/src/prompts/persona-resolver.ts +194 -0
  295. package/src/prompts/system-prompt.ts +44 -4
  296. package/src/prompts/templates/SOUL.md +10 -0
  297. package/src/prompts/templates/users/default.md +1 -0
  298. package/src/providers/provider-send-message.ts +3 -32
  299. package/src/providers/registry.ts +29 -179
  300. package/src/providers/types.ts +1 -1
  301. package/src/runtime/access-request-helper.ts +4 -0
  302. package/src/runtime/auth/__tests__/credential-service.test.ts +0 -1
  303. package/src/runtime/auth/__tests__/external-assistant-id.test.ts +13 -68
  304. package/src/runtime/auth/__tests__/guard-tests.test.ts +9 -50
  305. package/src/runtime/auth/external-assistant-id.ts +13 -59
  306. package/src/runtime/auth/route-policy.ts +17 -1
  307. package/src/runtime/auth/token-service.ts +43 -138
  308. package/src/runtime/channel-readiness-service.ts +1 -16
  309. package/src/runtime/gateway-client.ts +47 -4
  310. package/src/runtime/guardian-decision-types.ts +45 -4
  311. package/src/runtime/http-server.ts +31 -3
  312. package/src/runtime/middleware/error-handler.ts +1 -9
  313. package/src/runtime/routes/access-request-decision.ts +2 -2
  314. package/src/runtime/routes/app-management-routes.ts +2 -1
  315. package/src/runtime/routes/approval-strategies/guardian-callback-strategy.ts +219 -30
  316. package/src/runtime/routes/approval-strategies/guardian-text-engine-strategy.ts +37 -14
  317. package/src/runtime/routes/audio-routes.ts +40 -0
  318. package/src/runtime/routes/btw-routes.ts +0 -17
  319. package/src/runtime/routes/channel-readiness-routes.ts +9 -4
  320. package/src/runtime/routes/conversation-query-routes.ts +63 -1
  321. package/src/runtime/routes/conversation-routes.ts +4 -44
  322. package/src/runtime/routes/debug-routes.ts +12 -9
  323. package/src/runtime/routes/diagnostics-routes.ts +1 -477
  324. package/src/runtime/routes/guardian-approval-interception.ts +168 -11
  325. package/src/runtime/routes/guardian-approval-prompt.ts +6 -1
  326. package/src/runtime/routes/guardian-approval-reply-helpers.ts +103 -21
  327. package/src/runtime/routes/identity-routes.ts +19 -30
  328. package/src/runtime/routes/inbound-message-handler.ts +31 -1
  329. package/src/runtime/routes/inbound-stages/acl-enforcement.ts +64 -5
  330. package/src/runtime/routes/inbound-stages/background-dispatch.ts +52 -40
  331. package/src/runtime/routes/inbound-stages/secret-ingress-check.ts +4 -33
  332. package/src/runtime/routes/inbound-stages/transcribe-audio.test.ts +1 -1
  333. package/src/runtime/routes/integrations/twilio.ts +52 -10
  334. package/src/runtime/routes/integrations/vercel.ts +89 -0
  335. package/src/runtime/routes/log-export-routes.ts +5 -0
  336. package/src/runtime/routes/memory-item-routes.test.ts +3 -3
  337. package/src/runtime/routes/memory-item-routes.ts +46 -14
  338. package/src/runtime/routes/migration-rollback-routes.ts +209 -0
  339. package/src/runtime/routes/migration-routes.ts +17 -1
  340. package/src/runtime/routes/notification-routes.ts +58 -0
  341. package/src/runtime/routes/schedule-routes.ts +65 -0
  342. package/src/runtime/routes/secret-routes.ts +141 -10
  343. package/src/runtime/routes/settings-routes.ts +41 -1
  344. package/src/runtime/routes/tts-routes.ts +96 -0
  345. package/src/runtime/routes/upgrade-broadcast-routes.ts +26 -2
  346. package/src/runtime/routes/workspace-commit-routes.ts +62 -0
  347. package/src/runtime/routes/workspace-routes.test.ts +22 -1
  348. package/src/runtime/routes/workspace-routes.ts +1 -1
  349. package/src/runtime/routes/workspace-utils.ts +86 -2
  350. package/src/security/ces-credential-client.ts +75 -29
  351. package/src/security/ces-rpc-credential-backend.ts +86 -0
  352. package/src/security/credential-backend.ts +22 -92
  353. package/src/security/keychain-broker-client.ts +10 -2
  354. package/src/security/secure-keys.ts +113 -115
  355. package/src/skills/catalog-install.ts +6 -32
  356. package/src/skills/skill-memory.ts +1 -0
  357. package/src/subagent/manager.ts +2 -5
  358. package/src/telemetry/usage-telemetry-reporter.ts +4 -2
  359. package/src/tools/acp/spawn.ts +78 -1
  360. package/src/tools/calls/call-start.ts +1 -0
  361. package/src/tools/credentials/vault.ts +5 -3
  362. package/src/tools/executor.ts +0 -4
  363. package/src/tools/memory/definitions.ts +3 -2
  364. package/src/tools/memory/handlers.ts +10 -7
  365. package/src/tools/network/script-proxy/session-manager.ts +19 -4
  366. package/src/tools/network/web-fetch.ts +3 -1
  367. package/src/tools/skills/execute.ts +1 -1
  368. package/src/tools/terminal/safe-env.ts +1 -0
  369. package/src/tools/types.ts +0 -8
  370. package/src/util/browser.ts +15 -0
  371. package/src/util/errors.ts +0 -12
  372. package/src/util/platform.ts +4 -51
  373. package/src/workspace/git-service.ts +5 -2
  374. package/src/workspace/migrations/001-avatar-rename.ts +15 -0
  375. package/src/workspace/migrations/003-seed-device-id.ts +17 -1
  376. package/src/workspace/migrations/004-extract-collect-usage-data.ts +33 -0
  377. package/src/workspace/migrations/005-add-send-diagnostics.ts +3 -0
  378. package/src/workspace/migrations/006-services-config.ts +49 -0
  379. package/src/workspace/migrations/007-web-search-provider-rename.ts +27 -0
  380. package/src/workspace/migrations/008-voice-timeout-and-max-steps.ts +3 -0
  381. package/src/workspace/migrations/009-backfill-conversation-disk-view.ts +4 -0
  382. package/src/workspace/migrations/010-app-dir-rename.ts +78 -0
  383. package/src/workspace/migrations/011-backfill-installation-id.ts +11 -0
  384. package/src/workspace/migrations/012-rename-conversation-disk-view-dirs.ts +44 -0
  385. package/src/workspace/migrations/013-repair-conversation-disk-view.ts +5 -0
  386. package/src/workspace/migrations/015-migrate-credentials-to-keychain.ts +153 -0
  387. package/src/workspace/migrations/016-extract-feature-flags-to-protected.ts +156 -0
  388. package/src/workspace/migrations/016-migrate-credentials-from-keychain.ts +150 -0
  389. package/src/workspace/migrations/017-seed-persona-dirs.ts +96 -0
  390. package/src/workspace/migrations/018-rekey-compound-credential-keys.ts +184 -0
  391. package/src/workspace/migrations/019-scope-journal-to-guardian.ts +103 -0
  392. package/src/workspace/migrations/migrate-to-workspace-volume.ts +27 -5
  393. package/src/workspace/migrations/registry.ts +12 -0
  394. package/src/workspace/migrations/runner.ts +106 -2
  395. package/src/workspace/migrations/types.ts +4 -0
  396. package/src/workspace/provider-commit-message-generator.ts +12 -21
  397. package/src/__tests__/claude-code-skill-regression.test.ts +0 -206
  398. package/src/__tests__/claude-code-tool-profiles.test.ts +0 -99
  399. package/src/__tests__/diagnostics-export.test.ts +0 -288
  400. package/src/__tests__/local-gateway-health.test.ts +0 -209
  401. package/src/__tests__/provider-fail-open-selection.test.ts +0 -271
  402. package/src/__tests__/provider-failover-actual-provider.test.ts +0 -66
  403. package/src/__tests__/secret-ingress-handler.test.ts +0 -120
  404. package/src/__tests__/swarm-conversation-integration.test.ts +0 -358
  405. package/src/__tests__/swarm-dag-pathological.test.ts +0 -547
  406. package/src/__tests__/swarm-orchestrator.test.ts +0 -463
  407. package/src/__tests__/swarm-plan-validator.test.ts +0 -384
  408. package/src/__tests__/swarm-recursion.test.ts +0 -197
  409. package/src/__tests__/swarm-router-planner.test.ts +0 -234
  410. package/src/__tests__/swarm-tool.test.ts +0 -185
  411. package/src/__tests__/swarm-worker-backend.test.ts +0 -144
  412. package/src/__tests__/swarm-worker-runner.test.ts +0 -288
  413. package/src/commands/__tests__/cc-command-registry.test.ts +0 -396
  414. package/src/commands/cc-command-registry.ts +0 -248
  415. package/src/config/bundled-skills/claude-code/SKILL.md +0 -53
  416. package/src/config/bundled-skills/claude-code/TOOLS.json +0 -47
  417. package/src/config/bundled-skills/claude-code/tools/claude-code.ts +0 -12
  418. package/src/config/bundled-skills/orchestration/SKILL.md +0 -33
  419. package/src/config/bundled-skills/orchestration/TOOLS.json +0 -35
  420. package/src/config/bundled-skills/orchestration/tools/swarm-delegate.ts +0 -12
  421. package/src/config/schemas/swarm.ts +0 -82
  422. package/src/logfire.ts +0 -135
  423. package/src/memory/search/lexical.ts +0 -48
  424. package/src/providers/failover.ts +0 -186
  425. package/src/runtime/local-gateway-health.ts +0 -275
  426. package/src/security/secret-ingress.ts +0 -68
  427. package/src/swarm/backend-claude-code.ts +0 -225
  428. package/src/swarm/checkpoint.ts +0 -137
  429. package/src/swarm/graph-utils.ts +0 -53
  430. package/src/swarm/index.ts +0 -55
  431. package/src/swarm/limits.ts +0 -66
  432. package/src/swarm/orchestrator.ts +0 -424
  433. package/src/swarm/plan-validator.ts +0 -117
  434. package/src/swarm/router-planner.ts +0 -162
  435. package/src/swarm/router-prompts.ts +0 -39
  436. package/src/swarm/synthesizer.ts +0 -81
  437. package/src/swarm/types.ts +0 -72
  438. package/src/swarm/worker-backend.ts +0 -131
  439. package/src/swarm/worker-prompts.ts +0 -80
  440. package/src/swarm/worker-runner.ts +0 -170
  441. package/src/tools/claude-code/claude-code.ts +0 -610
  442. package/src/tools/swarm/delegate.ts +0 -205
@@ -1,3 +1,44 @@
1
+ import type { DrizzleDb } from "../db-connection.js";
2
+ import { downJobDeferrals } from "./001-job-deferrals.js";
3
+ import { downMemoryEntityRelationDedup } from "./004-entity-relation-dedup.js";
4
+ import { downMemoryItemsFingerprintScopeUnique } from "./005-fingerprint-scope-unique.js";
5
+ import { downMemoryItemsScopeSaltedFingerprints } from "./006-scope-salted-fingerprints.js";
6
+ import { downAssistantIdToSelf } from "./007-assistant-id-to-self.js";
7
+ import { downRemoveAssistantIdColumns } from "./008-remove-assistant-id-columns.js";
8
+ import { downLlmUsageEventsDropAssistantId } from "./009-llm-usage-events-drop-assistant-id.js";
9
+ import { downBackfillInboxThreadState } from "./014-backfill-inbox-thread-state.js";
10
+ import { downDropActiveSearchIndex } from "./015-drop-active-search-index.js";
11
+ import { downNotificationTablesSchema } from "./019-notification-tables-schema-migration.js";
12
+ import { downRenameChannelToVellum } from "./020-rename-macos-ios-channel-to-vellum.js";
13
+ import { downEmbeddingVectorBlob } from "./024-embedding-vector-blob.js";
14
+ import { downEmbeddingsNullableVectorJson } from "./026a-embeddings-nullable-vector-json.js";
15
+ import { downNormalizePhoneIdentities } from "./036-normalize-phone-identities.js";
16
+ import { downBackfillGuardianPrincipalId } from "./126-backfill-guardian-principal-id.js";
17
+ import { downGuardianPrincipalIdNotNull } from "./127-guardian-principal-id-not-null.js";
18
+ import { downContactsNotesColumn } from "./134-contacts-notes-column.js";
19
+ import { downBackfillContactInteractionStats } from "./135-backfill-contact-interaction-stats.js";
20
+ import { downDropAssistantIdColumns } from "./136-drop-assistant-id-columns.js";
21
+ import { downBackfillUsageCacheAccounting } from "./140-backfill-usage-cache-accounting.js";
22
+ import { downRenameVerificationTable } from "./141-rename-verification-table.js";
23
+ import { downRenameVerificationSessionIdColumn } from "./142-rename-verification-session-id-column.js";
24
+ import { downRenameGuardianVerificationValues } from "./143-rename-guardian-verification-values.js";
25
+ import { downRenameVoiceToPhone } from "./144-rename-voice-to-phone.js";
26
+ import { migrateDropAccountsTableDown } from "./145-drop-accounts-table.js";
27
+ import { migrateRemindersToSchedulesDown } from "./147-migrate-reminders-to-schedules.js";
28
+ import { migrateDropRemindersTableDown } from "./148-drop-reminders-table.js";
29
+ import { migrateOAuthAppsClientSecretPathDown } from "./150-oauth-apps-client-secret-path.js";
30
+ import {
31
+ migrateGuardianTimestampsEpochMsDown,
32
+ migrateGuardianTimestampsRebuildDown,
33
+ } from "./162-guardian-timestamps-epoch-ms.js";
34
+ import { migrateRenameGmailProviderKeyToGoogleDown } from "./169-rename-gmail-provider-key-to-google.js";
35
+ import { migrateRenameThreadStartersTableDown } from "./174-rename-thread-starters-table.js";
36
+ import { migrateDropCapabilityCardStateDown } from "./176-drop-capability-card-state.js";
37
+ import { migrateBackfillInlineAttachmentsToDiskDown } from "./180-backfill-inline-attachments-to-disk.js";
38
+ import { migrateRenameThreadStartersCheckpointsDown } from "./181-rename-thread-starters-checkpoints.js";
39
+ import { migrateBackfillAudioAttachmentMimeTypesDown } from "./191-backfill-audio-attachment-mime-types.js";
40
+ import { migrateAddSourceTypeColumnsDown } from "./193-add-source-type-columns.js";
41
+
1
42
  export interface MigrationRegistryEntry {
2
43
  /** The checkpoint key written to memory_checkpoints on completion. */
3
44
  key: string;
@@ -7,6 +48,8 @@ export interface MigrationRegistryEntry {
7
48
  dependsOn?: string[];
8
49
  /** Human-readable description for diagnostics and future authorship guidance. */
9
50
  description: string;
51
+ /** Reverse the migration. Must be idempotent — safe to re-run. */
52
+ down: (database: DrizzleDb) => void;
10
53
  }
11
54
 
12
55
  // ---------------------------------------------------------------------------
@@ -26,18 +69,21 @@ export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
26
69
  version: 1,
27
70
  description:
28
71
  "Reconcile legacy deferral history from attempts column into deferrals column",
72
+ down: downJobDeferrals,
29
73
  },
30
74
  {
31
75
  key: "migration_memory_entity_relations_dedup_v1",
32
76
  version: 2,
33
77
  description:
34
78
  "Deduplicate entity relation edges before enforcing the (source, target, relation) unique index",
79
+ down: downMemoryEntityRelationDedup,
35
80
  },
36
81
  {
37
82
  key: "migration_memory_items_fingerprint_scope_unique_v1",
38
83
  version: 3,
39
84
  description:
40
85
  "Replace column-level UNIQUE on fingerprint with compound (fingerprint, scope_id) unique index",
86
+ down: downMemoryItemsFingerprintScopeUnique,
41
87
  },
42
88
  {
43
89
  key: "migration_memory_items_scope_salted_fingerprints_v1",
@@ -45,12 +91,14 @@ export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
45
91
  dependsOn: ["migration_memory_items_fingerprint_scope_unique_v1"],
46
92
  description:
47
93
  "Recompute memory item fingerprints to include scope_id prefix after schema change",
94
+ down: downMemoryItemsScopeSaltedFingerprints,
48
95
  },
49
96
  {
50
97
  key: "migration_normalize_assistant_id_to_self_v1",
51
98
  version: 5,
52
99
  description:
53
100
  'Normalize all assistant_id values in scoped tables to the implicit "self" single-tenant identity',
101
+ down: downAssistantIdToSelf,
54
102
  },
55
103
  {
56
104
  key: "migration_remove_assistant_id_columns_v1",
@@ -58,6 +106,7 @@ export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
58
106
  dependsOn: ["migration_normalize_assistant_id_to_self_v1"],
59
107
  description:
60
108
  "Rebuild four tables to drop the assistant_id column after normalization",
109
+ down: downRemoveAssistantIdColumns,
61
110
  },
62
111
  {
63
112
  key: "migration_remove_assistant_id_lue_v1",
@@ -65,36 +114,42 @@ export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
65
114
  dependsOn: ["migration_normalize_assistant_id_to_self_v1"],
66
115
  description:
67
116
  "Remove assistant_id column from llm_usage_events (separate checkpoint from the four-table migration)",
117
+ down: downLlmUsageEventsDropAssistantId,
68
118
  },
69
119
  {
70
120
  key: "backfill_inbox_thread_state_from_bindings",
71
121
  version: 8,
72
122
  description:
73
123
  "Seed assistant_inbox_thread_state from external_conversation_bindings",
124
+ down: downBackfillInboxThreadState,
74
125
  },
75
126
  {
76
127
  key: "drop_active_search_index_v1",
77
128
  version: 9,
78
129
  description:
79
130
  "Drop old idx_memory_items_active_search so it can be recreated with updated covering columns",
131
+ down: downDropActiveSearchIndex,
80
132
  },
81
133
  {
82
134
  key: "migration_notification_tables_schema_v1",
83
135
  version: 10,
84
136
  description:
85
137
  "Drop legacy enum-based notification tables so they can be recreated with the new signal-contract schema",
138
+ down: downNotificationTablesSchema,
86
139
  },
87
140
  {
88
141
  key: "migration_rename_macos_ios_channel_to_vellum_v1",
89
142
  version: 11,
90
143
  description:
91
144
  "Rename macos and ios channel identifiers to vellum across all tables",
145
+ down: downRenameChannelToVellum,
92
146
  },
93
147
  {
94
148
  key: "migration_embedding_vector_blob_v1",
95
149
  version: 12,
96
150
  description:
97
151
  "Add vector_blob BLOB column to memory_embeddings and backfill from vector_json for compact binary storage",
152
+ down: downEmbeddingVectorBlob,
98
153
  },
99
154
  {
100
155
  key: "migration_embeddings_nullable_vector_json_v1",
@@ -102,18 +157,21 @@ export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
102
157
  dependsOn: ["migration_embedding_vector_blob_v1"],
103
158
  description:
104
159
  "Rebuild memory_embeddings to make vector_json nullable (pre-100 DBs had NOT NULL)",
160
+ down: downEmbeddingsNullableVectorJson,
105
161
  },
106
162
  {
107
163
  key: "migration_normalize_phone_identities_v1",
108
164
  version: 14,
109
165
  description:
110
166
  "Normalize phone-like identity fields to E.164 format across guardian bindings, verification challenges, canonical requests, ingress members, and rate limits",
167
+ down: downNormalizePhoneIdentities,
111
168
  },
112
169
  {
113
170
  key: "migration_backfill_guardian_principal_id_v3",
114
171
  version: 15,
115
172
  description:
116
173
  "Backfill guardianPrincipalId for existing channel_guardian_bindings and canonical_guardian_requests rows, expire unresolvable pending requests",
174
+ down: downBackfillGuardianPrincipalId,
117
175
  },
118
176
  {
119
177
  key: "migration_guardian_principal_id_not_null_v1",
@@ -121,18 +179,21 @@ export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
121
179
  dependsOn: ["migration_backfill_guardian_principal_id_v3"],
122
180
  description:
123
181
  "Enforce NOT NULL on channel_guardian_bindings.guardian_principal_id after backfill",
182
+ down: downGuardianPrincipalIdNotNull,
124
183
  },
125
184
  {
126
185
  key: "migration_contacts_notes_column_v1",
127
186
  version: 17,
128
187
  description:
129
188
  "Consolidate relationship/importance/response_expectation/preferred_tone into a single notes TEXT column, then drop the legacy columns",
189
+ down: downContactsNotesColumn,
130
190
  },
131
191
  {
132
192
  key: "backfill_contact_interaction_stats",
133
193
  version: 18,
134
194
  description:
135
195
  "Backfill contacts.last_interaction from the max lastSeenAt across each contact's channels",
196
+ down: downBackfillContactInteractionStats,
136
197
  },
137
198
  {
138
199
  key: "migration_drop_assistant_id_columns_v1",
@@ -140,48 +201,56 @@ export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
140
201
  dependsOn: ["migration_normalize_assistant_id_to_self_v1"],
141
202
  description:
142
203
  "Drop assistant_id columns from all 16 daemon tables after normalization to single-tenant identity",
204
+ down: downDropAssistantIdColumns,
143
205
  },
144
206
  {
145
207
  key: "migration_backfill_usage_cache_accounting_v1",
146
208
  version: 20,
147
209
  description:
148
210
  "Backfill historical Anthropic llm_usage_events rows from llm_request_logs with cache-aware pricing",
211
+ down: downBackfillUsageCacheAccounting,
149
212
  },
150
213
  {
151
214
  key: "migration_rename_verification_table_v1",
152
215
  version: 21,
153
216
  description:
154
217
  "Rename channel_guardian_verification_challenges table to channel_verification_sessions and update indexes",
218
+ down: downRenameVerificationTable,
155
219
  },
156
220
  {
157
221
  key: "migration_rename_verification_session_id_column_v1",
158
222
  version: 22,
159
223
  description:
160
224
  "Rename guardian_verification_session_id column in call_sessions to verification_session_id",
225
+ down: downRenameVerificationSessionIdColumn,
161
226
  },
162
227
  {
163
228
  key: "migration_rename_guardian_verification_values_v1",
164
229
  version: 23,
165
230
  description:
166
231
  "Rename persisted guardian_verification call_mode and guardian_voice_verification_* event_type values to drop the guardian_ prefix",
232
+ down: downRenameGuardianVerificationValues,
167
233
  },
168
234
  {
169
235
  key: "migration_rename_voice_to_phone_v1",
170
236
  version: 24,
171
237
  description:
172
238
  'Rename stored "voice" channel values to "phone" across all tables with channel text columns',
239
+ down: downRenameVoiceToPhone,
173
240
  },
174
241
  {
175
242
  key: "migration_drop_accounts_table_v1",
176
243
  version: 25,
177
244
  description:
178
245
  "Drop the unused legacy accounts table and its leftover indexes after account_manage removal",
246
+ down: migrateDropAccountsTableDown,
179
247
  },
180
248
  {
181
249
  key: "migration_reminders_to_schedules_v1",
182
250
  version: 26,
183
251
  description:
184
252
  "Copy all existing reminders into cron_jobs as one-shot schedules with correct status and field mapping",
253
+ down: migrateRemindersToSchedulesDown,
185
254
  },
186
255
  {
187
256
  key: "migration_drop_reminders_table_v1",
@@ -189,18 +258,21 @@ export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
189
258
  dependsOn: ["migration_reminders_to_schedules_v1"],
190
259
  description:
191
260
  "Drop the legacy reminders table and its index after data migration to cron_jobs",
261
+ down: migrateDropRemindersTableDown,
192
262
  },
193
263
  {
194
264
  key: "migration_oauth_apps_client_secret_path_v1",
195
265
  version: 28,
196
266
  description:
197
267
  "Add client_secret_credential_path column to oauth_apps and backfill existing rows with convention-based paths",
268
+ down: migrateOAuthAppsClientSecretPathDown,
198
269
  },
199
270
  {
200
271
  key: "migration_guardian_timestamps_epoch_ms_v1",
201
272
  version: 29,
202
273
  description:
203
274
  "Convert guardian table timestamps from ISO 8601 text to epoch ms integers for consistency with all other tables",
275
+ down: migrateGuardianTimestampsEpochMsDown,
204
276
  },
205
277
  {
206
278
  key: "migration_guardian_timestamps_rebuild_v1",
@@ -208,18 +280,21 @@ export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
208
280
  dependsOn: ["migration_guardian_timestamps_epoch_ms_v1"],
209
281
  description:
210
282
  "Rebuild guardian tables so timestamp columns have INTEGER affinity instead of TEXT",
283
+ down: migrateGuardianTimestampsRebuildDown,
211
284
  },
212
285
  {
213
286
  key: "migration_rename_gmail_provider_key_to_google_v1",
214
287
  version: 31,
215
288
  description:
216
289
  "Rename integration:gmail provider key to integration:google across oauth_providers, oauth_apps, and oauth_connections",
290
+ down: migrateRenameGmailProviderKeyToGoogleDown,
217
291
  },
218
292
  {
219
293
  key: "migration_rename_thread_starters_table_v1",
220
294
  version: 32,
221
295
  description:
222
296
  "Rename thread_starters table to conversation_starters and recreate indexes with new names",
297
+ down: migrateRenameThreadStartersTableDown,
223
298
  },
224
299
  {
225
300
  key: "migration_drop_capability_card_state_v1",
@@ -227,12 +302,14 @@ export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
227
302
  dependsOn: ["migration_rename_thread_starters_table_v1"],
228
303
  description:
229
304
  "Remove deleted capability-card rows, jobs, checkpoints, and category state",
305
+ down: migrateDropCapabilityCardStateDown,
230
306
  },
231
307
  {
232
308
  key: "migration_backfill_inline_attachments_v1",
233
309
  version: 34,
234
310
  description:
235
311
  "Backfill existing inline base64 attachments to on-disk storage and clear dataBase64",
312
+ down: migrateBackfillInlineAttachmentsToDiskDown,
236
313
  },
237
314
  {
238
315
  key: "migration_rename_thread_starters_checkpoints_v1",
@@ -240,12 +317,33 @@ export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
240
317
  dependsOn: ["migration_rename_thread_starters_table_v1"],
241
318
  description:
242
319
  "Rename checkpoint keys from thread_starters: to conversation_starters: prefix so renamed code paths find existing generation state",
320
+ down: migrateRenameThreadStartersCheckpointsDown,
321
+ },
322
+ {
323
+ key: "migration_backfill_audio_attachment_mime_types_v1",
324
+ version: 36,
325
+ description:
326
+ "Backfill correct MIME types for audio attachments stored as application/octet-stream due to missing extension map entries",
327
+ down: migrateBackfillAudioAttachmentMimeTypesDown,
328
+ },
329
+ {
330
+ key: "migration_add_source_type_columns_v1",
331
+ version: 37,
332
+ description:
333
+ "Add source_type and source_message_role columns to memory_items with backfill from verification_state and source messages",
334
+ down: migrateAddSourceTypeColumnsDown,
243
335
  },
244
336
  ];
245
337
 
338
+ export function getMaxMigrationVersion(): number {
339
+ return Math.max(...MIGRATION_REGISTRY.map((e) => e.version));
340
+ }
341
+
246
342
  export interface MigrationValidationResult {
247
343
  /** Keys of migrations whose checkpoint has value 'started' — started but never completed. */
248
344
  crashed: string[];
249
345
  /** Pairs where a completed migration's declared prerequisite is missing from checkpoints. */
250
346
  dependencyViolations: Array<{ migration: string; missingDependency: string }>;
347
+ /** Checkpoint keys present in the database but absent from the migration registry — likely from a newer version. */
348
+ unknownCheckpoints: string[];
251
349
  }
@@ -33,7 +33,9 @@ export function recoverCrashedMigrations(database: DrizzleDb): string[] {
33
33
  return [];
34
34
  }
35
35
 
36
- const crashed = rows.filter((r) => r.value === "started").map((r) => r.key);
36
+ const crashed = rows
37
+ .filter((r) => r.value === "started" || r.value === "rolling_back")
38
+ .map((r) => r.key);
37
39
  if (crashed.length === 0) return [];
38
40
 
39
41
  log.error(
@@ -83,7 +85,12 @@ export function withCrashRecovery(
83
85
  const existing = raw
84
86
  .query(`SELECT value FROM memory_checkpoints WHERE key = ?`)
85
87
  .get(checkpointKey) as { value: string } | null;
86
- if (existing && existing.value !== "started") return;
88
+ if (
89
+ existing &&
90
+ existing.value !== "started" &&
91
+ existing.value !== "rolling_back"
92
+ )
93
+ return;
87
94
 
88
95
  raw
89
96
  .query(
@@ -126,12 +133,14 @@ export function validateMigrationState(
126
133
  .all() as Array<{ key: string; value: string }>;
127
134
  } catch {
128
135
  // memory_checkpoints may not exist on a very old database; skip.
129
- return { crashed: [], dependencyViolations: [] };
136
+ return { crashed: [], dependencyViolations: [], unknownCheckpoints: [] };
130
137
  }
131
138
 
132
- // Any remaining 'started' checkpoints after recovery + migration execution
133
- // indicate a migration that was retried but failed again.
134
- const crashed = rows.filter((r) => r.value === "started").map((r) => r.key);
139
+ // Any remaining 'started' or 'rolling_back' checkpoints after recovery +
140
+ // migration execution indicate a migration that was retried but failed again.
141
+ const crashed = rows
142
+ .filter((r) => r.value === "started" || r.value === "rolling_back")
143
+ .map((r) => r.key);
135
144
  if (crashed.length > 0) {
136
145
  log.error(
137
146
  { crashed },
@@ -149,11 +158,14 @@ export function validateMigrationState(
149
158
  );
150
159
  }
151
160
 
152
- // Only rows whose value is NOT 'started' represent truly completed migrations.
153
- // In-progress/crashed checkpoints (value = 'started') must not count as applied
154
- // dependencies — the migration never finished, so its postconditions are unmet.
161
+ // Only rows whose value is NOT 'started' or 'rolling_back' represent truly
162
+ // completed migrations. In-progress/crashed checkpoints must not count as
163
+ // applied dependencies — the migration never finished, so its postconditions
164
+ // are unmet.
155
165
  const completed = new Set(
156
- rows.filter((r) => r.value !== "started").map((r) => r.key),
166
+ rows
167
+ .filter((r) => r.value !== "started" && r.value !== "rolling_back")
168
+ .map((r) => r.key),
157
169
  );
158
170
 
159
171
  const dependencyViolations: Array<{
@@ -191,5 +203,119 @@ export function validateMigrationState(
191
203
  );
192
204
  }
193
205
 
194
- return { crashed, dependencyViolations };
206
+ // Detect checkpoints that exist in the database but have no corresponding
207
+ // registry entry — these are from a newer version of the daemon.
208
+ //
209
+ // The memory_checkpoints table is a general-purpose key-value store also
210
+ // used by non-migration subsystems (e.g., "identity:intro:text",
211
+ // "conversation_starters:item_count_at_last_gen"). Filter to only keys
212
+ // that follow migration naming conventions before comparing against the
213
+ // registry to avoid false-positive warnings.
214
+ const registryKeys = new Set(MIGRATION_REGISTRY.map((e) => e.key));
215
+ const isMigrationKey = (k: string): boolean =>
216
+ k.startsWith("migration_") ||
217
+ k.startsWith("backfill_") ||
218
+ k.startsWith("drop_");
219
+ const unknownCheckpoints = [...completed].filter(
220
+ (k) => isMigrationKey(k) && !registryKeys.has(k),
221
+ );
222
+
223
+ if (unknownCheckpoints.length > 0) {
224
+ log.warn(
225
+ { unknownCheckpoints },
226
+ `Database contains ${unknownCheckpoints.length} migration checkpoint(s) from a newer version. Data may be incompatible.`,
227
+ );
228
+ }
229
+
230
+ return { crashed, dependencyViolations, unknownCheckpoints };
231
+ }
232
+
233
+ /**
234
+ * Roll back all completed memory (database) migrations with version > targetVersion.
235
+ *
236
+ * Iterates eligible migrations in reverse version order. For each:
237
+ * 1. Marks the checkpoint as `"rolling_back"` for crash recovery.
238
+ * 2. Calls `entry.down(database)` — each down() manages its own transactions.
239
+ * (`down` is required on `MigrationRegistryEntry` at the type level.)
240
+ * 3. Deletes the checkpoint from `memory_checkpoints`.
241
+ *
242
+ * **Usage**: Pass the target version number you want to roll back *to*. All
243
+ * migrations with a higher version number that have been applied will be
244
+ * reversed. For example, `rollbackMemoryMigration(db, 5)` rolls back all
245
+ * applied migrations with version > 5.
246
+ *
247
+ * **Checkpoint state**: Each rolled-back migration's checkpoint is deleted
248
+ * from `memory_checkpoints`. If the process crashes mid-rollback, the
249
+ * `"rolling_back"` marker is detected and cleared by
250
+ * `recoverCrashedMigrations` on the next startup.
251
+ *
252
+ * **Warning — data loss**: Some down() migrations may not fully restore the
253
+ * original state (e.g., DROP TABLE migrations recreate the table but cannot
254
+ * recover the original data). Review each migration's down() implementation
255
+ * before calling this function programmatically.
256
+ *
257
+ * **Important**: Stop the assistant before running rollbacks. Rolling back
258
+ * migrations while the assistant is running may cause schema mismatches,
259
+ * query failures, or data corruption.
260
+ *
261
+ * @param database The Drizzle database instance.
262
+ * @param targetVersion Roll back to this version (exclusive — all migrations
263
+ * with version > targetVersion are reversed).
264
+ * @returns The list of rolled-back migration keys.
265
+ */
266
+ export function rollbackMemoryMigration(
267
+ database: DrizzleDb,
268
+ targetVersion: number,
269
+ ): string[] {
270
+ const raw = getSqliteFrom(database);
271
+
272
+ // Read completed checkpoints to determine which migrations have been applied.
273
+ let rows: Array<{ key: string; value: string }>;
274
+ try {
275
+ rows = raw
276
+ .query(`SELECT key, value FROM memory_checkpoints`)
277
+ .all() as Array<{ key: string; value: string }>;
278
+ } catch {
279
+ return [];
280
+ }
281
+
282
+ const completedKeys = new Set(
283
+ rows
284
+ .filter((r) => r.value !== "started" && r.value !== "rolling_back")
285
+ .map((r) => r.key),
286
+ );
287
+
288
+ // Find registry entries with version > targetVersion that have completed checkpoints.
289
+ const toRollback = MIGRATION_REGISTRY.filter(
290
+ (entry) => entry.version > targetVersion && completedKeys.has(entry.key),
291
+ ).sort((a, b) => b.version - a.version); // reverse version order
292
+
293
+ const rolledBack: string[] = [];
294
+
295
+ for (const entry of toRollback) {
296
+ // Mark as rolling_back for crash recovery — if the process crashes here,
297
+ // recoverCrashedMigrations will clear this checkpoint on next startup.
298
+ raw
299
+ .query(
300
+ `UPDATE memory_checkpoints SET value = 'rolling_back', updated_at = ? WHERE key = ?`,
301
+ )
302
+ .run(Date.now(), entry.key);
303
+
304
+ // Execute the down migration — let it manage its own transaction lifecycle.
305
+ // Many down() functions call BEGIN/COMMIT internally or use PRAGMA statements
306
+ // that are no-ops inside a transaction.
307
+ entry.down(database);
308
+
309
+ // Delete the checkpoint after down() succeeds — outside any transaction
310
+ // so it's not affected by down()'s internal transaction management.
311
+ raw.query(`DELETE FROM memory_checkpoints WHERE key = ?`).run(entry.key);
312
+
313
+ log.info(
314
+ { key: entry.key, version: entry.version },
315
+ `Rolled back migration "${entry.key}" (version ${entry.version})`,
316
+ );
317
+ rolledBack.push(entry.key);
318
+ }
319
+
320
+ return rolledBack;
195
321
  }
@@ -101,6 +101,15 @@ export function isQdrantBreakerOpen(): boolean {
101
101
  return breakerState !== "closed";
102
102
  }
103
103
 
104
+ /**
105
+ * Returns true when the breaker is open and the cooldown has elapsed,
106
+ * meaning the next call to `withQdrantBreaker` will transition to half-open.
107
+ * Use this to allow a single probe job through when embed jobs are otherwise skipped.
108
+ */
109
+ export function shouldAllowQdrantProbe(): boolean {
110
+ return breakerState === "open" && Date.now() - openedAt >= COOLDOWN_MS;
111
+ }
112
+
104
113
  /** @internal Test-only: reset circuit breaker state */
105
114
  export function _resetQdrantBreaker(): void {
106
115
  breakerState = "closed";
@@ -4,6 +4,7 @@ import {
4
4
  existsSync,
5
5
  mkdirSync,
6
6
  readFileSync,
7
+ symlinkSync,
7
8
  unlinkSync,
8
9
  writeFileSync,
9
10
  } from "node:fs";
@@ -12,7 +13,7 @@ import { dirname, join } from "node:path";
12
13
 
13
14
  import type { Subprocess } from "bun";
14
15
 
15
- import { getQdrantUrlEnv } from "../config/env.js";
16
+ import { getQdrantReadyzTimeoutMs, getQdrantUrlEnv } from "../config/env.js";
16
17
  import { getLogger } from "../util/logger.js";
17
18
  import { getDataDir } from "../util/platform.js";
18
19
 
@@ -47,6 +48,8 @@ export interface QdrantManagerConfig {
47
48
  */
48
49
  export class QdrantManager {
49
50
  private process: Subprocess | null = null;
51
+ private stderrBuffer = "";
52
+ private stderrDrained: Promise<void> = Promise.resolve();
50
53
  private readonly url: string;
51
54
  private readonly host: string;
52
55
  private readonly port: number;
@@ -67,7 +70,8 @@ export class QdrantManager {
67
70
 
68
71
  this.readyzPollIntervalMs =
69
72
  config.readyzPollIntervalMs ?? READYZ_POLL_INTERVAL_MS;
70
- this.readyzTimeoutMs = config.readyzTimeoutMs ?? READYZ_TIMEOUT_MS;
73
+ this.readyzTimeoutMs =
74
+ config.readyzTimeoutMs ?? getQdrantReadyzTimeoutMs() ?? READYZ_TIMEOUT_MS;
71
75
  this.shutdownGraceMs = config.shutdownGraceMs ?? SHUTDOWN_GRACE_MS;
72
76
 
73
77
  // External mode only if QDRANT_URL is explicitly set
@@ -92,13 +96,15 @@ export class QdrantManager {
92
96
  await this.installBinary(binaryPath);
93
97
  }
94
98
 
99
+ const spawnPath = this.ensureVellumSymlink(binaryPath);
100
+
95
101
  log.info(
96
- { binaryPath, storagePath: this.storagePath, port: this.port },
102
+ { binaryPath: spawnPath, storagePath: this.storagePath, port: this.port },
97
103
  "Starting Qdrant",
98
104
  );
99
105
 
100
- this.process = Bun.spawn({
101
- cmd: [binaryPath],
106
+ const proc = Bun.spawn({
107
+ cmd: [spawnPath],
102
108
  env: {
103
109
  ...process.env,
104
110
  QDRANT__SERVICE__HOST: this.host,
@@ -109,8 +115,10 @@ export class QdrantManager {
109
115
  QDRANT__LOG_LEVEL: "WARN",
110
116
  },
111
117
  stdout: "ignore",
112
- stderr: "ignore",
118
+ stderr: "pipe",
113
119
  });
120
+ this.process = proc;
121
+ this.drainStderrFrom(proc.stderr);
114
122
 
115
123
  if (this.process.pid) {
116
124
  this.writePid(this.process.pid);
@@ -150,6 +158,7 @@ export class QdrantManager {
150
158
  }
151
159
 
152
160
  this.process = null;
161
+ this.stderrBuffer = "";
153
162
  this.cleanupPid();
154
163
  log.info("Qdrant stopped");
155
164
  }
@@ -259,6 +268,15 @@ export class QdrantManager {
259
268
  private async waitForReady(): Promise<void> {
260
269
  const start = Date.now();
261
270
  while (Date.now() - start < this.readyzTimeoutMs) {
271
+ // Fail fast if the managed process exited before becoming ready
272
+ if (this.process != null && this.process.exitCode != null) {
273
+ await this.stderrDrained;
274
+ const stderr = this.stderrBuffer.trim();
275
+ throw new Error(
276
+ `Qdrant process exited with code ${this.process.exitCode} before becoming ready` +
277
+ (stderr ? `\nstderr:\n${stderr}` : ""),
278
+ );
279
+ }
262
280
  try {
263
281
  const res = await fetch(`${this.url}/readyz`);
264
282
  if (res.ok) return;
@@ -267,11 +285,32 @@ export class QdrantManager {
267
285
  }
268
286
  await Bun.sleep(this.readyzPollIntervalMs);
269
287
  }
288
+ const stderr = this.stderrBuffer.trim();
270
289
  throw new Error(
271
- `Qdrant did not become ready within ${this.readyzTimeoutMs}ms at ${this.url}`,
290
+ `Qdrant did not become ready within ${this.readyzTimeoutMs}ms at ${this.url}` +
291
+ (stderr ? `\nstderr:\n${stderr}` : ""),
272
292
  );
273
293
  }
274
294
 
295
+ private drainStderrFrom(stream: ReadableStream<Uint8Array>): void {
296
+ const reader = stream.getReader();
297
+ const decoder = new TextDecoder();
298
+ this.stderrDrained = (async () => {
299
+ try {
300
+ for (;;) {
301
+ const { done, value } = await reader.read();
302
+ if (done) break;
303
+ this.stderrBuffer += decoder.decode(value, { stream: true });
304
+ if (this.stderrBuffer.length > 4096) {
305
+ this.stderrBuffer = this.stderrBuffer.slice(-4096);
306
+ }
307
+ }
308
+ } catch {
309
+ // Stream closed or error — expected during shutdown
310
+ }
311
+ })();
312
+ }
313
+
275
314
  private getBinaryPath(): string {
276
315
  return join(getDataDir(), "qdrant", "bin", "qdrant");
277
316
  }
@@ -314,4 +353,22 @@ export class QdrantManager {
314
353
  }
315
354
  }
316
355
  }
356
+
357
+ /**
358
+ * Ensures a `vellum-qdrant` symlink exists next to the real binary so that
359
+ * `lsof` reports "vellum-qdrant" in the COMMAND column, making the process
360
+ * discoverable by tools that scan for "vellum" in process names.
361
+ */
362
+ private ensureVellumSymlink(binaryPath: string): string {
363
+ const symlinkPath = join(dirname(binaryPath), "vellum-qdrant");
364
+ if (!existsSync(symlinkPath)) {
365
+ try {
366
+ symlinkSync(binaryPath, symlinkPath);
367
+ } catch {
368
+ // Fall back to the real binary if symlink creation fails
369
+ return binaryPath;
370
+ }
371
+ }
372
+ return symlinkPath;
373
+ }
317
374
  }