@vellumai/assistant 0.7.1 → 0.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (535) hide show
  1. package/ARCHITECTURE.md +32 -49
  2. package/Dockerfile +1 -0
  3. package/README.md +1 -2
  4. package/__tests__/permissions/gateway-threshold-reader.test.ts +9 -3
  5. package/bun.lock +26 -26
  6. package/docs/architecture/security.md +20 -0
  7. package/docs/plugins.md +7 -9
  8. package/knip.json +1 -0
  9. package/node_modules/@vellumai/gateway-client/src/index.ts +1 -0
  10. package/node_modules/@vellumai/gateway-client/src/ipc-client.ts +39 -1
  11. package/node_modules/@vellumai/gateway-client/src/types.ts +11 -0
  12. package/node_modules/@vellumai/service-contracts/package.json +2 -0
  13. package/node_modules/@vellumai/service-contracts/src/__tests__/contracts.test.ts +4 -0
  14. package/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +107 -0
  15. package/node_modules/@vellumai/service-contracts/src/index.ts +5 -1
  16. package/node_modules/@vellumai/service-contracts/src/ingress.ts +24 -0
  17. package/node_modules/@vellumai/service-contracts/src/twilio-ingress.ts +84 -0
  18. package/node_modules/@vellumai/skill-host-contracts/src/assistant-event.ts +9 -0
  19. package/node_modules/@vellumai/twilio-client/bun.lock +24 -0
  20. package/node_modules/@vellumai/twilio-client/package.json +18 -0
  21. package/node_modules/@vellumai/twilio-client/src/__tests__/twilio-client.test.ts +128 -0
  22. package/node_modules/@vellumai/twilio-client/src/index.ts +179 -0
  23. package/node_modules/@vellumai/twilio-client/tsconfig.json +20 -0
  24. package/openapi.yaml +565 -12
  25. package/package.json +6 -3
  26. package/src/__tests__/app-builder-tool-scripts.test.ts +3 -3
  27. package/src/__tests__/app-bundler.test.ts +170 -1
  28. package/src/__tests__/app-control-flow.test.ts +374 -0
  29. package/src/__tests__/app-control-no-global-cgevent.test.ts +98 -0
  30. package/src/__tests__/app-control-tool-schemas.test.ts +621 -0
  31. package/src/__tests__/app-executors.test.ts +30 -43
  32. package/src/__tests__/approval-routes-http.test.ts +23 -6
  33. package/src/__tests__/assistant-event-hub-machine-name.test.ts +146 -0
  34. package/src/__tests__/assistant-event-hub-targeted.test.ts +257 -0
  35. package/src/__tests__/assistant-event-hub.test.ts +109 -2
  36. package/src/__tests__/assistant-event.test.ts +10 -0
  37. package/src/__tests__/assistant-events-sse-hardening.test.ts +7 -2
  38. package/src/__tests__/assistant-feature-flags-integration.test.ts +11 -7
  39. package/src/__tests__/background-shell-host-bash.test.ts +14 -15
  40. package/src/__tests__/bootstrap-turn-cleanup.test.ts +44 -0
  41. package/src/__tests__/btw-routes.test.ts +13 -4
  42. package/src/__tests__/call-controller.test.ts +49 -1
  43. package/src/__tests__/call-domain.test.ts +0 -2
  44. package/src/__tests__/call-routes-http.test.ts +0 -2
  45. package/src/__tests__/channel-readiness-service.test.ts +59 -1
  46. package/src/__tests__/checker.test.ts +3 -4
  47. package/src/__tests__/config-loader-backfill.test.ts +90 -155
  48. package/src/__tests__/config-loader-platform-defaults.test.ts +196 -0
  49. package/src/__tests__/config-schema-cmd.test.ts +0 -1
  50. package/src/__tests__/config-set-platform-guard.test.ts +48 -4
  51. package/src/__tests__/config-watcher-cleanup-throttle.test.ts +2 -2
  52. package/src/__tests__/config-watcher.test.ts +2 -2
  53. package/src/__tests__/conversation-app-control-instantiation.test.ts +392 -0
  54. package/src/__tests__/conversation-app-control-lifecycle.test.ts +237 -0
  55. package/src/__tests__/conversation-init.benchmark.test.ts +0 -2
  56. package/src/__tests__/conversation-lifecycle.test.ts +36 -0
  57. package/src/__tests__/conversation-process-app-control-preactivation.test.ts +283 -0
  58. package/src/__tests__/conversation-routes-disk-view.test.ts +6 -0
  59. package/src/__tests__/conversation-routes-guardian-reply.test.ts +120 -72
  60. package/src/__tests__/conversation-routes-slash-commands.test.ts +1 -0
  61. package/src/__tests__/conversation-slash-commands.test.ts +0 -4
  62. package/src/__tests__/conversation-surfaces-action-delivery.test.ts +202 -0
  63. package/src/__tests__/conversation-surfaces-app-control.test.ts +317 -0
  64. package/src/__tests__/credential-execution-feature-gates.test.ts +5 -12
  65. package/src/__tests__/credential-execution-managed-contract.test.ts +3 -131
  66. package/src/__tests__/credentials-cli.test.ts +5 -12
  67. package/src/__tests__/cu-unified-flow.test.ts +185 -23
  68. package/src/__tests__/daemon-credential-client.test.ts +101 -19
  69. package/src/__tests__/db-schedule-syntax-migration.test.ts +2 -0
  70. package/src/__tests__/dynamic-skill-workflow-prompt.test.ts +0 -1
  71. package/src/__tests__/gateway-only-enforcement.test.ts +0 -1
  72. package/src/__tests__/guardian-verification-voice-binding.test.ts +0 -2
  73. package/src/__tests__/handlers-skills-memory-v2-reseed.test.ts +0 -2
  74. package/src/__tests__/handlers-user-message-approval-consumption.test.ts +0 -1
  75. package/src/__tests__/heartbeat-service.test.ts +718 -1
  76. package/src/__tests__/helpers/call-route-handler.ts +7 -1
  77. package/src/__tests__/host-app-control-proxy.test.ts +602 -0
  78. package/src/__tests__/host-app-control-routes.test.ts +263 -0
  79. package/src/__tests__/host-bash-proxy.test.ts +246 -47
  80. package/src/__tests__/host-bash-routes.test.ts +294 -0
  81. package/src/__tests__/host-browser-proxy.test.ts +24 -22
  82. package/src/__tests__/host-browser-routes.test.ts +39 -13
  83. package/src/__tests__/host-cu-proxy.test.ts +41 -52
  84. package/src/__tests__/host-cu-routes-targeted.test.ts +300 -0
  85. package/src/__tests__/host-file-edit-tool.test.ts +47 -1
  86. package/src/__tests__/host-file-proxy-targeted.test.ts +339 -0
  87. package/src/__tests__/host-file-proxy.test.ts +37 -43
  88. package/src/__tests__/host-file-read-tool.test.ts +17 -0
  89. package/src/__tests__/host-file-routes-targeted.test.ts +262 -0
  90. package/src/__tests__/host-file-write-tool.test.ts +42 -1
  91. package/src/__tests__/host-proxy-base.test.ts +312 -0
  92. package/src/__tests__/host-shell-tool.test.ts +22 -4
  93. package/src/__tests__/host-transfer-proxy-targeted.test.ts +583 -0
  94. package/src/__tests__/host-transfer-proxy.test.ts +121 -22
  95. package/src/__tests__/host-transfer-routes-targeted.test.ts +447 -0
  96. package/src/__tests__/http-user-message-parity.test.ts +1 -0
  97. package/src/__tests__/identity-intro-cache.test.ts +29 -0
  98. package/src/__tests__/identity-routes.test.ts +103 -1
  99. package/src/__tests__/init-feature-flag-overrides.test.ts +26 -3
  100. package/src/__tests__/inline-command-runner.test.ts +0 -1
  101. package/src/__tests__/inline-skill-load-permissions.test.ts +5 -11
  102. package/src/__tests__/integration-status.test.ts +85 -5
  103. package/src/__tests__/intent-routing.test.ts +0 -1
  104. package/src/__tests__/jobs-store-qdrant-breaker.test.ts +95 -5
  105. package/src/__tests__/lifecycle-memory-v2-seed.test.ts +17 -0
  106. package/src/__tests__/managed-skill-lifecycle.test.ts +0 -1
  107. package/src/__tests__/mcp-auth-routes.test.ts +197 -0
  108. package/src/__tests__/mcp-cli.test.ts +338 -2
  109. package/src/__tests__/memory-jobs-worker-lanes.test.ts +188 -0
  110. package/src/__tests__/migration-import-commit-http.test.ts +108 -2
  111. package/src/__tests__/mock-gateway-ipc.ts +1 -0
  112. package/src/__tests__/oauth-cli.test.ts +0 -2
  113. package/src/__tests__/oauth2-gateway-transport.test.ts +0 -1
  114. package/src/__tests__/persistence-secret-redaction.test.ts +299 -0
  115. package/src/__tests__/platform-bash-auto-approve.test.ts +5 -9
  116. package/src/__tests__/prechat-onboarding-contract.test.ts +3 -1
  117. package/src/__tests__/process-message-background-slack.test.ts +2 -0
  118. package/src/__tests__/provider-commit-message-generator.test.ts +0 -1
  119. package/src/__tests__/public-ingress-urls.test.ts +97 -0
  120. package/src/__tests__/require-fresh-approval.test.ts +0 -1
  121. package/src/__tests__/retry-backoff.test.ts +87 -0
  122. package/src/__tests__/runtime-events-sse.test.ts +10 -6
  123. package/src/__tests__/sanitize-config-for-transfer.test.ts +24 -2
  124. package/src/__tests__/schedule-retry.test.ts +715 -0
  125. package/src/__tests__/script-proxy-mitm-handler.test.ts +1 -1
  126. package/src/__tests__/secret-ingress-http.test.ts +1 -0
  127. package/src/__tests__/send-endpoint-busy.test.ts +3 -0
  128. package/src/__tests__/shell-tool-proxy-mode.test.ts +0 -1
  129. package/src/__tests__/skill-feature-flags.test.ts +43 -41
  130. package/src/__tests__/skill-load-feature-flag.test.ts +13 -14
  131. package/src/__tests__/skill-load-inline-command.test.ts +0 -51
  132. package/src/__tests__/skill-load-inline-includes.test.ts +0 -43
  133. package/src/__tests__/skill-projection.benchmark.test.ts +0 -1
  134. package/src/__tests__/skill-script-runner-sandbox.test.ts +0 -1
  135. package/src/__tests__/slack-channel-config.test.ts +9 -14
  136. package/src/__tests__/system-prompt-ask-mode.test.ts +0 -1
  137. package/src/__tests__/system-prompt.test.ts +0 -1
  138. package/src/__tests__/telegram-config.test.ts +0 -1
  139. package/src/__tests__/test-preload.ts +8 -0
  140. package/src/__tests__/tool-approval-handler.test.ts +3 -4
  141. package/src/__tests__/tool-audit-listener.test.ts +48 -0
  142. package/src/__tests__/tool-execute-pipeline.test.ts +0 -1
  143. package/src/__tests__/tool-execution-abort-cleanup.test.ts +0 -1
  144. package/src/__tests__/tool-executor-lifecycle-events.test.ts +0 -1
  145. package/src/__tests__/tool-executor.test.ts +0 -1
  146. package/src/__tests__/twilio-config.test.ts +3 -16
  147. package/src/__tests__/twilio-routes.test.ts +3 -5
  148. package/src/__tests__/twilio-validation.test.ts +93 -0
  149. package/src/__tests__/vellum-self-knowledge-inline-command.test.ts +1 -4
  150. package/src/__tests__/verification-control-plane-policy.test.ts +2 -4
  151. package/src/__tests__/voice-ingress-preflight.test.ts +19 -0
  152. package/src/__tests__/workspace-migration-006-services-config.test.ts +3 -2
  153. package/src/__tests__/workspace-migration-backfill-installation-id.test.ts +1 -5
  154. package/src/__tests__/workspace-migration-down-functions.test.ts +8 -8
  155. package/src/__tests__/workspace-migration-unify-llm-callsite-configs.test.ts +10 -6
  156. package/src/backup/__tests__/paths.test.ts +0 -22
  157. package/src/backup/__tests__/restore.test.ts +51 -151
  158. package/src/backup/paths.ts +2 -18
  159. package/src/backup/restore.ts +107 -231
  160. package/src/bundler/app-bundler.ts +51 -3
  161. package/src/calls/relay-server.ts +4 -44
  162. package/src/calls/twilio-config.ts +2 -17
  163. package/src/calls/twilio-rest.ts +33 -105
  164. package/src/calls/twilio-routes.ts +11 -12
  165. package/src/channels/types.ts +8 -7
  166. package/src/cli/commands/__tests__/backup.test.ts +6 -277
  167. package/src/cli/commands/__tests__/gateway.test.ts +288 -0
  168. package/src/cli/commands/__tests__/memory-v2.test.ts +4 -0
  169. package/src/cli/commands/__tests__/webhooks.test.ts +0 -1
  170. package/src/cli/commands/backup.ts +6 -331
  171. package/src/cli/commands/clients.ts +36 -37
  172. package/src/cli/commands/contacts.ts +73 -0
  173. package/src/cli/commands/conversations.ts +2 -5
  174. package/src/cli/commands/credentials.ts +15 -7
  175. package/src/cli/commands/domain.ts +66 -15
  176. package/src/cli/commands/gateway.ts +183 -0
  177. package/src/cli/commands/keys.ts +9 -6
  178. package/src/cli/commands/mcp.ts +116 -156
  179. package/src/cli/commands/memory-v2.ts +296 -1
  180. package/src/cli/commands/platform/__tests__/callback-routes-list.test.ts +0 -1
  181. package/src/cli/commands/platform/__tests__/connect.test.ts +0 -2
  182. package/src/cli/commands/platform/__tests__/disconnect.test.ts +0 -2
  183. package/src/cli/commands/platform/__tests__/status.test.ts +13 -15
  184. package/src/cli/commands/platform/disconnect.ts +5 -4
  185. package/src/cli/commands/platform/index.ts +0 -18
  186. package/src/cli/lib/daemon-credential-client.ts +110 -28
  187. package/src/cli/program.ts +2 -0
  188. package/src/config/assistant-feature-flags.ts +67 -10
  189. package/src/config/bundled-skills/acp/SKILL.md +6 -0
  190. package/src/config/bundled-skills/acp/TOOLS.json +1 -22
  191. package/src/config/bundled-skills/app-builder/SKILL.md +14 -109
  192. package/src/config/bundled-skills/app-builder/TOOLS.json +1 -28
  193. package/src/config/bundled-skills/app-builder/tools/app-create.ts +1 -10
  194. package/src/config/bundled-skills/app-control/SKILL.md +75 -0
  195. package/src/config/bundled-skills/app-control/TOOLS.json +299 -0
  196. package/src/config/bundled-skills/app-control/tools/app-control-click.ts +12 -0
  197. package/src/config/bundled-skills/app-control/tools/app-control-combo.ts +12 -0
  198. package/src/config/bundled-skills/app-control/tools/app-control-drag.ts +12 -0
  199. package/src/config/bundled-skills/app-control/tools/app-control-observe.ts +12 -0
  200. package/src/config/bundled-skills/app-control/tools/app-control-press.ts +12 -0
  201. package/src/config/bundled-skills/app-control/tools/app-control-sequence.ts +12 -0
  202. package/src/config/bundled-skills/app-control/tools/app-control-start.ts +12 -0
  203. package/src/config/bundled-skills/app-control/tools/app-control-stop.ts +12 -0
  204. package/src/config/bundled-skills/app-control/tools/app-control-type.ts +12 -0
  205. package/src/config/bundled-skills/computer-use/SKILL.md +6 -0
  206. package/src/config/bundled-skills/computer-use/TOOLS.json +67 -43
  207. package/src/config/bundled-skills/contacts/TOOLS.json +0 -16
  208. package/src/config/bundled-skills/document/TOOLS.json +0 -8
  209. package/src/config/bundled-skills/followups/TOOLS.json +0 -12
  210. package/src/config/bundled-skills/image-studio/SKILL.md +4 -0
  211. package/src/config/bundled-skills/image-studio/TOOLS.json +0 -4
  212. package/src/config/bundled-skills/media-processing/TOOLS.json +0 -24
  213. package/src/config/bundled-skills/messaging/TOOLS.json +0 -40
  214. package/src/config/bundled-skills/phone-calls/TOOLS.json +0 -12
  215. package/src/config/bundled-skills/phone-calls/references/TROUBLESHOOTING.md +19 -4
  216. package/src/config/bundled-skills/playbooks/TOOLS.json +0 -16
  217. package/src/config/bundled-skills/schedule/TOOLS.json +14 -14
  218. package/src/config/bundled-skills/sequences/TOOLS.json +0 -36
  219. package/src/config/bundled-skills/settings/SKILL.md +4 -0
  220. package/src/config/bundled-skills/settings/TOOLS.json +0 -12
  221. package/src/config/bundled-skills/skill-management/SKILL.md +6 -0
  222. package/src/config/bundled-skills/skill-management/TOOLS.json +0 -8
  223. package/src/config/bundled-skills/subagent/SKILL.md +6 -2
  224. package/src/config/bundled-skills/subagent/TOOLS.json +0 -20
  225. package/src/config/bundled-skills/transcribe/SKILL.md +4 -0
  226. package/src/config/bundled-skills/transcribe/TOOLS.json +0 -4
  227. package/src/config/bundled-tool-registry.ts +21 -0
  228. package/src/config/env-registry.ts +0 -2
  229. package/src/config/env.ts +19 -12
  230. package/src/config/feature-flag-registry.json +21 -133
  231. package/src/config/loader.ts +73 -99
  232. package/src/config/sanitize-for-transfer.ts +2 -0
  233. package/src/config/schemas/__tests__/memory-lifecycle.test.ts +80 -0
  234. package/src/config/schemas/__tests__/memory-v2.test.ts +7 -4
  235. package/src/config/schemas/calls.ts +0 -9
  236. package/src/config/schemas/heartbeat.ts +63 -0
  237. package/src/config/schemas/ingress.ts +10 -6
  238. package/src/config/schemas/llm.ts +5 -10
  239. package/src/config/schemas/memory-lifecycle.ts +77 -24
  240. package/src/config/schemas/memory-v2.ts +48 -4
  241. package/src/config/schemas/platform.ts +6 -0
  242. package/src/config/schemas/services.ts +1 -15
  243. package/src/config/schemas/skills.ts +0 -6
  244. package/src/config/seed-inference-profiles.ts +1 -1
  245. package/src/contacts/contact-store.ts +0 -30
  246. package/src/contacts/contacts-write.ts +0 -27
  247. package/src/context/window-manager.ts +1 -2
  248. package/src/credential-execution/feature-gates.ts +10 -10
  249. package/src/credential-execution/process-manager.ts +12 -41
  250. package/src/daemon/__tests__/conversation-tool-setup.test.ts +126 -5
  251. package/src/daemon/bootstrap-turn-cleanup.ts +45 -0
  252. package/src/daemon/config-watcher.ts +4 -3
  253. package/src/daemon/conversation-agent-loop-handlers.ts +21 -3
  254. package/src/daemon/conversation-agent-loop.ts +32 -28
  255. package/src/daemon/conversation-lifecycle.ts +8 -1
  256. package/src/daemon/conversation-process.ts +16 -11
  257. package/src/daemon/conversation-runtime-assembly.ts +2 -2
  258. package/src/daemon/conversation-surfaces.ts +125 -4
  259. package/src/daemon/conversation-tool-setup.ts +16 -55
  260. package/src/daemon/conversation.ts +21 -2
  261. package/src/daemon/doordash-steps.ts +1 -1
  262. package/src/daemon/handlers/shared.ts +4 -1
  263. package/src/daemon/host-app-control-proxy.ts +293 -0
  264. package/src/daemon/host-bash-proxy.ts +84 -74
  265. package/src/daemon/host-browser-proxy.ts +67 -82
  266. package/src/daemon/host-cu-proxy.ts +81 -86
  267. package/src/daemon/host-file-proxy.ts +93 -69
  268. package/src/daemon/host-proxy-base.ts +294 -0
  269. package/src/daemon/host-proxy-preactivation.ts +82 -0
  270. package/src/daemon/host-transfer-proxy.ts +247 -129
  271. package/src/daemon/lifecycle.ts +115 -117
  272. package/src/daemon/message-protocol.ts +3 -8
  273. package/src/daemon/message-types/contacts.ts +23 -1
  274. package/src/daemon/message-types/conversations.ts +11 -8
  275. package/src/daemon/message-types/host-app-control.ts +150 -0
  276. package/src/daemon/message-types/host-bash.ts +4 -0
  277. package/src/daemon/message-types/host-cu.ts +2 -0
  278. package/src/daemon/message-types/host-file.ts +4 -0
  279. package/src/daemon/message-types/host-transfer.ts +3 -0
  280. package/src/daemon/message-types/schedules.ts +8 -3
  281. package/src/daemon/message-types/skills.ts +2 -2
  282. package/src/daemon/process-message.ts +18 -1
  283. package/src/daemon/shutdown-handlers.ts +0 -3
  284. package/src/daemon/tool-setup-types.ts +51 -0
  285. package/src/daemon/tool-side-effects.ts +1 -1
  286. package/src/events/tool-audit-listener.ts +2 -1
  287. package/src/heartbeat/__tests__/heartbeat-feed-event.test.ts +15 -7
  288. package/src/heartbeat/__tests__/heartbeat-run-store.test.ts +216 -0
  289. package/src/heartbeat/heartbeat-run-store.ts +236 -0
  290. package/src/heartbeat/heartbeat-service.ts +280 -49
  291. package/src/home/__tests__/post-connect-feed.test.ts +99 -0
  292. package/src/home/__tests__/relationship-state-writer.test.ts +11 -9
  293. package/src/home/__tests__/suggested-prompts.test.ts +89 -0
  294. package/src/home/post-connect-feed.ts +68 -0
  295. package/src/home/relationship-state-writer.ts +17 -92
  296. package/src/home/suggested-prompts.ts +46 -10
  297. package/src/inbound/public-ingress-urls.ts +32 -34
  298. package/src/ipc/__tests__/route-error-envelope.test.ts +80 -0
  299. package/src/ipc/assistant-server.ts +14 -1
  300. package/src/ipc/cli-client.ts +32 -1
  301. package/src/live-voice/live-voice-metrics.ts +10 -10
  302. package/src/mcp/__tests__/mcp-auth-orchestrator.test.ts +304 -0
  303. package/src/mcp/mcp-auth-orchestrator.ts +213 -0
  304. package/src/mcp/mcp-auth-state.ts +133 -0
  305. package/src/mcp/mcp-oauth-provider.ts +19 -0
  306. package/src/memory/__tests__/jobs-store-job-classes.test.ts +24 -0
  307. package/src/memory/__tests__/qdrant-client-sentinel.test.ts +49 -0
  308. package/src/memory/__tests__/sparse-tokenize.test.ts +66 -0
  309. package/src/memory/anisotropy.test.ts +247 -0
  310. package/src/memory/anisotropy.ts +443 -0
  311. package/src/memory/auto-analysis-constants.ts +17 -0
  312. package/src/memory/auto-analysis-guard.ts +5 -15
  313. package/src/memory/canonical-guardian-store.ts +7 -7
  314. package/src/memory/context-search/__tests__/agent-runner-redaction.test.ts +122 -0
  315. package/src/memory/context-search/agent-protocol.ts +6 -6
  316. package/src/memory/context-search/agent-runner.ts +32 -7
  317. package/src/memory/context-search/sources/memory-v2.ts +17 -5
  318. package/src/memory/conversation-crud.ts +1 -1
  319. package/src/memory/conversation-key-store.ts +2 -15
  320. package/src/memory/db-init.ts +4 -0
  321. package/src/memory/embedding-backend.ts +9 -21
  322. package/src/memory/graph/__tests__/conversation-graph-memory-v2-routing.test.ts +49 -4
  323. package/src/memory/graph/conversation-graph-memory.ts +1 -24
  324. package/src/memory/graph/graph-search.ts +8 -0
  325. package/src/memory/graph/retriever.ts +28 -0
  326. package/src/memory/graph/tools.ts +1 -1
  327. package/src/memory/jobs/__tests__/embed-concept-page.test.ts +8 -2
  328. package/src/memory/jobs/embed-concept-page.ts +28 -2
  329. package/src/memory/jobs/embed-pkb-file.test.ts +2 -2
  330. package/src/memory/jobs-store.ts +66 -22
  331. package/src/memory/jobs-worker.ts +112 -63
  332. package/src/memory/memory-v2-activation-log-store.ts +1 -1
  333. package/src/memory/migrations/237-heartbeat-runs.ts +45 -0
  334. package/src/memory/migrations/238-schedule-retry-policy.ts +20 -0
  335. package/src/memory/migrations/index.ts +5 -0
  336. package/src/memory/migrations/registry.ts +8 -0
  337. package/src/memory/pkb/pkb-search.ts +7 -0
  338. package/src/memory/qdrant-client.ts +50 -20
  339. package/src/memory/schema/infrastructure.ts +15 -0
  340. package/src/memory/search/semantic.ts +7 -0
  341. package/src/memory/sparse-tokenize.ts +49 -0
  342. package/src/memory/v2/__tests__/activation.test.ts +77 -95
  343. package/src/memory/v2/__tests__/injection.test.ts +43 -21
  344. package/src/memory/v2/__tests__/sim.test.ts +166 -6
  345. package/src/memory/v2/__tests__/sparse-bm25.test.ts +292 -0
  346. package/src/memory/v2/__tests__/static-context.test.ts +0 -1
  347. package/src/memory/v2/activation.ts +69 -88
  348. package/src/memory/v2/consolidation-job.ts +3 -5
  349. package/src/memory/v2/constants.ts +7 -0
  350. package/src/memory/v2/injection.ts +86 -53
  351. package/src/memory/v2/prompts/consolidation.ts +312 -91
  352. package/src/memory/v2/qdrant.ts +99 -1
  353. package/src/memory/v2/sim.ts +126 -16
  354. package/src/memory/v2/skill-qdrant.ts +12 -3
  355. package/src/memory/v2/skill-store.ts +16 -1
  356. package/src/memory/v2/sparse-bm25.ts +245 -0
  357. package/src/memory/v2/static-context.ts +6 -5
  358. package/src/messaging/providers/gmail/types.ts +0 -49
  359. package/src/messaging/providers/slack/adapter.ts +1 -31
  360. package/src/messaging/providers/slack/types.ts +0 -32
  361. package/src/notifications/README.md +10 -10
  362. package/src/notifications/broadcaster.ts +1 -1
  363. package/src/notifications/guardian-question-mode.ts +5 -5
  364. package/src/oauth/connect-orchestrator.ts +4 -0
  365. package/src/oauth/credential-token-resolver.ts +1 -3
  366. package/src/oauth/manual-token-connection.ts +0 -4
  367. package/src/outbound-proxy/index.ts +1 -37
  368. package/src/outbound-proxy/logging.ts +1 -1
  369. package/src/outbound-proxy/policy.ts +6 -5
  370. package/src/outbound-proxy/router.ts +2 -1
  371. package/src/permissions/approval-policy.test.ts +6 -275
  372. package/src/permissions/approval-policy.ts +0 -51
  373. package/src/permissions/checker.test.ts +0 -1
  374. package/src/permissions/checker.ts +3 -17
  375. package/src/permissions/gateway-threshold-reader.ts +2 -0
  376. package/src/permissions/prompter.ts +34 -1
  377. package/src/permissions/secret-prompter.ts +6 -2
  378. package/src/prompts/bootstrap-cleanup.ts +27 -0
  379. package/src/prompts/system-prompt.ts +3 -18
  380. package/src/prompts/templates/SOUL.md +13 -1
  381. package/src/providers/speech-to-text/provider-catalog.ts +7 -8
  382. package/src/runtime/assistant-event-hub.ts +118 -96
  383. package/src/runtime/assistant-event.ts +1 -0
  384. package/src/runtime/auth/__tests__/middleware.test.ts +11 -56
  385. package/src/runtime/auth/middleware.ts +0 -96
  386. package/src/runtime/auth/route-policy.ts +19 -0
  387. package/src/runtime/btw-sidechain.ts +2 -3
  388. package/src/runtime/channel-invite-transport.ts +2 -48
  389. package/src/runtime/channel-invite-transports/email.ts +1 -1
  390. package/src/runtime/channel-invite-transports/slack.ts +1 -1
  391. package/src/runtime/channel-invite-transports/telegram.ts +1 -1
  392. package/src/runtime/channel-invite-transports/voice.ts +1 -1
  393. package/src/runtime/channel-invite-transports/whatsapp.ts +1 -1
  394. package/src/runtime/channel-invite-types.ts +54 -0
  395. package/src/runtime/channel-readiness-service.ts +32 -13
  396. package/src/runtime/http-server.ts +3 -329
  397. package/src/runtime/http-types.ts +0 -5
  398. package/src/runtime/migrations/__tests__/vbundle-import-parity.test.ts +413 -0
  399. package/src/runtime/migrations/__tests__/vbundle-import-policy.test.ts +260 -0
  400. package/src/runtime/migrations/__tests__/vbundle-import-version-compat.test.ts +189 -0
  401. package/src/runtime/migrations/__tests__/vbundle-streaming-importer.test.ts +153 -1
  402. package/src/runtime/migrations/__tests__/vbundle-symlink-importer.test.ts +451 -0
  403. package/src/runtime/migrations/__tests__/vbundle-symlink-streaming-importer.test.ts +0 -0
  404. package/src/runtime/migrations/__tests__/vbundle-symlink-streaming.test.ts +515 -0
  405. package/src/runtime/migrations/__tests__/vbundle-symlink-tar.test.ts +437 -0
  406. package/src/runtime/migrations/__tests__/vbundle-symlink-walker.test.ts +319 -0
  407. package/src/runtime/migrations/__tests__/vbundle-validator-v1-schema.test.ts +51 -1
  408. package/src/runtime/migrations/migration-transport.ts +7 -7
  409. package/src/runtime/migrations/vbundle-builder.ts +327 -60
  410. package/src/runtime/migrations/vbundle-import-analyzer.ts +4 -4
  411. package/src/runtime/migrations/vbundle-import-policy.ts +172 -0
  412. package/src/runtime/migrations/vbundle-importer.ts +245 -68
  413. package/src/runtime/migrations/vbundle-streaming-importer.ts +326 -35
  414. package/src/runtime/migrations/vbundle-streaming-validator.ts +157 -4
  415. package/src/runtime/migrations/vbundle-tar-stream.ts +15 -6
  416. package/src/runtime/migrations/vbundle-validator.ts +114 -0
  417. package/src/runtime/pending-interactions.ts +35 -9
  418. package/src/runtime/routes/__tests__/backup-routes.test.ts +22 -150
  419. package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +98 -0
  420. package/src/runtime/routes/__tests__/gateway-log-routes.test.ts +242 -0
  421. package/src/runtime/routes/__tests__/heartbeat-routes.test.ts +112 -0
  422. package/src/runtime/routes/approval-interception-types.ts +13 -0
  423. package/src/runtime/routes/approval-strategies/guardian-text-engine-strategy.ts +1 -1
  424. package/src/runtime/routes/backup-routes.ts +15 -38
  425. package/src/runtime/routes/btw-routes.ts +14 -37
  426. package/src/runtime/routes/client-routes.ts +1 -0
  427. package/src/runtime/routes/contact-prompt-routes.ts +183 -0
  428. package/src/runtime/routes/conversation-query-routes.ts +36 -1
  429. package/src/runtime/routes/conversation-routes.ts +30 -13
  430. package/src/runtime/routes/document-pdf-renderer.ts +165 -0
  431. package/src/runtime/routes/documents-routes.ts +30 -0
  432. package/src/runtime/routes/errors.ts +19 -4
  433. package/src/runtime/routes/events-routes.ts +12 -6
  434. package/src/runtime/routes/gateway-log-routes.ts +79 -0
  435. package/src/runtime/routes/guardian-approval-interception.ts +2 -8
  436. package/src/runtime/routes/heartbeat-routes.ts +103 -38
  437. package/src/runtime/routes/host-app-control-routes.ts +134 -0
  438. package/src/runtime/routes/host-bash-routes.ts +36 -6
  439. package/src/runtime/routes/host-browser-routes.ts +108 -13
  440. package/src/runtime/routes/host-cu-routes.ts +44 -14
  441. package/src/runtime/routes/host-file-routes.ts +33 -10
  442. package/src/runtime/routes/host-transfer-routes.ts +64 -24
  443. package/src/runtime/routes/http-adapter.ts +1 -0
  444. package/src/runtime/routes/identity-intro-cache.ts +30 -0
  445. package/src/runtime/routes/identity-routes.ts +15 -43
  446. package/src/runtime/routes/inbound-message-handler.ts +1 -9
  447. package/src/runtime/routes/inbound-stages/acl-enforcement.ts +0 -7
  448. package/src/runtime/routes/inbound-stages/edit-intercept.ts +0 -8
  449. package/src/runtime/routes/inbound-stages/transcribe-audio.test.ts +0 -20
  450. package/src/runtime/routes/inbound-stages/transcribe-audio.ts +5 -13
  451. package/src/runtime/routes/index.ts +8 -0
  452. package/src/runtime/routes/mcp-auth-routes.ts +132 -0
  453. package/src/runtime/routes/memory-item-routes.ts +10 -12
  454. package/src/runtime/routes/memory-v2-routes.ts +441 -1
  455. package/src/runtime/routes/migration-routes.ts +96 -0
  456. package/src/runtime/routes/schedule-routes.ts +7 -0
  457. package/src/runtime/verification-templates.ts +4 -7
  458. package/src/schedule/integration-status.ts +66 -2
  459. package/src/schedule/recurrence-engine.ts +4 -1
  460. package/src/schedule/retry-backoff.ts +18 -0
  461. package/src/schedule/retry-policy.ts +82 -0
  462. package/src/schedule/schedule-recovery.ts +64 -0
  463. package/src/schedule/schedule-store.ts +106 -2
  464. package/src/schedule/scheduler-types.ts +25 -0
  465. package/src/schedule/scheduler.ts +63 -38
  466. package/src/security/oauth-callback-registry.ts +8 -0
  467. package/src/sequence/analytics.ts +5 -5
  468. package/src/sequence/engine.ts +1 -1
  469. package/src/skills/catalog-files.ts +2 -8
  470. package/src/skills/include-graph.ts +5 -5
  471. package/src/skills/remote-skill-policy.ts +5 -5
  472. package/src/skills/skill-file-provider.ts +1 -1
  473. package/src/skills/skill-file-types.ts +13 -0
  474. package/src/skills/skillssh-audit-types.ts +28 -0
  475. package/src/skills/skillssh-registry.ts +8 -21
  476. package/src/telemetry/types.ts +2 -0
  477. package/src/telemetry/usage-telemetry-reporter.test.ts +21 -0
  478. package/src/telemetry/usage-telemetry-reporter.ts +1 -0
  479. package/src/tools/app-control/skill-proxy-bridge.ts +28 -0
  480. package/src/tools/apps/executors.ts +56 -69
  481. package/src/tools/browser/__tests__/browser-status.test.ts +21 -18
  482. package/src/tools/browser/browser-execution.ts +2 -2
  483. package/src/tools/browser/cdp-client/__tests__/factory.test.ts +55 -4
  484. package/src/tools/browser/cdp-client/cdp-inspect/__tests__/ws-transport.test.ts +12 -6
  485. package/src/tools/browser/cdp-client/factory.ts +23 -24
  486. package/src/tools/browser/cdp-client/index.ts +1 -14
  487. package/src/tools/computer-use/definitions.ts +42 -20
  488. package/src/tools/executor.ts +2 -0
  489. package/src/tools/host-filesystem/edit.ts +26 -0
  490. package/src/tools/host-filesystem/read.ts +26 -0
  491. package/src/tools/host-filesystem/transfer.ts +31 -1
  492. package/src/tools/host-filesystem/write.ts +26 -0
  493. package/src/tools/host-terminal/host-shell.ts +58 -0
  494. package/src/tools/schedule/create.ts +6 -0
  495. package/src/tools/schedule/list.ts +2 -0
  496. package/src/tools/schedule/update.ts +10 -0
  497. package/src/tools/shared/filesystem/file-ops-service.ts +2 -0
  498. package/src/tools/shared/filesystem/path-policy.ts +25 -1
  499. package/src/tools/skills/load.ts +0 -32
  500. package/src/tools/tool-approval-handler.ts +1 -5
  501. package/src/tools/types.ts +4 -0
  502. package/src/usage/pricing.ts +1 -1
  503. package/src/workspace/hatched-date.ts +86 -0
  504. package/src/workspace/migrations/003-seed-device-id.ts +1 -1
  505. package/src/workspace/migrations/006-services-config.ts +8 -5
  506. package/src/workspace/migrations/016-extract-feature-flags-to-protected.ts +3 -9
  507. package/src/workspace/migrations/021-move-signals-to-workspace.ts +4 -10
  508. package/src/workspace/migrations/022-move-hooks-to-workspace.ts +4 -10
  509. package/src/workspace/migrations/023-move-config-files-to-workspace.ts +4 -11
  510. package/src/workspace/migrations/024-move-runtime-files-to-workspace.ts +3 -10
  511. package/src/workspace/migrations/040-seed-latency-callsite-defaults.ts +3 -2
  512. package/src/workspace/migrations/050-seed-main-agent-opus-callsite.ts +2 -1
  513. package/src/workspace/migrations/059-move-pid-to-workspace.ts +3 -8
  514. package/src/workspace/migrations/061-move-backup-key-to-workspace.ts +3 -8
  515. package/src/workspace/migrations/AGENTS.md +1 -1
  516. package/src/workspace/migrations/migrate-to-workspace-volume.ts +4 -10
  517. package/src/workspace/migrations/utils.ts +21 -0
  518. package/src/__tests__/host-browser-e2e-cloud.test.ts +0 -443
  519. package/src/__tests__/host-browser-e2e-self-hosted-capability.test.ts +0 -226
  520. package/src/__tests__/host-browser-ws-events-e2e.test.ts +0 -427
  521. package/src/__tests__/twilio-rest.test.ts +0 -34
  522. package/src/backup/__tests__/backup-key.test.ts +0 -152
  523. package/src/backup/__tests__/backup-worker.test.ts +0 -782
  524. package/src/backup/__tests__/offsite-writer.test.ts +0 -641
  525. package/src/backup/__tests__/stream-crypt.test.ts +0 -228
  526. package/src/backup/backup-key.ts +0 -137
  527. package/src/backup/backup-worker.ts +0 -472
  528. package/src/backup/offsite-writer.ts +0 -222
  529. package/src/backup/stream-crypt.ts +0 -263
  530. package/src/daemon/message-types/pairing.ts +0 -58
  531. package/src/outbound-proxy/config.ts +0 -20
  532. package/src/outbound-proxy/health.ts +0 -18
  533. package/src/outbound-proxy/types.ts +0 -150
  534. package/src/runtime/capability-tokens.ts +0 -190
  535. package/src/signals/mcp-reload.ts +0 -18
@@ -73,19 +73,31 @@ export interface QdrantSearchResult {
73
73
  payload: QdrantPointPayload;
74
74
  }
75
75
 
76
+ /**
77
+ * Strip a trailing `:sparse-v<digits>` segment from a sentinel string.
78
+ *
79
+ * Legacy v1 sentinels (pre-decouple) included the sparse encoder version in
80
+ * the embedding-model identity. The suffix is no longer written, but stored
81
+ * sentinels written by older daemons may still carry it — strip it before
82
+ * comparing identities so existing collections aren't unnecessarily rebuilt.
83
+ */
84
+ export function stripLegacySparseSuffix(sentinel: string): string {
85
+ return sentinel.replace(/:sparse-v\d+$/, "");
86
+ }
87
+
76
88
  let _instance: VellumQdrantClient | null = null;
77
89
 
78
90
  export function getQdrantClient(): VellumQdrantClient {
79
91
  if (!_instance) {
80
92
  throw new Error(
81
- "Qdrant client not initialized. Call initQdrantClient() first."
93
+ "Qdrant client not initialized. Call initQdrantClient() first.",
82
94
  );
83
95
  }
84
96
  return _instance;
85
97
  }
86
98
 
87
99
  export function initQdrantClient(
88
- config: QdrantClientConfig
100
+ config: QdrantClientConfig,
89
101
  ): VellumQdrantClient {
90
102
  _instance = new VellumQdrantClient(config);
91
103
  return _instance;
@@ -140,12 +152,28 @@ export class VellumQdrantClient {
140
152
  const dimMismatch =
141
153
  currentSize != null && currentSize !== this.vectorSize;
142
154
 
143
- // Check model identity via a sentinel point that stores the embedding model
155
+ // Check model identity via a sentinel point that stores the embedding model.
156
+ //
157
+ // Legacy sentinels included a ":sparse-v<N>" suffix that conflated the
158
+ // sparse encoder version with the dense model identity. Sparse vectors
159
+ // are upserted in place and never require collection recreation, so a
160
+ // stored value matching the current dense identity after stripping any
161
+ // legacy sparse suffix is treated as compatible. Compatible-but-legacy
162
+ // sentinels are rewritten to the new format below to clean up gradually.
144
163
  let modelMismatch = false;
164
+ let needsSentinelRewrite = false;
145
165
  if (this.embeddingModel) {
146
166
  const sentinel = await this.readSentinel();
147
- if (sentinel && sentinel !== this.embeddingModel) {
148
- modelMismatch = true;
167
+ if (sentinel) {
168
+ const normalizedStored = stripLegacySparseSuffix(sentinel);
169
+ const normalizedCurrent = stripLegacySparseSuffix(
170
+ this.embeddingModel,
171
+ );
172
+ if (normalizedStored !== normalizedCurrent) {
173
+ modelMismatch = true;
174
+ } else if (sentinel !== this.embeddingModel) {
175
+ needsSentinelRewrite = true;
176
+ }
149
177
  }
150
178
  }
151
179
 
@@ -156,7 +184,7 @@ export class VellumQdrantClient {
156
184
  currentSize,
157
185
  expectedSize: this.vectorSize,
158
186
  },
159
- "Qdrant collection uses unnamed vectors (legacy) — deleting and recreating with named vectors. Embeddings will be re-indexed."
187
+ "Qdrant collection uses unnamed vectors (legacy) — deleting and recreating with named vectors. Embeddings will be re-indexed.",
160
188
  );
161
189
  await this.client.deleteCollection(this.collection);
162
190
  migrated = true;
@@ -169,7 +197,7 @@ export class VellumQdrantClient {
169
197
  expectedSize: this.vectorSize,
170
198
  modelMismatch,
171
199
  },
172
- "Qdrant collection incompatible (dimension or model change) — deleting and recreating. Embeddings will be regenerated on demand."
200
+ "Qdrant collection incompatible (dimension or model change) — deleting and recreating. Embeddings will be regenerated on demand.",
173
201
  );
174
202
  await this.client.deleteCollection(this.collection);
175
203
  migrated = true;
@@ -178,12 +206,15 @@ export class VellumQdrantClient {
178
206
  if (await this.ensurePayloadIndexesSafe()) {
179
207
  this.collectionReady = true;
180
208
  }
209
+ if (needsSentinelRewrite && this.embeddingModel) {
210
+ await this.writeSentinel(this.embeddingModel);
211
+ }
181
212
  return { migrated: false };
182
213
  }
183
214
  } catch (err) {
184
215
  log.warn(
185
216
  { err },
186
- "Failed to verify collection compatibility, assuming compatible"
217
+ "Failed to verify collection compatibility, assuming compatible",
187
218
  );
188
219
  if (await this.ensurePayloadIndexesSafe()) {
189
220
  this.collectionReady = true;
@@ -197,7 +228,7 @@ export class VellumQdrantClient {
197
228
 
198
229
  log.info(
199
230
  { collection: this.collection, vectorSize: this.vectorSize },
200
- "Creating Qdrant collection with named vectors (dense + sparse)"
231
+ "Creating Qdrant collection with named vectors (dense + sparse)",
201
232
  );
202
233
 
203
234
  try {
@@ -253,7 +284,7 @@ export class VellumQdrantClient {
253
284
  this.collectionReady = true;
254
285
  log.info(
255
286
  { collection: this.collection },
256
- "Qdrant collection created with payload indexes"
287
+ "Qdrant collection created with payload indexes",
257
288
  );
258
289
  }
259
290
 
@@ -265,7 +296,7 @@ export class VellumQdrantClient {
265
296
  targetId: string,
266
297
  vector: number[],
267
298
  payload: Omit<QdrantPointPayload, "target_type" | "target_id">,
268
- sparseVector?: QdrantSparseVector
299
+ sparseVector?: QdrantSparseVector,
269
300
  ): Promise<string> {
270
301
  await this.ensureCollection();
271
302
 
@@ -320,7 +351,7 @@ export class VellumQdrantClient {
320
351
  async search(
321
352
  vector: number[],
322
353
  limit: number,
323
- filter?: Record<string, unknown>
354
+ filter?: Record<string, unknown>,
324
355
  ): Promise<QdrantSearchResult[]> {
325
356
  await this.ensureCollection();
326
357
 
@@ -348,7 +379,7 @@ export class VellumQdrantClient {
348
379
  return results.map((result) => ({
349
380
  id: typeof result.id === "string" ? result.id : String(result.id),
350
381
  score: result.score,
351
- payload: (result.payload as unknown) as QdrantPointPayload,
382
+ payload: result.payload as unknown as QdrantPointPayload,
352
383
  }));
353
384
  }
354
385
 
@@ -357,7 +388,7 @@ export class VellumQdrantClient {
357
388
  limit: number,
358
389
  targetTypes: Array<"segment" | "item" | "summary" | "media">,
359
390
  excludeMessageIds?: string[],
360
- scopeIds?: string[]
391
+ scopeIds?: string[],
361
392
  ): Promise<QdrantSearchResult[]> {
362
393
  const mustConditions: Array<Record<string, unknown>> = [
363
394
  {
@@ -434,7 +465,7 @@ export class VellumQdrantClient {
434
465
  const queryParams = {
435
466
  prefetch: [
436
467
  {
437
- query: (denseVector as unknown) as number[],
468
+ query: denseVector as unknown as number[],
438
469
  using: "dense",
439
470
  limit: effectivePrefetchLimit,
440
471
  },
@@ -469,7 +500,7 @@ export class VellumQdrantClient {
469
500
  return (results.points ?? []).map((point) => ({
470
501
  id: typeof point.id === "string" ? point.id : String(point.id),
471
502
  score: point.score ?? 0,
472
- payload: (point.payload as unknown) as QdrantPointPayload,
503
+ payload: point.payload as unknown as QdrantPointPayload,
473
504
  }));
474
505
  }
475
506
 
@@ -594,7 +625,7 @@ export class VellumQdrantClient {
594
625
  } catch (err) {
595
626
  log.warn(
596
627
  { err, collection: this.collection },
597
- "Failed to delete Qdrant collection"
628
+ "Failed to delete Qdrant collection",
598
629
  );
599
630
  return false;
600
631
  }
@@ -762,8 +793,7 @@ export class VellumQdrantClient {
762
793
  ...(offset !== undefined ? { offset } : {}),
763
794
  });
764
795
  for (const point of result.points) {
765
- const id =
766
- typeof point.id === "string" ? point.id : String(point.id);
796
+ const id = typeof point.id === "string" ? point.id : String(point.id);
767
797
  const payload = (point.payload ?? {}) as Record<string, unknown>;
768
798
  out.push({ id, payload });
769
799
  }
@@ -788,7 +818,7 @@ export class VellumQdrantClient {
788
818
 
789
819
  private async findByTarget(
790
820
  targetType: string,
791
- targetId: string
821
+ targetId: string,
792
822
  ): Promise<string | null> {
793
823
  try {
794
824
  const results = await this.client.scroll(this.collection, {
@@ -19,6 +19,8 @@ export const cronJobs = sqliteTable("cron_jobs", {
19
19
  lastRunAt: integer("last_run_at"),
20
20
  lastStatus: text("last_status"), // 'ok' | 'error'
21
21
  retryCount: integer("retry_count").notNull().default(0),
22
+ maxRetries: integer("max_retries").notNull().default(3),
23
+ retryBackoffMs: integer("retry_backoff_ms").notNull().default(60000),
22
24
  createdBy: text("created_by").notNull(), // 'agent' | 'user'
23
25
  mode: text("mode").notNull().default("execute"), // 'notify' | 'execute'
24
26
  routingIntent: text("routing_intent").notNull().default("all_channels"), // 'single_channel' | 'multi_channel' | 'all_channels'
@@ -54,6 +56,19 @@ export const cronRuns = sqliteTable("cron_runs", {
54
56
  export const scheduleJobs = cronJobs;
55
57
  export const scheduleRuns = cronRuns;
56
58
 
59
+ export const heartbeatRuns = sqliteTable("heartbeat_runs", {
60
+ id: text("id").primaryKey(),
61
+ scheduledFor: integer("scheduled_for").notNull(),
62
+ startedAt: integer("started_at"),
63
+ finishedAt: integer("finished_at"),
64
+ durationMs: integer("duration_ms"),
65
+ status: text("status").notNull(), // 'pending' | 'running' | 'ok' | 'error' | 'timeout' | 'skipped' | 'missed' | 'superseded'
66
+ skipReason: text("skip_reason"), // 'disabled' | 'outside_active_hours' | 'overlap'
67
+ error: text("error"),
68
+ conversationId: text("conversation_id"),
69
+ createdAt: integer("created_at").notNull(),
70
+ });
71
+
57
72
  export const sharedAppLinks = sqliteTable("shared_app_links", {
58
73
  id: text("id").primaryKey(),
59
74
  shareToken: text("share_token").notNull().unique(),
@@ -1,5 +1,7 @@
1
1
  import { inArray } from "drizzle-orm";
2
2
 
3
+ import { getConfig } from "../../config/loader.js";
4
+ import { isMemoryV2ReadActive } from "../context-search/sources/memory-v2.js";
3
5
  import { getDb } from "../db-connection.js";
4
6
  import { withQdrantBreaker } from "../qdrant-circuit-breaker.js";
5
7
  import type {
@@ -55,6 +57,11 @@ export async function semanticSearch(
55
57
  ): Promise<Candidate[]> {
56
58
  if (limit <= 0) return [];
57
59
 
60
+ // v2 owns the read path when both gates are on; the v1 `memory` collection
61
+ // is in active retirement, and routing semantic recall there would re-enter
62
+ // the same corrupted sparse segments that can OOM-crash Qdrant.
63
+ if (isMemoryV2ReadActive(getConfig())) return [];
64
+
58
65
  const qdrant = getQdrantClient();
59
66
 
60
67
  // Overfetch to account for items filtered out post-query (invalidated, excluded, etc.)
@@ -0,0 +1,49 @@
1
+ // ---------------------------------------------------------------------------
2
+ // Sparse-vector tokenization primitives
3
+ // ---------------------------------------------------------------------------
4
+ //
5
+ // Shared by both the legacy TF-only encoder in `embedding-backend.ts`
6
+ // (`generateSparseEmbedding`) and the BM25 encoder in `v2/sparse-bm25.ts`.
7
+ //
8
+ // Lives in its own module so consumers of the BM25 encoder don't transitively
9
+ // depend on `embedding-backend.ts` for these primitives — that matters
10
+ // because many tests mock `embedding-backend.js` wholesale via
11
+ // `mock.module(...)`, and a missing export from the mock would break any
12
+ // transitive importer of these helpers.
13
+
14
+ import { stemmer } from "stemmer";
15
+
16
+ /** Hashed-vocabulary size for sparse encoders. */
17
+ export const SPARSE_VOCAB_SIZE = 30_000;
18
+
19
+ /** Tokenize text into lowercase alphanumeric tokens (Unicode-aware). */
20
+ export function tokenize(text: string): string[] {
21
+ return text.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [];
22
+ }
23
+
24
+ /**
25
+ * Tokenize and apply Porter stemming so morphological variants collapse to a
26
+ * shared bucket (e.g. `running`/`runs`/`ran` → `run`, `supplements` →
27
+ * `supplement`). Used only by the BM25 sparse channel in
28
+ * `v2/sparse-bm25.ts`; both the document-side and query-side encoders call
29
+ * this so doc and query tokens land in the same hash buckets.
30
+ *
31
+ * Other callers (workspace context-search, the legacy TF-only
32
+ * `generateSparseEmbedding`) intentionally keep the non-stemmed `tokenize()`
33
+ * because they predate this and rebuilding their on-disk indexes is out of
34
+ * scope here.
35
+ */
36
+ export function tokenizeStemmed(text: string): string[] {
37
+ return tokenize(text).map((token) => stemmer(token));
38
+ }
39
+
40
+ /** Hash a token to a stable index in [0, vocabSize). */
41
+ export function tokenHash(token: string, vocabSize: number): number {
42
+ // FNV-1a 32-bit hash for speed
43
+ let hash = 0x811c9dc5;
44
+ for (let i = 0; i < token.length; i++) {
45
+ hash ^= token.charCodeAt(i);
46
+ hash = Math.imul(hash, 0x01000193) >>> 0;
47
+ }
48
+ return hash % vocabSize;
49
+ }
@@ -148,7 +148,6 @@ const {
148
148
  computeSkillActivation,
149
149
  selectCandidates,
150
150
  selectInjections,
151
- selectSkillCandidates,
152
151
  selectSkillInjections,
153
152
  spreadActivation,
154
153
  } = await import("../activation.js");
@@ -193,6 +192,7 @@ function makeConfig(
193
192
  epsilon: number;
194
193
  dense_weight: number;
195
194
  sparse_weight: number;
195
+ ann_candidate_limit: number | null;
196
196
  }> = {},
197
197
  ): AssistantConfig {
198
198
  return {
@@ -205,6 +205,7 @@ function makeConfig(
205
205
  epsilon: 0.01,
206
206
  dense_weight: 1.0,
207
207
  sparse_weight: 0.0,
208
+ ann_candidate_limit: null,
208
209
  ...overrides,
209
210
  },
210
211
  },
@@ -342,7 +343,13 @@ describe("selectCandidates", () => {
342
343
  expect(out.fromAnn).toEqual(new Set(["alice-vscode", "delta-recipe"]));
343
344
  });
344
345
 
345
- test("ANN top-K limit equals 50 and runs without slug restriction", async () => {
346
+ test("ANN candidate query honors `config.memory.v2.ann_candidate_limit` and runs without slug restriction", async () => {
347
+ // Default `ann_candidate_limit: null` → unlimited, so the unlimited
348
+ // sentinel (1_000_000 — a Qdrant-safe stand-in for "every page") is
349
+ // passed to both channels. We don't pin to `MAX_SAFE_INTEGER` here:
350
+ // Qdrant's sparse `SearchContext` pre-allocates `limit * 16` bytes,
351
+ // and `MAX_SAFE_INTEGER` triggers a ~144 PB alloc that SIGABRTs the
352
+ // Qdrant process — so the constant deliberately undercuts it.
346
353
  stageHybridResponse([{ slug: "alpha", denseScore: 0.5, sparseScore: 1 }]);
347
354
  await selectCandidates({
348
355
  priorState: null,
@@ -351,10 +358,25 @@ describe("selectCandidates", () => {
351
358
  nowText: "",
352
359
  config: makeConfig(),
353
360
  });
354
- // Both channels (dense + sparse) ran with limit=50 and no filter.
355
361
  expect(state.queryCalls).toHaveLength(2);
356
362
  for (const call of state.queryCalls) {
357
- expect(call.limit).toBe(50);
363
+ expect(call.limit).toBe(1_000_000);
364
+ expect(call.filter).toBeUndefined();
365
+ }
366
+
367
+ // Explicit override flows through both channels verbatim.
368
+ state.queryCalls.length = 0;
369
+ stageHybridResponse([{ slug: "beta", denseScore: 0.5, sparseScore: 1 }]);
370
+ await selectCandidates({
371
+ priorState: null,
372
+ userText: "hello",
373
+ assistantText: "",
374
+ nowText: "",
375
+ config: makeConfig({ ann_candidate_limit: 25 }),
376
+ });
377
+ expect(state.queryCalls).toHaveLength(2);
378
+ for (const call of state.queryCalls) {
379
+ expect(call.limit).toBe(25);
358
380
  expect(call.filter).toBeUndefined();
359
381
  }
360
382
  });
@@ -682,15 +704,59 @@ describe("spreadActivation", () => {
682
704
  expect(out.final.get("bob")).toBeCloseTo(0.9, 6);
683
705
  });
684
706
 
685
- test("missing predecessor activation contributes 0 to the numerator", () => {
686
- // Edge alice→bob: bob has predecessor alice. alice is not in
687
- // `ownActivation`, so it contributes 0 to the numerator while the
688
- // denominator still counts the structural predecessor.
707
+ test("predecessors not in the candidate set are dropped from both numerator and denominator", () => {
708
+ // Edge alice→bob: bob has structural predecessor alice, but alice is not
709
+ // in `ownActivation`. With the new formula she contributes nothing
710
+ // hop1 has no active predecessors so the whole hop drops out of both
711
+ // sides of the ratio. Bob therefore stays at his own activation.
689
712
  const edges = buildEdgeIndex([["alice", "bob"]]);
690
713
  const own = new Map([["bob", 0.6]]);
691
714
  const out = spreadActivation(own, edges, 0.5, 2);
692
- // numerator = 0.6 + 0.5*0 = 0.6. denominator = 1 + 0.5*1 = 1.5.
693
- expect(out.final.get("bob")).toBeCloseTo(0.4, 6);
715
+ expect(out.final.get("bob")).toBeCloseTo(0.6, 6);
716
+ });
717
+
718
+ test("L_2 norm over multiple active predecessors rewards strong outliers more than avg would", () => {
719
+ // bob has 4 predecessors in the candidate set: one strong, three weak.
720
+ // L_2 = √((0.8² + 0.1² + 0.1² + 0.1²) / 4) = √(0.1675) ≈ 0.40927
721
+ // Plain avg of the same set = 0.275, so L_2 lifts bob more than avg
722
+ // would — the design goal of preferring quality over quantity.
723
+ const edges = buildEdgeIndex([
724
+ ["a1", "bob"],
725
+ ["a2", "bob"],
726
+ ["a3", "bob"],
727
+ ["a4", "bob"],
728
+ ]);
729
+ const own = new Map([
730
+ ["a1", 0.8],
731
+ ["a2", 0.1],
732
+ ["a3", 0.1],
733
+ ["a4", 0.1],
734
+ ["bob", 0.0],
735
+ ]);
736
+ const out = spreadActivation(own, edges, 0.5, 2);
737
+ const rms = Math.sqrt((0.8 * 0.8 + 3 * 0.1 * 0.1) / 4);
738
+ // numerator = 0 + 0.5 · rms
739
+ // denominator = 1 + 0.5
740
+ expect(out.final.get("bob")).toBeCloseTo((0.5 * rms) / 1.5, 6);
741
+ });
742
+
743
+ test("high-in-degree hub with mostly-inactive predecessors stays near A_o", () => {
744
+ // 100 structural predecessors point at hub; only one (`pred0`) is in
745
+ // the candidate set. The old formula would crush hub by the structural
746
+ // count (denominator ≈ 51); the new formula folds the empty bulk out
747
+ // and the L_2 averages over the single active predecessor only.
748
+ const rawEdges: Array<[string, string]> = [];
749
+ for (let i = 0; i < 100; i++) rawEdges.push([`pred${i}`, "hub"]);
750
+ const edges = buildEdgeIndex(rawEdges);
751
+ const own = new Map([
752
+ ["hub", 0.6],
753
+ ["pred0", 0.5],
754
+ ]);
755
+ const out = spreadActivation(own, edges, 0.5, 2);
756
+ // hop1 active = {pred0}, L_2([0.5]) = 0.5.
757
+ // numerator = 0.6 + 0.5 · 0.5 = 0.85
758
+ // denominator = 1 + 0.5 = 1.5
759
+ expect(out.final.get("hub")).toBeCloseTo(0.85 / 1.5, 6);
694
760
  });
695
761
 
696
762
  test("empty own-activation map returns empty result", () => {
@@ -829,7 +895,7 @@ describe("selectInjections", () => {
829
895
  });
830
896
 
831
897
  // ---------------------------------------------------------------------------
832
- // selectSkillCandidates
898
+ // computeSkillActivation
833
899
  // ---------------------------------------------------------------------------
834
900
 
835
901
  /** Stage a single hybrid response on the skills queues (payload key = `id`). */
@@ -848,90 +914,6 @@ function stageSkillHybridResponse(
848
914
  });
849
915
  }
850
916
 
851
- describe("selectSkillCandidates", () => {
852
- test("returns hit ids from the skills collection", async () => {
853
- stageSkillHybridResponse([
854
- { id: "example-skill-a", denseScore: 0.5, sparseScore: 1 },
855
- { id: "example-skill-b", denseScore: 0.3, sparseScore: 1 },
856
- ]);
857
- const out = await selectSkillCandidates({
858
- userText: "user said hello",
859
- assistantText: "",
860
- nowText: "",
861
- config: makeConfig(),
862
- topK: 10,
863
- });
864
- expect(out).toEqual(new Set(["example-skill-a", "example-skill-b"]));
865
- });
866
-
867
- test("empty turn text short-circuits without backend calls", async () => {
868
- const out = await selectSkillCandidates({
869
- userText: "",
870
- assistantText: "",
871
- nowText: "",
872
- config: makeConfig(),
873
- topK: 10,
874
- });
875
- expect(out.size).toBe(0);
876
- expect(state.embedCalls).toHaveLength(0);
877
- expect(state.queryCalls).toHaveLength(0);
878
- });
879
-
880
- test("topK=0 short-circuits without backend calls", async () => {
881
- const out = await selectSkillCandidates({
882
- userText: "anything",
883
- assistantText: "anything",
884
- nowText: "anything",
885
- config: makeConfig(),
886
- topK: 0,
887
- });
888
- expect(out.size).toBe(0);
889
- expect(state.embedCalls).toHaveLength(0);
890
- expect(state.queryCalls).toHaveLength(0);
891
- });
892
-
893
- test("forwards topK and queries the skills collection unrestricted", async () => {
894
- stageSkillHybridResponse([
895
- { id: "example-skill-a", denseScore: 0.5, sparseScore: 1 },
896
- ]);
897
- await selectSkillCandidates({
898
- userText: "hello",
899
- assistantText: "",
900
- nowText: "",
901
- config: makeConfig(),
902
- topK: 7,
903
- });
904
- // Both channels (dense + sparse) ran with limit=7 and no slug/id filter,
905
- // against the dedicated skills collection.
906
- expect(state.queryCalls).toHaveLength(2);
907
- for (const call of state.queryCalls) {
908
- expect(call.collection).toBe("memory_v2_skills");
909
- expect(call.limit).toBe(7);
910
- expect(call.filter).toBeUndefined();
911
- }
912
- });
913
-
914
- test("embeds concatenated turn text exactly once", async () => {
915
- stageSkillHybridResponse([]);
916
- await selectSkillCandidates({
917
- userText: "user line",
918
- assistantText: "assistant line",
919
- nowText: "now line",
920
- config: makeConfig(),
921
- topK: 5,
922
- });
923
- expect(state.embedCalls).toHaveLength(1);
924
- expect(state.embedCalls[0].inputs).toEqual([
925
- "user line\nassistant line\nnow line",
926
- ]);
927
- expect(state.sparseCalls).toEqual(["user line\nassistant line\nnow line"]);
928
- });
929
- });
930
-
931
- // ---------------------------------------------------------------------------
932
- // computeSkillActivation
933
- // ---------------------------------------------------------------------------
934
-
935
917
  describe("computeSkillActivation", () => {
936
918
  test("empty candidates short-circuits without backend calls", async () => {
937
919
  const out = await computeSkillActivation({
@@ -14,12 +14,12 @@
14
14
  *
15
15
  * Hermetic by design: the embedding backend, qdrant client, and `getConfig`
16
16
  * are mocked at the module level so the suite never reaches a real backend.
17
- * The skill activation pipeline (`selectSkillCandidates`,
18
- * `computeSkillActivation`, `selectSkillInjections`) and the skill-store
19
- * lookup (`getSkillCapability`) are also mocked at the module level so each
20
- * test can stage its skill slate without touching the dedicated skills
21
- * Qdrant collection. The activation-store uses an in-memory SQLite database
22
- * so writes are real but contained.
17
+ * The skill activation pipeline (`computeSkillActivation`,
18
+ * `selectSkillInjections`) and the skill-store helpers (`getAllSkillIds`,
19
+ * `getSkillCapability`) are also mocked at the module level so each test can
20
+ * stage its skill slate without touching the dedicated skills Qdrant
21
+ * collection. The activation-store uses an in-memory SQLite database so
22
+ * writes are real but contained.
23
23
  *
24
24
  * Tests use a temp workspace (mkdtemp) and never touch `~/.vellum/`. Sample
25
25
  * page content uses generic placeholders (Alice, Bob, etc.) per the cross-
@@ -127,18 +127,19 @@ mock.module("@qdrant/js-client-rest", () => ({
127
127
  // Skill pipeline mocks
128
128
  // ---------------------------------------------------------------------------
129
129
  //
130
- // The skill side of the per-turn pipeline (`selectSkillCandidates`,
131
- // `computeSkillActivation`, `selectSkillInjections`) has its own dedicated
132
- // Qdrant collection and embedding round-trips. Rather than threading staged
133
- // hits through that whole pipeline for every test, we mock the three skill
134
- // helpers and the synchronous `getSkillCapability` lookup at the module level
135
- // and let each test stage a `topNow` ordering and the matching `SkillEntry`
136
- // content directly.
130
+ // The skill side of the per-turn pipeline (`computeSkillActivation`,
131
+ // `selectSkillInjections`) has its own dedicated Qdrant collection and
132
+ // embedding round-trips. Rather than threading staged hits through that whole
133
+ // pipeline for every test, we mock the two activation helpers and the two
134
+ // skill-store helpers (`getAllSkillIds` for the candidate pool,
135
+ // `getSkillCapability` for content lookup) at the module level and let each
136
+ // test stage a `topNow` ordering and the matching `SkillEntry` content
137
+ // directly.
137
138
 
138
139
  const skillState = {
139
140
  /** Ordered ids `selectSkillInjections.topNow` returns this turn. */
140
141
  topSkillIds: [] as string[],
141
- /** id → SkillEntry used by `getSkillCapability`. */
142
+ /** id → SkillEntry used by `getSkillCapability` and `getAllSkillIds`. */
142
143
  entries: new Map<string, SkillEntry>(),
143
144
  };
144
145
 
@@ -149,7 +150,6 @@ mock.module("../activation.js", () => ({
149
150
  // activation map are inputs to `selectSkillInjections`, not anything the
150
151
  // injection logic introspects. Stub them to empty so the test stays focused
151
152
  // on the wiring, not the pipeline internals (covered in activation.test.ts).
152
- selectSkillCandidates: async () => new Set<string>(),
153
153
  computeSkillActivation: async () => ({
154
154
  activation: new Map<string, number>(),
155
155
  breakdown: new Map(),
@@ -160,6 +160,7 @@ mock.module("../activation.js", () => ({
160
160
  }));
161
161
 
162
162
  mock.module("../skill-store.js", () => ({
163
+ getAllSkillIds: () => [...skillState.entries.keys()],
163
164
  getSkillCapability: (id: string) => skillState.entries.get(id) ?? null,
164
165
  }));
165
166
 
@@ -331,6 +332,13 @@ function stageTurn(
331
332
  hits: Array<{ slug: string; denseScore?: number; sparseScore?: number }>,
332
333
  channels = 4,
333
334
  ): void {
335
+ // Clear any leftovers from a prior turn before staging this one so unused
336
+ // staged responses can't bleed into the next injection. The activation
337
+ // pipeline now skips the embedding round-trip for empty texts (turn 1's
338
+ // assistantMessage), so consumed-channel counts vary per turn — staging
339
+ // exclusively is the only way multi-turn tests stay aligned.
340
+ state.queryResponses.dense.length = 0;
341
+ state.queryResponses.sparse.length = 0;
334
342
  for (let i = 0; i < channels; i++) {
335
343
  state.queryResponses.dense.push({
336
344
  points: hits
@@ -399,11 +407,13 @@ describe("injectMemoryV2Block", () => {
399
407
 
400
408
  expect(result.toInject).toEqual(["alice-vscode"]);
401
409
  expect(result.block).not.toBeNull();
402
- expect(result.block).toContain("<memory>");
410
+ // `block` is the unwrapped inner content; the caller adds the
411
+ // `<memory>...</memory>` wrapper exactly once at injection time.
412
+ expect(result.block).not.toContain("<memory>");
413
+ expect(result.block).not.toContain("</memory>");
403
414
  expect(result.block).not.toContain("## What I Remember Right Now");
404
415
  expect(result.block).toContain("### alice-vscode");
405
416
  expect(result.block).toContain("VS Code");
406
- expect(result.block).toContain("</memory>");
407
417
 
408
418
  // State persisted: alice's activation is above epsilon and recorded;
409
419
  // everInjected captured the new slug + currentTurn.
@@ -652,13 +662,24 @@ describe("injectMemoryV2Block", () => {
652
662
  expect(persisted!.everInjected).toEqual([
653
663
  { slug: "phantom-slug", turn: 1 },
654
664
  ]);
665
+
666
+ // Activation log marks the slug `page_missing` (not `injected`) so a
667
+ // stale Qdrant / edge-index entry pointing at a vanished page is
668
+ // visible in telemetry instead of masquerading as a successful inject.
669
+ expect(telemetryState.recordCalls.length).toBe(1);
670
+ const row = telemetryState.recordCalls[0] as {
671
+ concepts: Array<{ slug: string; status: string }>;
672
+ };
673
+ const phantom = row.concepts.find((c) => c.slug === "phantom-slug");
674
+ expect(phantom).toBeDefined();
675
+ expect(phantom!.status).toBe("page_missing");
655
676
  });
656
677
 
657
678
  // ---------------------------------------------------------------------------
658
679
  // Skill subsection rendering
659
680
  // ---------------------------------------------------------------------------
660
681
 
661
- test("renders a skill-only block in the same `<memory>` wrapper as concept-page-only blocks", async () => {
682
+ test("renders a skill-only block alongside concept-page-only blocks", async () => {
662
683
  // No concept-page candidates this turn — the candidate query and the three
663
684
  // simBatch queries all return empty. The skill pipeline is mocked to
664
685
  // surface a single skill.
@@ -687,10 +708,11 @@ describe("injectMemoryV2Block", () => {
687
708
 
688
709
  expect(result.toInject).toEqual([]);
689
710
  expect(result.block).not.toBeNull();
690
- // Same outer wrapping as concept-page-only blocks.
691
- expect(result.block).toContain("<memory>");
711
+ // `block` is the unwrapped inner content; the caller adds the
712
+ // `<memory>...</memory>` wrapper exactly once at injection time.
713
+ expect(result.block).not.toContain("<memory>");
714
+ expect(result.block).not.toContain("</memory>");
692
715
  expect(result.block).not.toContain("## What I Remember Right Now");
693
- expect(result.block).toContain("</memory>");
694
716
  // No concept-page sections; skills subsection present with the right
695
717
  // bullet shape and the unconditional `→ use skill_load to activate` suffix.
696
718
  expect(result.block).not.toContain("### alice-vscode");