@vellumai/assistant 0.3.4 → 0.3.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (506) hide show
  1. package/Dockerfile +2 -0
  2. package/README.md +88 -2
  3. package/eslint.config.mjs +31 -0
  4. package/package.json +1 -1
  5. package/scripts/ipc/check-swift-decoder-drift.ts +4 -1
  6. package/scripts/ipc/generate-swift.ts +31 -2
  7. package/src/__tests__/__snapshots__/ipc-snapshot.test.ts.snap +438 -1
  8. package/src/__tests__/approval-conversation-turn.test.ts +214 -0
  9. package/src/__tests__/approval-hardcoded-copy-guard.test.ts +41 -0
  10. package/src/__tests__/approval-message-composer.test.ts +253 -0
  11. package/src/__tests__/browser-manager.test.ts +1 -0
  12. package/src/__tests__/call-conversation-messages.test.ts +130 -0
  13. package/src/__tests__/call-domain.test.ts +12 -2
  14. package/src/__tests__/call-orchestrator.test.ts +799 -249
  15. package/src/__tests__/call-pointer-messages.test.ts +148 -0
  16. package/src/__tests__/call-recovery.test.ts +3 -0
  17. package/src/__tests__/call-routes-http.test.ts +32 -2
  18. package/src/__tests__/call-store.test.ts +3 -0
  19. package/src/__tests__/channel-approval-routes.test.ts +1277 -98
  20. package/src/__tests__/channel-approval.test.ts +37 -0
  21. package/src/__tests__/channel-approvals.test.ts +36 -50
  22. package/src/__tests__/channel-guardian.test.ts +630 -22
  23. package/src/__tests__/channel-readiness-service.test.ts +324 -0
  24. package/src/__tests__/checker.test.ts +14 -7
  25. package/src/__tests__/clarification-resolver.test.ts +44 -24
  26. package/src/__tests__/commit-message-enrichment-service.test.ts +9 -4
  27. package/src/__tests__/computer-use-session-working-dir.test.ts +8 -0
  28. package/src/__tests__/config-schema.test.ts +14 -8
  29. package/src/__tests__/context-window-manager.test.ts +30 -2
  30. package/src/__tests__/contradiction-checker.test.ts +20 -5
  31. package/src/__tests__/credential-security-invariants.test.ts +7 -2
  32. package/src/__tests__/daemon-lifecycle.test.ts +13 -12
  33. package/src/__tests__/db-migration-rollback.test.ts +752 -0
  34. package/src/__tests__/dictation-mode-detection.test.ts +63 -0
  35. package/src/__tests__/dynamic-skill-workflow-prompt.test.ts +2 -0
  36. package/src/__tests__/entity-search.test.ts +615 -0
  37. package/src/__tests__/fuzzy-match-property.test.ts +5 -5
  38. package/src/__tests__/guardian-action-store.test.ts +123 -0
  39. package/src/__tests__/guardian-action-sweep.test.ts +277 -0
  40. package/src/__tests__/guardian-dispatch.test.ts +389 -0
  41. package/src/__tests__/guardian-question-copy.test.ts +47 -0
  42. package/src/__tests__/handlers-telegram-config.test.ts +4 -2
  43. package/src/__tests__/handlers-twilio-config.test.ts +533 -0
  44. package/src/__tests__/intent-routing.test.ts +2 -0
  45. package/src/__tests__/ipc-snapshot.test.ts +291 -1
  46. package/src/__tests__/memory-upsert-concurrency.test.ts +828 -0
  47. package/src/__tests__/messaging-send-tool.test.ts +65 -0
  48. package/src/__tests__/model-intents.test.ts +96 -0
  49. package/src/__tests__/no-direct-anthropic-sdk-imports.test.ts +42 -0
  50. package/src/__tests__/oauth2-gateway-transport.test.ts +130 -0
  51. package/src/__tests__/onboarding-starter-tasks.test.ts +2 -0
  52. package/src/__tests__/provider-commit-message-generator.test.ts +89 -13
  53. package/src/__tests__/provider-error-scenarios.test.ts +621 -0
  54. package/src/__tests__/provider-fail-open-selection.test.ts +119 -0
  55. package/src/__tests__/qdrant-manager.test.ts +27 -20
  56. package/src/__tests__/relay-server.test.ts +779 -40
  57. package/src/__tests__/run-orchestrator-assistant-events.test.ts +6 -0
  58. package/src/__tests__/run-orchestrator.test.ts +42 -4
  59. package/src/__tests__/runtime-runs-http.test.ts +17 -1
  60. package/src/__tests__/runtime-runs.test.ts +16 -0
  61. package/src/__tests__/schedule-store.test.ts +18 -4
  62. package/src/__tests__/scheduler-recurrence.test.ts +13 -4
  63. package/src/__tests__/session-abort-tool-results.test.ts +6 -0
  64. package/src/__tests__/session-agent-loop.test.ts +857 -0
  65. package/src/__tests__/session-conflict-gate.test.ts +6 -0
  66. package/src/__tests__/session-pre-run-repair.test.ts +6 -0
  67. package/src/__tests__/session-profile-injection.test.ts +6 -0
  68. package/src/__tests__/session-provider-retry-repair.test.ts +6 -0
  69. package/src/__tests__/session-queue.test.ts +6 -0
  70. package/src/__tests__/session-runtime-assembly.test.ts +321 -13
  71. package/src/__tests__/session-slash-known.test.ts +6 -0
  72. package/src/__tests__/session-slash-queue.test.ts +6 -0
  73. package/src/__tests__/session-slash-unknown.test.ts +6 -0
  74. package/src/__tests__/session-surfaces-task-progress.test.ts +2 -0
  75. package/src/__tests__/session-tool-setup-app-refresh.test.ts +1 -0
  76. package/src/__tests__/session-tool-setup-memory-scope.test.ts +1 -0
  77. package/src/__tests__/session-tool-setup-side-effect-flag.test.ts +1 -0
  78. package/src/__tests__/session-workspace-injection.test.ts +6 -0
  79. package/src/__tests__/session-workspace-tool-tracking.test.ts +6 -0
  80. package/src/__tests__/skills.test.ts +2 -0
  81. package/src/__tests__/sms-messaging-provider.test.ts +126 -0
  82. package/src/__tests__/starter-task-flow.test.ts +2 -0
  83. package/src/__tests__/swarm-dag-pathological.test.ts +535 -0
  84. package/src/__tests__/system-prompt.test.ts +2 -0
  85. package/src/__tests__/task-management-tools.test.ts +2 -2
  86. package/src/__tests__/task-runner.test.ts +14 -4
  87. package/src/__tests__/terminal-tools.test.ts +25 -19
  88. package/src/__tests__/tool-execution-abort-cleanup.test.ts +545 -0
  89. package/src/__tests__/tool-executor-shell-integration.test.ts +11 -11
  90. package/src/__tests__/tool-executor.test.ts +23 -24
  91. package/src/__tests__/trust-store.test.ts +3 -3
  92. package/src/__tests__/twilio-rest.test.ts +29 -0
  93. package/src/__tests__/twilio-routes-elevenlabs.test.ts +3 -0
  94. package/src/__tests__/twilio-routes-twiml.test.ts +11 -0
  95. package/src/__tests__/twilio-routes.test.ts +167 -11
  96. package/src/__tests__/twitter-cli-error-shaping.test.ts +2 -2
  97. package/src/__tests__/user-reference.test.ts +2 -0
  98. package/src/__tests__/voice-quality.test.ts +222 -0
  99. package/src/__tests__/web-search.test.ts +46 -30
  100. package/src/__tests__/work-item-output.test.ts +110 -0
  101. package/src/agent/loop.ts +1 -1
  102. package/src/agent-heartbeat/agent-heartbeat-service.ts +2 -10
  103. package/src/amazon/client.ts +1418 -0
  104. package/src/amazon/request-extractor.ts +135 -0
  105. package/src/amazon/session.ts +109 -0
  106. package/src/autonomy/autonomy-store.ts +5 -5
  107. package/src/browser-extension-relay/client.ts +124 -0
  108. package/src/browser-extension-relay/protocol.ts +63 -0
  109. package/src/browser-extension-relay/server.ts +177 -0
  110. package/src/bundler/app-bundler.ts +3 -3
  111. package/src/bundler/bundle-signer.ts +1 -1
  112. package/src/bundler/signature-verifier.ts +1 -1
  113. package/src/calls/call-conversation-messages.ts +33 -0
  114. package/src/calls/call-domain.ts +114 -10
  115. package/src/calls/call-orchestrator.ts +268 -59
  116. package/src/calls/call-pointer-messages.ts +53 -0
  117. package/src/calls/call-recovery.ts +3 -8
  118. package/src/calls/call-store.ts +69 -87
  119. package/src/calls/elevenlabs-config.ts +3 -2
  120. package/src/calls/guardian-action-sweep.ts +105 -0
  121. package/src/calls/guardian-dispatch.ts +203 -0
  122. package/src/calls/guardian-question-copy.ts +133 -0
  123. package/src/calls/relay-server.ts +466 -8
  124. package/src/calls/speaker-identification.ts +1 -1
  125. package/src/calls/twilio-config.ts +22 -14
  126. package/src/calls/twilio-provider.ts +6 -4
  127. package/src/calls/twilio-rest.ts +308 -7
  128. package/src/calls/twilio-routes.ts +65 -12
  129. package/src/calls/types.ts +3 -1
  130. package/src/channels/types.ts +25 -0
  131. package/src/cli/amazon.ts +815 -0
  132. package/src/cli/config-commands.ts +2 -2
  133. package/src/cli/core-commands.ts +4 -3
  134. package/src/cli/influencer.ts +244 -0
  135. package/src/cli/map.ts +89 -6
  136. package/src/cli.ts +1 -1
  137. package/src/config/agent-schema.ts +171 -0
  138. package/src/config/bundled-skills/amazon/SKILL.md +127 -0
  139. package/src/config/bundled-skills/amazon/icon.svg +13 -0
  140. package/src/config/bundled-skills/api-mapping/SKILL.md +78 -0
  141. package/src/config/bundled-skills/browser/SKILL.md +1 -0
  142. package/src/config/bundled-skills/browser/TOOLS.json +17 -0
  143. package/src/config/bundled-skills/browser/tools/browser-wait-for-download.ts +25 -0
  144. package/src/config/bundled-skills/doordash/SKILL.md +51 -51
  145. package/src/config/bundled-skills/email-setup/SKILL.md +14 -5
  146. package/src/config/bundled-skills/google-oauth-setup/SKILL.md +183 -0
  147. package/src/config/bundled-skills/influencer/SKILL.md +144 -0
  148. package/src/config/bundled-skills/knowledge-graph/SKILL.md +15 -0
  149. package/src/config/bundled-skills/knowledge-graph/TOOLS.json +56 -0
  150. package/src/config/bundled-skills/knowledge-graph/tools/graph-query.ts +185 -0
  151. package/src/config/bundled-skills/macos-automation/icon.svg +12 -0
  152. package/src/config/bundled-skills/media-processing/SKILL.md +176 -0
  153. package/src/config/bundled-skills/media-processing/TOOLS.json +230 -0
  154. package/src/config/bundled-skills/media-processing/__tests__/concurrency-pool.test.ts +77 -0
  155. package/src/config/bundled-skills/media-processing/__tests__/cost-tracker.test.ts +69 -0
  156. package/src/config/bundled-skills/media-processing/__tests__/preprocess.test.ts +303 -0
  157. package/src/config/bundled-skills/media-processing/services/concurrency-pool.ts +55 -0
  158. package/src/config/bundled-skills/media-processing/services/cost-tracker.ts +86 -0
  159. package/src/config/bundled-skills/media-processing/services/gemini-map.ts +339 -0
  160. package/src/config/bundled-skills/media-processing/services/preprocess.ts +551 -0
  161. package/src/config/bundled-skills/media-processing/services/processing-pipeline.ts +259 -0
  162. package/src/config/bundled-skills/media-processing/services/reduce.ts +197 -0
  163. package/src/config/bundled-skills/media-processing/tools/analyze-keyframes.ts +136 -0
  164. package/src/config/bundled-skills/media-processing/tools/extract-keyframes.ts +59 -0
  165. package/src/config/bundled-skills/media-processing/tools/generate-clip.ts +195 -0
  166. package/src/config/bundled-skills/media-processing/tools/ingest-media.ts +197 -0
  167. package/src/config/bundled-skills/media-processing/tools/media-diagnostics.ts +143 -0
  168. package/src/config/bundled-skills/media-processing/tools/media-status.ts +75 -0
  169. package/src/config/bundled-skills/media-processing/tools/query-media-events.ts +65 -0
  170. package/src/config/bundled-skills/messaging/SKILL.md +33 -8
  171. package/src/config/bundled-skills/messaging/tools/messaging-analyze-style.ts +4 -7
  172. package/src/config/bundled-skills/messaging/tools/messaging-reply.ts +2 -1
  173. package/src/config/bundled-skills/messaging/tools/messaging-send.ts +5 -1
  174. package/src/config/bundled-skills/phone-calls/SKILL.md +88 -23
  175. package/src/config/bundled-skills/twitter/SKILL.md +19 -3
  176. package/src/config/bundled-skills/twitter/icon.svg +14 -0
  177. package/src/config/bundled-tool-registry.ts +310 -0
  178. package/src/config/calls-schema.ts +181 -0
  179. package/src/config/core-schema.ts +309 -0
  180. package/src/config/defaults.ts +28 -3
  181. package/src/config/env-registry.ts +162 -0
  182. package/src/config/env.ts +175 -0
  183. package/src/config/loader.ts +6 -6
  184. package/src/config/memory-schema.ts +528 -0
  185. package/src/config/sandbox-schema.ts +55 -0
  186. package/src/config/schema.ts +158 -1133
  187. package/src/config/skill-state.ts +1 -1
  188. package/src/config/skills-schema.ts +32 -0
  189. package/src/config/skills.ts +35 -24
  190. package/src/config/system-prompt.ts +131 -56
  191. package/src/config/templates/IDENTITY.md +2 -2
  192. package/src/config/templates/SOUL.md +1 -1
  193. package/src/config/types.ts +1 -0
  194. package/src/config/user-reference.ts +4 -9
  195. package/src/config/vellum-skills/catalog.json +6 -7
  196. package/src/config/vellum-skills/chatgpt-import/tools/chatgpt-import.ts +5 -1
  197. package/src/config/vellum-skills/slack-oauth-setup/SKILL.md +4 -3
  198. package/src/config/vellum-skills/sms-setup/SKILL.md +216 -0
  199. package/src/config/vellum-skills/twilio-setup/SKILL.md +40 -8
  200. package/src/context/window-manager.ts +27 -7
  201. package/src/daemon/approval-generators.ts +186 -0
  202. package/src/daemon/approved-devices-store.ts +140 -0
  203. package/src/daemon/assistant-attachments.ts +1 -1
  204. package/src/daemon/classifier.ts +35 -32
  205. package/src/daemon/config-watcher.ts +1 -1
  206. package/src/daemon/daemon-control.ts +217 -0
  207. package/src/daemon/handlers/apps.ts +2 -3
  208. package/src/daemon/handlers/config-channels.ts +158 -0
  209. package/src/daemon/handlers/config-inbox.ts +540 -0
  210. package/src/daemon/handlers/config-ingress.ts +231 -0
  211. package/src/daemon/handlers/config-integrations.ts +258 -0
  212. package/src/daemon/handlers/config-model.ts +143 -0
  213. package/src/daemon/handlers/config-parental.ts +163 -0
  214. package/src/daemon/handlers/config-scheduling.ts +172 -0
  215. package/src/daemon/handlers/config-slack.ts +92 -0
  216. package/src/daemon/handlers/config-telegram.ts +301 -0
  217. package/src/daemon/handlers/config-tools.ts +177 -0
  218. package/src/daemon/handlers/config-trust.ts +104 -0
  219. package/src/daemon/handlers/config-twilio.ts +1080 -0
  220. package/src/daemon/handlers/config.ts +53 -1689
  221. package/src/daemon/handlers/diagnostics.ts +1 -1
  222. package/src/daemon/handlers/dictation.ts +180 -0
  223. package/src/daemon/handlers/documents.ts +18 -32
  224. package/src/daemon/handlers/identity.ts +14 -23
  225. package/src/daemon/handlers/index.ts +11 -0
  226. package/src/daemon/handlers/misc.ts +3 -5
  227. package/src/daemon/handlers/pairing.ts +98 -0
  228. package/src/daemon/handlers/sessions.ts +56 -5
  229. package/src/daemon/handlers/shared.ts +6 -1
  230. package/src/daemon/handlers/skills.ts +1 -1
  231. package/src/daemon/handlers/twitter-auth.ts +2 -0
  232. package/src/daemon/handlers/work-items.ts +17 -9
  233. package/src/daemon/handlers/workspace-files.ts +4 -3
  234. package/src/daemon/install-cli-launchers.ts +113 -0
  235. package/src/daemon/ipc-contract/apps.ts +356 -0
  236. package/src/daemon/ipc-contract/browser.ts +74 -0
  237. package/src/daemon/ipc-contract/computer-use.ts +151 -0
  238. package/src/daemon/ipc-contract/diagnostics.ts +56 -0
  239. package/src/daemon/ipc-contract/documents.ts +74 -0
  240. package/src/daemon/ipc-contract/inbox.ts +209 -0
  241. package/src/daemon/ipc-contract/integrations.ts +284 -0
  242. package/src/daemon/ipc-contract/memory.ts +48 -0
  243. package/src/daemon/ipc-contract/messages.ts +211 -0
  244. package/src/daemon/ipc-contract/pairing.ts +45 -0
  245. package/src/daemon/ipc-contract/parental-control.ts +95 -0
  246. package/src/daemon/ipc-contract/schedules.ts +97 -0
  247. package/src/daemon/ipc-contract/sessions.ts +315 -0
  248. package/src/daemon/ipc-contract/shared.ts +42 -0
  249. package/src/daemon/ipc-contract/skills.ts +120 -0
  250. package/src/daemon/ipc-contract/subagents.ts +58 -0
  251. package/src/daemon/ipc-contract/surfaces.ts +250 -0
  252. package/src/daemon/ipc-contract/trust.ts +60 -0
  253. package/src/daemon/ipc-contract/work-items.ts +225 -0
  254. package/src/daemon/ipc-contract/workspace.ts +113 -0
  255. package/src/daemon/ipc-contract-inventory.json +70 -0
  256. package/src/daemon/ipc-contract-inventory.ts +55 -29
  257. package/src/daemon/ipc-contract.ts +229 -2426
  258. package/src/daemon/ipc-protocol.ts +1 -1
  259. package/src/daemon/ipc-validate.ts +7 -0
  260. package/src/daemon/lifecycle.ts +97 -377
  261. package/src/daemon/pairing-store.ts +177 -0
  262. package/src/daemon/providers-setup.ts +43 -0
  263. package/src/daemon/ride-shotgun-handler.ts +68 -3
  264. package/src/daemon/server.ts +66 -46
  265. package/src/daemon/session-agent-loop-handlers.ts +421 -0
  266. package/src/daemon/session-agent-loop.ts +117 -275
  267. package/src/daemon/session-dynamic-profile.ts +1 -1
  268. package/src/daemon/session-history.ts +1 -1
  269. package/src/daemon/session-media-retry.ts +1 -1
  270. package/src/daemon/session-messaging.ts +37 -2
  271. package/src/daemon/session-notifiers.ts +5 -25
  272. package/src/daemon/session-process.ts +99 -59
  273. package/src/daemon/session-queue-manager.ts +96 -4
  274. package/src/daemon/session-runtime-assembly.ts +199 -10
  275. package/src/daemon/session-surfaces.ts +19 -4
  276. package/src/daemon/session-tool-setup.ts +30 -30
  277. package/src/daemon/session-workspace.ts +1 -1
  278. package/src/daemon/session.ts +35 -2
  279. package/src/daemon/shutdown-handlers.ts +122 -0
  280. package/src/daemon/trace-emitter.ts +1 -1
  281. package/src/daemon/watch-handler.ts +36 -33
  282. package/src/doordash/cart-queries.ts +787 -0
  283. package/src/doordash/client.ts +144 -127
  284. package/src/doordash/order-queries.ts +85 -0
  285. package/src/doordash/queries.ts +10 -1308
  286. package/src/doordash/search-queries.ts +203 -0
  287. package/src/doordash/session.ts +3 -2
  288. package/src/doordash/store-queries.ts +246 -0
  289. package/src/doordash/types.ts +367 -0
  290. package/src/email/providers/agentmail.ts +2 -1
  291. package/src/email/providers/index.ts +3 -2
  292. package/src/email/service.ts +3 -2
  293. package/src/errors.ts +43 -0
  294. package/src/home-base/prebuilt/seed.ts +1 -1
  295. package/src/hooks/cli.ts +6 -5
  296. package/src/hooks/config.ts +6 -8
  297. package/src/hooks/discovery.ts +6 -5
  298. package/src/hooks/manager.ts +4 -3
  299. package/src/hooks/runner.ts +2 -2
  300. package/src/hooks/templates.ts +5 -5
  301. package/src/inbound/public-ingress-urls.ts +6 -4
  302. package/src/index.ts +4 -2
  303. package/src/influencer/client.ts +1104 -0
  304. package/src/instrument.ts +4 -3
  305. package/src/logfire.ts +4 -3
  306. package/src/memory/admin.ts +25 -35
  307. package/src/memory/attachments-store.ts +4 -7
  308. package/src/memory/channel-delivery-store.ts +30 -1
  309. package/src/memory/channel-guardian-store.ts +202 -2
  310. package/src/memory/clarification-resolver.ts +37 -33
  311. package/src/memory/conflict-store.ts +67 -61
  312. package/src/memory/contradiction-checker.ts +141 -117
  313. package/src/memory/conversation-store.ts +335 -51
  314. package/src/memory/db-connection.ts +27 -4
  315. package/src/memory/db-init.ts +265 -4
  316. package/src/memory/db.ts +14 -1
  317. package/src/memory/embedding-backend.ts +27 -5
  318. package/src/memory/embedding-ollama.ts +2 -1
  319. package/src/memory/entity-extractor.ts +38 -35
  320. package/src/memory/guardian-action-store.ts +430 -0
  321. package/src/memory/inbox-escalation-projection.ts +59 -0
  322. package/src/memory/inbox-thread-store.ts +218 -0
  323. package/src/memory/ingress-invite-store.ts +338 -0
  324. package/src/memory/ingress-member-store.ts +350 -0
  325. package/src/memory/items-extractor.ts +91 -97
  326. package/src/memory/job-handlers/index-maintenance.ts +3 -3
  327. package/src/memory/job-handlers/media-processing.ts +69 -0
  328. package/src/memory/job-handlers/summarization.ts +32 -26
  329. package/src/memory/job-utils.ts +3 -10
  330. package/src/memory/jobs-store.ts +8 -10
  331. package/src/memory/jobs-worker.ts +55 -36
  332. package/src/memory/media-store.ts +759 -0
  333. package/src/memory/migrations/001-job-deferrals.ts +45 -0
  334. package/src/memory/migrations/002-tool-invocations-fk.ts +43 -0
  335. package/src/memory/migrations/003-memory-fts-backfill.ts +24 -0
  336. package/src/memory/migrations/004-entity-relation-dedup.ts +87 -0
  337. package/src/memory/migrations/005-fingerprint-scope-unique.ts +80 -0
  338. package/src/memory/migrations/006-scope-salted-fingerprints.ts +62 -0
  339. package/src/memory/migrations/007-assistant-id-to-self.ts +254 -0
  340. package/src/memory/migrations/008-remove-assistant-id-columns.ts +208 -0
  341. package/src/memory/migrations/009-llm-usage-events-drop-assistant-id.ts +83 -0
  342. package/src/memory/migrations/010-ext-conv-bindings-channel-chat-unique.ts +56 -0
  343. package/src/memory/migrations/011-call-sessions-provider-sid-dedup.ts +63 -0
  344. package/src/memory/migrations/012-call-sessions-add-initiated-from.ts +19 -0
  345. package/src/memory/migrations/013-guardian-action-tables.ts +68 -0
  346. package/src/memory/migrations/014-backfill-inbox-thread-state.ts +76 -0
  347. package/src/memory/migrations/015-drop-active-search-index.ts +27 -0
  348. package/src/memory/migrations/016-memory-segments-indexes.ts +11 -0
  349. package/src/memory/migrations/017-memory-items-indexes.ts +10 -0
  350. package/src/memory/migrations/018-remaining-table-indexes.ts +13 -0
  351. package/src/memory/migrations/index.ts +24 -0
  352. package/src/memory/migrations/registry.ts +79 -0
  353. package/src/memory/migrations/validate-migration-state.ts +69 -0
  354. package/src/memory/qdrant-manager.ts +49 -8
  355. package/src/memory/query-builder.ts +1 -1
  356. package/src/memory/raw-query.ts +119 -0
  357. package/src/memory/recall-cache.ts +4 -1
  358. package/src/memory/retriever.ts +165 -47
  359. package/src/memory/schema-migration.ts +25 -984
  360. package/src/memory/schema.ts +228 -7
  361. package/src/memory/search/entity.ts +205 -31
  362. package/src/memory/search/lexical.ts +81 -52
  363. package/src/memory/search/ranking.ts +27 -23
  364. package/src/memory/search/semantic.ts +157 -19
  365. package/src/memory/search/types.ts +24 -0
  366. package/src/memory/shared-app-links-store.ts +4 -5
  367. package/src/memory/validation.ts +19 -0
  368. package/src/messaging/draft-store.ts +5 -6
  369. package/src/messaging/provider-types.ts +2 -0
  370. package/src/messaging/providers/sms/adapter.ts +201 -0
  371. package/src/messaging/providers/sms/client.ts +93 -0
  372. package/src/messaging/providers/sms/types.ts +7 -0
  373. package/src/messaging/providers/telegram-bot/adapter.ts +2 -5
  374. package/src/messaging/providers/whatsapp/adapter.ts +136 -0
  375. package/src/messaging/providers/whatsapp/client.ts +67 -0
  376. package/src/messaging/style-analyzer.ts +5 -4
  377. package/src/messaging/thread-summarizer.ts +61 -69
  378. package/src/messaging/triage-engine.ts +62 -71
  379. package/src/migrations/config-merge.ts +53 -0
  380. package/src/migrations/data-layout.ts +68 -0
  381. package/src/migrations/data-merge.ts +33 -0
  382. package/src/migrations/hooks-merge.ts +90 -0
  383. package/src/migrations/index.ts +6 -0
  384. package/src/migrations/log.ts +23 -0
  385. package/src/migrations/skills-merge.ts +33 -0
  386. package/src/migrations/workspace-layout.ts +79 -0
  387. package/src/permissions/checker.ts +133 -11
  388. package/src/permissions/prompter.ts +14 -0
  389. package/src/permissions/shell-identity.ts +31 -1
  390. package/src/permissions/trust-store.ts +21 -1
  391. package/src/providers/anthropic/client.ts +4 -4
  392. package/src/providers/failover.ts +2 -2
  393. package/src/providers/model-intents.ts +70 -0
  394. package/src/providers/ollama/client.ts +2 -1
  395. package/src/providers/provider-send-message.ts +176 -0
  396. package/src/providers/registry.ts +71 -30
  397. package/src/providers/retry.ts +35 -1
  398. package/src/providers/types.ts +12 -1
  399. package/src/runtime/approval-conversation-turn.ts +97 -0
  400. package/src/runtime/approval-message-composer.ts +253 -0
  401. package/src/runtime/channel-approval-parser.ts +36 -2
  402. package/src/runtime/channel-approvals.ts +11 -24
  403. package/src/runtime/channel-guardian-service.ts +88 -21
  404. package/src/runtime/channel-readiness-service.ts +418 -0
  405. package/src/runtime/channel-readiness-types.ts +35 -0
  406. package/src/runtime/channel-retry-sweep.ts +184 -0
  407. package/src/runtime/guardian-context-resolver.ts +108 -0
  408. package/src/runtime/http-server.ts +275 -717
  409. package/src/runtime/http-types.ts +59 -3
  410. package/src/runtime/middleware/auth.ts +116 -0
  411. package/src/runtime/middleware/error-handler.ts +33 -0
  412. package/src/runtime/middleware/twilio-validation.ts +127 -0
  413. package/src/runtime/routes/app-routes.ts +1 -1
  414. package/src/runtime/routes/call-routes.ts +51 -7
  415. package/src/runtime/routes/channel-delivery-routes.ts +170 -0
  416. package/src/runtime/routes/channel-guardian-routes.ts +1191 -0
  417. package/src/runtime/routes/channel-inbound-routes.ts +1152 -0
  418. package/src/runtime/routes/channel-route-shared.ts +144 -0
  419. package/src/runtime/routes/channel-routes.ts +32 -1588
  420. package/src/runtime/routes/conversation-routes.ts +50 -7
  421. package/src/runtime/routes/events-routes.ts +2 -2
  422. package/src/runtime/routes/identity-routes.ts +126 -0
  423. package/src/runtime/routes/pairing-routes.ts +143 -0
  424. package/src/runtime/routes/run-routes.ts +15 -1
  425. package/src/runtime/run-orchestrator.ts +86 -35
  426. package/src/schedule/schedule-store.ts +36 -32
  427. package/src/schedule/scheduler.ts +3 -3
  428. package/src/security/encrypted-store.ts +5 -7
  429. package/src/security/oauth2.ts +45 -15
  430. package/src/security/parental-control-store.ts +183 -0
  431. package/src/security/secret-allowlist.ts +4 -3
  432. package/src/security/secret-scanner.ts +5 -5
  433. package/src/security/secure-keys.ts +1 -1
  434. package/src/security/token-manager.ts +3 -2
  435. package/src/services/vercel-deploy.ts +6 -2
  436. package/src/skills/tool-manifest.ts +3 -3
  437. package/src/skills/vellum-catalog-remote.ts +75 -16
  438. package/src/slack/slack-webhook.ts +2 -1
  439. package/src/swarm/orchestrator.ts +92 -1
  440. package/src/swarm/router-planner.ts +6 -9
  441. package/src/swarm/worker-prompts.ts +9 -12
  442. package/src/tasks/task-compiler.ts +19 -28
  443. package/src/tasks/task-runner.ts +1 -1
  444. package/src/tools/assets/materialize.ts +2 -2
  445. package/src/tools/assets/search.ts +15 -14
  446. package/src/tools/browser/__tests__/auth-detector.test.ts +1 -0
  447. package/src/tools/browser/auto-navigate.ts +1 -0
  448. package/src/tools/browser/browser-execution.ts +10 -1
  449. package/src/tools/browser/browser-manager.ts +119 -4
  450. package/src/tools/browser/network-recorder.ts +5 -0
  451. package/src/tools/calls/call-start.ts +1 -0
  452. package/src/tools/credentials/broker.ts +11 -2
  453. package/src/tools/credentials/metadata-store.ts +18 -14
  454. package/src/tools/credentials/post-connect-hooks.ts +61 -0
  455. package/src/tools/credentials/vault.ts +49 -23
  456. package/src/tools/execution-target.ts +11 -1
  457. package/src/tools/executor.ts +68 -9
  458. package/src/tools/host-terminal/cli-discover.ts +1 -1
  459. package/src/tools/network/script-proxy/http-forwarder.ts +1 -1
  460. package/src/tools/network/script-proxy/mitm-handler.ts +1 -1
  461. package/src/tools/network/script-proxy/server.ts +1 -1
  462. package/src/tools/network/script-proxy/session-manager.ts +6 -5
  463. package/src/tools/network/web-fetch.ts +18 -2
  464. package/src/tools/network/web-search.ts +8 -4
  465. package/src/tools/reminder/reminder-store.ts +14 -15
  466. package/src/tools/schedule/create.ts +1 -0
  467. package/src/tools/schedule/list.ts +2 -1
  468. package/src/tools/shared/filesystem/file-ops-service.ts +5 -7
  469. package/src/tools/skills/skill-script-runner.ts +24 -9
  470. package/src/tools/skills/skill-tool-factory.ts +1 -0
  471. package/src/tools/tasks/work-item-enqueue.ts +2 -2
  472. package/src/tools/terminal/evaluate-typescript.ts +21 -12
  473. package/src/tools/terminal/parser.ts +50 -0
  474. package/src/tools/types.ts +2 -0
  475. package/src/tools/watcher/delete.ts +6 -0
  476. package/src/tools/weather/service.ts +1 -1
  477. package/src/twitter/client.ts +190 -24
  478. package/src/twitter/router.ts +1 -1
  479. package/src/twitter/session.ts +4 -3
  480. package/src/util/clipboard.ts +1 -1
  481. package/src/util/errors.ts +65 -8
  482. package/src/util/fs.ts +40 -0
  483. package/src/util/json.ts +10 -0
  484. package/src/util/log-redact.ts +189 -0
  485. package/src/util/logger.ts +19 -17
  486. package/src/util/object.ts +3 -0
  487. package/src/util/platform.ts +105 -363
  488. package/src/util/pricing.ts +1 -1
  489. package/src/util/promise-guard.ts +1 -1
  490. package/src/util/retry.ts +19 -0
  491. package/src/util/row-mapper.ts +79 -0
  492. package/src/util/silently.ts +21 -0
  493. package/src/watcher/engine.ts +5 -1
  494. package/src/watcher/provider-types.ts +20 -0
  495. package/src/watcher/providers/github.ts +156 -0
  496. package/src/watcher/providers/gmail.ts +1 -0
  497. package/src/watcher/providers/google-calendar.ts +1 -0
  498. package/src/watcher/providers/linear.ts +460 -0
  499. package/src/watcher/providers/slack.ts +1 -0
  500. package/src/work-items/work-item-runner.ts +1 -1
  501. package/src/workspace/git-service.ts +1 -1
  502. package/src/workspace/provider-commit-message-generator.ts +51 -22
  503. package/src/__tests__/call-bridge.test.ts +0 -517
  504. package/src/__tests__/session-process-bridge.test.ts +0 -244
  505. package/src/calls/call-bridge.ts +0 -168
  506. package/src/config/vellum-skills/google-oauth-setup/SKILL.md +0 -199
@@ -1,15 +1,16 @@
1
- import Anthropic from '@anthropic-ai/sdk';
2
1
  import { and, eq, sql } from 'drizzle-orm';
3
2
  import { v4 as uuid } from 'uuid';
4
3
  import { getConfig } from '../config/loader.js';
5
4
  import type { MemoryExtractionConfig } from '../config/types.js';
6
5
  import { getLogger } from '../util/logger.js';
7
6
  import { truncate } from '../util/truncate.js';
7
+ import { getConfiguredProvider, createTimeout, extractToolUse, userMessage } from '../providers/provider-send-message.js';
8
8
  import { computeMemoryFingerprint } from './fingerprint.js';
9
9
  import { enqueueMemoryJob } from './jobs-store.js';
10
10
  import { extractTextFromStoredMessageContent } from './message-content.js';
11
11
  import { getDb } from './db.js';
12
12
  import { memoryItemConflicts, memoryItems, memoryItemSources, messages } from './schema.js';
13
+ import { clampUnitInterval } from './validation.js';
13
14
 
14
15
  const log = getLogger('memory-items-extractor');
15
16
 
@@ -114,108 +115,105 @@ async function extractItemsWithLLM(
114
115
  extractionConfig: MemoryExtractionConfig,
115
116
  scopeId: string,
116
117
  ): Promise<ExtractedItem[]> {
117
- const config = getConfig();
118
- const apiKey = config.apiKeys.anthropic ?? process.env.ANTHROPIC_API_KEY;
119
- if (!apiKey) {
120
- log.debug('No Anthropic API key available for LLM extraction, falling back to pattern-based');
118
+ const provider = getConfiguredProvider();
119
+ if (!provider) {
120
+ log.debug('Configured provider unavailable for LLM extraction, falling back to pattern-based');
121
121
  return extractItemsPatternBased(text, scopeId);
122
122
  }
123
123
 
124
124
  try {
125
- const client = new Anthropic({ apiKey });
126
- const abortController = new AbortController();
127
- let timer: ReturnType<typeof setTimeout>;
128
- const apiCall = client.messages.create({
129
- model: extractionConfig.model,
130
- max_tokens: 1024,
131
- system: EXTRACTION_SYSTEM_PROMPT,
132
- tools: [{
133
- name: 'store_memory_items',
134
- description: 'Store extracted memory items from the message',
135
- input_schema: {
136
- type: 'object' as const,
137
- properties: {
138
- items: {
139
- type: 'array',
125
+ const { signal, cleanup } = createTimeout(15000);
126
+
127
+ try {
128
+ const response = await provider.sendMessage(
129
+ [userMessage(text)],
130
+ [{
131
+ name: 'store_memory_items',
132
+ description: 'Store extracted memory items from the message',
133
+ input_schema: {
134
+ type: 'object' as const,
135
+ properties: {
140
136
  items: {
141
- type: 'object',
142
- properties: {
143
- kind: {
144
- type: 'string',
145
- enum: [...VALID_KINDS],
146
- description: 'Category of memory item',
147
- },
148
- subject: {
149
- type: 'string',
150
- description: 'Short label (2-8 words) for what this is about',
151
- },
152
- statement: {
153
- type: 'string',
154
- description: 'Full factual statement to remember (1-2 sentences)',
155
- },
156
- confidence: {
157
- type: 'number',
158
- description: 'Confidence that this is accurate (0.0-1.0)',
159
- },
160
- importance: {
161
- type: 'number',
162
- description: 'How valuable this is to remember (0.0-1.0)',
137
+ type: 'array',
138
+ items: {
139
+ type: 'object',
140
+ properties: {
141
+ kind: {
142
+ type: 'string',
143
+ enum: [...VALID_KINDS],
144
+ description: 'Category of memory item',
145
+ },
146
+ subject: {
147
+ type: 'string',
148
+ description: 'Short label (2-8 words) for what this is about',
149
+ },
150
+ statement: {
151
+ type: 'string',
152
+ description: 'Full factual statement to remember (1-2 sentences)',
153
+ },
154
+ confidence: {
155
+ type: 'number',
156
+ description: 'Confidence that this is accurate (0.0-1.0)',
157
+ },
158
+ importance: {
159
+ type: 'number',
160
+ description: 'How valuable this is to remember (0.0-1.0)',
161
+ },
163
162
  },
163
+ required: ['kind', 'subject', 'statement', 'confidence', 'importance'],
164
164
  },
165
- required: ['kind', 'subject', 'statement', 'confidence', 'importance'],
166
165
  },
167
166
  },
167
+ required: ['items'],
168
+ },
169
+ }],
170
+ EXTRACTION_SYSTEM_PROMPT,
171
+ {
172
+ config: {
173
+ model: extractionConfig.model,
174
+ max_tokens: 1024,
175
+ tool_choice: { type: 'tool' as const, name: 'store_memory_items' },
168
176
  },
169
- required: ['items'],
177
+ signal,
170
178
  },
171
- }],
172
- tool_choice: { type: 'tool' as const, name: 'store_memory_items' },
173
- messages: [{ role: 'user' as const, content: text }],
174
- }, { signal: abortController.signal });
175
- // Swallow the abort rejection that fires when the timeout wins the race
176
- apiCall.catch(() => {});
177
- const response = await Promise.race([
178
- apiCall.finally(() => clearTimeout(timer)),
179
- new Promise<never>((_, reject) => {
180
- timer = setTimeout(() => {
181
- abortController.abort();
182
- reject(new Error('LLM extraction timeout'));
183
- }, 15000);
184
- }),
185
- ]);
186
-
187
- const toolBlock = response.content.find((b) => b.type === 'tool_use');
188
- if (!toolBlock || toolBlock.type !== 'tool_use') {
189
- log.warn('No tool_use block in LLM extraction response, falling back to pattern-based');
190
- return extractItemsPatternBased(text, scopeId);
179
+ );
180
+ cleanup();
181
+
182
+ const toolBlock = extractToolUse(response);
183
+ if (!toolBlock) {
184
+ log.warn('No tool_use block in LLM extraction response, falling back to pattern-based');
185
+ return extractItemsPatternBased(text, scopeId);
186
+ }
187
+
188
+ const input = toolBlock.input as { items?: LLMExtractedItem[] };
189
+ if (!Array.isArray(input.items)) {
190
+ log.warn('Invalid items in LLM extraction response, falling back to pattern-based');
191
+ return extractItemsPatternBased(text, scopeId);
192
+ }
193
+
194
+ const items: ExtractedItem[] = [];
195
+ for (const raw of input.items) {
196
+ if (!VALID_KINDS.has(raw.kind)) continue;
197
+ if (!raw.subject || !raw.statement) continue;
198
+ const subject = truncate(String(raw.subject), 80, '');
199
+ const statement = truncate(String(raw.statement), 500, '');
200
+ const confidence = clampUnitInterval(parseScore(raw.confidence, 0.5));
201
+ const importance = clampUnitInterval(parseScore(raw.importance, 0.5));
202
+ const fingerprint = computeMemoryFingerprint(scopeId, raw.kind, subject, statement);
203
+ items.push({
204
+ kind: raw.kind as MemoryItemKind,
205
+ subject,
206
+ statement,
207
+ confidence,
208
+ importance,
209
+ fingerprint,
210
+ });
211
+ }
212
+
213
+ return deduplicateItems(items);
214
+ } finally {
215
+ cleanup();
191
216
  }
192
-
193
- const input = toolBlock.input as { items?: LLMExtractedItem[] };
194
- if (!Array.isArray(input.items)) {
195
- log.warn('Invalid items in LLM extraction response, falling back to pattern-based');
196
- return extractItemsPatternBased(text, scopeId);
197
- }
198
-
199
- const items: ExtractedItem[] = [];
200
- for (const raw of input.items) {
201
- if (!VALID_KINDS.has(raw.kind)) continue;
202
- if (!raw.subject || !raw.statement) continue;
203
- const subject = truncate(String(raw.subject), 80, '');
204
- const statement = truncate(String(raw.statement), 500, '');
205
- const confidence = clamp(parseScore(raw.confidence, 0.5), 0, 1);
206
- const importance = clamp(parseScore(raw.importance, 0.5), 0, 1);
207
- const fingerprint = computeMemoryFingerprint(scopeId, raw.kind, subject, statement);
208
- items.push({
209
- kind: raw.kind as MemoryItemKind,
210
- subject,
211
- statement,
212
- confidence,
213
- importance,
214
- fingerprint,
215
- });
216
- }
217
-
218
- return deduplicateItems(items);
219
217
  } catch (err) {
220
218
  const message = err instanceof Error ? err.message : String(err);
221
219
  log.warn({ err: message }, 'LLM extraction failed, falling back to pattern-based');
@@ -287,8 +285,8 @@ export async function extractAndUpsertMemoryItemsForMessage(messageId: string, s
287
285
  db.update(memoryItems)
288
286
  .set({
289
287
  status: effectiveStatus,
290
- confidence: Math.max(existing.confidence, item.confidence),
291
- importance: Math.max(existing.importance ?? 0, item.importance),
288
+ confidence: clampUnitInterval(Math.max(existing.confidence, item.confidence)),
289
+ importance: clampUnitInterval(Math.max(existing.importance ?? 0, item.importance)),
292
290
  lastSeenAt: Math.max(existing.lastSeenAt, seenAt),
293
291
  verificationState: promotedState,
294
292
  })
@@ -441,10 +439,6 @@ function parseScore(value: unknown, fallback: number): number {
441
439
  return Number.isFinite(n) ? n : fallback;
442
440
  }
443
441
 
444
- function clamp(value: number, min: number, max: number): number {
445
- return Math.min(max, Math.max(min, value));
446
- }
447
-
448
442
  /** Returns true if the given memory item is the candidate in an unresolved conflict. */
449
443
  function hasPendingConflict(itemId: string): boolean {
450
444
  const db = getDb();
@@ -1,6 +1,6 @@
1
1
  import { eq } from 'drizzle-orm';
2
2
  import { getLogger } from '../../util/logger.js';
3
- import { getDb } from '../db.js';
3
+ import { getDb, rawExec } from '../db.js';
4
4
  import { enqueueMemoryJob, type MemoryJob } from '../jobs-store.js';
5
5
  import { asString, BackendUnavailableError } from '../job-utils.js';
6
6
  import { getQdrantClient } from '../qdrant-client.js';
@@ -10,8 +10,8 @@ const log = getLogger('memory-jobs-worker');
10
10
 
11
11
  export function rebuildIndexJob(): void {
12
12
  const db = getDb();
13
- db.run(/*sql*/ `DELETE FROM memory_segment_fts`);
14
- db.run(/*sql*/ `
13
+ rawExec(/*sql*/ `DELETE FROM memory_segment_fts`);
14
+ rawExec(/*sql*/ `
15
15
  INSERT INTO memory_segment_fts(segment_id, text)
16
16
  SELECT id, text FROM memory_segments
17
17
  `);
@@ -0,0 +1,69 @@
1
+ import { getLogger } from '../../util/logger.js';
2
+ import { asString } from '../job-utils.js';
3
+ import { getMediaAssetById, updateMediaAssetStatus } from '../media-store.js';
4
+ import type { MemoryJob } from '../jobs-store.js';
5
+ import {
6
+ runPipeline,
7
+ type PipelineStageName,
8
+ type StageHandler,
9
+ } from '../../config/bundled-skills/media-processing/services/processing-pipeline.js';
10
+ import { preprocessForAsset } from '../../config/bundled-skills/media-processing/tools/extract-keyframes.js';
11
+ import { mapSegmentsForAsset } from '../../config/bundled-skills/media-processing/tools/analyze-keyframes.js';
12
+ import { reduceForAsset } from '../../config/bundled-skills/media-processing/tools/query-media-events.js';
13
+
14
+ const log = getLogger('media-processing-job');
15
+
16
+ export async function mediaProcessingJob(job: MemoryJob): Promise<void> {
17
+ const mediaAssetId = asString(job.payload.mediaAssetId);
18
+ if (!mediaAssetId) {
19
+ log.warn({ jobId: job.id }, 'Missing mediaAssetId in job payload');
20
+ return;
21
+ }
22
+
23
+ const asset = getMediaAssetById(mediaAssetId);
24
+ if (!asset) {
25
+ log.warn({ jobId: job.id, mediaAssetId }, 'Media asset not found');
26
+ return;
27
+ }
28
+
29
+ if (asset.mediaType !== 'video') {
30
+ log.info(
31
+ { assetId: mediaAssetId, mediaType: asset.mediaType },
32
+ 'Skipping media processing pipeline — only video assets are supported',
33
+ );
34
+ updateMediaAssetStatus(mediaAssetId, 'indexed');
35
+ return;
36
+ }
37
+
38
+ const handlers: Record<PipelineStageName, StageHandler> = {
39
+ preprocess: { execute: async (assetId, onProgress) => { await preprocessForAsset(assetId, {}, onProgress); } },
40
+ map: { execute: async (assetId, onProgress) => { await mapSegmentsForAsset(assetId, {
41
+ systemPrompt: 'Describe what you see in these video frames. For each frame, note: subjects present, actions occurring, scene context, and any text visible.',
42
+ outputSchema: { type: 'object', properties: { frames: { type: 'array', items: { type: 'object', properties: { timestamp: { type: 'number' }, subjects: { type: 'array', items: { type: 'string' } }, actions: { type: 'array', items: { type: 'string' } }, scene: { type: 'string' }, text: { type: 'string' } } } } } }
43
+ }, onProgress); } },
44
+ reduce: { execute: async (assetId, onProgress) => { await reduceForAsset(assetId, {
45
+ systemPrompt: 'Summarize the video content based on the structured observations.',
46
+ }, onProgress); } },
47
+ };
48
+
49
+ const result = await runPipeline(mediaAssetId, handlers, {
50
+ onProgress: (msg) => log.info({ mediaAssetId }, msg),
51
+ });
52
+
53
+ log.info(
54
+ {
55
+ mediaAssetId,
56
+ completedStages: result.completedStages,
57
+ failedStage: result.failedStage,
58
+ cancelled: result.cancelled,
59
+ },
60
+ 'Media processing pipeline finished',
61
+ );
62
+
63
+ if (result.failedStage) {
64
+ throw new Error(`Media processing failed at stage ${result.failedStage}: ${result.failureReason}`);
65
+ }
66
+ if (result.cancelled) {
67
+ throw new Error(`Media processing cancelled for asset ${mediaAssetId}`);
68
+ }
69
+ }
@@ -1,9 +1,9 @@
1
- import Anthropic from '@anthropic-ai/sdk';
2
1
  import { and, desc, eq, gte, isNull, lt } from 'drizzle-orm';
3
2
  import { v4 as uuid } from 'uuid';
4
3
  import type { AssistantConfig } from '../../config/types.js';
5
4
  import { estimateTextTokens } from '../../context/token-estimator.js';
6
5
  import { getLogger } from '../../util/logger.js';
6
+ import { getConfiguredProvider, createTimeout, extractText, userMessage } from '../../providers/provider-send-message.js';
7
7
  import { getConversationMemoryScopeId } from '../conversation-store.js';
8
8
  import { getDb } from '../db.js';
9
9
  import { enqueueMemoryJob, type MemoryJob } from '../jobs-store.js';
@@ -230,9 +230,9 @@ async function summarizeWithLLM(
230
230
  return buildFallbackSummary(existingSummary, newContent, label);
231
231
  }
232
232
 
233
- const apiKey = config.apiKeys.anthropic ?? process.env.ANTHROPIC_API_KEY;
234
- if (!apiKey) {
235
- log.debug({ label }, 'No Anthropic API key available for summarization, using fallback');
233
+ const provider = getConfiguredProvider();
234
+ if (!provider) {
235
+ log.debug({ label }, 'Configured provider unavailable for summarization, using fallback');
236
236
  return buildFallbackSummary(existingSummary, newContent, label);
237
237
  }
238
238
 
@@ -247,30 +247,36 @@ async function summarizeWithLLM(
247
247
  userParts.push('### New Data', newContent);
248
248
 
249
249
  try {
250
- const client = new Anthropic({ apiKey });
251
- const response = await Promise.race([
252
- client.messages.create({
253
- model: summarizationConfig.model,
254
- max_tokens: SUMMARY_MAX_TOKENS,
255
- system: systemPrompt,
256
- messages: [{ role: 'user' as const, content: userParts.join('\n') }],
257
- }),
258
- new Promise<never>((_, reject) =>
259
- setTimeout(() => reject(new Error('Summarization LLM timeout')), SUMMARY_LLM_TIMEOUT_MS),
260
- ),
261
- ]);
262
-
263
- const textBlock = response.content.find((b) => b.type === 'text');
264
- if (textBlock && textBlock.type === 'text' && textBlock.text.trim().length > 0) {
265
- log.debug(
266
- { label, inputTokens: response.usage.input_tokens, outputTokens: response.usage.output_tokens },
267
- 'LLM summarization completed',
250
+ const { signal, cleanup } = createTimeout(SUMMARY_LLM_TIMEOUT_MS);
251
+ try {
252
+ const response = await provider.sendMessage(
253
+ [userMessage(userParts.join('\n'))],
254
+ undefined,
255
+ systemPrompt,
256
+ {
257
+ config: {
258
+ model: summarizationConfig.model,
259
+ max_tokens: SUMMARY_MAX_TOKENS,
260
+ },
261
+ signal,
262
+ },
268
263
  );
269
- return textBlock.text.trim();
270
- }
264
+ cleanup();
271
265
 
272
- log.warn({ label }, 'LLM summarization returned empty text, using fallback');
273
- return buildFallbackSummary(existingSummary, newContent, label);
266
+ const text = extractText(response);
267
+ if (text.length > 0) {
268
+ log.debug(
269
+ { label, inputTokens: response.usage.inputTokens, outputTokens: response.usage.outputTokens },
270
+ 'LLM summarization completed',
271
+ );
272
+ return text;
273
+ }
274
+
275
+ log.warn({ label }, 'LLM summarization returned empty text, using fallback');
276
+ return buildFallbackSummary(existingSummary, newContent, label);
277
+ } finally {
278
+ cleanup();
279
+ }
274
280
  } catch (err) {
275
281
  const message = err instanceof Error ? err.message : String(err);
276
282
  log.warn({ err: message, label }, 'LLM summarization failed, using fallback');
@@ -6,18 +6,11 @@ import { getDb } from './db.js';
6
6
  import { getQdrantClient } from './qdrant-client.js';
7
7
  import { memoryEmbeddings } from './schema.js';
8
8
  import type { AssistantConfig } from '../config/types.js';
9
+ import { BackendUnavailableError } from '../util/errors.js';
9
10
 
10
- const log = getLogger('memory-jobs-worker');
11
-
12
- // ── Sentinel error ─────────────────────────────────────────────────
11
+ export { BackendUnavailableError };
13
12
 
14
- /** Sentinel error: the embedding backend is not configured yet. */
15
- export class BackendUnavailableError extends Error {
16
- constructor(reason: string) {
17
- super(reason);
18
- this.name = 'BackendUnavailableError';
19
- }
20
- }
13
+ const log = getLogger('memory-jobs-worker');
21
14
 
22
15
  // ── Error classification for LLM / API errors ─────────────────────
23
16
 
@@ -1,6 +1,6 @@
1
1
  import { and, asc, eq, lte, notInArray, inArray } from 'drizzle-orm';
2
2
  import { v4 as uuid } from 'uuid';
3
- import { getDb } from './db.js';
3
+ import { getDb, rawGet, rawAll } from './db.js';
4
4
  import { memoryJobs } from './schema.js';
5
5
  import { truncate } from '../util/truncate.js';
6
6
 
@@ -20,7 +20,8 @@ export type MemoryJobType =
20
20
  | 'build_conversation_summary'
21
21
  | 'backfill'
22
22
  | 'rebuild_index'
23
- | 'delete_qdrant_vectors';
23
+ | 'delete_qdrant_vectors'
24
+ | 'media_processing';
24
25
 
25
26
  const EMBED_JOB_TYPES: MemoryJobType[] = ['embed_segment', 'embed_item', 'embed_summary'];
26
27
 
@@ -109,9 +110,8 @@ export function enqueueResolvePendingConflictsForMessageJob(
109
110
  throw new Error('enqueueResolvePendingConflictsForMessageJob requires a non-empty messageId');
110
111
  }
111
112
  const normalizedScopeId = scopeId.trim() || 'default';
112
- // Dedup check always uses root db since tx doesn't expose $client
113
- const raw = (getDb() as unknown as { $client: { query: (q: string) => { get: (...params: unknown[]) => unknown } } }).$client;
114
- const existing = raw.query(`
113
+ // Dedup check always uses root db since tx doesn't expose raw client
114
+ const existing = rawGet<{ id: string }>(`
115
115
  SELECT id
116
116
  FROM memory_jobs
117
117
  WHERE type = 'resolve_pending_conflicts_for_message'
@@ -120,7 +120,7 @@ export function enqueueResolvePendingConflictsForMessageJob(
120
120
  AND COALESCE(json_extract(payload, '$.scopeId'), 'default') = ?
121
121
  ORDER BY created_at ASC
122
122
  LIMIT 1
123
- `).get(normalizedMessageId, normalizedScopeId) as { id: string } | null;
123
+ `, normalizedMessageId, normalizedScopeId);
124
124
  if (existing?.id) return existing.id;
125
125
 
126
126
  return enqueueMemoryJob('resolve_pending_conflicts_for_message', {
@@ -365,13 +365,11 @@ export function resetRunningJobsToPending(): number {
365
365
  }
366
366
 
367
367
  export function getMemoryJobCounts(): Record<string, number> {
368
- const db = getDb();
369
- const raw = (db as unknown as { $client: { query: (q: string) => { all: () => unknown[] } } }).$client;
370
- const rows = raw.query(`
368
+ const rows = rawAll<{ status: string; c: number }>(`
371
369
  SELECT status, COUNT(*) AS c
372
370
  FROM memory_jobs
373
371
  GROUP BY status
374
- `).all() as Array<{ status: string; c: number }>;
372
+ `);
375
373
  const counts: Record<string, number> = { pending: 0, running: 0, completed: 0, failed: 0 };
376
374
  for (const row of rows) {
377
375
  counts[row.status] = row.c;
@@ -1,7 +1,7 @@
1
1
  import type { AssistantConfig } from '../config/types.js';
2
2
  import { getConfig } from '../config/loader.js';
3
3
  import { getLogger } from '../util/logger.js';
4
- import { getDb } from './db.js';
4
+ import { rawRun } from './db.js';
5
5
  import {
6
6
  claimMemoryJobs,
7
7
  completeMemoryJob,
@@ -29,6 +29,7 @@ import { checkContradictionsJob, cleanupStaleSupersededItemsJob } from './job-ha
29
29
  import { buildConversationSummaryJob, buildGlobalSummaryJob } from './job-handlers/summarization.js';
30
30
  import { backfillJob, backfillEntityRelationsJob } from './job-handlers/backfill.js';
31
31
  import { rebuildIndexJob, deleteQdrantVectorsJob } from './job-handlers/index-maintenance.js';
32
+ import { mediaProcessingJob } from './job-handlers/media-processing.js';
32
33
 
33
34
  // Re-export public utilities consumed by tests and other modules
34
35
  export { currentWeekWindow } from './job-utils.js';
@@ -111,38 +112,56 @@ export async function runMemoryJobsOnce(
111
112
  let processed = 0;
112
113
  const typeGroups = [...jobsByType.values()];
113
114
 
114
- // Run type groups concurrently (up to workerConcurrency at a time).
115
- // Within each group, jobs are processed sequentially.
116
- for (let i = 0; i < typeGroups.length; i += concurrency) {
117
- const groupChunk = typeGroups.slice(i, i + concurrency);
118
- const groupResults = await Promise.allSettled(
119
- groupChunk.map(async (group) => {
120
- let groupProcessed = 0;
121
- for (const job of group) {
122
- try {
123
- await processJob(job, config);
124
- completeMemoryJob(job.id);
125
- bumpMemoryVersion();
126
- groupProcessed += 1;
127
- } catch (err) {
128
- try {
129
- handleJobError(job, err);
130
- } catch (handlerErr) {
131
- log.error({ err: handlerErr, jobId: job.id, type: job.type }, 'handleJobError itself threw, job left in running status');
132
- }
133
- }
115
+ // Run type groups concurrently using a task pool (up to workerConcurrency
116
+ // active at a time). Unlike the old wave approach, a new group starts as
117
+ // soon as any slot frees up no waiting for an entire wave to finish.
118
+ const processGroup = async (group: MemoryJob[]): Promise<number> => {
119
+ let groupProcessed = 0;
120
+ for (const job of group) {
121
+ try {
122
+ await processJob(job, config);
123
+ completeMemoryJob(job.id);
124
+ bumpMemoryVersion();
125
+ groupProcessed += 1;
126
+ } catch (err) {
127
+ try {
128
+ handleJobError(job, err);
129
+ } catch (handlerErr) {
130
+ log.error({ err: handlerErr, jobId: job.id, type: job.type }, 'handleJobError itself threw, job left in running status');
134
131
  }
135
- return groupProcessed;
136
- }),
137
- );
138
- for (const result of groupResults) {
132
+ }
133
+ }
134
+ return groupProcessed;
135
+ };
136
+
137
+ if (typeGroups.length <= concurrency) {
138
+ // Fast path: all groups fit within the concurrency limit
139
+ const results = await Promise.allSettled(typeGroups.map(processGroup));
140
+ for (const result of results) {
139
141
  if (result.status === 'fulfilled') {
140
142
  processed += result.value;
143
+ } else {
144
+ log.error({ err: result.reason }, 'Memory job group rejected unexpectedly — jobs in this batch may have been dropped');
141
145
  }
142
- // Errors within groups are already handled per-job above;
143
- // a rejected group promise would only come from an unexpected
144
- // error in the loop itself, which is unlikely.
145
146
  }
147
+ } else {
148
+ // Task pool: maintain `concurrency` in-flight groups at all times
149
+ let nextIdx = 0;
150
+
151
+ const startNext = (): Promise<void> | undefined => {
152
+ if (nextIdx >= typeGroups.length) return undefined;
153
+ const group = typeGroups[nextIdx++]!;
154
+ return processGroup(group).then(
155
+ (count) => { processed += count; },
156
+ (err) => { log.error({ err }, 'Memory job group rejected unexpectedly — jobs in this batch may have been dropped'); },
157
+ ).then(() => startNext());
158
+ };
159
+
160
+ const workers = Array.from(
161
+ { length: Math.min(concurrency, typeGroups.length) },
162
+ () => startNext()!,
163
+ );
164
+ await Promise.all(workers);
146
165
  }
147
166
  if (enableScheduledCleanup) {
148
167
  maybeEnqueueScheduledCleanupJobs(config);
@@ -229,6 +248,9 @@ async function processJob(job: MemoryJob, config: AssistantConfig): Promise<void
229
248
  case 'delete_qdrant_vectors':
230
249
  await deleteQdrantVectorsJob(job);
231
250
  return;
251
+ case 'media_processing':
252
+ await mediaProcessingJob(job);
253
+ return;
232
254
  default:
233
255
  throw new Error(`Unknown memory job type: ${(job as { type: string }).type}`);
234
256
  }
@@ -292,16 +314,13 @@ export function sweepStaleItems(config: AssistantConfig): number {
292
314
  if (now - lastStaleSweepMs < STALE_SWEEP_INTERVAL_MS) return 0;
293
315
  lastStaleSweepMs = now;
294
316
 
295
- const db = getDb();
296
- const raw = (db as unknown as { $client: { query: (q: string) => { run: (...params: unknown[]) => { changes: number } } } }).$client;
297
-
298
317
  let totalMarked = 0;
299
318
  for (const [kind, maxAgeDays] of Object.entries(freshness.maxAgeDays)) {
300
319
  if (maxAgeDays <= 0) continue;
301
320
  // Mark invalid if: past 2x window, no access in the shield period, and not already invalid
302
321
  const cutoffMs = now - maxAgeDays * 2 * 86_400_000;
303
322
  const shieldCutoffMs = now - freshness.reinforcementShieldDays * 86_400_000;
304
- const result = raw.query(`
323
+ const changes = rawRun(`
305
324
  UPDATE memory_items
306
325
  SET invalid_at = ?
307
326
  WHERE kind = ?
@@ -309,10 +328,10 @@ export function sweepStaleItems(config: AssistantConfig): number {
309
328
  AND invalid_at IS NULL
310
329
  AND last_seen_at < ?
311
330
  AND (access_count = 0 OR COALESCE(last_used_at, 0) < ?)
312
- `).run(now, kind, cutoffMs, shieldCutoffMs);
313
- if (result.changes > 0) {
314
- log.info({ kind, marked: result.changes, cutoffMs }, 'Marked stale memory items as invalid');
315
- totalMarked += result.changes;
331
+ `, now, kind, cutoffMs, shieldCutoffMs);
332
+ if (changes > 0) {
333
+ log.info({ kind, marked: changes, cutoffMs }, 'Marked stale memory items as invalid');
334
+ totalMarked += changes;
316
335
  }
317
336
  }
318
337
  return totalMarked;